748 files changed, 133208 insertions, 35765 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
new file mode 100644
index 0000000000..34e1c8b846
--- /dev/null
+++ b/.appveyor.yml
@@ -0,0 +1,91 @@
+version: 1.0.{build}
+
+clone_depth: 50
+
+environment:
+  compiler: mingw
+
+  matrix:
+  - target: i686-w64-mingw32
+    compiler_path: mingw32
+    mingw_prefix: mingw-w64-i686
+    hardening:
+  - target: x86_64-w64-mingw32
+    compiler_path: mingw64
+    mingw_prefix: mingw-w64-x86_64
+    # hardening doesn't work with mingw-w64-x86_64-gcc, because it's gcc 8
+    hardening: --disable-gcc-hardening
+
+install:
+- ps: >-
+    Function Execute-Command ($commandPath)
+    {
+        & $commandPath $args 2>&1
+        if ( $LastExitCode -ne 0 ) {
+            $host.SetShouldExit( $LastExitCode )
+        }
+    }
+    Function Execute-Bash ()
+    {
+        Execute-Command 'c:\msys64\usr\bin\bash' '-e' '-c' $args
+    }
+    <# mingw packages start with ${env:mingw_prefix}
+     # unprefixed packages are from MSYS2, which is like Cygwin. Avoid them.
+     #
+     # Use pacman --debug to show package downloads and install locations
+     #>
+    Execute-Command "C:\msys64\usr\bin\pacman" -Sy --verbose --needed --noconfirm ${env:mingw_prefix}-libevent ${env:mingw_prefix}-pkg-config ${env:mingw_prefix}-xz ${env:mingw_prefix}-zstd ;
+
+build_script:
+- ps: >-
+    if ($env:compiler -eq "mingw") {
+            <# use the MSYS2 compiler and user binaries to build and install #>
+            $oldpath = ${env:Path} -split ';'
+            $buildpath = @("C:\msys64\${env:compiler_path}\bin", "C:\msys64\usr\bin") + $oldpath
+            $env:Path = @($buildpath) -join ';'
+            $env:build = @("${env:APPVEYOR_BUILD_FOLDER}", $env:target) -join '\'
+            Set-Location "${env:APPVEYOR_BUILD_FOLDER}"
+            Execute-Bash 'autoreconf -i'
+            mkdir "${env:build}"
+            Set-Location "${env:build}"
+            Execute-Bash "which ${env:target}-gcc"
+            Execute-Bash "${env:target}-gcc --version"
+            <# compile for mingw
+             # mingw zstd doesn't come with a pkg-config file, so we manually
+             # configure its flags. liblzma just works.
+             #>
+            Execute-Bash "ZSTD_CFLAGS='-L/${env:compiler_path}/include' ZSTD_LIBS='-L/${env:compiler_path}/lib -lzstd' ../configure --prefix=/${env:compiler_path} --build=${env:target} --host=${env:target} --with-openssl-dir=/${env:compiler_path} --disable-asciidoc --enable-fatal-warnings ${env:hardening}"
+            Execute-Bash "V=1 make -j2"
+            Execute-Bash "V=1 make -j2 install"
+     }
+
+test_script:
+- ps: >-
+    if ($env:compiler -eq "mingw") {
+            <# use the MSYS2 compiler binaries to make check #>
+            $oldpath = ${env:Path} -split ';'
+            $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath
+            $env:Path = $buildpath -join ';'
+            Set-Location "${env:build}"
+            Execute-Bash "VERBOSE=1 make -j2 check"
+    }
+
+on_finish:
+- ps: >-
+    if ($env:compiler -eq "mingw") {
+            <# use the MSYS2 user binaries to archive failures #>
+            $oldpath = ${env:Path} -split ';'
+            $buildpath = @("C:\msys64\usr\bin") + $oldpath
+            $env:Path = @($buildpath) -join ';'
+            Set-Location "${env:build}"
+            <# store logs as appveyor artifacts: see the artifacts tab #>
+            Execute-Bash "7z a logs.zip config.log || true"
+            Execute-Bash "7z a logs.zip test-suite.log || true"
+            Execute-Bash "appveyor PushArtifact logs.zip || true"
+            Execute-Bash "tail -1000 config.log || true"
+            Execute-Bash "cat test-suite.log || true"
+    }
+
+# notify the IRC channel of any failures
+on_failure:
+- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
diff --git a/.gitignore b/.gitignore
index dc6738c079..b3b13694af 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
 .#*
 *~
 *.swp
+*.swo
 # C stuff
 *.o
 *.obj
@@ -18,6 +19,8 @@
 .dirstamp
 *.trs
 *.log
+# Calltool stuff
+.*.graph
 # Stuff made by our makefiles
 *.bak
 # Python droppings
@@ -38,9 +41,11 @@ uptime-*.json
 /Makefile
 /Makefile.in
 /aclocal.m4
+/ar-lib
 /autom4te.cache
 /build-stamp
 /compile
+/config.rust
 /configure
 /Doxyfile
 /orconfig.h
@@ -51,6 +56,7 @@ uptime-*.json
 /config.guess
 /config.sub
 /conftest*
+/link_rust.sh
 /micro-revision.*
 /patch-stamp
 /stamp-h
@@ -67,6 +73,7 @@ uptime-*.json
 /Tor*Bundle.dmg
 /tor-*-win32.exe
 /coverage_html/
+/callgraph/
 
 # /contrib/
 /contrib/dist/tor.sh
@@ -94,11 +101,6 @@ uptime-*.json
 /doc/tor.html
 /doc/tor.html.in
 /doc/tor.1.xml
-/doc/tor-fw-helper.1
-/doc/tor-fw-helper.1.in
-/doc/tor-fw-helper.html
-/doc/tor-fw-helper.html.in
-/doc/tor-fw-helper.1.xml
 /doc/tor-gencert.1
 /doc/tor-gencert.1.in
 /doc/tor-gencert.html
@@ -127,6 +129,9 @@ uptime-*.json
 /src/Makefile
 /src/Makefile.in
 
+# /src/trace
+/src/trace/libor-trace.a
+
 # /src/common/
 /src/common/Makefile
 /src/common/Makefile.in
@@ -172,6 +177,12 @@ uptime-*.json
 /src/or/libtor-testing.a
 /src/or/libtor.lib
 
+# /src/rust
+/src/rust/.cargo/config
+/src/rust/.cargo/registry
+/src/rust/target
+/src/rust/registry
+
 # /src/test
 /src/test/Makefile
 /src/test/Makefile.in
@@ -183,6 +194,7 @@ uptime-*.json
 /src/test/test-child
 /src/test/test-memwipe
 /src/test/test-ntor-cl
+/src/test/test-hs-ntor-cl
 /src/test/test-switch-id
 /src/test/test-timers
 /src/test/test_workqueue
@@ -191,12 +203,18 @@ uptime-*.json
 /src/test/test-bt-cl.exe
 /src/test/test-child.exe
 /src/test/test-ntor-cl.exe
+/src/test/test-hs-ntor-cl.exe
 /src/test/test-memwipe.exe
 /src/test/test-switch-id.exe
 /src/test/test-timers.exe
 /src/test/test_workqueue.exe
 
+# /src/test/fuzz
+/src/test/fuzz/fuzz-*
+/src/test/fuzz/lf-fuzz-*
+
 # /src/tools/
+/src/tools/libtorrunner.a
 /src/tools/tor-checkkey
 /src/tools/tor-resolve
 /src/tools/tor-cov-resolve
@@ -214,12 +232,6 @@ uptime-*.json
 /src/trunnel/libor-trunnel-testing.a
 /src/trunnel/libor-trunnel.a
 
-# /src/tools/tor-fw-helper/
-/src/tools/tor-fw-helper/tor-fw-helper
-/src/tools/tor-fw-helper/tor-fw-helper.exe
-/src/tools/tor-fw-helper/Makefile
-/src/tools/tor-fw-helper/Makefile.in
-
 # /src/win32/
 /src/win32/Makefile
 /src/win32/Makefile.in
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000..d2d0d55dd4
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,45 @@
+before_script:
+    - apt-get update -qq
+    - apt-get upgrade -qy
+
+build:
+  script:
+    - apt-get install -qy --fix-missing automake build-essential
+      libevent-dev libssl-dev zlib1g-dev
+      libseccomp-dev liblzma-dev libscrypt-dev
+    - ./autogen.sh
+    - ./configure --disable-asciidoc --enable-fatal-warnings
+      --disable-silent-rules
+    - make check || (e=$?; cat test-suite.log; exit $e)
+    - make install
+
+update:
+  only:
+    - schedules
+  script: 
+    - "apt-get install -y --fix-missing git openssh-client"
+    
+    # Run ssh-agent (inside the build environment)
+    - eval $(ssh-agent -s)
+
+    # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
+    - ssh-add <(echo "$DEPLOY_KEY")
+
+    # For Docker builds disable host key checking. Be aware that by adding that
+    # you are susceptible to man-in-the-middle attacks.
+    # WARNING: Use this only with the Docker executor, if you use it with shell
+    # you will overwrite your user's SSH config.
+    - mkdir -p ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+    # In order to properly check the server's host key, assuming you created the
+    # SSH_SERVER_HOSTKEYS variable previously, uncomment the following two lines
+    # instead.
+    - mkdir -p ~/.ssh
+    - '[[ -f /.dockerenv ]] && echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts'
+    - echo "merging from torgit"
+    - git config --global user.email "labadmin@oniongit.eu"
+    - git config --global user.name "gitadmin"
+    - "mkdir tor"
+    - "cd tor" 
+    - git clone --bare https://git.torproject.org/tor.git
+    - git push --mirror git@oniongit.eu:network/tor.git
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000..8f80405650
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/ext/rust"]
+	path = src/ext/rust
+	url = https://git.torproject.org/tor-rust-dependencies
diff --git a/.travis.yml b/.travis.yml
index 456b5abc56..ed805aac31 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,13 @@ language: c
 
 cache:
   ccache: true
+  ## cargo: true
+  directories:
+    - $HOME/.cargo
+    ## caching CARGO_TARGET_DIR actually slows down the build over time,
+    ## because old build products are never deleted.
+    ## where we point CARGO_TARGET_DIR in all our cargo invocations
+    #- $TRAVIS_BUILD_DIR/src/rust/target
 
 compiler:
   - gcc
@@ -28,6 +35,12 @@ env:
     ## We don't list default variable values, because we set the defaults
     ## in global (or the default is unset)
     -
+    ## We turn off hardening for Rust builds, because they are incompatible,
+    ## and it's going to take a while for them to be fixed. See:
+    ##   https:/trac.torproject.org/projects/tor/ticket/25386
+    ##   https:/trac.torproject.org/projects/tor/ticket/26398
+    ## TOR_RUST_DEPENDENCIES is spelt RUST_DEPENDENCIES in 0.3.2
+    - RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS=""
 
 matrix:
   ## include creates builds with gcc, linux, sudo: false
@@ -36,10 +49,16 @@ matrix:
     - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
     ## We only want to check these build option combinations once
     ## (they shouldn't vary by compiler or OS)
-    ## We run coverage with hardening off, which seems like enough
+    ## We run rust and coverage with hardening off, which seems like enough
     # - env: HARDENING_OPTIONS=""
     ## We check asciidoc with distcheck, to make sure we remove doc products
     - env: DISTCHECK="yes" ASCIIDOC_OPTIONS=""
+    ## Check rust online with distcheck, to make sure we remove rust products
+    ## But without hardening (see above)
+    - env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" HARDENING_OPTIONS=""
+    ## Check disable module dirauth with and without rust
+    - env: MODULES_OPTIONS="--disable-module-dirauth" RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS=""
+    - env: MODULES_OPTIONS="--disable-module-dirauth"
 
   ## Uncomment to allow the build to report success (with non-required
   ## sub-builds continuing to run) if all required sub-builds have
@@ -67,6 +86,12 @@ matrix:
     - compiler: gcc
       os: linux
       env:
+    ## offline rust builds for gcc on Linux are redundant, because we do an
+    ## online rust build for gcc on Linux
+    - compiler: gcc
+      os: linux
+      ## TOR_RUST_DEPENDENCIES is spelt RUST_DEPENDENCIES in 0.3.2
+      env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS=""
 
 ## We don't need sudo. (The "apt:" stanza after this allows us to not need
 ## sudo; otherwise, we would need it for getting dependencies.)
@@ -97,8 +122,11 @@ addons:
       - zlib1g-dev
       ## Optional dependencies
       - libcap-dev
+      - liblzma-dev
       - libscrypt-dev
       - libseccomp-dev
+      ## zstd doesn't exist in Ubuntu Trusty
+      #- libzstd
       ## Conditional build dependencies
       ## Always installed, so we don't need sudo
       - asciidoc
@@ -117,6 +145,8 @@ addons:
       #- zlib
       ## Optional dependencies
       - libscrypt
+      - xz
+      - zstd
       ## Required build dependencies
       ## Tor needs pkg-config to find some dependencies at build time
       - pkg-config
@@ -132,6 +162,10 @@ addons:
 ## Default is Xcode 9.4 on macOS 10.13 as of August 2018
 #osx_image: xcode9.4
 
+before_install:
+  ## Create empty rust directories for non-Rust builds, so caching succeeds
+  - if [[ "$RUST_OPTIONS" == "" ]]; then mkdir -p $HOME/.cargo $TRAVIS_BUILD_DIR/src/rust/target; fi
+
 install:
   ## If we're on OSX, configure ccache (ccache is automatically installed and configured on Linux)
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export PATH="/usr/local/opt/ccache/libexec:$PATH"; fi
@@ -142,14 +176,28 @@ install:
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi
   ## If we're on OSX, and using asciidoc, configure asciidoc
   - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export XML_CATALOG_FILES="/usr/local/etc/xml/catalog"; fi
+  ## If we're using Rust, download rustup
+  - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
+  ## Install the stable channels of rustc and cargo and setup our toolchain environment
+  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain stable; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
+  ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
+  - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
   ##
   ## Finally, list installed package versions
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then dpkg-query --show; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --versions; fi
+  ## Get some info about rustup, rustc and cargo
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustup; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustc; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which cargo; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi
 
 script:
   - ./autogen.sh
-  - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $OPENSSL_OPTIONS --enable-fatal-warnings --disable-silent-rules"
+  - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules"
   - echo "Configure flags are $CONFIGURE_FLAGS"
   - ./configure $CONFIGURE_FLAGS
   ## We run `make check` because that's what https://jenkins.torproject.org does.
@@ -170,6 +218,9 @@ after_failure:
 before_cache:
   ## Delete all gcov files.
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then make reset-gcov; fi
+  ## Delete the cargo registry before caching .cargo, because it's cheaper to
+  ## download the registry and throw it away, rather than caching it
+  - rm -rf $HOME/.cargo/registry
 
 notifications:
   irc:
diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 100644
index 0000000000..3569f45a88
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1,39 @@
+Contributing to Tor
+-------------------
+
+### Getting started
+
+Welcome!
+
+We have a bunch of documentation about how to develop Tor in the
+doc/HACKING/ directory.  We recommend that you start with
+doc/HACKING/README.1st.md , and then go from there.  It will tell
+you how to find your way around the source code, how to get
+involved with the Tor community, how to write patches, and much
+more!
+
+You don't have to be a C developer to help with Tor: have a look
+at https://www.torproject.org/getinvolved/volunteer !
+
+The Tor Project is committed to fostering a inclusive community
+where people feel safe to engage, share their points of view, and
+participate. For the latest version of our Code of Conduct, please
+see
+
+https://gitweb.torproject.org/community/policies.git/plain/code_of_conduct.txt
+
+
+
+### License issues
+
+Tor is distributed under the license terms in the LICENSE -- in
+brief, the "3-clause BSD license".  If you send us code to
+distribute with Tor, it needs to be code that we can distribute
+under those terms.  Please don't send us patches unless you agree
+to allow this.
+
+Some compatible licenses include:
+
+  - 3-clause BSD
+  - 2-clause BSD
+  - CC0 Public Domain Dedication
diff --git a/ChangeLog b/ChangeLog
index 7a10a7ce96..60583ac7f3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,6975 @@
-Changes in version 0.2.9.5-rc - 2016-1?-??
+Changes in version 0.3.3.7 - 2018-06-12
+  Tor 0.3.3.7 backports several changes from the 0.3.4.x series, including
+  fixes for bugs affecting compatibility and stability.
+
+  o Directory authority changes:
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26351.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.2-alpha):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.4.2-alpha):
+    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
+      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously
+      miscounting the total number of circuits for these field values.)
+      Fixes bug 26121; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (path selection, backport from 0.3.4.1-alpha):
+    - Only select relays when they have the descriptors we prefer to use
+      for them. This change fixes a bug where we could select a relay
+      because it had _some_ descriptor, but reject it later with a
+      nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
+
+
+Changes in version 0.3.4.2-alpha - 2018-06-12
+  Tor 0.3.4.2-alpha fixes several minor bugs in the previous alpha
+  release, and forward-ports an authority-only security fix from 0.3.3.6.
+
+  o Directory authority changes:
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Major bugfixes (security, directory authority, denial-of-service, also in 0.3.3.6):
+    - Fix a bug that could have allowed an attacker to force a directory
+      authority to use up all its RAM by passing it a maliciously
+      crafted protocol versions string. Fixes bug 25517; bugfix on
+      0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005.
+
+  o Minor features (continuous integration):
+    - Add the necessary configuration files for continuous integration
+      testing on Windows, via the Appveyor platform. Closes ticket
+      25549. Patches from Marcin Cieślak and Isis Lovecruft.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26351.
+
+  o Minor bugfixes (compatibility, openssl):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+    - Fix compilation when using OpenSSL 1.1.0 with the "no-deprecated"
+      flag enabled. Fixes bug 26156; bugfix on 0.3.4.1-alpha.
+    - Avoid a compiler warning when casting the return value of
+      smartlist_len() to double with DEBUG_SMARTLIST enabled. Fixes bug
+      26283; bugfix on 0.2.4.10-alpha.
+
+  o Minor bugfixes (control port):
+    - Do not count 0-length RELAY_COMMAND_DATA cells as valid data in
+      CIRC_BW events. Previously, such cells were counted entirely in
+      the OVERHEAD field. Now they are not. Fixes bug 26259; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
+      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously
+      miscounting the total number of circuits for these field values.)
+      Fixes bug 26121; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (hardening):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (onion services):
+    - Fix a bug that blocked the creation of ephemeral v3 onion
+      services. Fixes bug 25939; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (test coverage tools):
+    - Update our "cov-diff" script to handle output from the latest
+      version of gcov, and to remove extraneous timestamp information
+      from its output. Fixes bugs 26101 and 26102; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.3.6 - 2018-05-22
+  Tor 0.3.3.6 is the first stable release in the 0.3.3 series. It
+  backports several important fixes from the 0.3.4.1-alpha.
+
+  The Tor 0.3.3 series includes controller support and other
+  improvements for v3 onion services, official support for embedding Tor
+  within other applications, and our first non-trivial module written in
+  the Rust programming language. (Rust is still not enabled by default
+  when building Tor.) And as usual, there are numerous other smaller
+  bugfixes, features, and improvements.
+
+  Below are the changes since 0.3.3.5-rc. For a list of all changes
+  since 0.3.2.10, see the ReleaseNotes file.
+
+  o Major bugfixes (directory authorities, security, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (security, directory authority, denial-of-service):
+    - Fix a bug that could have allowed an attacker to force a directory
+      authority to use up all its RAM by passing it a maliciously
+      crafted protocol versions string. Fixes bug 25517; bugfix on
+      0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005.
+
+  o Major bugfixes (crash, backport from 0.3.4.1-alpha):
+    - Avoid a rare assertion failure in the circuit build timeout code
+      if we fail to allow any circuits to actually complete. Fixes bug
+      25733; bugfix on 0.2.2.2-alpha.
+
+  o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha):
+    - Avoid a crash when testing router reachability on a router that
+      could have an ed25519 ID, but which does not. Fixes bug 25415;
+      bugfix on 0.3.3.2-alpha.
+
+  o Major bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Major bugfixes (relay, denial of service, backport from 0.3.4.1-alpha):
+    - Impose a limit on circuit cell queue size. The limit can be
+      controlled by a consensus parameter. Fixes bug 25226; bugfix
+      on 0.2.4.14-alpha.
+
+  o Minor features (compatibility, backport from 0.3.4.1-alpha):
+    - Avoid some compilation warnings with recent versions of LibreSSL.
+      Closes ticket 26006.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
+      database. Closes ticket 26104.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (documentation, backport from 0.3.4.1-alpha):
+    - Stop saying in the manual that clients cache ipv4 dns answers from
+      exit relays. We haven't used them since 0.2.6.3-alpha, and in
+      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
+      we forgot to say so in the man page. Fixes bug 26052; bugfix
+      on 0.3.2.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha):
+    - Avoid a crash when running with DirPort set but ORPort tuned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Documentation (backport from 0.3.4.1-alpha):
+    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
+      ticket 25857. Patch from "CTassisF".
+
+
+Changes in version 0.3.4.1-alpha - 2018-05-17
+  Tor 0.3.4.1-alpha is the first release in the 0.3.4.x series. It
+  includes refactoring to begin reducing Tor's binary size and idle CPU
+  usage on mobile, along with prep work for new bandwidth scanners,
+  improvements to the experimental "vanguards" feature, and numerous
+  other small features and bugfixes.
+
+  o New system requirements:
+    - Tor no longer tries to support old operating systems without
+      mmap() or some local equivalent. Apparently, compilation on such
+      systems has been broken for some time, without anybody noticing or
+      complaining. Closes ticket 25398.
+
+  o Major feature (directory authority, modularization):
+    - The directory authority subsystem has been modularized. The code
+      is now located in src/or/dirauth/, and is compiled in by default.
+      To disable the module, the configure option
+      --disable-module-dirauth has been added. This module may be
+      disabled by default in some future release. Closes ticket 25610.
+
+  o Major features (main loop, CPU usage):
+    - When Tor is disabled (via DisableNetwork or via hibernation), it
+      no longer needs to run any per-second events. This change should
+      make it easier for mobile applications to disable Tor while the
+      device is sleeping, or Tor is not running. Closes ticket 26063.
+    - Tor no longer enables all of its periodic events by default.
+      Previously, Tor would enable all possible main loop events,
+      regardless of whether it needed them. Furthermore, many of these
+      events are now disabled with Tor is hibernating or DisableNetwork
+      is set. This is a big step towards reducing client CPU usage by
+      reducing the amount of wake-ups the daemon does. Closes ticket
+      25376 and 25762.
+    - The bandwidth-limitation logic has been refactored so that
+      bandwidth calculations are performed on-demand, rather than every
+      TokenBucketRefillInterval milliseconds. This change should improve
+      the granularity of our bandwidth calculations, and limit the
+      number of times that the Tor process needs to wake up when it is
+      idle. Closes ticket 25373.
+    - Move responsibility for many operations from a once-per-second
+      callback to a callback that is only scheduled as needed. Moving
+      this functionality has allowed us to disable the callback when
+      Tor's network is disabled. Once enough items are removed from our
+      once-per-second callback, we can eliminate it entirely to conserve
+      CPU when idle. The functionality removed includes: closing
+      connections, circuits, and channels (ticket 25932); consensus
+      voting (25937); flushing log callbacks (25951); honoring delayed
+      SIGNEWNYM requests (25949); rescanning the consensus cache
+      (25931); saving the state file to disk (25948); warning relay
+      operators about unreachable ports (25952); and keeping track of
+      Tor's uptime (26009).
+
+  o Major bugfixes (directory authorities, security):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (crash):
+    - Avoid a rare assertion failure in the circuit build timeout code
+      if we fail to allow any circuits to actually complete. Fixes bug
+      25733; bugfix on 0.2.2.2-alpha.
+
+  o Major bugfixes (directory authority):
+    - Avoid a crash when testing router reachability on a router that
+      could have an ed25519 ID, but which does not. Fixes bug 25415;
+      bugfix on 0.3.3.2-alpha.
+
+  o Major bugfixes (onion service):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Major bugfixes (protover, voting):
+    - Revise Rust implementation of protover to use a more memory-
+      efficient voting algorithm and corresponding data structures, thus
+      avoiding a potential (but small impact) DoS attack where specially
+      crafted protocol strings would expand to several potential
+      megabytes in memory. In the process, several portions of code were
+      revised to be methods on new, custom types, rather than functions
+      taking interchangeable types, thus increasing type safety of the
+      module. Custom error types and handling were added as well, in
+      order to facilitate better error dismissal/handling in outside
+      crates and avoid mistakenly passing an internal error string to C
+      over the FFI boundary. Many tests were added, and some previous
+      differences between the C and Rust implementations have been
+      remedied. Fixes bug 24031; bugfix on 0.3.3.1-alpha.
+
+  o Major bugfixes (relay, denial of service):
+    - Impose a limit on circuit cell queue size. The limit can be
+      controlled by a consensus parameter. Fixes bug 25226; bugfix
+      on 0.2.4.14-alpha.
+
+  o Minor features (accounting):
+    - When Tor becomes dormant, it now uses a scheduled event to wake up
+      at the right time. Previously, we would use the per-second timer
+      to check whether to wake up, but we no longer have any per-second
+      timers enabled when the network is disabled. Closes ticket 26064.
+
+  o Minor features (code quality):
+    - Add optional spell-checking for the Tor codebase, using the
+      "misspell" program. To use this feature, run "make check-typos".
+      Closes ticket 25024.
+
+  o Minor features (compatibility):
+    - Tor now detects versions of OpenSSL 1.1.0 and later compiled with
+      the no-deprecated option, and builds correctly with them. Closes
+      tickets 19429, 19981, and 25353.
+    - Avoid some compilation warnings with recent versions of LibreSSL.
+      Closes ticket 26006.
+
+  o Minor features (compression, zstd):
+    - When running with zstd, Tor now considers using advanced functions
+      that the zstd maintainers have labeled as potentially unstable. To
+      prevent breakage, Tor will only use this functionality when the
+      runtime version of the zstd library matches the version with which
+      Tor was compiled. Closes ticket 25162.
+
+  o Minor features (configuration):
+    - The "DownloadSchedule" options have been renamed to end with
+      "DownloadInitialDelay". The old names are still allowed, but will
+      produce a warning. Comma-separated lists are still permitted for
+      these options, but all values after the first are ignored (as they
+      have been since 0.2.9). Closes ticket 23354.
+
+  o Minor features (continuous integration):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (control port):
+    - Introduce GETINFO "current-time/{local,utc}" to return the local
+      and UTC times respectively in ISO format. This helps a controller
+      like Tor Browser detect a time-related error. Closes ticket 25511.
+      Patch by Neel Chauhan.
+    - Introduce new fields to the CIRC_BW event. There are two new
+      fields in each of the read and written directions. The DELIVERED
+      fields report the total valid data on the circuit, as measured by
+      the payload sizes of verified and error-checked relay command
+      cells. The OVERHEAD fields report the total unused bytes in each
+      of these cells. Closes ticket 25903.
+
+  o Minor features (directory authority):
+    - Directory authorities now open their key-pinning files as O_SYNC,
+      to limit their chances of accidentally writing partial lines.
+      Closes ticket 23909.
+
+  o Minor features (directory authority, forward compatibility):
+    - Make the lines of the measured bandwidth file able to contain
+      their entries in any order. Previously, the node_id entry needed
+      to come first. Closes ticket 26004.
+
+  o Minor features (entry guards):
+    - Introduce a new torrc option NumPrimaryGuards for controlling the
+      number of primary guards. Closes ticket 25843.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
+      database. Closes ticket 26104.
+
+  o Minor features (performance):
+    - Avoid a needless call to malloc() when processing an incoming
+      relay cell. Closes ticket 24914.
+    - Make our timing-wheel code run a tiny bit faster on 32-bit
+      platforms, by preferring 32-bit math to 64-bit. Closes
+      ticket 24688.
+    - Avoid a needless malloc()/free() pair every time we handle an ntor
+      handshake. Closes ticket 25150.
+
+  o Minor features (testing):
+    - Add a unit test for voting_schedule_get_start_of_next_interval().
+      Closes ticket 26014, and helps make unit test coverage
+      more deterministic.
+    - A new unittests module specifically for testing the functions in
+      the (new-ish) bridges.c module has been created with new
+      unittests, raising the code coverage percentages. Closes 25425.
+    - We now have improved testing for addressmap_get_virtual_address()
+      function. This should improve our test coverage, and make our test
+      coverage more deterministic. Closes ticket 25993.
+
+  o Minor features (timekeeping, circuit scheduling):
+    - When keeping track of how busy each circuit have been recently on
+      a given connection, use coarse-grained monotonic timers rather
+      than gettimeofday(). This change should marginally increase
+      accuracy and performance. Implements part of ticket 25927.
+
+  o Minor bugfixes (bandwidth management):
+    - Consider ourselves "low on write bandwidth" if we have exhausted
+      our write bandwidth some time in the last second. This was the
+      documented behavior before, but the actual behavior was to change
+      this value every TokenBucketRefillInterval. Fixes bug 25828;
+      bugfix on 0.2.3.5-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Add a missing lock acquisition in the shutdown code of the control
+      subsystem. Fixes bug 25675; bugfix on 0.2.7.3-rc. Found by
+      Coverity; this is CID 1433643.
+
+  o Minor bugfixes (circuit path selection):
+    - Don't count path selection failures as circuit build failures.
+      This change should eliminate cases where Tor blames its guard or
+      the network for situations like insufficient microdescriptors
+      and/or overly restrictive torrc settings. Fixes bug 25705; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (client):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (code style):
+    - Fixed multiple includes of transports.h in src/or/connection.c
+      Fixes bug 25261; bugfix on 0.2.5.1-alpha.
+    - Remove the unused variable n_possible from the function
+      channel_get_for_extend(). Fixes bug 25645; bugfix on 0.2.4.4-alpha
+
+  o Minor bugfixes (control interface):
+    - Respond with more human-readable error messages to GETINFO exit-
+      policy/* requests. Also, let controller know if an error is
+      transient (response code 551) or not (response code 552). Fixes
+      bug 25852; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Make CIRC_BW event reflect the total of all data sent on a
+      circuit, including padding and dropped cells. Also fix a mis-
+      counting bug when STREAM_BW events were enabled. Fixes bug 25400;
+      bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness, client):
+    - Upon receiving a malformed connected cell, stop processing the cell
+      immediately.  Previously we would mark the connection for close, but
+      continue processing the cell as if the connection were open. Fixes bug
+      26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (directory client):
+    - When unverified-consensus is verified, rename it to cached-
+      consenus. Fixes bug 4187; bugfix on 0.2.0.3-alpha.
+    - Fixed launching a certificate fetch always during the scheduled
+      periodic consensus fetch by fetching only in those cases when
+      consensus are waiting for certs. Fixes bug 24740; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Stop saying in the manual that clients cache ipv4 dns answers from
+      exit relays. We haven't used them since 0.2.6.3-alpha, and in
+      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
+      we forgot to say so in the man page. Fixes bug 26052; bugfix
+      on 0.3.2.6-alpha.
+
+  o Minor bugfixes (error reporting):
+    - Improve tolerance for directory authorities with skewed clocks.
+      Previously, an authority with a clock more than 60 seconds ahead
+      could cause a client with a correct clock to warn that the
+      client's clock was behind. Now the clocks of a majority of
+      directory authorities have to be ahead of the client before this
+      warning will occur. Fixes bug 25756; bugfix on 0.2.2.25-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (onion service):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (path selection):
+    - Only select relays when they have the descriptors we prefer to use
+      for them. This change fixes a bug where we could select a relay
+      because it had _some_ descriptor, but reject it later with a
+      nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (portability):
+    - Do not align mmap length, as it is not required by POSIX, and the
+      getpagesize function is deprecated. Fixes bug 25399; bugfix
+      on 0.1.1.23.
+
+  o Minor bugfixes (portability, FreeBSD):
+    - In have_enough_mem_for_dircache(), the variable DIRCACHE_MIN_MEM_MB
+      does not stringify on FreeBSD, so we switch to tor_asprintf().
+      Fixes bug 20887; bugfix on 0.2.8.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (relay statistics):
+    - When a relay is collecting internal statistics about how many
+      create cell requests it has seen of each type, accurately count
+      the requests from relays that temporarily fall out of the
+      consensus. (To be extra conservative, we were already ignoring
+      requests from clients in our counts, and we continue ignoring them
+      here.) Fixes bug 24910; bugfix on 0.2.4.17-rc.
+
+  o Minor bugfixes (relay, crash):
+    - Avoid a crash when running with DirPort set but ORPort tuned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (restart-in-process):
+    - When shutting down, Tor now clears all the flags in the control.c
+      module. This should prevent a bug where authentication cookies are
+      not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (testing):
+    - When testing workqueue event-cancellation, make sure that we
+      actually cancel an event, and that cancel each event with equal
+      probability. (It was previously possible, though extremely
+      unlikely, for our event-canceling test not to cancel any events.)
+      Fixes bug 26008; bugfix on 0.2.6.3-alpha.
+    - Repeat part of the test in test_client_pick_intro() a number of
+      times, to give it consistent coverage. Fixes bug 25996; bugfix
+      on 0.3.2.1-alpha.
+    - Remove randomness from the hs_common/responsible_hsdirs test, so
+      that it always takes the same path through the function it tests.
+      Fixes bug 25997; bugfix on 0.3.2.1-alpha.
+    - Change the behavior of the "channel/outbound" test so that it
+      never causes a 10-second rollover for the EWMA circuitmux code.
+      Previously, this behavior would happen randomly, and result in
+      fluctuating test coverage. Fixes bug 25994; bugfix
+      on 0.3.3.1-alpha.
+    - Use X509_new() to allocate certificates that will be freed later
+      with X509_free(). Previously, some parts of the unit tests had
+      used tor_malloc_zero(), which is incorrect, and which caused test
+      failures on Windows when they were built with extra hardening.
+      Fixes bugs 25943 and 25944; bugfix on 0.2.8.1-alpha. Patch by
+      Marcin Cieślak.
+    - While running the circuit_timeout test, fix the PRNG to a
+      deterministic AES stream, so that the test coverage from this test
+      will itself be deterministic. Fixes bug 25995; bugfix
+      on 0.2.2.2-alpha.
+
+  o Minor bugfixes (vanguards):
+    - Allow the last hop in a vanguard circuit to be the same as our
+      first, to prevent the adversary from influencing guard node choice
+      by choice of last hop. Also prevent the creation of A - B - A
+      paths, or A - A paths, which are forbidden by relays. Fixes bug
+      25870; bugfix on 0.3.3.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove duplicate code in parse_{c,s}method_line and bootstrap
+      their functionalities into a single function. Fixes bug 6236;
+      bugfix on 0.2.3.6-alpha.
+    - We remove the PortForwsrding and PortForwardingHelper options,
+      related functions, and the port_forwarding tests. These options
+      were used by the now-deprecated Vidalia to help ordinary users
+      become Tor relays or bridges. Closes ticket 25409. Patch by
+      Neel Chauhan.
+    - In order to make the OR and dir checking function in router.c less
+      confusing we renamed some functions and
+      consider_testing_reachability() has been split into
+      router_should_check_reachability() and
+      router_do_reachability_checks(). Also we improved the documentation
+      in some functions. Closes ticket 18918.
+    - Initial work to isolate Libevent usage to a handful of modules in
+      our codebase, to simplify our call structure, and so that we can
+      more easily change event loops in the future if needed. Closes
+      ticket 23750.
+    - Introduce a function to call getsockname() and return tor_addr_t,
+      to save a little complexity throughout the codebase. Closes
+      ticket 18105.
+    - Make hsdir_index in node_t a hsdir_index_t rather than a pointer
+      as hsdir_index is always present. Also, we move hsdir_index_t into
+      or.h. Closes ticket 23094. Patch by Neel Chauhan.
+    - Merge functions used for describing nodes and suppress the
+      functions that do not allocate memory for the output buffer
+      string. NODE_DESC_BUF_LEN constant and format_node_description()
+      function cannot be used externally from router.c module anymore.
+      Closes ticket 25432. Patch by valentecaio.
+    - Our main loop has been simplified so that all important operations
+      happen inside events. Previously, some operations had to happen
+      outside the event loop, to prevent infinite sequences of event
+      activations. Closes ticket 25374.
+    - Put a SHA1 public key digest in hs_service_intro_point_t, and use
+      it in register_intro_circ() and service_intro_point_new(). This
+      prevents the digest from being re-calculated each time. Closes
+      ticket 23107. Patch by Neel Chauhan.
+    - Refactor token-bucket implementations to use a common backend.
+      Closes ticket 25766.
+    - Remove extern declaration of stats_n_seconds_working variable from
+      main, protecting its accesses with get_uptime() and reset_uptime()
+      functions. Closes ticket 25081, patch by “valentecaio”.
+    - Remove our previous logic for "cached gettimeofday()" -- our
+      coarse monotonic timers are fast enough for this purpose, and far
+      less error-prone. Implements part of ticket 25927.
+    - Remove the return value for fascist_firewall_choose_address_base(),
+      and sister functions such as fascist_firewall_choose_address_node()
+      and fascist_firewall_choose_address_rs(). Also, while we're here,
+      initialize the ap argument as leaving it uninitialized can pose a
+      security hazard. Closes ticket 24734. Patch by Neel Chauhan.
+    - Rename two fields of connection_t struct. timestamp_lastwritten is
+      renamed to timestamp_last_write_allowed and timestamp_lastread is
+      renamed to timestamp_last_read_allowed. Closes ticket 24714, patch
+      by "valentecaio".
+    - Since Tor requires C99, remove our old workaround code for libc
+      implementations where free(NULL) doesn't work. Closes ticket 24484.
+    - Use our standard rate-limiting code to deal with excessive
+      libevent failures, rather than the hand-rolled logic we had
+      before. Closes ticket 26016.
+    - We remove the return value of node_get_prim_orport() and
+      node_get_prim_dirport(), and introduce node_get_prim_orport() in
+      node_ipv6_or_preferred() and node_ipv6_dir_preferred() in order to
+      check for a null address. Closes ticket 23873. Patch by
+      Neel Chauhan.
+    - We switch to should_record_bridge_info() in
+      geoip_note_client_seen() and options_need_geoip_info() instead of
+      accessing the configuration values directly. Fixes bug 25290;
+      bugfix on 0.2.1.6-alpha. Patch by Neel Chauhan.
+
+  o Deprecated features:
+    - As we are not recommending 0.2.5 anymore, we require relays that
+      once had an ed25519 key associated with their RSA key to always
+      have that key, instead of allowing them to drop back to a version
+      that didn't support ed25519. This means they need to use a new RSA
+      key if the want to downgrade to an older version of tor without
+      ed25519. Closes ticket 20522.
+
+  o Documentation:
+    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
+      ticket 25857. Patch from "CTassisF".
+
+  o Removed features:
+    - Directory authorities will no longer support voting according to
+      any consensus method before consensus method 25. This keeps
+      authorities compatible with all authorities running 0.2.9.8 and
+      later, and does not break any clients or relays. Implements ticket
+      24378 and proposal 290.
+    - The PortForwarding and PortForwardingHelper features have been
+      removed. The reasoning is, given that implementations of NAT
+      traversal protocols within common consumer grade routers are
+      frequently buggy, and that the target audience for a NAT punching
+      feature is a perhaps less-technically-inclined relay operator,
+      when the helper fails to setup traversal the problems are usually
+      deep, ugly, and very router specific, making them horrendously
+      impossible for technical support to reliable assist with, and thus
+      resulting in frustration all around. Unfortunately, relay
+      operators who would like to run relays behind NATs will need to
+      become more familiar with the port forwarding configurations on
+      their local router. Closes 25409.
+    - The TestingEnableTbEmptyEvent option has been removed. It was used
+      in testing simulations to measure how often connection buckets
+      were emptied, in order to improve our scheduling, but it has not
+      been actively used in years. Closes ticket 25760.
+    - The old "round-robin" circuit multiplexer (circuitmux)
+      implementation has been removed, along with a fairly large set of
+      code that existed to support it. It has not been the default
+      circuitmux since we introduced the "EWMA" circuitmux in 0.2.4.x,
+      but it still required an unreasonable amount of memory and CPU.
+      Closes ticket 25268.
+
+
+Changes in version 0.3.3.5-rc - 2018-04-15
+  Tor 0.3.3.5-rc fixes various bugs in earlier versions of Tor,
+  including some that could affect reliability or correctness.
+
+  This is the first release candidate in the 0.3.3 series. If we find no
+  new bugs or regression here, then the first stable 0.3.3 release will
+  be nearly identical to this one.
+
+  o Major bugfixes (security, protover, voting):
+    - Revise Rust implementation of protover to use a more memory-
+      efficient voting algorithm and corresponding data structures, thus
+      avoiding a potential memory-based DoS attack where specially
+      crafted protocol strings would expand to fill available memory.
+      Fixes bug 24031; bugfix on 0.3.3.1-alpha.
+
+  o Major bugfixes (performance, load balancing):
+    - Directory authorities no longer vote in favor of the Guard flag
+      for relays without directory support. Starting in Tor
+      0.3.0.1-alpha, clients have been avoiding using such relays in the
+      Guard position, leading to increasingly broken load balancing for
+      the 5%-or-so of Guards that don't advertise directory support.
+      Fixes bug 22310; bugfix on 0.3.0.6.
+
+  o Minor feature (continuous integration):
+    - Update the Travis CI configuration to use the stable Rust channel,
+      now that we have decided to require that. Closes ticket 25714.
+
+  o Minor features (config options):
+    - Change the way the default value for MaxMemInQueues is calculated.
+      We now use 40% of the hardware RAM if the system has 8 GB RAM or
+      more. Otherwise we use the former value of 75%. Closes
+      ticket 24782.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 25718.
+
+  o Minor bugfixes (client):
+    - When using a listed relay as a bridge, and also using
+      microdescriptors, and considering that relay as a non-bridge in a
+      circuit, treat its microdescriptor as a valid source of
+      information about that relay. This change should prevent a non-
+      fatal assertion error. Fixes bug 25691; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the correct operation of the RESOLVE command, which had
+      been broken since we added the ability to enable/disable DNS on
+      specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (distribution, compilation, rust):
+    - Build correctly when the rust dependencies submodule is loaded,
+      but the TOR_RUST_DEPENDENCIES environment variable is not set.
+      Fixes bug 25679; bugfix on 0.3.3.1-alpha.
+    - Actually include all of our Rust source in our source
+      distributions. (Previously, a few of the files were accidentally
+      omitted.) Fixes bug 25732; bugfix on 0.3.3.2-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document that the PerConnBW{Rate,Burst} options will fall back to
+      their corresponding consensus parameters only if those parameters
+      are set. Previously we had claimed that these values would always
+      be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha.
+    - Revert a misformatting issue in the ExitPolicy documentation.
+      Fixes bug 25582; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (exit relay DNS retries):
+    - Re-attempt timed-out DNS queries 3 times before failure, since our
+      timeout is 5 seconds for them, but clients wait 10-15. Also allow
+      slightly more timeouts per resolver when an exit has multiple
+      resolvers configured. Fixes bug 21394; bugfix on 0.3.1.9.
+
+  o Minor bugfixes (onion services):
+    - Re-instate counting the client HSDir fetch circuits against the
+      MaxClientCircuitsPending rate limit. Fixes bug 24989; bugfix
+      on 0.3.3.1-alpha.
+    - Remove underscores from the _HSLayer{2,3}Nodes options. This
+      expert-user configuration can now be enabled as HSLayer{2,3}Nodes.
+      Fixes bug 25581; bugfix on 0.3.3.1-alpha
+
+  o Code simplification and refactoring:
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+
+  o Documentation (manpage, denial of service):
+    - Provide more detail about the denial-of-service options, by
+      listing each mitigation and explaining how they relate. Closes
+      ticket 25248.
+
+
+Changes in version 0.3.3.4-alpha - 2018-03-29
+  Tor 0.3.3.4-alpha includes various bugfixes for issues found during
+  the alpha testing of earlier releases in its series. We are
+  approaching a stable 0.3.3.4-alpha release: more testing is welcome!
+
+  o New system requirements:
+    - When built with Rust, Tor now depends on version 0.2.39 of the
+      libc crate. Closes tickets 25310 and 25664.
+
+  o Major bugfixes (relay, connection):
+    - If we have failed to connect to a relay and received a connection
+      refused, timeout, or similar error (at the TCP level), do not try
+      that same address/port again for 60 seconds after the failure has
+      occurred. Fixes bug 24767; bugfix on 0.0.6.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 8 2018 Maxmind GeoLite2
+      Country database. Closes ticket 25469.
+
+  o Minor features (log messages):
+    - Improve log message in the out-of-memory handler to include
+      information about memory usage from the different compression
+      backends. Closes ticket 25372.
+
+  o Minor features (sandbox):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor bugfixes (C correctness):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (channel, client):
+    - Better identify client connection when reporting to the geoip
+      client cache. Fixes bug 24904; bugfix on 0.3.1.7.
+
+  o Minor bugfixes (compilation):
+    - Fix a C99 compliance issue in our configuration script that caused
+      compilation issues when compiling Tor with certain versions of
+      xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (controller, reliability):
+    - Avoid a (nonfatal) assertion failure when extending a one-hop
+      circuit from the controller to become a multihop circuit. Fixes
+      bug 24903; bugfix on 0.2.5.2-alpha.
+
+  o Major bugfixes (networking):
+    - Tor will no longer reject IPv6 address strings from Tor Browser
+      when they are passed as hostnames in SOCKS5 requests. Fixes bug
+      25036, bugfix on Tor 0.3.1.2.
+
+  o Minor bugfixes (networking):
+    - string_is_valid_hostname() will not consider IP strings to be
+      valid hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5.
+
+  o Minor bugfixes (onion service v3):
+    - Avoid an assertion failure when the next onion service
+      descriptor rotation type is out of sync with the consensus's
+      valid-after time. Instead, log a warning message with extra
+      information, so we can better hunt down the cause of this
+      assertion. Fixes bug 25306; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      onion service introduction point creation finishing within 5
+      seconds of real clock time. Fixes bug 25450; bugfix
+      on 0.3.1.3-alpha.
+    - Rust crates are now automatically detected and tested. Previously,
+      some crates were not tested by `make test-rust` due to a static
+      string in the `src/test/test_rust.sh` script specifying which
+      crates to test. Fixes bug 25560; bugfix on 0.3.3.3-alpha.
+
+  o Minor bugfixes (testing, benchmarks):
+    - Fix a crash when running benchmark tests on win32 systems. The
+      crash was due to a mutex that wasn't initialized before logging
+      and options were initialized. Fixes bug 25479; bugfix
+      on 0.3.3.3-alpha.
+
+  o Minor bugfixes (warnings, ipv6):
+    - Avoid a bug warning that could occur when trying to connect to a
+      relay over IPv6. This warning would occur on a Tor instance that
+      downloads router descriptors, but prefers to use microdescriptors.
+      Fixes bug 25213; bugfix on 0.3.3.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove the old (deterministic) directory retry logic entirely:
+      We've used exponential backoff exclusively for some time. Closes
+      ticket 23814.
+
+  o Documentation:
+    - Improved the documentation of AccountingStart parameter. Closes
+      ticket 23635.
+    - Update the documentation for "Log" to include the current list of
+      logging domains. Closes ticket 25378.
+
+
+Changes in version 0.3.1.10 - 2018-03-03
+  Tor 0.3.1.10 backports a number of bugfixes, including important fixes for
+  security issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.3.1.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  Please note: according to our release calendar, Tor 0.3.1 will no
+  longer be supported after 1 July 2018. If you will be running Tor
+  after that date, you should make sure to plan to upgrade to the latest
+  stable version, or downgrade to 0.2.9 (which will receive long-term
+  support).
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Minor features (linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor feature (relay statistics, backport from 0.3.2.6-alpha):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (fallback directory mirrors, backport from 0.3.2.9):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection, backport from 0.3.2.9):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Major bugfixes (bootstrapping, backport from 0.3.2.5-alpha):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.7-rc):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (directory cache, backport from 0.3.2.5-alpha):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (entry guards, backport from 0.3.2.3-alpha):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (memory usage, backport from 0.3.2.8-rc):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer, backport from 0.3.2.5-alpha):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (path selection, backport from 0.3.2.4-alpha):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (portability, msvc, backport from 0.3.2.9):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cyptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay, partial backport):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+
+Changes in version 0.2.9.15 - 2018-03-03
+  Tor 0.2.9.15 backports important security and stability bugfixes from
+  later Tor releases.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.2.9.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+
+  o Major bugfixes (onion services, retry behavior):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Minor feature (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor bugfix (channel connection):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (OSX):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (portability, msvc):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+
+Changes in version 0.3.2.10 - 2018-03-03
+  Tor 0.3.2.10 is the second stable release in the 0.3.2 series. It
+  backports a number of bugfixes, including important fixes for security
+  issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  Additionally, it backports a fix for a bug whose severity we have
+  upgraded: Bug 24700, which was fixed in 0.3.3.2-alpha, can be remotely
+  triggered in order to crash relays with a use-after-free pattern. As
+  such, we are now tracking that bug as TROVE-2018-002 and
+  CVE-2018-0491, and backporting it to earlier releases. This bug
+  affected versions 0.3.2.1-alpha through 0.3.2.9, as well as version
+  0.3.3.1-alpha.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  Relays running 0.3.2.x SHOULD upgrade to one of the versions released
+  today, for the fix to TROVE-2018-002. Directory authorities should
+  also upgrade. (Relays on earlier versions might want to update too for
+  the DoS mitigations.)
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major bugfixes (scheduler, KIST, denial-of-service, backport from 0.3.3.2-alpha):
+    - Avoid adding the same channel twice in the KIST scheduler pending
+      list, which could lead to remote denial-of-service use-after-free
+      attacks against relays. Fixes bug 24700; bugfix on 0.3.2.1-alpha.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+    - New-style (v3) onion services now obey the "max rendezvous circuit
+      attempts" logic. Previously they would make as many rendezvous
+      circuit attempts as they could fit in the MAX_REND_TIMEOUT second
+      window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (scheduler, consensus, backport from 0.3.3.2-alpha):
+    - The scheduler subsystem was failing to promptly notice changes in
+      consensus parameters, making it harder to switch schedulers
+      network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (logging, diagnostic, backport from 0.3.3.2-alpha):
+    - When logging a failure to create an onion service's descriptor,
+      also log what the problem with the descriptor was. Diagnostic
+      for ticket 24972.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (build, rust, backport from 0.3.3.1-alpha):
+    - When building with Rust on OSX, link against libresolv, to work
+      around the issue at https://github.com/rust-lang/rust/issues/46797.
+      Fixes bug 24652; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.3.2-alpha):
+    - Remove a BUG() statement when a client fetches an onion descriptor
+      that has a lower revision counter than the one in its cache. This
+      can happen in normal circumstances due to HSDir desync. Fixes bug
+      24976; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.3.1-alpha):
+    - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug
+      25005; bugfix on 0.3.2.7-rc.
+
+  o Minor bugfixes (v3 onion services, backport from 0.3.3.2-alpha):
+    - Look at the "HSRend" protocol version, not the "HSDir" protocol
+      version, when deciding whether a consensus entry can support the
+      v3 onion service protocol as a rendezvous point. Fixes bug 25105;
+      bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+  o Documentation (backport from 0.3.3.1-alpha)
+    - Document that operators who run more than one relay or bridge are
+      expected to set MyFamily and ContactInfo correctly. Closes
+      ticket 24526.
+
+
+Changes in version 0.3.3.3-alpha - 2018-03-03
+  Tor 0.3.3.3-alpha is the third alpha release for the 0.3.3.x series.
+  It includes an important security fix for a remote crash attack
+  against directory authorities tracked as TROVE-2018-001.
+
+  Additionally, with this release, we are upgrading the severity of a
+  bug fixed in 0.3.3.2-alpha. Bug 24700, which was fixed in
+  0.3.3.2-alpha, can be remotely triggered in order to crash relays with
+  a use-after-free pattern. As such, we are now tracking that bug as
+  TROVE-2018-002 and CVE-2018-0491. This bug affected versions
+  0.3.2.1-alpha through 0.3.2.9, as well as 0.3.3.1-alpha.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  Relays running 0.3.2.x should upgrade to one of the versions released
+  today, for the fix to TROVE-2018-002. Directory authorities should
+  also upgrade. (Relays on earlier versions might want to update too for
+  the DoS mitigations.)
+
+  o Major bugfixes (denial-of-service, directory authority):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Minor features (compatibility, OpenSSL):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (logging):
+    - Clarify the log messages produced when getrandom() or a related
+      entropy-generation mechanism gives an error. Closes ticket 25120.
+
+  o Minor features (testing):
+    - Add a "make test-rust" target to run the rust tests only. Closes
+      ticket 25071.
+
+  o Minor bugfixes (denial-of-service):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (DoS mitigation):
+    - Add extra safety checks when refilling the circuit creation bucket
+      to ensure we never set a value above the allowed maximum burst.
+      Fixes bug 25202; bugfix on 0.3.3.2-alpha.
+    - When a new consensus arrives, don't update our DoS-mitigation
+      parameters if we aren't a public relay. Fixes bug 25223; bugfix
+      on 0.3.3.2-alpha.
+
+  o Minor bugfixes (man page, SocksPort):
+    - Remove dead code from the old "SocksSocket" option, and rename
+      SocksSocketsGroupWritable to UnixSocksGroupWritable. The old option
+      still works, but is deprecated. Fixes bug 24343; bugfix on 0.2.6.3.
+
+  o Minor bugfixes (performance):
+    - Reduce the number of circuits that will be opened at once during
+      the circuit build timeout phase. This is done by increasing the
+      idle timeout to 3 minutes, and lowering the maximum number of
+      concurrent learning circuits to 10. Fixes bug 24769; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (spec conformance):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (spec conformance, rust):
+    - Resolve a denial-of-service issue caused by an infinite loop in
+      the rust protover code. Fixes bug 25250, bugfix on 0.3.3.1-alpha.
+      Also tracked as TROVE-2018-003.
+
+  o Code simplification and refactoring:
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+
+Changes in version 0.3.3.2-alpha - 2018-02-10
+  Tor 0.3.3.2-alpha is the second alpha in the 0.3.3.x series. It
+  introduces a mechanism to handle the high loads that many relay
+  operators have been reporting recently. It also fixes several bugs in
+  older releases. If this new code proves reliable, we plan to backport
+  it to older supported release series.
+
+  o Major features (denial-of-service mitigation):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (netflow padding):
+    - Stop adding unneeded channel padding right after we finish
+      flushing to a connection that has been trying to flush for many
+      seconds. Instead, treat all partial or complete flushes as
+      activity on the channel, which will defer the time until we need
+      to add padding. This fix should resolve confusing and scary log
+      messages like "Channel padding timeout scheduled 221453ms in the
+      past." Fixes bug 22212; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (protocol versions):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (scheduler, consensus):
+    - The scheduler subsystem was failing to promptly notice changes in
+      consensus parameters, making it harder to switch schedulers
+      network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (directory authority):
+    - When directory authorities are unable to add signatures to a
+      pending consensus, log the reason why. Closes ticket 24849.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (logging, diagnostic):
+    - When logging a failure to create an onion service's descriptor,
+      also log what the problem with the descriptor was. Diagnostic for
+      ticket 24972.
+
+  o Minor bugfix (channel connection):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (all versions of Tor):
+    - Use the "misspell" tool to detect and fix typos throughout the
+      source code. Fixes bug 23650; bugfix on various versions of Tor.
+      Patch from Deepesh Pathak.
+
+  o Minor bugfixes (circuit, cannibalization):
+    - Don't cannibalize preemptively-built circuits if we no longer
+      recognize their first hop. This situation can happen if our Guard
+      relay went off the consensus after the circuit was created. Fixes
+      bug 24469; bugfix on 0.0.6.
+
+  o Minor bugfixes (correctness):
+    - Remove a nonworking, unnecessary check to see whether a circuit
+      hop's identity digest was set when the circuit failed. Fixes bug
+      24927; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (logging):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+    - Fix a (mostly harmless) race condition when invoking
+      LOG_PROTOCOL_WARN message from a subthread while the torrc options
+      are changing. Fixes bug 23954; bugfix on 0.1.1.9-alpha.
+
+  o Minor bugfixes (onion services):
+    - Remove a BUG() statement when a client fetches an onion descriptor
+      that has a lower revision counter than the one in its cache. This
+      can happen in normal circumstances due to HSDir desync. Fixes bug
+      24976; bugfix on 0.3.2.1-alpha.
+    - If we are configured to offer a single onion service, don't log
+      long-term established one hop rendezvous points in the heartbeat.
+      Fixes bug 25116; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (performance):
+    - Avoid calling protocol_list_supports_protocol() from inside tight
+      loops when running with cached routerinfo_t objects. Instead,
+      summarize the relevant protocols as flags in the routerinfo_t, as
+      we do for routerstatus_t objects. This change simplifies our code
+      a little, and saves a large amount of short-term memory allocation
+      operations. Fixes bug 25008; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (Rust FFI):
+    - Fix a minor memory leak which would happen whenever the C code
+      would call the Rust implementation of
+      protover_get_supported_protocols(). This was due to the C version
+      returning a static string, whereas the Rust version newly allocated
+      a CString to pass across the FFI boundary. Consequently, the C
+      code was not expecting to need to free() what it was given. Fixes
+      bug 25127; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (scheduler, KIST):
+    - Avoid adding the same channel twice in the KIST scheduler pending
+      list, which would waste CPU cycles. Fixes bug 24700; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (unit test, monotonic time):
+    - Increase a constant (1msec to 10msec) in the monotonic time test
+      that makes sure the nsec/usec/msec times read are synchronized.
+      This change was needed to accommodate slow systems like armel or
+      when the clock_gettime() is not a VDSO on the running kernel.
+      Fixes bug 25113; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (v3 onion services):
+    - Look at the "HSRend" protocol version, not the "HSDir" protocol
+      version, when deciding whether a consensus entry can support the
+      v3 onion service protocol as a rendezvous point. Fixes bug 25105;
+      bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove the unused nodelist_recompute_all_hsdir_indices(). Closes
+      ticket 25108.
+    - Remove a series of counters used to track circuit extend attempts
+      and connection status but that in reality we aren't using for
+      anything other than stats logged by a SIGUSR1 signal. Closes
+      ticket 25163.
+
+  o Documentation (man page):
+    - The HiddenServiceVersion torrc option accepts only one number:
+      either version 2 or 3. Closes ticket 25026; bugfix
+      on 0.3.2.2-alpha.
+
+
+Changes in version 0.3.3.1-alpha - 2018-01-25
+  Tor 0.3.3.1-alpha is the first release in the 0.3.3.x series. It adds
+  several new features to Tor, including several improvements to
+  bootstrapping, and support for an experimental "vanguards" feature to
+  resist guard discovery attacks. This series also includes better
+  support for applications that need to embed Tor or manage v3
+  onion services.
+
+  o Major features (embedding):
+    - There is now a documented stable API for programs that need to
+      embed Tor. See tor_api.h for full documentation and known bugs.
+      Closes ticket 23684.
+    - Tor now has support for restarting in the same process.
+      Controllers that run Tor using the "tor_api.h" interface can now
+      restart Tor after Tor has exited. This support is incomplete,
+      however: we fixed crash bugs that prevented it from working at
+      all, but many bugs probably remain, including a possibility of
+      security issues. Implements ticket 24581.
+
+  o Major features (IPv6, directory documents):
+    - Add consensus method 27, which adds IPv6 ORPorts to the microdesc
+      consensus. This information makes it easier for IPv6 clients to
+      bootstrap and choose reachable entry guards. Implements ticket 23826.
+    - Add consensus method 28, which removes IPv6 ORPorts from
+      microdescriptors. Now that the consensus contains IPv6 ORPorts, they
+      are redundant in microdescs. This change will be used by Tor clients
+      on 0.2.8.x and later. (That is to say, with all Tor clients that
+      have IPv6 bootstrap and guard support.) Implements ticket 23828.
+    - Expand the documentation for AuthDirHasIPv6Connectivity when it is
+      set by different numbers of authorities. Fixes 23870
+      on 0.2.4.1-alpha.
+
+  o Major features (onion service v3, control port):
+    - The control port now supports commands and events for v3 onion
+      services. It is now possible to create ephemeral v3 services using
+      ADD_ONION. Additionally, several events (HS_DESC, HS_DESC_CONTENT,
+      CIRC and CIRC_MINOR) and commands (GETINFO, HSPOST, ADD_ONION and
+      DEL_ONION) have been extended to support v3 onion services. Closes
+      ticket 20699; implements proposal 284.
+
+  o Major features (onion services):
+    - Provide torrc options to pin the second and third hops of onion
+      service circuits to a list of nodes. The option HSLayer2Guards
+      pins the second hop, and the option HSLayer3Guards pins the third
+      hop. These options are for use in conjunction with experiments
+      with "vanguards" for preventing guard enumeration attacks. Closes
+      ticket 13837.
+
+  o Major features (rust, portability, experimental):
+    - Tor now ships with an optional implementation of one of its
+      smaller modules (protover.c) in the Rust programming language. To
+      try it out, install a Rust build environment, and configure Tor
+      with "--enable-rust --enable-cargo-online-mode". This should not
+      cause any user-visible changes, but should help us gain more
+      experience with Rust, and plan future Rust integration work.
+      Implementation by Chelsea Komlo. Closes ticket 22840.
+
+  o Minor features (storage, configuration):
+    - Users can store cached directory documents somewhere other than
+      the DataDirectory by using the CacheDirectory option. Similarly,
+      the storage location for relay's keys can be overridden with the
+      KeyDirectory option. Closes ticket 22703.
+
+  o Major features (v3 onion services, ipv6):
+    - When v3 onion service clients send introduce cells, they now
+      include the IPv6 address of the rendezvous point, if it has one.
+      Current v3 onion services running 0.3.2 ignore IPv6 addresses, but
+      in future Tor versions, IPv6-only v3 single onion services will be
+      able to use IPv6 addresses to connect directly to the rendezvous
+      point. Closes ticket 23577. Patch by Neel Chauhan.
+
+  o Major bugfixes (onion services, retry behavior):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+    - New-style (v3) onion services now obey the "max rendezvous circuit
+      attempts" logic. Previously they would make as many rendezvous
+      circuit attempts as they could fit in the MAX_REND_TIMEOUT second
+      window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha.
+
+  o Major bugfixes (relays):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Minor feature (IPv6):
+    - Make IPv6-only clients wait for microdescs for relays, even if we
+      were previously using descriptors (or were using them as a bridge)
+      and have a cached descriptor for them. Implements ticket 23827.
+    - When a consensus has IPv6 ORPorts, make IPv6-only clients use
+      them, rather than waiting to download microdescriptors.
+      Implements ticket 23827.
+
+  o Minor features (cleanup):
+    - Tor now deletes the CookieAuthFile and ExtORPortCookieAuthFile
+      when it stops. Closes ticket 23271.
+
+  o Minor features (defensive programming):
+    - Most of the functions in Tor that free objects have been replaced
+      with macros that free the objects and set the corresponding
+      pointers to NULL. This change should help prevent a large class of
+      dangling pointer bugs. Closes ticket 24337.
+    - Where possible, the tor_free() macro now only evaluates its input
+      once. Part of ticket 24337.
+    - Check that microdesc ed25519 ids are non-zero in
+      node_get_ed25519_id() before returning them. Implements ticket
+      24001, patch by "aruna1234".
+
+  o Minor features (embedding):
+    - Tor can now start with a preauthenticated control connection
+      created by the process that launched it. This feature is meant for
+      use by programs that want to launch and manage a Tor process
+      without allowing other programs to manage it as well. For more
+      information, see the __OwningControllerFD option documented in
+      control-spec.txt. Closes ticket 23900.
+    - On most errors that would cause Tor to exit, it now tries to
+      return from the tor_main() function, rather than calling the
+      system exit() function. Most users won't notice a difference here,
+      but it should be significant for programs that run Tor inside
+      a separate thread: they should now be able to survive Tor's exit
+      conditions rather than having Tor shut down the entire process.
+      Closes ticket 23848.
+    - Applications that want to embed Tor can now tell Tor not to
+      register any of its own POSIX signal handlers, using the
+      __DisableSignalHandlers option. Closes ticket 24588.
+
+  o Minor features (fallback directory list):
+    - Avoid selecting fallbacks that change their IP addresses too
+      often. Select more fallbacks by ignoring the Guard flag, and
+      allowing lower cutoffs for the Running and V2Dir flags. Also allow
+      a lower bandwidth, and a higher number of fallbacks per operator
+      (5% of the list). Implements ticket 24785.
+    - Update the fallback whitelist and blacklist based on opt-ins and
+      relay changes. Closes tickets 22321, 24678, 22527, 24135,
+      and 24695.
+
+  o Minor features (fallback directory mirror configuration):
+    - Add a nickname to each fallback in a C comment. This makes it
+      easier for operators to find their relays, and allows stem to use
+      nicknames to identify fallbacks. Implements ticket 24600.
+    - Add a type and version header to the fallback directory mirror
+      file. Also add a delimiter to the end of each fallback entry. This
+      helps external parsers like stem and Relay Search. Implements
+      ticket 24725.
+    - Add an extrainfo cache flag for each fallback in a C comment. This
+      allows stem to use fallbacks to fetch extra-info documents, rather
+      than using authorities. Implements ticket 22759.
+    - Add the generateFallbackDirLine.py script for automatically
+      generating fallback directory mirror lines from relay fingerprints.
+      No more typos! Add the lookupFallbackDirContact.py script for
+      automatically looking up operator contact info from relay
+      fingerprints. Implements ticket 24706, patch by teor and atagar.
+    - Reject any fallback directory mirror that serves an expired
+      consensus. Implements ticket 20942, patch by "minik".
+    - Remove commas and equals signs from external string inputs to the
+      fallback list. This avoids format confusion attacks. Implements
+      ticket 24726.
+    - Remove the "weight=10" line from fallback directory mirror
+      entries. Ticket 24681 will maintain the current fallback weights
+      by changing Tor's default fallback weight to 10. Implements
+      ticket 24679.
+    - Stop logging excessive information about fallback netblocks.
+      Implements ticket 24791.
+
+  o Minor features (forward-compatibility):
+    - If a relay supports some link authentication protocol that we do
+      not recognize, then include that relay's ed25519 key when telling
+      other relays to extend to it. Previously, we treated future
+      versions as if they were too old to support ed25519 link
+      authentication. Closes ticket 20895.
+
+  o Minor features (heartbeat):
+    - Add onion service information to our heartbeat logs, displaying
+      stats about the activity of configured onion services. Closes
+      ticket 24896.
+
+  o Minor features (instrumentation, development):
+    - Add the MainloopStats option to allow developers to get
+      instrumentation information from the main event loop via the
+      heartbeat messages. We hope to use this to improve Tor's behavior
+      when it's trying to sleep. Closes ticket 24605.
+
+  o Minor features (log messages):
+    - Improve a warning message that happens when we fail to re-parse an
+      old router because of an expired certificate. Closes ticket 20020.
+    - Make the log more quantitative when we hit MaxMemInQueues
+      threshold exposing some values. Closes ticket 24501.
+
+  o Minor features (logging, android):
+    - Added support for the Android logging subsystem. Closes
+      ticket 24362.
+
+  o Minor features (performance):
+    - Support predictive circuit building for onion service circuits
+      with multiple layers of guards. Closes ticket 23101.
+    - Use stdatomic.h where available, rather than mutexes, to implement
+      atomic_counter_t. Closes ticket 23953.
+
+  o Minor features (performance, 32-bit):
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when calculating the timestamp in milliseconds for channel padding
+      computations. Implements ticket 24613.
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when timestamping cells and buffer chunks for OOM calculations.
+      Implements ticket 24374.
+
+  o Minor features (performance, OSX, iOS):
+    - Use the mach_approximate_time() function (when available) to
+      implement coarse monotonic time. Having a coarse time function
+      should avoid a large number of system calls, and improve
+      performance slightly, especially under load. Closes ticket 24427.
+
+  o Minor features (performance, windows):
+    - Improve performance on Windows Vista and Windows 7 by adjusting
+      TCP send window size according to the recommendation from
+      SIO_IDEAL_SEND_BACKLOG_QUERY. Closes ticket 22798. Patch
+      from Vort.
+
+  o Major features (relay):
+    - Implement an option, ReducedExitPolicy, to allow an Tor exit relay
+      operator to use a more reasonable ("reduced") exit policy, rather
+      than the default one. If you want to run an exit node without
+      thinking too hard about which ports to allow, this one is for you.
+      Closes ticket 13605. Patch from Neel Chauhan.
+
+  o Minor features (testing, debugging, embedding):
+    - For development purposes, Tor now has a mode in which it runs for
+      a few seconds, then stops, and starts again without exiting the
+      process. This mode is meant to help us debug various issues with
+      ticket 23847. To use this feature, compile with
+      --enable-restart-debugging, and set the TOR_DEBUG_RESTART
+      environment variable. This is expected to crash a lot, and is
+      really meant for developers only. It will likely be removed in a
+      future release. Implements ticket 24583.
+
+  o Minor bugfix (network IPv6 test):
+    - Tor's test scripts now check if "ping -6 ::1" works when the user
+      runs "make test-network-all". Fixes bug 24677; bugfix on
+      0.2.9.3-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (build, rust):
+    - Fix output of autoconf checks to display success messages for Rust
+      dependencies and a suitable rustc compiler version. Fixes bug
+      24612; bugfix on 0.3.1.3-alpha.
+    - When building with Rust on OSX, link against libresolv, to work
+      around the issue at https://github.com/rust-lang/rust/issues/46797.
+      Fixes bug 24652; bugfix on 0.3.1.1-alpha.
+    - Don't pass the --quiet option to cargo: it seems to suppress some
+      errors, which is not what we want to do when building. Fixes bug
+      24518; bugfix on 0.3.1.7.
+    - Build correctly when building from outside Tor's source tree with
+      the TOR_RUST_DEPENDENCIES option set. Fixes bug 22768; bugfix
+      on 0.3.1.7.
+
+  o Minor bugfixes (directory authorities, IPv6):
+    - When creating a routerstatus (vote) from a routerinfo (descriptor),
+      set the IPv6 address to the unspecified IPv6 address, and
+      explicitly initialize the port to zero. Fixes bug 24488; bugfix
+      on 0.2.4.1-alpha.
+
+  o Minor bugfixes (fallback directory mirrors):
+    - Make updateFallbackDirs.py search harder for python. (Some OSs
+      don't put it in /usr/bin.) Fixes bug 24708; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hibernation, bandwidth accounting, shutdown):
+    - When hibernating, close connections normally and allow them to
+      flush. Fixes bug 23571; bugfix on 0.2.4.7-alpha. Also fixes
+      bug 7267.
+    - Do not attempt to launch self-reachability tests when entering
+      hibernation. Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+    - Resolve several bugs related to descriptor fetching on bridge
+      clients with bandwidth accounting enabled. (This combination is
+      not recommended!) Fixes a case of bug 12062; bugfix
+      on 0.2.0.3-alpha.
+    - When hibernating, do not attempt to launch DNS checks. Fixes a
+      case of bug 12062; bugfix on 0.1.2.2-alpha.
+    - When hibernating, do not try to upload or download descriptors.
+      Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (IPv6, bridges):
+    - Tor now always sets IPv6 preferences for bridges. Fixes bug 24573;
+      bugfix on 0.2.8.2-alpha.
+    - Tor now sets IPv6 address in the routerstatus as well as in the
+      router descriptors when updating addresses for a bridge. Closes
+      ticket 24572; bugfix on 0.2.4.5-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - When running with the sandbox enabled, reload configuration files
+      correctly even when %include was used. Previously we would crash.
+      Fixes bug 22605; bugfix on 0.3.1. Patch from Daniel Pinto.
+
+  o Minor bugfixes (memory leaks):
+    - Avoid possible at-exit memory leaks related to use of Libevent's
+      event_base_once() function. (This function tends to leak memory if
+      the event_base is closed before the event fires.) Fixes bug 24584;
+      bugfix on 0.2.8.1-alpha.
+    - Fix a harmless memory leak in tor-resolve. Fixes bug 24582; bugfix
+      on 0.2.1.1-alpha.
+
+  o Minor bugfixes (OSX):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (performance, fragile-hardening):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (performance, timeouts):
+    - Consider circuits for timeout as soon as they complete a hop. This
+      is more accurate than applying the timeout in
+      circuit_expire_building() because that function is only called
+      once per second, which is now too slow for typical timeouts on the
+      current network. Fixes bug 23114; bugfix on 0.2.2.2-alpha.
+    - Use onion service circuits (and other circuits longer than 3 hops)
+      to calculate a circuit build timeout. Previously, Tor only
+      calculated its build timeout based on circuits that planned to be
+      exactly 3 hops long. With this change, we include measurements
+      from all circuits at the point where they complete their third
+      hop. Fixes bug 23100; bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (testing):
+    - Give out Exit flags in bootstrapping networks. Fixes bug 24137;
+      bugfix on 0.2.3.1-alpha.
+    - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug
+      25005; bugfix on 0.3.2.7-rc.
+
+  o Code simplification and refactoring:
+    - Remove /usr/athena from search path in configure.ac. Closes
+      ticket 24363.
+    - Remove duplicate code in node_has_curve25519_onion_key() and
+      node_get_curve25519_onion_key(), and add a check for a zero
+      microdesc curve25519 onion key. Closes ticket 23966, patch by
+      "aruna1234" and teor.
+    - Rewrite channel_rsa_id_group_set_badness to reduce temporary
+      memory allocations with large numbers of OR connections (e.g.
+      relays). Closes ticket 24119.
+    - Separate the function that deletes ephemeral files when Tor
+      stops gracefully.
+    - Small changes to Tor's buf_t API to make it suitable for use as a
+      general-purpose safe string constructor. Closes ticket 22342.
+    - Switch -Wnormalized=id to -Wnormalized=nfkc in configure.ac to
+      avoid source code identifier confusion. Closes ticket 24467.
+    - The tor_git_revision[] constant no longer needs to be redeclared
+      by everything that links against the rest of Tor. Done as part of
+      ticket 23845, to simplify our external API.
+    - We make extend_info_from_node() use node_get_curve25519_onion_key()
+      introduced in ticket 23577 to access the curve25519 public keys
+      rather than accessing it directly. Closes ticket 23760. Patch by
+      Neel Chauhan.
+    - Add a function to log channels' scheduler state changes to aid
+      debugging efforts. Closes ticket 24531.
+
+  o Documentation:
+    - Add documentation on how to build tor with Rust dependencies
+      without having to be online. Closes ticket 22907; bugfix
+      on 0.3.0.3-alpha.
+    - Clarify the behavior of RelayBandwidth{Rate,Burst} with client
+      traffic. Closes ticket 24318.
+    - Document that OutboundBindAddress doesn't apply to DNS requests.
+      Closes ticket 22145. Patch from Aruna Maurya.
+    - Document that operators who run more than one relay or bridge are
+      expected to set MyFamily and ContactInfo correctly. Closes
+      ticket 24526.
+
+  o Code simplification and refactoring (channels):
+    - Remove the incoming and outgoing channel queues. These were never
+      used, but still took up a step in our fast path.
+    - The majority of the channel unit tests have been rewritten and the
+      code coverage has now been raised to 83.6% for channel.c. Closes
+      ticket 23709.
+    - Remove other dead code from the channel subsystem: All together,
+      this cleanup has removed more than 1500 lines of code overall and
+      adding very little except for unit test.
+
+  o Code simplification and refactoring (circuit rendezvous):
+    - Split the client-side rendezvous circuit lookup into two
+      functions: one that returns only established circuits and another
+      that returns all kinds of circuits. Closes ticket 23459.
+
+  o Code simplification and refactoring (controller):
+    - Make most of the variables in networkstatus_getinfo_by_purpose()
+      const. Implements ticket 24489.
+
+
+Changes in version 0.3.2.9 - 2018-01-09
+  Tor 0.3.2.9 is the first stable release in the 0.3.2 series.
+
+  The 0.3.2 series includes our long-anticipated new onion service
+  design, with numerous security features. (For more information, see
+  our blog post at https://blog.torproject.org/fall-harvest.) We also
+  have a new circuit scheduler algorithm for improved performance on
+  relays everywhere (see https://blog.torproject.org/kist-and-tell),
+  along with many smaller features and bugfixes.
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.2.8-rc. For a list of all
+  changes since 0.3.1, see the ReleaseNotes file.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 5 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (compilation):
+    - Resolve a few shadowed-variable warnings in the onion service
+      code. Fixes bug 24634; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (portability, msvc):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.2.8-rc - 2017-12-21
+  Tor 0.3.2.8-rc fixes a pair of bugs in the KIST and KISTLite
+  schedulers that had led servers under heavy load to overload their
+  outgoing connections. All relay operators running earlier 0.3.2.x
+  versions should upgrade. This version also includes a mitigation for
+  over-full DESTROY queues leading to out-of-memory conditions: if it
+  works, we will soon backport it to earlier release series.
+
+  This is the second release candidate in the 0.3.2 series. If we find
+  no new bugs or regression here, then the first stable 0.3.2 release
+  will be nearly identical to this.
+
+  o Major bugfixes (KIST, scheduler):
+    - The KIST scheduler did not correctly account for data already
+      enqueued in each connection's send socket buffer, particularly in
+      cases when the TCP/IP congestion window was reduced between
+      scheduler calls. This situation lead to excessive per-connection
+      buffering in the kernel, and a potential memory DoS. Fixes bug
+      24665; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (hidden service v3):
+    - Bump hsdir_spread_store parameter from 3 to 4 in order to increase
+      the probability of reaching a service for a client missing
+      microdescriptors. Fixes bug 24425; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (scheduler, KIST):
+    - Use a sane write limit for KISTLite when writing onto a connection
+      buffer instead of using INT_MAX and shoving as much as it can.
+      Because the OOM handler cleans up circuit queues, we are better
+      off at keeping them in that queue instead of the connection's
+      buffer. Fixes bug 24671; bugfix on 0.3.2.1-alpha.
+
+
+Changes in version 0.3.2.7-rc - 2017-12-14
+  Tor 0.3.2.7-rc fixes various bugs in earlier versions of Tor,
+  including some that could affect reliability or correctness.
+
+  This is the first release candidate in the 0.3.2 series. If we find no
+  new bugs or regression here, then the first stable 0.3.2. release will
+  be nearly identical to this.
+
+  o Major bugfixes (circuit prediction):
+    - Fix circuit prediction logic so that a client doesn't treat a port
+      as being "handled" by a circuit if that circuit already has
+      isolation settings on it. This change should make Tor clients more
+      responsive by improving their chances of having a pre-created
+      circuit ready for use when a request arrives. Fixes bug 18859;
+      bugfix on 0.2.3.3-alpha.
+
+  o Minor features (logging):
+    - Provide better warnings when the getrandom() syscall fails. Closes
+      ticket 24500.
+
+  o Minor features (portability):
+    - Tor now compiles correctly on arm64 with libseccomp-dev installed.
+      (It doesn't yet work with the sandbox enabled.) Closes
+      ticket 24424.
+
+  o Minor bugfixes (bridge clients, bootstrap):
+    - Retry directory downloads when we get our first bridge descriptor
+      during bootstrap or while reconnecting to the network. Keep
+      retrying every time we get a bridge descriptor, until we have a
+      reachable bridge. Fixes part of bug 24367; bugfix on 0.2.0.3-alpha.
+    - Stop delaying bridge descriptor fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when we
+      have at least one reachable bridge. Fixes part of bug 24367;
+      bugfix on 0.2.0.3-alpha.
+    - Stop delaying directory fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when
+      all our bridges are definitely unreachable. Fixes part of bug
+      24367; bugfix on 0.2.0.3-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (correctness):
+    - Fix several places in our codebase where a C compiler would be
+      likely to eliminate a check, based on assuming that undefined
+      behavior had not happened elsewhere in the code. These cases are
+      usually a sign of redundant checking or dubious arithmetic. Found
+      by Georg Koppen using the "STACK" tool from Wang, Zeldovich,
+      Kaashoek, and Solar-Lezama. Fixes bug 24423; bugfix on various
+      Tor versions.
+
+  o Minor bugfixes (onion service v3):
+    - Fix a race where an onion service would launch a new intro circuit
+      after closing an old one, but fail to register it before freeing
+      the previously closed circuit. This bug was making the service
+      unable to find the established intro circuit and thus not upload
+      its descriptor, thus making a service unavailable for up to 24
+      hours. Fixes bug 23603; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (scheduler, KIST):
+    - Properly set the scheduler state of an unopened channel in the
+      KIST scheduler main loop. This prevents a harmless but annoying
+      log warning. Fixes bug 24502; bugfix on 0.3.2.4-alpha.
+    - Avoid a possible integer overflow when computing the available
+      space on the TCP buffer of a channel. This had no security
+      implications; but could make KIST allow too many cells on a
+      saturated connection. Fixes bug 24590; bugfix on 0.3.2.1-alpha.
+    - Downgrade to "info" a harmless warning about the monotonic time
+      moving backwards: This can happen on platform not supporting
+      monotonic time. Fixes bug 23696; bugfix on 0.3.2.1-alpha.
+
+
+Changes in version 0.3.2.6-alpha - 2017-12-01
+  This version of Tor is the latest in the 0.3.2 alpha series. It
+  includes fixes for several important security issues. All Tor users
+  should upgrade to this release, or to one of the other releases coming
+  out today.
+
+  o Major bugfixes (security):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor feature (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (directory authority):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor bugfixes (client):
+    - By default, do not enable storage of client-side DNS values. These
+      values were unused by default previously, but they should not have
+      been cached at all. Fixes bug 24050; bugfix on 0.2.6.3-alpha.
+
+
+Changes in version 0.3.1.9 - 2017-12-01:
+  Tor 0.3.1.9 backports important security and stability fixes from the
+  0.3.2 development series. All Tor users should upgrade to this
+  release, or to another of the releases coming out today.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (bridge):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.2.5-alpha):
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+
+Changes in version 0.3.0.13 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.9.14 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.8.17 - 2017-12-01
+  Tor 0.2.8.17 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please upgrade with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path through
+      ourselves, even in the case where we have somehow lost the version of
+      our descriptor appearing in the consensus. Fixes part of bug 21534;
+      bugfix on 0.2.0.1-alpha. This issue is also tracked as TROVE-2017-012
+      and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.5.16 - 2017-12-01
+  Tor 0.2.5.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.3.2.5-alpha - 2017-11-22
+  Tor 0.3.2.5-alpha is the fifth alpha release in the 0.3.2.x series. It
+  fixes several stability and reliability bugs, including a fix for
+  intermittent bootstrapping failures that some people have been seeing
+  since the 0.3.0.x series.
+
+  Please test this alpha out -- many of these fixes will soon be
+  backported to stable Tor versions if no additional bugs are found
+  in them.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (directory authority):
+    - Make the "Exit" flag assignment only depend on whether the exit
+      policy allows connections to ports 80 and 443. Previously relays
+      would get the Exit flag if they allowed connections to one of
+      these ports and also port 6667. Resolves ticket 23637.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor features (logging):
+    - Downgrade a pair of log messages that could occur when an exit's
+      resolver gave us an unusual (but not forbidden) response. Closes
+      ticket 24097.
+    - Improve the message we log when re-enabling circuit build timeouts
+      after having received a consensus. Closes ticket 20963.
+
+  o Minor bugfixes (compilation):
+    - Fix a memory leak warning in one of the libevent-related
+      configuration tests that could occur when manually specifying
+      -fsanitize=address. Fixes bug 24279; bugfix on 0.3.0.2-alpha.
+      Found and patched by Alex Xu.
+    - When detecting OpenSSL on Windows from our configure script, make
+      sure to try linking with the ws2_32 library. Fixes bug 23783;
+      bugfix on 0.3.2.2-alpha.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (control port, onion services):
+    - Report "FAILED" instead of "UPLOAD_FAILED" "FAILED" for the
+      HS_DESC event when a service is not able to upload a descriptor.
+      Fixes bug 24230; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (directory cache):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Only log once if we notice that KIST support is gone. Fixes bug
+      24158; bugfix on 0.3.2.1-alpha.
+    - Suppress a log notice when relay descriptors arrive. We already
+      have a bootstrap progress for this so no need to log notice
+      everytime tor receives relay descriptors. Microdescriptors behave
+      the same. Fixes bug 23861; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (onion services):
+    - The introduction circuit was being timed out too quickly while
+      waiting for the rendezvous circuit to complete. Keep the intro
+      circuit around longer instead of timing out and reopening new ones
+      constantly. Fixes bug 23681; bugfix on 0.2.4.8-alpha.
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+    - Silence a warning about failed v3 onion descriptor uploads that
+      can happen naturally under certain edge cases. Fixes part of bug
+      23662; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (tests):
+    - Fix a memory leak in one of the bridge-distribution test cases.
+      Fixes bug 24345; bugfix on 0.3.2.3-alpha.
+    - Fix a bug in our fuzzing mock replacement for crypto_pk_checksig(),
+      to correctly handle cases where a caller gives it an RSA key of
+      under 160 bits. (This is not actually a bug in Tor itself, but
+      rather in our fuzzing code.) Fixes bug 24247; bugfix on
+      0.3.0.3-alpha. Found by OSS-Fuzz as issue 4177.
+
+  o Documentation:
+    - Add notes in man page regarding OS support for the various
+      scheduler types. Attempt to use less jargon in the scheduler
+      section. Closes ticket 24254.
+
+
+Changes in version 0.3.2.4-alpha - 2017-11-08
+  Tor 0.3.2.4-alpha is the fourth alpha release in the 0.3.2.x series.
+  It fixes several stability and reliability bugs, especially including
+  a major reliability issue that has been plaguing fast exit relays in
+  recent months.
+
+  o Major bugfixes (exit relays, DNS):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (scheduler, channel):
+    - Stop processing scheduled channels if they closed while flushing
+      cells. This can happen if the write on the connection fails
+      leading to the channel being closed while in the scheduler loop.
+      Fixes bug 23751; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (logging, scheduler):
+    - Introduce a SCHED_BUG() function to log extra information about
+      the scheduler state if we ever catch a bug in the scheduler.
+      Closes ticket 23753.
+
+  o Minor features (removed deprecations):
+    - The ClientDNSRejectInternalAddresses flag can once again be set in
+      non-testing Tor networks, so long as they do not use the default
+      directory authorities. This change also removes the deprecation of
+      this flag from 0.2.9.2-alpha. Closes ticket 21031.
+
+  o Minor features (testing):
+    - Our fuzzing tests now test the encrypted portions of v3 onion
+      service descriptors. Implements more of 21509.
+
+  o Minor bugfixes (directory client):
+    - On failure to download directory information, delay retry attempts
+      by a random amount based on the "decorrelated jitter" algorithm.
+      Our previous delay algorithm tended to produce extra-long delays
+      too easily. Fixes bug 23816; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (IPv6, v3 single onion services):
+    - Remove buggy code for IPv6-only v3 single onion services, and
+      reject attempts to configure them. This release supports IPv4,
+      dual-stack, and IPv6-only v3 onion services; and IPv4 and dual-
+      stack v3 single onion services. Fixes bug 23820; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, relay):
+    - Give only a protocol warning when the ed25519 key is not
+      consistent between the descriptor and microdescriptor of a relay.
+      This can happen, for instance, if the relay has been flagged
+      NoEdConsensus. Fixes bug 24025; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (manpage, onion service):
+    - Document that the HiddenServiceNumIntroductionPoints option is
+      0-10 for v2 services and 0-20 for v3 services. Fixes bug 24115;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a minor memory leak at exit in the KIST scheduler. This bug
+      should have no user-visible impact. Fixes bug 23774; bugfix
+      on 0.3.2.1-alpha.
+    - Fix a memory leak when decrypting a badly formatted v3 onion
+      service descriptor. Fixes bug 24150; bugfix on 0.3.2.1-alpha.
+      Found by OSS-Fuzz; this is OSS-Fuzz issue 3994.
+
+  o Minor bugfixes (onion services):
+    - Cache some needed onion service client information instead of
+      constantly computing it over and over again. Fixes bug 23623;
+      bugfix on 0.3.2.1-alpha.
+    - Properly retry HSv3 descriptor fetches when missing required
+      directory information. Fixes bug 23762; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (relay, crash):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix a spurious fuzzing-only use of an uninitialized value. Found
+      by Brian Carpenter. Fixes bug 24082; bugfix on 0.3.0.3-alpha.
+    - Test that IPv6-only clients can use microdescriptors when running
+      "make test-network-all". Requires chutney master 61c28b9 or later.
+      Closes ticket 24109.
+
+
+Changes in version 0.3.2.3-alpha - 2017-10-27
+  Tor 0.3.2.3-alpha is the third release in the 0.3.2 series. It fixes
+  numerous small bugs in earlier versions of 0.3.2.x, and adds a new
+  directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Minor features (bridge):
+    - Bridge relays can now set the BridgeDistribution config option to
+      add a "bridge-distribution-request" line to their bridge
+      descriptor, which tells BridgeDB how they'd like their bridge
+      address to be given out. (Note that as of Oct 2017, BridgeDB does
+      not yet implement this feature.) As a side benefit, this feature
+      provides a way to distinguish bridge descriptors from non-bridge
+      descriptors. Implements tickets 18329.
+
+  o Minor features (client, entry guards):
+    - Improve log messages when missing descriptors for primary guards.
+      Resolves ticket 23670.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bridge):
+    - Overwrite the bridge address earlier in the process of retrieving
+      its descriptor, to make sure we reach it on the configured
+      address. Fixes bug 20532; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document better how to read gcov, and what our gcov postprocessing
+      scripts do. Fixes bug 23739; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (entry guards):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service client):
+    - When handling multiple SOCKS request for the same .onion address,
+      only fetch the service descriptor once.
+    - When a descriptor fetch fails with a non-recoverable error, close
+      all pending SOCKS requests for that .onion. Fixes bug 23653;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Always regenerate missing hidden service public key files. Prior
+      to this, if the public key was deleted from disk, it wouldn't get
+      recreated. Fixes bug 23748; bugfix on 0.3.2.2-alpha. Patch
+      from "cathugger".
+    - Make sure that we have a usable ed25519 key when the intro point
+      relay supports ed25519 link authentication. Fixes bug 24002;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (hidden service, v2):
+    - When reloading configured hidden services, copy all information
+      from the old service object. Previously, some data was omitted,
+      causing delays in descriptor upload, and other bugs. Fixes bug
+      23790; bugfix on 0.2.1.9-alpha.
+
+  o Minor bugfixes (memory safety, defensive programming):
+    - Clear the target address when node_get_prim_orport() returns
+      early. Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a BUG warning when receiving a dubious CREATE cell while an
+      option transition is in progress. Fixes bug 23952; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Adjust the GitLab CI configuration to more closely match that of
+      Travis CI. Fixes bug 23757; bugfix on 0.3.2.2-alpha.
+    - Prevent scripts/test/coverage from attempting to move gcov output
+      to the root directory. Fixes bug 23741; bugfix on 0.2.5.1-alpha.
+    - When running unit tests as root, skip a test that would fail
+      because it expects a permissions error. This affects some
+      continuous integration setups. Fixes bug 23758; bugfix
+      on 0.3.2.2-alpha.
+    - Stop unconditionally mirroring the tor repository in GitLab CI.
+      This prevented developers from enabling GitLab CI on master. Fixes
+      bug 23755; bugfix on 0.3.2.2-alpha.
+    - Fix the hidden service v3 descriptor decoding fuzzing to use the
+      latest decoding API correctly. Fixes bug 21509; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (warnings):
+    - When we get an HTTP request on a SOCKS port, tell the user about
+      the new HTTPTunnelPort option. Previously, we would give a "Tor is
+      not an HTTP Proxy" message, which stopped being true when
+      HTTPTunnelPort was introduced. Fixes bug 23678; bugfix
+      on 0.3.2.1-alpha.
+
+
+Changes in version 0.2.5.15 - 2017-10-25
+  Tor 0.2.5.15 backports a collection of bugfixes from later Tor release
+  series. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.8.16 - 2017-10-25
+  Tor 0.2.8.16 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.13 - 2017-10-25
+  Tor 0.2.9.13 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.0.12 - 2017-10-25
+  Tor 0.3.0.12 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.1.8 - 2017-10-25
+  Tor 0.3.1.8 is the second stable release in the 0.3.1 series.
+  It includes several bugfixes, including a bugfix for a crash issue
+  that had affected relays under memory pressure. It also adds
+  a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.2-alpha):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+
+  o Minor bugfixes (compression, backport from 0.3.2.2-alpha):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.2.1-alpha):
+    - Remove the length limit on HTTP status lines that authorities can
+      send in their replies. Fixes bug 23499; bugfix on 0.3.1.6-rc.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.2.2-alpha):
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+
+
+Changes in version 0.3.2.2-alpha - 2017-09-29
+  Tor 0.3.2.2-alpha is the second release in the 0.3.2 series. This
+  release fixes several minor bugs in the new scheduler and next-
+  generation onion services; both features were newly added in the 0.3.2
+  series. Other fixes in this alpha include several fixes for non-fatal
+  tracebacks which would appear in logs.
+
+  With the aim to stabilise the 0.3.2 series by 15 December 2017, this
+  alpha does not contain any substantial new features. Minor features
+  include better testing and logging.
+
+  The following comprises the complete list of changes included
+  in 0.3.2.2-alpha:
+
+  o Major bugfixes (relay, crash, assertion failure):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Major bugfixes (scheduler):
+    - If a channel is put into the scheduler's pending list, then it
+      starts closing, and then if the scheduler runs before it finishes
+      closing, the scheduler will get stuck trying to flush its cells
+      while the lower layers refuse to cooperate. Fix that race
+      condition by giving the scheduler an escape method. Fixes bug
+      23676; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (build, compilation):
+    - The "check-changes" feature is now part of the "make check" tests;
+      we'll use it to try to prevent misformed changes files from
+      accumulating. Closes ticket 23564.
+    - Tor builds should now fail if there are any mismatches between the
+      C type representing a configuration variable and the C type the
+      data-driven parser uses to store a value there. Previously, we
+      needed to check these by hand, which sometimes led to mistakes.
+      Closes ticket 23643.
+
+  o Minor features (directory authorities):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (hidden service, circuit, logging):
+    - Improve logging of many callsite in the circuit subsystem to print
+      the circuit identifier(s).
+    - Log when we cleanup an intro point from a service so we know when
+      and for what reason it happened. Closes ticket 23604.
+
+  o Minor features (logging):
+    - Log more circuit information whenever we are about to try to
+      package a relay cell on a circuit with a nonexistent n_chan.
+      Attempt to diagnose ticket 8185.
+    - Improve info-level log identification of particular circuits, to
+      help with debugging. Closes ticket 23645.
+
+  o Minor features (relay):
+    - When choosing which circuits can be expired as unused, consider
+      circuits from clients even if those clients used regular CREATE
+      cells to make them; and do not consider circuits from relays even
+      if they were made with CREATE_FAST. Part of ticket 22805.
+
+  o Minor features (robustness):
+    - Change several fatal assertions when flushing buffers into non-
+      fatal assertions, to prevent any recurrence of 23690.
+
+  o Minor features (spec conformance, bridge, diagnostic):
+    - When handling the USERADDR command on an ExtOrPort, warn when the
+      transports provides a USERADDR with no port. In a future version,
+      USERADDR commands of this format may be rejected. Detects problems
+      related to ticket 23080.
+
+  o Minor features (testing):
+    - Add a unit test to make sure that our own generated platform
+      string will be accepted by directory authorities. Closes
+      ticket 22109.
+
+  o Minor bugfixes (bootstrapping):
+    - When warning about state file clock skew, report the correct
+      direction for the detected skew. Fixes bug 23606; bugfix
+      on 0.2.8.1-alpha.
+    - Avoid an assertion failure when logging a state file clock skew
+      very early in bootstrapping. Fixes bug 23607; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (build, compilation):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+    - When searching for OpenSSL, don't accept any OpenSSL library that
+      lacks TLSv1_1_method(): Tor doesn't build with those versions.
+      Additionally, look in /usr/local/opt/openssl, if it's present.
+      These changes together repair the default build on OSX systems
+      with Homebrew installed. Fixes bug 23602; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (compression):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Fix manpage to not refer to the obsolete (and misspelled)
+      UseEntryGuardsAsDirectoryGuards parameter in the description of
+      NumDirectoryGuards. Fixes bug 23611; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (hidden service v3):
+    - Don't log an assertion failure when we can't find the right
+      information to extend to an introduction point. In rare cases,
+      this could happen, causing a warning, even though tor would
+      recover gracefully. Fixes bug 23159; bugfix on 0.3.2.1-alpha.
+    - Pad RENDEZVOUS cell up to the size of the legacy cell which is
+      much bigger so the rendezvous point can't distinguish which hidden
+      service protocol is being used. Fixes bug 23420; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (hidden service, relay):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (scheduler):
+    - When switching schedulers due to a consensus change, we didn't
+      give the new scheduler a chance to react to the consensus. Fix
+      that. Fixes bug 23537; bugfix on 0.3.2.1-alpha.
+    - Make the KISTSchedRunInterval option a non negative value. With
+      this, the way to disable KIST through the consensus is to set it
+      to 0. Fixes bug 23539; bugfix on 0.3.2.1-alpha.
+    - Only notice log the selected scheduler when we switch scheduler
+      types. Fixes bug 23552; bugfix on 0.3.2.1-alpha.
+    - Avoid a compilation warning on macOS in scheduler_ev_add() caused
+      by a different tv_usec data type. Fixes bug 23575; bugfix
+      on 0.3.2.1-alpha.
+    - Make a hard exit if tor is unable to pick a scheduler which can
+      happen if the user specifies a scheduler type that is not
+      supported and not other types in Schedulers. Fixes bug 23581;
+      bugfix on 0.3.2.1-alpha.
+    - Properly initialize the scheduler last run time counter so it is
+      not 0 at the first tick. Fixes bug 23696; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Capture and detect several "Result does not fit" warnings in unit
+      tests on platforms with 32-bit time_t. Fixes bug 21800; bugfix
+      on 0.2.9.3-alpha.
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+    - The removal of some old scheduler options caused some tests to
+      fail on BSD systems. Assume current behavior is correct and make
+      the tests pass again. Fixes bug 23566; bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove various ways of testing circuits and connections for
+      "clientness"; instead, favor channel_is_client(). Part of
+      ticket 22805.
+
+  o Deprecated features:
+    - The ReachableDirAddresses and ClientPreferIPv6DirPort options are
+      now deprecated; they do not apply to relays, and they have had no
+      effect on clients since 0.2.8.x. Closes ticket 19704.
+
+  o Documentation:
+    - HiddenServiceVersion man page entry wasn't mentioning the now
+      supported version 3. Fixes ticket 23580; bugfix on 0.3.2.1-alpha.
+    - Clarify that the Address option is entirely about setting an
+      advertised IPv4 address. Closes ticket 18891.
+    - Clarify the manpage's use of the term "address" to clarify what
+      kind of address is intended. Closes ticket 21405.
+    - Document that onion service subdomains are allowed, and ignored.
+      Closes ticket 18736.
+
+
+Changes in version 0.3.2.1-alpha - 2017-09-18
+  Tor 0.3.2.1-alpha is the first release in the 0.3.2.x series. It
+  includes support for our next-generation ("v3") onion service
+  protocol, and adds a new circuit scheduler for more responsive
+  forwarding decisions from relays. There are also numerous other small
+  features and bugfixes here.
+
+  Below are the changes since Tor 0.3.1.7.
+
+  o Major feature (scheduler, channel):
+    - Tor now uses new schedulers to decide which circuits should
+      deliver cells first, in order to improve congestion at relays. The
+      first type is called "KIST" ("Kernel Informed Socket Transport"),
+      and is only available on Linux-like systems: it uses feedback from
+      the kernel to prevent the kernel's TCP buffers from growing too
+      full. The second new scheduler type is called "KISTLite": it
+      behaves the same as KIST, but runs on systems without kernel
+      support for inspecting TCP implementation details. The old
+      scheduler is still available, under the name "Vanilla". To change
+      the default scheduler preference order, use the new "Schedulers"
+      option. (The default preference order is "KIST,KISTLite,Vanilla".)
+
+      Matt Traudt implemented KIST, based on research by Rob Jansen,
+      John Geddes, Christ Wacek, Micah Sherr, and Paul Syverson. For
+      more information, see the design paper at
+      http://www.robgjansen.com/publications/kist-sec2014.pdf and the
+      followup implementation paper at https://arxiv.org/abs/1709.01044.
+      Closes ticket 12541.
+
+  o Major features (next-generation onion services):
+    - Tor now supports the next-generation onion services protocol for
+      clients and services! As part of this release, the core of
+      proposal 224 has been implemented and is available for
+      experimentation and testing by our users. This newer version of
+      onion services ("v3") features many improvements over the legacy
+      system, including:
+
+      a) Better crypto (replaced SHA1/DH/RSA1024
+      with SHA3/ed25519/curve25519)
+
+      b) Improved directory protocol, leaking much less information to
+      directory servers.
+
+      c) Improved directory protocol, with smaller surface for
+      targeted attacks.
+
+      d) Better onion address security against impersonation.
+
+      e) More extensible introduction/rendezvous protocol.
+
+      f) A cleaner and more modular codebase.
+
+      You can identify a next-generation onion address by its length:
+      they are 56 characters long, as in
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad.onion".
+
+      In the future, we will release more options and features for v3
+      onion services, but we first need a testing period, so that the
+      current codebase matures and becomes more robust. Planned features
+      include: offline keys, advanced client authorization, improved
+      guard algorithms, and statistics. For full details, see
+      proposal 224.
+
+      Legacy ("v2") onion services will still work for the foreseeable
+      future, and will remain the default until this new codebase gets
+      tested and hardened. Service operators who want to experiment with
+      the new system can use the 'HiddenServiceVersion 3' torrc
+      directive along with the regular onion service configuration
+      options. We will publish a blog post about this new feature
+      soon! Enjoy!
+
+  o Major bugfixes (usability, control port):
+    - Report trusted clock skew indications as bootstrap errors, so
+      controllers can more easily alert users when their clocks are
+      wrong. Fixes bug 23506; bugfix on 0.1.2.6-alpha.
+
+  o Minor features (bug detection):
+    - Log a warning message with a stack trace for any attempt to call
+      get_options() during option validation. This pattern has caused
+      subtle bugs in the past. Closes ticket 22281.
+
+  o Minor features (client):
+    - You can now use Tor as a tunneled HTTP proxy: use the new
+      HTTPTunnelPort option to open a port that accepts HTTP CONNECT
+      requests. Closes ticket 22407.
+    - Add an extra check to make sure that we always use the newer guard
+      selection code for picking our guards. Closes ticket 22779.
+    - When downloading (micro)descriptors, don't split the list into
+      multiple requests unless we want at least 32 descriptors.
+      Previously, we split at 4, not 32, which led to significant
+      overhead in HTTP request size and degradation in compression
+      performance. Closes ticket 23220.
+
+  o Minor features (command line):
+    - Add a new commandline option, --key-expiration, which prints when
+      the current signing key is going to expire. Implements ticket
+      17639; patch by Isis Lovecruft.
+
+  o Minor features (control port):
+    - If an application tries to use the control port as an HTTP proxy,
+      respond with a meaningful "This is the Tor control port" message,
+      and log the event. Closes ticket 1667. Patch from Ravi
+      Chandra Padmala.
+    - Provide better error message for GETINFO desc/(id|name) when not
+      fetching router descriptors. Closes ticket 5847. Patch by
+      Kevin Butler.
+    - Add GETINFO "{desc,md}/download-enabled", to inform the controller
+      whether Tor will try to download router descriptors and
+      microdescriptors respectively. Closes ticket 22684.
+    - Added new GETINFO targets "ip-to-country/{ipv4,ipv6}-available",
+      so controllers can tell whether the geoip databases are loaded.
+      Closes ticket 23237.
+    - Adds a timestamp field to the CIRC_BW and STREAM_BW bandwidth
+      events. Closes ticket 19254. Patch by "DonnchaC".
+
+  o Minor features (development support):
+    - Developers can now generate a call-graph for Tor using the
+      "calltool" python program, which post-processes object dumps. It
+      should work okay on many Linux and OSX platforms, and might work
+      elsewhere too. To run it, install calltool from
+      https://gitweb.torproject.org/user/nickm/calltool.git and run
+      "make callgraph". Closes ticket 19307.
+
+  o Minor features (ed25519):
+    - Add validation function to checks for torsion components in
+      ed25519 public keys, used by prop224 client-side code. Closes
+      ticket 22006. Math help by Ian Goldberg.
+
+  o Minor features (exit relay, DNS):
+    - Improve the clarity and safety of the log message from evdns when
+      receiving an apparently spoofed DNS reply. Closes ticket 3056.
+
+  o Minor features (integration, hardening):
+    - Add a new NoExec option to prevent Tor from running other
+      programs. When this option is set to 1, Tor will never try to run
+      another program, regardless of the settings of
+      PortForwardingHelper, ClientTransportPlugin, or
+      ServerTransportPlugin. Once NoExec is set, it cannot be disabled
+      without restarting Tor. Closes ticket 22976.
+
+  o Minor features (logging):
+    - Improve the warning message for specifying a relay by nickname.
+      The previous message implied that nickname registration was still
+      part of the Tor network design, which it isn't. Closes
+      ticket 20488.
+    - If the sandbox filter fails to load, suggest to the user that
+      their kernel might not support seccomp2. Closes ticket 23090.
+
+  o Minor features (portability):
+    - Check at configure time whether uint8_t is the same type as
+      unsigned char. Lots of existing code already makes this
+      assumption, and there could be strict aliasing issues if the
+      assumption is violated. Closes ticket 22410.
+
+  o Minor features (relay, configuration):
+    - Reject attempts to use relative file paths when RunAsDaemon is
+      set. Previously, Tor would accept these, but the directory-
+      changing step of RunAsDaemon would give strange and/or confusing
+      results. Closes ticket 22731.
+
+  o Minor features (startup, safety):
+    - When configured to write a PID file, Tor now exits if it is unable
+      to do so. Previously, it would warn and continue. Closes
+      ticket 20119.
+
+  o Minor features (static analysis):
+    - The BUG() macro has been changed slightly so that Coverity no
+      longer complains about dead code if the bug is impossible. Closes
+      ticket 23054.
+
+  o Minor features (testing):
+    - The default chutney network tests now include tests for the v3
+      hidden service design. Make sure you have the latest version of
+      chutney if you want to run these. Closes ticket 22437.
+    - Add a unit test to verify that we can parse a hardcoded v2 hidden
+      service descriptor. Closes ticket 15554.
+
+  o Minor bugfixes (certificate handling):
+    - Fix a time handling bug in Tor certificates set to expire after
+      the year 2106. Fixes bug 23055; bugfix on 0.3.0.1-alpha. Found by
+      Coverity as CID 1415728.
+
+  o Minor bugfixes (client, usability):
+    - Refrain from needlessly rejecting SOCKS5-with-hostnames and
+      SOCKS4a requests that contain IP address strings, even when
+      SafeSocks in enabled, as this prevents user from connecting to
+      known IP addresses without relying on DNS for resolving. SafeSocks
+      still rejects SOCKS connections that connect to IP addresses when
+      those addresses are _not_ encoded as hostnames. Fixes bug 22461;
+      bugfix on Tor 0.2.6.2-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Call htons() in extend_cell_format() for encoding a 16-bit value.
+      Previously we used ntohs(), which happens to behave the same on
+      all the platforms we support, but which isn't really correct.
+      Fixes bug 23106; bugfix on 0.2.4.8-alpha.
+    - For defense-in-depth, make the controller's write_escaped_data()
+      function robust to extremely long inputs. Fixes bug 19281; bugfix
+      on 0.1.1.1-alpha. Reported by Guido Vranken.
+
+  o Minor bugfixes (compilation):
+    - Fix unused-variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (consensus expiry):
+    - Check for adequate directory information correctly. Previously, Tor
+      would reconsider whether it had sufficient directory information
+      every 2 minutes. Fixes bug 23091; bugfix on 0.2.0.19-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Directory servers now include a "Date:" http header for response
+      codes other than 200. Clients starting with a skewed clock and a
+      recent consensus were getting "304 Not modified" responses from
+      directory authorities, so without the Date header, the client
+      would never hear about a wrong clock. Fixes bug 23499; bugfix
+      on 0.0.8rc1.
+    - Make clients wait for 6 seconds before trying to download a
+      consensus from an authority. Fixes bug 17750; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (DoS-resistance):
+    - If future code asks if there are any running bridges, without
+      checking if bridges are enabled, log a BUG warning rather than
+      crashing. Fixes bug 23524; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (format strictness):
+    - Restrict several data formats to decimal. Previously, the
+      BuildTimeHistogram entries in the state file, the "bw=" entries in
+      the bandwidth authority file, and the process IDs passed to the
+      __OwningControllerProcess option could all be specified in hex or
+      octal as well as in decimal. This was not an intentional feature.
+      Fixes bug 22802; bugfixes on 0.2.2.1-alpha, 0.2.2.2-alpha,
+      and 0.2.2.28-beta.
+
+  o Minor bugfixes (heartbeat):
+    - If we fail to write a heartbeat message, schedule a retry for the
+      minimum heartbeat interval number of seconds in the future. Fixes
+      bug 19476; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, logging):
+    - Fix some messages on unexpected errors from the seccomp2 library.
+      Fixes bug 22750; bugfix on 0.2.5.1-alpha. Patch from "cypherpunks".
+
+  o Minor bugfixes (logging):
+    - Remove duplicate log messages regarding opening non-local
+      SocksPorts upon parsing config and opening listeners at startup.
+      Fixes bug 4019; bugfix on 0.2.3.3-alpha.
+    - Use a more comprehensible log message when telling the user
+      they've excluded every running exit node. Fixes bug 7890; bugfix
+      on 0.2.2.25-alpha.
+    - When logging the number of descriptors we intend to download per
+      directory request, do not log a number higher than then the number
+      of descriptors we're fetching in total. Fixes bug 19648; bugfix
+      on 0.1.1.8-alpha.
+    - When warning about a directory owned by the wrong user, log the
+      actual name of the user owning the directory. Previously, we'd log
+      the name of the process owner twice. Fixes bug 23487; bugfix
+      on 0.2.9.1-alpha.
+    - The tor specification says hop counts are 1-based, so fix two log
+      messages that mistakenly logged 0-based hop counts. Fixes bug
+      18982; bugfix on 0.2.6.2-alpha and 0.2.4.5-alpha. Patch by teor.
+      Credit to Xiaofan Li for reporting this issue.
+
+  o Minor bugfixes (portability):
+    - Stop using the PATH_MAX variable, which is not defined on GNU
+      Hurd. Fixes bug 23098; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (relay):
+    - When uploading our descriptor for the first time after startup,
+      report the reason for uploading as "Tor just started" rather than
+      leaving it blank. Fixes bug 22885; bugfix on 0.2.3.4-alpha.
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (tests):
+    - Fix a broken unit test for the OutboundAddress option: the parsing
+      function was never returning an error on failure. Fixes bug 23366;
+      bugfix on 0.3.0.3-alpha.
+    - Fix a signed-integer overflow in the unit tests for
+      dir/download_status_random_backoff, which was untriggered until we
+      fixed bug 17750. Fixes bug 22924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (usability, control port):
+    - Stop making an unnecessary routerlist check in NETINFO clock skew
+      detection; this was preventing clients from reporting NETINFO clock
+      skew to controllers. Fixes bug 23532; bugfix on 0.2.4.4-alpha.
+
+  o Code simplification and refactoring:
+    - Extract the code for handling newly-open channels into a separate
+      function from the general code to handle channel state
+      transitions. This change simplifies our callgraph, reducing the
+      size of the largest strongly connected component by roughly a
+      factor of two. Closes ticket 22608.
+    - Remove dead code for largely unused statistics on the number of
+      times we've attempted various public key operations. Fixes bug
+      19871; bugfix on 0.1.2.4-alpha. Fix by Isis Lovecruft.
+    - Remove several now-obsolete functions for asking about old
+      variants directory authority status. Closes ticket 22311; patch
+      from "huyvq".
+    - Remove some of the code that once supported "Named" and "Unnamed"
+      routers. Authorities no longer vote for these flags. Closes
+      ticket 22215.
+    - Rename the obsolete malleable hybrid_encrypt functions used in TAP
+      and old hidden services, to indicate that they aren't suitable for
+      new protocols or formats. Closes ticket 23026.
+    - Replace our STRUCT_OFFSET() macro with offsetof(). Closes ticket
+      22521. Patch from Neel Chauhan.
+    - Split the enormous circuit_send_next_onion_skin() function into
+      multiple subfunctions. Closes ticket 22804.
+    - Split the portions of the buffer.c module that handle particular
+      protocols into separate modules. Part of ticket 23149.
+    - Use our test macros more consistently, to produce more useful
+      error messages when our unit tests fail. Add coccinelle patches to
+      allow us to re-check for test macro uses. Closes ticket 22497.
+
+  o Deprecated features:
+    - Deprecate HTTPProxy/HTTPProxyAuthenticator config options. They
+      only applies to direct unencrypted HTTP connections to your
+      directory server, which your Tor probably isn't using. Closes
+      ticket 20575.
+
+  o Documentation:
+    - Clarify in the manual that "Sandbox 1" is only supported on Linux
+      kernels. Closes ticket 22677.
+    - Document all values of PublishServerDescriptor in the manpage.
+      Closes ticket 15645.
+    - Improve the documentation for the directory port part of the
+      DirAuthority line. Closes ticket 20152.
+    - Restore documentation for the authorities' "approved-routers"
+      file. Closes ticket 21148.
+
+  o Removed features:
+    - The AllowDotExit option has been removed as unsafe. It has been
+      deprecated since 0.2.9.2-alpha. Closes ticket 23426.
+    - The ClientDNSRejectInternalAddresses flag can no longer be set on
+      non-testing networks. It has been deprecated since 0.2.9.2-alpha.
+      Closes ticket 21031.
+    - The controller API no longer includes an AUTHDIR_NEWDESCS event:
+      nobody was using it any longer. Closes ticket 22377.
+
+
+Changes in version 0.2.8.15 - 2017-09-18
+  Tor 0.2.8.15 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Note that Tor 0.2.8.x will no longer be supported after 1 Jan
+  2018.  We suggest that you upgrade to the latest stable release if
+  possible.  If you can't, we recommend that you upgrade at least to
+  0.2.9, which will be supported until 2020.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.9.12 - 2017-09-18
+  Tor 0.2.9.12 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Major features (security, backport from 0.3.0.2-alpha):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (crash, directory connections, backport from 0.3.0.5-rc):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (DNS, backport from 0.3.0.2-alpha):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (code style, backport from 0.3.1.3-alpha):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.1-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt support
+      on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions the mingw C header files, avoid
+      float-conversion warnings when calling the C functions isfinite(),
+      isnan(), and signbit(). Fixes bug 22801; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for running tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an
+      empty body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, backport from 0.3.0.5-rc):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.3.0.11 - 2017-09-18
+  Tor 0.3.0.11 backports a collection of bugfixes from Tor the 0.3.1
+  series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Minor features (code style, backport from 0.3.1.7):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (logging, relay, backport from 0.3.1.6-rc):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+
+Changes in version 0.3.1.7 - 2017-09-18
+  Tor 0.3.1.7 is the first stable release in the 0.3.1 series.
+
+  With the 0.3.1 series, Tor now serves and downloads directory
+  information in more compact formats, to save on bandwidth overhead. It
+  also contains a new padding system to resist netflow-based traffic
+  analysis, and experimental support for building parts of Tor in Rust
+  (though no parts of Tor are in Rust yet). There are also numerous
+  small features, bugfixes on earlier release series, and groundwork for
+  the hidden services revamp of 0.3.2.
+
+  This release also includes a fix for TROVE-2017-008, a security bug
+  that affects hidden services running with the SafeLogging option
+  disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.1.6-rc. For a list of all
+  changes since 0.3.0, see the ReleaseNotes file.
+
+  o Major bugfixes (security, hidden services, loggging):
+    - Fix a bug where we could log uninitialized stack when a certain
+      hidden service error occurred while SafeLogging was disabled.
+      Fixes bug #23490; bugfix on 0.2.7.2-alpha. This is also tracked as
+      TROVE-2017-008 and CVE-2017-0380.
+
+  o Minor features (defensive programming):
+    - Create a pair of consensus parameters, nf_pad_tor2web and
+      nf_pad_single_onion, to disable netflow padding in the consensus
+      for non-anonymous connections in case the overhead is high. Closes
+      ticket 17857.
+
+  o Minor features (diagnostic):
+    - Add a stack trace to the bug warnings that can be logged when
+      trying to send an outgoing relay cell with n_chan == 0. Diagnostic
+      attempt for bug 23105.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (controller):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Inform the geoip and rephist modules about all requests, even on
+      relays that are only fetching microdescriptors. Fixes a bug
+      related to 21585; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Fix a channelpadding unit test failure on slow systems by using
+      mocked time instead of actual time. Fixes bug 23077; bugfix
+      on 0.3.1.1-alpha.
+
+
+Changes in version 0.3.1.6-rc - 2017-09-05
+  Tor 0.3.1.6-rc fixes a few small bugs and annoyances in the 0.3.1
+  release series, including a bug that produced weird behavior on
+  Windows directory caches.
+
+  This is the first release candidate in the Tor 0.3.1 series. If we
+  find no new bugs or regressions here, the first stable 0.3.1 release
+  will be nearly identical to it.
+
+  o Major bugfixes (windows, directory cache):
+    - On Windows, do not try to delete cached consensus documents and
+      diffs before they are unmapped from memory--Windows won't allow
+      that. Instead, allow the consensus cache directory to grow larger,
+      to hold files that might need to stay around longer. Fixes bug
+      22752; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (directory authority):
+    - Improve the message that authorities report to relays that present
+      RSA/Ed25519 keypairs that conflict with previously pinned keys.
+      Closes ticket 22348.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 3 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (testing):
+    - Add more tests for compression backend initialization. Closes
+      ticket 22286.
+
+  o Minor bugfixes (directory cache):
+    - Fix a memory leak when recovering space in the consensus cache.
+      Fixes bug 23139; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Increase the number of circuits that a service is allowed to
+      open over a specific period of time. The value was lower than it
+      should be (8 vs 12) in the normal case of 3 introduction points.
+      Fixes bug 22159; bugfix on 0.3.0.5-rc.
+    - Fix a BUG warning during HSv3 descriptor decoding that could be
+      cause by a specially crafted descriptor. Fixes bug 23233; bugfix
+      on 0.3.0.1-alpha. Bug found by "haxxpop".
+    - Rate-limit the log messages if we exceed the maximum number of
+      allowed intro circuits. Fixes bug 22159; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging, relay):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+  o Minor bugfixes (relay):
+    - When a relay is not running as a directory cache, it will no
+      longer generate compressed consensuses and consensus diff
+      information. Previously, this was a waste of disk and CPU. Fixes
+      bug 23275; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (robustness, error handling):
+    - Improve our handling of the cases where OpenSSL encounters a
+      memory error while encoding keys and certificates. We haven't
+      observed these errors in the wild, but if they do happen, we now
+      detect and respond better. Fixes bug 19418; bugfix on all versions
+      of Tor. Reported by Guido Vranken.
+
+  o Minor bugfixes (stability):
+    - Avoid crashing on a double-free when unable to load or process an
+      included file. Fixes bug 23155; bugfix on 0.3.1.1-alpha. Found
+      with the clang static analyzer.
+
+  o Minor bugfixes (testing):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+    - Port the hs_ntor handshake test to work correctly with recent
+      versions of the pysha3 module. Fixes bug 23071; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (Windows service):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.0.10 - 2017-08-02
+   Tor 0.3.0.10 backports a collection of small-to-medium bugfixes
+   from the current Tor alpha series. OpenBSD users and TPROXY users
+   should upgrade; others are probably okay sticking with 0.3.0.9.
+
+  o Major features (build system, continuous integration, backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xbar" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (backport from 0.3.1.5-alpha):
+    - Update geoip and geoip6 to the July 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.2-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation warnings, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.1-alpha):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.1.5-alpha)
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.1.5-alpha - 2017-08-01
+  Tor 0.3.1.5-alpha improves the performance of consensus diff
+  calculation, fixes a crash bug on older versions of OpenBSD, and fixes
+  several other bugs. If no serious bugs are found in this version, the
+  next version will be a release candidate.
+
+  This release also marks the end of support for the Tor 0.2.4.x,
+  0.2.6.x, and 0.2.7.x release series. Those releases will receive no
+  further bug or security fixes. Anyone still running or distributing
+  one of those versions should upgrade.
+
+  o Major features (build system, continuous integration):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major bugfixes (openbsd, denial-of-service):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xbar" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Major bugfixes (relay, performance):
+    - Perform circuit handshake operations at a higher priority than we
+      use for consensus diff creation and compression. This should
+      prevent circuits from starving when a relay or bridge receives a
+      new consensus, especially on lower-powered machines. Fixes bug
+      22883; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (bridge authority):
+    - Add "fingerprint" lines to the networkstatus-bridges file produced
+      by bridge authorities. Closes ticket 22207.
+
+  o Minor features (directory cache, consensus diff):
+    - Add a new MaxConsensusAgeForDiffs option to allow directory cache
+      operators with low-resource environments to adjust the number of
+      consensuses they'll store and generate diffs from. Most cache
+      operators should leave it unchanged. Helps to work around
+      bug 22883.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (relay, performance):
+    - Always start relays with at least two worker threads, to prevent
+      priority inversion on slow tasks. Part of the fix for bug 22883.
+    - Allow background work to be queued with different priorities, so
+      that a big pile of slow low-priority jobs will not starve out
+      higher priority jobs. This lays the groundwork for a fix for
+      bug 22883.
+
+  o Minor bugfixes (build system, rust):
+    - Fix a problem where Rust toolchains were not being found when
+      building without --enable-cargo-online-mode, due to setting the
+      $HOME environment variable instead of $CARGO_HOME. Fixes bug
+      22830; bugfix on 0.3.1.1-alpha. Fix by Chelsea Komlo.
+
+  o Minor bugfixes (compatibility, zstd):
+    - Write zstd epilogues correctly when the epilogue requires
+      reallocation of the output buffer, even with zstd 1.3.0.
+      (Previously, we worked on 1.2.0 and failed with 1.3.0). Fixes bug
+      22927; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (compilation warnings):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug
+      22915; bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - Compile correctly when both openssl 1.1.0 and libscrypt are
+      detected. Previously this would cause an error. Fixes bug 22892;
+      bugfix on 0.3.1.1-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (coverity build support):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (error reporting, windows):
+    - When formatting Windows error messages, use the English format to
+      avoid codepage issues. Fixes bug 22520; bugfix on 0.1.2.8-alpha.
+      Patch from "Vort".
+
+  o Minor bugfixes (file limits, osx):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a small memory leak when validating a configuration that uses
+      two or more AF_UNIX sockets for the same port type. Fixes bug
+      23053; bugfix on 0.2.6.3-alpha. This is CID 1415725.
+
+  o Minor bugfixes (unit tests):
+    - test_consdiff_base64cmp would fail on OS X because while OS X
+      follows the standard of (less than zero/zero/greater than zero),
+      it doesn't follow the convention of (-1/0/+1). Make the test
+      comply with the standard. Fixes bug 22870; bugfix on 0.3.1.1-alpha.
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.1.4-alpha - 2017-06-29
+  Tor 0.3.1.4-alpha fixes a path selection bug that would allow a client
+  to use a guard that was in the same network family as a chosen exit
+  relay. This is a security regression; all clients running earlier
+  versions of 0.3.0.x or 0.3.1.x should upgrade to 0.3.0.9
+  or 0.3.1.4-alpha.
+
+  This release also fixes several other bugs introduced in 0.3.0.x
+  and 0.3.1.x, including others that can affect bandwidth usage
+  and correctness.
+
+  o New dependencies:
+    - To build with zstd and lzma support, Tor now requires the
+      pkg-config tool at build time. (This requirement was new in
+      0.3.1.1-alpha, but was not noted at the time. Noting it here to
+      close ticket 22623.)
+
+  o Major bugfixes (path selection, security):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (compression, zstd):
+    - Correctly detect a full buffer when decompressing a large zstd-
+      compressed input. Previously, we would sometimes treat a full
+      buffer as an error. Fixes bug 22628; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (directory protocol):
+    - Ensure that we send "304 Not modified" as HTTP status code when a
+      client is attempting to fetch a consensus or consensus diff, and
+      the best one we can send them is one they already have. Fixes bug
+      22702; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (entry guards):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (bug mitigation, diagnostics, logging):
+    - Avoid an assertion failure, and log a better error message, when
+      unable to remove a file from the consensus cache on Windows.
+      Attempts to mitigate and diagnose bug 22752.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compression):
+    - When compressing or decompressing a buffer, check for a failure to
+      create a compression object. Fixes bug 22626; bugfix
+      on 0.3.1.1-alpha.
+    - When decompressing a buffer, check for extra data after the end of
+      the compressed data. Fixes bug 22629; bugfix on 0.3.1.1-alpha.
+    - When decompressing an object received over an anonymous directory
+      connection, if we have already decompressed it using an acceptable
+      compression method, do not reject it for looking like an
+      unacceptable compression method. Fixes part of bug 22670; bugfix
+      on 0.3.1.1-alpha.
+    - When serving directory votes compressed with zlib, do not claim to
+      have compressed them with zstd. Fixes bug 22669; bugfix
+      on 0.3.1.1-alpha.
+    - When spooling compressed data to an output buffer, don't try to
+      spool more data when there is no more data to spool and we are not
+      trying to flush the input. Previously, we would sometimes launch
+      compression requests with nothing to do, which interferes with our
+      22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha.
+
+  o Minor bugfixes (defensive programming):
+    - Detect and break out of infinite loops in our compression code. We
+      don't think that any such loops exist now, but it's best to be
+      safe. Closes ticket 22672.
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+    - Fix a crash in the LZMA module, when the sandbox was enabled, and
+      liblzma would allocate more than 16 MB of memory. We solve this by
+      bumping the mprotect() limit in the sandbox module from 16 MB to
+      20 MB. Fixes bug 22751; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - When decompressing, do not warn if we fail to decompress using a
+      compression method that we merely guessed. Fixes part of bug
+      22670; bugfix on 0.1.1.14-alpha.
+    - When decompressing, treat mismatch between content-encoding and
+      actual compression type as a protocol warning. Fixes part of bug
+      22670; bugfix on 0.1.1.9-alpha.
+    - Downgrade "assigned_to_cpuworker failed" message to info-level
+      severity. In every case that can reach it, either a better warning
+      has already been logged, or no warning is warranted. Fixes bug
+      22356; bugfix on 0.2.6.3-alpha.
+    - Demote a warn that was caused by libevent delays to info if
+      netflow padding is less than 4.5 seconds late, or to notice
+      if it is more (4.5 seconds is the amount of time that a netflow
+      record might be emitted after, if we chose the maximum timeout).
+      Fixes bug 22212; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (process behavior):
+    - When exiting because of an error, always exit with a nonzero exit
+      status. Previously, we would fail to report an error in our exit
+      status in cases related to __OwningControllerProcess failure,
+      lockfile contention, and Ed25519 key initialization. Fixes bug
+      22720; bugfix on versions 0.2.1.6-alpha, 0.2.2.28-beta, and
+      0.2.7.2-alpha respectively. Reported by "f55jwk4f"; patch
+      from "huyvq".
+
+  o Documentation:
+    - Add a manpage description for the key-pinning-journal file. Closes
+      ticket 22347.
+    - Correctly note that bandwidth accounting values are stored in the
+      state file, and the bw_accounting file is now obsolete. Closes
+      ticket 16082.
+    - Document more of the files in the Tor data directory, including
+      cached-extrainfo, secret_onion_key{,_ntor}.old, hidserv-stats,
+      approved-routers, sr-random, and diff-cache. Found while fixing
+      ticket 22347.
+
+
+Changes in version 0.3.0.9 - 2017-06-29
+  Tor 0.3.0.9 fixes a path selection bug that would allow a client
+  to use a guard that was in the same network family as a chosen exit
+  relay. This is a security regression; all clients running earlier
+  versions of 0.3.0.x or 0.3.1.x should upgrade to 0.3.0.9 or
+  0.3.1.4-alpha.
+
+  This release also backports several other bugfixes from the 0.3.1.x
+  series.
+
+  o Major bugfixes (path selection, security, backport from 0.3.1.4-alpha):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.1-alpha):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.4-alpha):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (defensive programming, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+
+Changes in version 0.3.1.3-alpha - 2017-06-08
+  Tor 0.3.1.3-alpha fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.1.3-alpha also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (torrc, crash):
+    - Fix a crash bug when using %include in torrc. Fixes bug 22417;
+      bugfix on 0.3.1.1-alpha. Patch by Daniel Pinto.
+
+  o Minor features (code style):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (diagnostic):
+    - Add logging messages to try to diagnose a rare bug that seems to
+      generate RSA->Ed25519 cross-certificates dated in the 1970s. We
+      think this is happening because of incorrect system clocks, but
+      we'd like to know for certain. Diagnostic for bug 22466.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Check for libzstd >= 1.1, because older versions lack the
+      necessary streaming API. Fixes bug 22413; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (link handshake):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (storage directories):
+    - Always check for underflows in the cached storage directory usage.
+      If the usage does underflow, re-calculate it. Also, avoid a
+      separate underflow when the usage is not known. Fixes bug 22424;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (unit tests):
+    - The unit tests now pass on systems where localhost is misconfigured
+      to some IPv4 address other than 127.0.0.1. Fixes bug 6298; bugfix
+      on 0.0.9pre2.
+
+  o Documentation:
+    - Clarify the manpage for the (deprecated) torify script. Closes
+      ticket 6892.
+
+Changes in version 0.3.0.8 - 2017-06-08
+  Tor 0.3.0.8 fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.0.8 also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management, backport from 0.3.1.3-alpha):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha):
+    - Stop rejecting v3 hidden service descriptors because their size
+      did not match an old padding rule. Fixes bug 22447; bugfix on
+      0.3.0.1-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (configuration, backport from 0.3.1.1-alpha):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (link handshake, backport from 0.3.1.3-alpha):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport from
+    0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.2.9.11 - 2017-06-08
+  Tor 0.2.9.11 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005. (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  Tor 0.2.9.11 also backports fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (future-proofing, backport from 0.3.0.7):
+    - Tor no longer refuses to download microdescriptors or descriptors if
+      they are listed as "published in the future".  This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor features (directory authorities, backport from 0.3.0.4-rc)
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port, backport from 0.3.0.6):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.0.7):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport
+    from 0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+Changes in version 0.2.8.14 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.7.8 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.2.6.12 - 2017-06-08
+  Tor 0.2.6.12 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.5.14 - 2017-06-08
+  Tor 0.2.5.14 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.4.29 - 2017-06-08
+  Tor 0.2.4.29 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.3.1.2-alpha - 2017-05-26
+  Tor 0.3.1.2-alpha is the second release in the 0.3.1.x series. It
+  fixes a few bugs found while testing 0.3.1.1-alpha, including a
+  memory corruption bug that affected relay stability.
+
+  o Major bugfixes (crash, relay):
+    - Fix a memory-corruption bug in relays that set MyFamily.
+      Previously, they would double-free MyFamily elements when making
+      the next descriptor or when changing their configuration. Fixes
+      bug 22368; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Log a better message when a directory authority replies to an
+      upload with an unexpected status code. Fixes bug 11121; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (memory leak, directory authority):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.3.1.1-alpha - 2017-05-22
+  Tor 0.3.1.1-alpha is the first release in the 0.3.1.x series. It
+  reduces the bandwidth usage for Tor's directory protocol, adds some
+  basic padding to resist netflow-based traffic analysis and to serve as
+  the basis of other padding in the future, and adds rust support to the
+  build system.
+
+  It also contains numerous other small features and improvements to
+  security, correctness, and performance.
+
+  Below are the changes since 0.3.0.7.
+
+  o Major features (directory protocol):
+    - Tor relays and authorities can now serve clients an abbreviated
+      version of the consensus document, containing only the changes
+      since an older consensus document that the client holds. Clients
+      now request these documents when available. When both client and
+      server use this new protocol, they will use far less bandwidth (up
+      to 94% less) to keep the client's consensus up-to-date. Implements
+      proposal 140; closes ticket 13339. Based on work by Daniel Martí.
+    - Tor can now compress directory traffic with lzma or with zstd
+      compression algorithms, which can deliver better bandwidth
+      performance. Because lzma is computationally expensive, it's only
+      used for documents that can be compressed once and served many
+      times. Support for these algorithms requires that tor is built
+      with the libzstd and/or liblzma libraries available. Implements
+      proposal 278; closes ticket 21662.
+    - Relays now perform the more expensive compression operations, and
+      consensus diff generation, in worker threads. This separation
+      avoids delaying the main thread when a new consensus arrives.
+
+  o Major features (experimental):
+    - Tor can now build modules written in Rust. To turn this on, pass
+      the "--enable-rust" flag to the configure script. It's not time to
+      get excited yet: currently, there is no actual Rust functionality
+      beyond some simple glue code, and a notice at startup to tell you
+      that Rust is running. Still, we hope that programmers and
+      packagers will try building Tor with Rust support, so that we can
+      find issues and solve portability problems. Closes ticket 22106.
+
+  o Major features (traffic analysis resistance):
+    - Connections between clients and relays now send a padding cell in
+      each direction every 1.5 to 9.5 seconds (tunable via consensus
+      parameters). This padding will not resist specialized
+      eavesdroppers, but it should be enough to make many ISPs' routine
+      network flow logging less useful in traffic analysis against
+      Tor users.
+
+      Padding is negotiated using Tor's link protocol, so both relays
+      and clients must upgrade for this to take effect. Clients may
+      still send padding despite the relay's version by setting
+      ConnectionPadding 1 in torrc, and may disable padding by setting
+      ConnectionPadding 0 in torrc. Padding may be minimized for mobile
+      users with the torrc option ReducedConnectionPadding. Implements
+      Proposal 251 and Section 2 of Proposal 254; closes ticket 16861.
+    - Relays will publish 24 hour totals of padding and non-padding cell
+      counts to their extra-info descriptors, unless PaddingStatistics 0
+      is set in torrc. These 24 hour totals are also rounded to
+      multiples of 10000.
+
+  o Major bugfixes (connection usage):
+    - We use NETINFO cells to try to determine if both relays involved
+      in a connection will agree on the canonical status of that
+      connection. We prefer the connections where this is the case for
+      extend cells, and try to close connections where relays disagree
+      on their canonical status early. Also, we now prefer the oldest
+      valid connection for extend cells. These two changes should reduce
+      the number of long-term connections that are kept open between
+      relays. Fixes bug 17604; bugfix on 0.2.5.5-alpha.
+    - Relays now log hourly statistics (look for
+      "channel_check_for_duplicates" lines) on the total number of
+      connections to other relays. If the number of connections per
+      relay is unexpectedly large, this log message is at notice level.
+      Otherwise it is at info.
+
+  o Major bugfixes (entry guards):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (linux TPROXY support):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Minor features (security, windows):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (config options):
+    - Allow "%include" directives in torrc configuration files. These
+      directives import the settings from other files, or from all the
+      files in a directory. Closes ticket 1922. Code by Daniel Pinto.
+    - Make SAVECONF return an error when overwriting a torrc that has
+      includes. Using SAVECONF with the FORCE option will allow it to
+      overwrite torrc even if includes are used. Related to ticket 1922.
+    - Add "GETINFO config-can-saveconf" to tell controllers if SAVECONF
+      will work without the FORCE option. Related to ticket 1922.
+
+  o Minor features (controller):
+    - Warn the first time that a controller requests data in the long-
+      deprecated 'GETINFO network-status' format. Closes ticket 21703.
+
+  o Minor features (defaults):
+    - The default value for UseCreateFast is now 0: clients which
+      haven't yet received a consensus document will now use a proper
+      ntor handshake to talk to their directory servers whenever they
+      can. Closes ticket 21407.
+    - Onion key rotation and expiry intervals are now defined as a
+      network consensus parameter, per proposal 274. The default
+      lifetime of an onion key is increased from 7 to 28 days. Old onion
+      keys will expire after 7 days by default. This change will make
+      consensus diffs much smaller, and save significant bandwidth.
+      Closes ticket 21641.
+
+  o Minor features (fallback directory list):
+    - Update the fallback directory mirror whitelist and blacklist based
+      on operator emails. Closes task 21121.
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (hidden services, logging):
+    - Log a message when a hidden service descriptor has fewer
+      introduction points than specified in
+      HiddenServiceNumIntroductionPoints. Closes tickets 21598.
+    - Log a message when a hidden service reaches its introduction point
+      circuit limit, and when that limit is reset. Follow up to ticket
+      21594; closes ticket 21622.
+    - Warn user if multiple entries in EntryNodes and at least one
+      HiddenService are used together. Pinning EntryNodes along with a
+      hidden service can be possibly harmful; for instance see ticket
+      14917 or 21155. Closes ticket 21155.
+
+  o Minor features (linux seccomp2 sandbox):
+    - We now have a document storage backend compatible with the Linux
+      seccomp2 sandbox. This backend is used for consensus documents and
+      diffs between them; in the long term, we'd like to use it for
+      unparseable directory material too. Closes ticket 21645
+    - Increase the maximum allowed size passed to mprotect(PROT_WRITE)
+      from 1MB to 16MB. This was necessary with the glibc allocator in
+      order to allow worker threads to allocate more memory -- which in
+      turn is necessary because of our new use of worker threads for
+      compression. Closes ticket 22096.
+
+  o Minor features (logging):
+    - Log files are no longer created world-readable by default.
+      (Previously, most distributors would store the logs in a non-
+      world-readable location to prevent inappropriate access. This
+      change is an extra precaution.) Closes ticket 21729; patch
+      from toralf.
+
+  o Minor features (performance):
+    - Our Keccak (SHA-3) implementation now accesses memory more
+      efficiently, especially on little-endian systems. Closes
+      ticket 21737.
+    - Add an O(1) implementation of channel_find_by_global_id(), to
+      speed some controller functions.
+
+  o Minor features (relay, configuration):
+    - The MyFamily option may now be repeated as many times as desired,
+      for relays that want to configure large families. Closes ticket
+      4998; patch by Daniel Pinto.
+
+  o Minor features (safety):
+    - Add an explicit check to extrainfo_parse_entry_from_string() for
+      NULL inputs. We don't believe this can actually happen, but it may
+      help silence a warning from the Clang analyzer. Closes
+      ticket 21496.
+
+  o Minor features (testing):
+    - Add a "--disable-memory-sentinels" feature to help with fuzzing.
+      When Tor is compiled with this option, we disable a number of
+      redundant memory-safety failsafes that are intended to stop bugs
+      from becoming security issues. This makes it easier to hunt for
+      bugs that would be security issues without the failsafes turned
+      on. Closes ticket 21439.
+    - Add a general event-tracing instrumentation support to Tor. This
+      subsystem will enable developers and researchers to add fine-
+      grained instrumentation to their Tor instances, for use when
+      examining Tor network performance issues. There are no trace
+      events yet, and event-tracing is off by default unless enabled at
+      compile time. Implements ticket 13802.
+    - Improve our version parsing tests: add tests for typical version
+      components, add tests for invalid versions, including numeric
+      range and non-numeric prefixes. Unit tests 21278, 21450, and
+      21507. Partially implements 21470.
+
+  o Minor bugfixes (bandwidth accounting):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (code correctness):
+    - Accurately identify client connections by their lack of peer
+      authentication. This means that we bail out earlier if asked to
+      extend to a client. Follow-up to 21407. Fixes bug 21406; bugfix
+      on 0.2.4.23.
+
+  o Minor bugfixes (configuration):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection lifespan):
+    - Allow more control over how long TLS connections are kept open:
+      unify CircuitIdleTimeout and PredictedPortsRelevanceTime into a
+      single option called CircuitsAvailableTimeout. Also, allow the
+      consensus to control the default values for both this preference
+      and the lifespan of relay-to-relay connections. Fixes bug 17592;
+      bugfix on 0.2.5.5-alpha.
+    - Increase the initial circuit build timeout testing frequency, to
+      help ensure that ReducedConnectionPadding clients finish learning
+      a timeout before their orconn would expire. The initial testing
+      rate was set back in the days of TAP and before the Tor Browser
+      updater, when we had to be much more careful about new clients
+      making lots of circuits. With this change, a circuit build timeout
+      is learned in about 15-20 minutes, instead of 100-120 minutes.
+
+  o Minor bugfixes (controller):
+    - GETINFO onions/current and onions/detached no longer respond with
+      551 on empty lists. Fixes bug 21329; bugfix on 0.2.7.1-alpha.
+    - Trigger HS descriptor events on the control port when the client
+      fails to pick a hidden service directory for a hidden service.
+      This can happen if all the hidden service directories are in
+      ExcludeNodes, or they have all been queried within the last 15
+      minutes. Fixes bug 22042; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+    - Prevent the shared randomness subsystem from asserting when
+      initialized by a bridge authority with an incomplete configuration
+      file. Fixes bug 21586; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (exit-side DNS):
+    - Fix an untriggerable assertion that checked the output of a
+      libevent DNS error, so that the assertion actually behaves as
+      expected. Fixes bug 22244; bugfix on 0.2.0.20-rc. Found by Andrey
+      Karpov using PVS-Studio.
+
+  o Minor bugfixes (fallback directories):
+    - Make the usage example in updateFallbackDirs.py actually work, and
+      explain what it does. Fixes bug 22270; bugfix on 0.3.0.3-alpha.
+    - Decrease the guard flag average required to be a fallback. This
+      allows us to keep relays that have their guard flag removed when
+      they restart. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+    - Decrease the minimum number of fallbacks to 100. Fixes bug 20913;
+      bugfix on 0.2.8.1-alpha.
+    - Make sure fallback directory mirrors have the same address, port,
+      and relay identity key for at least 30 days before they are
+      selected. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop printing a cryptic warning when a hidden service gets a
+      request to connect to a virtual port that it hasn't configured.
+      Fixes bug 16706; bugfix on 0.2.6.3-alpha.
+    - Simplify hidden service descriptor creation by using an existing
+      flag to check if an introduction point is established. Fixes bug
+      21599; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak at exit from the backtrace handler code.
+      Fixes bug 21788; bugfix on 0.2.5.2-alpha. Patch from Daniel Pinto.
+
+  o Minor bugfixes (protocol, logging):
+    - Downgrade a log statement about unexpected relay cells from "bug"
+      to "protocol warning", because there is at least one use case
+      where it can be triggered by a buggy tor implementation. Fixes bug
+      21293; bugfix on 0.1.1.14-alpha.
+
+  o Minor bugfixes (testing):
+    - Use unbuffered I/O for utility functions around the
+      process_handle_t type. This fixes unit test failures reported on
+      OpenBSD and FreeBSD. Fixes bug 21654; bugfix on 0.2.3.1-alpha.
+    - Make display of captured unit test log messages consistent. Fixes
+      bug 21510; bugfix on 0.2.9.3-alpha.
+    - Make test-network.sh always call chutney's test-network.sh.
+      Previously, this only worked on systems which had bash installed,
+      due to some bash-specific code in the script. Fixes bug 19699;
+      bugfix on 0.3.0.4-rc. Follow-up to ticket 21581.
+
+  o Minor bugfixes (voting consistency):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (windows, relay):
+    - Resolve "Failure from drain_fd: No error" warnings on Windows
+      relays. Fixes bug 21540; bugfix on 0.2.6.3-alpha.
+
+  o Code simplification and refactoring:
+    - Break up the 630-line function connection_dir_client_reached_eof()
+      into a dozen smaller functions. This change should help
+      maintainability and readability of the client directory code.
+    - Isolate our use of the openssl headers so that they are only
+      included from our crypto wrapper modules, and from tests that
+      examine those modules' internals. Closes ticket 21841.
+    - Simplify our API to launch directory requests, making it more
+      extensible and less error-prone. Now it's easier to add extra
+      headers to directory requests. Closes ticket 21646.
+    - Our base64 decoding functions no longer overestimate the output
+      space that they need when parsing unpadded inputs. Closes
+      ticket 17868.
+    - Remove unused "ROUTER_ADDED_NOTIFY_GENERATOR" internal value.
+      Resolves ticket 22213.
+    - The logic that directory caches use to spool request to clients,
+      serving them one part at a time so as not to allocate too much
+      memory, has been refactored for consistency. Previously there was
+      a separate spooling implementation per type of spoolable data. Now
+      there is one common spooling implementation, with extensible data
+      types. Closes ticket 21651.
+    - Tor's compression module now supports multiple backends. Part of
+      the implementation for proposal 278; closes ticket 21663.
+
+  o Documentation:
+    - Clarify the behavior of the KeepAliveIsolateSOCKSAuth sub-option.
+      Closes ticket 21873.
+    - Correct documentation about the default DataDirectory value.
+      Closes ticket 21151.
+    - Document the default behavior of NumEntryGuards and
+      NumDirectoryGuards correctly. Fixes bug 21715; bugfix
+      on 0.3.0.1-alpha.
+    - Document key=value pluggable transport arguments for Bridge lines
+      in torrc. Fixes bug 20341; bugfix on 0.2.5.1-alpha.
+    - Note that bandwidth-limiting options don't affect TCP headers or
+      DNS. Closes ticket 17170.
+
+  o Removed features (configuration options, all in ticket 22060):
+    - These configuration options are now marked Obsolete, and no longer
+      have any effect: AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ExcludeSingleHopRelays, FastFirstHopPK,
+      TLSECGroup, WarnUnsafeSocks. They were first marked as deprecated
+      in 0.2.9.2-alpha and have now been removed. The previous default
+      behavior is now always chosen; the previous (less secure) non-
+      default behavior is now unavailable.
+    - CloseHSClientCircuitsImmediatelyOnTimeout and
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout were deprecated in
+      0.2.9.2-alpha and now have been removed. HS circuits never close
+      on circuit build timeout; they have a longer timeout period.
+    - {Control,DNS,Dir,Socks,Trans,NATD,OR}ListenAddress were deprecated
+      in 0.2.9.2-alpha and now have been removed. Use the ORPort option
+      (and others) to configure listen-only and advertise-only addresses.
+
+  o Removed features (tools):
+    - We've removed the tor-checkkey tool from src/tools. Long ago, we
+      used it to help people detect RSA keys that were generated by
+      versions of Debian affected by CVE-2008-0166. But those keys have
+      been out of circulation for ages, and this tool is no longer
+      required. Closes ticket 21842.
+
+
+Changes in version 0.3.0.7 - 2017-05-15
+  Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions
+  of Tor 0.3.0.x, where an attacker could cause a Tor relay process
+  to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade;
+  clients are not affected.
+
+  o Major bugfixes (hidden service directory, security):
+    - Fix an assertion failure in the hidden service directory code, which
+      could be used by an attacker to remotely cause a Tor relay process to
+      exit. Relays running earlier versions of Tor 0.3.0.x should upgrade.
+      should upgrade. This security issue is tracked as TROVE-2017-002.
+      Fixes bug 22246; bugfix on 0.3.0.1-alpha.
+
+  o Minor features:
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (future-proofing):
+    - Tor no longer refuses to download microdescriptors or descriptors
+      if they are listed as "published in the future". This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.0.6 - 2017-04-26
+  Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series.
+
+  With the 0.3.0 series, clients and relays now use Ed25519 keys to
+  authenticate their link connections to relays, rather than the old
+  RSA1024 keys that they used before. (Circuit crypto has been
+  Curve25519-authenticated since 0.2.4.8-alpha.) We have also replaced
+  the guard selection and replacement algorithm to behave more robustly
+  in the presence of unreliable networks, and to resist guard-
+  capture attacks.
+
+  This series also includes numerous other small features and bugfixes,
+  along with more groundwork for the upcoming hidden-services revamp.
+
+  Per our stable release policy, we plan to support the Tor 0.3.0
+  release series for at least the next nine months, or for three months
+  after the first stable release of the 0.3.1 series: whichever is
+  longer. If you need a release with long-term support, we recommend
+  that you stay with the 0.2.9 series.
+
+  Below are the changes since 0.3.0.5-rc. For a list of all changes
+  since 0.2.9, see the ReleaseNotes file.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (crash prevention):
+    - Fix a (currently untriggerable, but potentially dangerous) crash
+      bug when base32-encoding inputs whose sizes are not a multiple of
+      5. Fixes bug 21894; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.5-rc - 2017-04-05
+  Tor 0.3.0.5-rc fixes a few remaining bugs, large and small, in the
+  0.3.0 release series.
+
+  This is the second release candidate in the Tor 0.3.0 series, and has
+  much fewer changes than the first. If we find no new bugs or
+  regressions here, the first stable 0.3.0 release will be nearly
+  identical to it.
+
+  o Major bugfixes (crash, directory connections):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (guard selection):
+    - Fix a guard selection bug where Tor would refuse to bootstrap in
+      some cases if the user swapped a bridge for another bridge in
+      their configuration file. Fixes bug 21771; bugfix on 0.3.0.1-alpha.
+      Reported by "torvlnt33r".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 7 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (compilation):
+    - Fix a warning when compiling hs_service.c. Previously, it had no
+      exported symbols when compiled for libor.a, resulting in a
+      compilation warning from clang. Fixes bug 21825; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Make hidden services check for failed intro point connections,
+      even when they have exceeded their intro point creation limit.
+      Fixes bug 21596; bugfix on 0.2.7.2-alpha. Reported by Alec Muffett.
+    - Make hidden services with 8 to 10 introduction points check for
+      failed circuits immediately after startup. Previously, they would
+      wait for 5 minutes before performing their first checks. Fixes bug
+      21594; bugfix on 0.2.3.9-alpha. Reported by Alec Muffett.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a memory leak when using GETCONF on a port option. Fixes bug
+      21682; bugfix on 0.3.0.3-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (tests):
+    - Run the entry_guard_parse_from_state_full() test with the time set
+      to a specific date. (The guard state that this test was parsing
+      contained guards that had expired since the test was first
+      written.) Fixes bug 21799; bugfix on 0.3.0.1-alpha.
+
+  o Documentation:
+    - Update the description of the directory server options in the
+      manual page, to clarify that a relay no longer needs to set
+      DirPort in order to be a directory cache. Closes ticket 21720.
+
+
+
+Changes in version 0.2.8.13 - 2017-03-03
+  Tor 0.2.8.13 backports a security fix from later Tor
+  releases.  Anybody running Tor 0.2.8.12 or earlier should upgrade to this
+  this release, if for some reason they cannot upgrade to a later
+  release series, and if they build Tor with the --enable-expensive-hardening
+  option.
+
+  Note that support for Tor 0.2.8.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.8.x series after 1 Jan 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.7.7 - 2017-03-03
+  Tor 0.2.7.7 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.7.6 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.7.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.7.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.6.11 - 2017-03-03
+  Tor 0.2.6.11 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.6.10 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.6.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.6.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+
+Changes in version 0.2.5.13 - 2017-03-03
+  Tor 0.2.5.13 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.5.13 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.5.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.5.x series after 1 May 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (crypto error-handling, backport from 0.2.7.2-alpha):
+    - Check for failures from crypto_early_init, and refuse to continue.
+      A previous typo meant that we could keep going with an
+      uninitialized crypto library, and would have OpenSSL initialize
+      its own PRNG. Fixes bug 16360; bugfix on 0.2.5.2-alpha, introduced
+      when implementing ticket 4900. Patch by "teor".
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.4.28 - 2017-03-03
+  Tor 0.2.4.28 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.4.27 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.4.x is ending soon: we will not issue
+  any fixes for the Tor 0.2.4.x series after 1 August 2017.  If you need
+  a Tor release series with long-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (DoS-resistance, backport from 0.2.7.1-alpha):
+    - Make it harder for attackers to overload hidden services with
+      introductions, by blocking multiple introduction requests on the
+      same circuit. Resolves ticket 15515.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.3.0.4-rc - 2017-03-01
+  Tor 0.3.0.4-rc fixes some remaining bugs, large and small, in the
+  0.3.0 release series, and introduces a few reliability features to
+  keep them from coming back.
+
+  This is the first release candidate in the Tor 0.3.0 series. If we
+  find no new bugs or regressions here, the first stable 0.3.0 release
+  will be nearly identical to it.
+
+  o Major bugfixes (bridges):
+    - When the same bridge is configured multiple times with the same
+      identity, but at different address:port combinations, treat those
+      bridge instances as separate guards. This fix restores the ability
+      of clients to configure the same bridge with multiple pluggable
+      transports. Fixes bug 21027; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service directory v3):
+    - Stop crashing on a failed v3 hidden service descriptor lookup
+      failure. Fixes bug 21471; bugfixes on 0.3.0.1-alpha.
+
+  o Major bugfixes (parsing):
+    - When parsing a malformed content-length field from an HTTP
+      message, do not read off the end of the buffer. This bug was a
+      potential remote denial-of-service attack against Tor clients and
+      relays. A workaround was released in October 2016, to prevent this
+      bug from crashing Tor. This is a fix for the underlying issue,
+      which should no longer matter (if you applied the earlier patch).
+      Fixes bug 20894; bugfix on 0.2.0.16-alpha. Bug found by fuzzing
+      using AFL (http://lcamtuf.coredump.cx/afl/).
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor feature (protocol versioning):
+    - Add new protocol version for proposal 224. HSIntro now advertises
+      version "3-4" and HSDir version "1-2". Fixes ticket 20656.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (reliability, crash):
+    - Try better to detect problems in buffers where they might grow (or
+      think they have grown) over 2 GB in size. Diagnostic for
+      bug 21369.
+
+  o Minor features (testing):
+    - During 'make test-network-all', if tor logs any warnings, ask
+      chutney to output them. Requires a recent version of chutney with
+      the 21572 patch. Implements 21570.
+
+  o Minor bugfixes (certificate expiration time):
+    - Avoid using link certificates that don't become valid till some
+      time in the future. Fixes bug 21420; bugfix on 0.2.4.11-alpha
+
+  o Minor bugfixes (code correctness):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+    - Remove a redundant check for the UseEntryGuards option from the
+      options_transition_affects_guards() function. Fixes bug 21492;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Allow relays to use directory mirrors without a DirPort: these
+      relays need to be contacted over their ORPorts using a begindir
+      connection. Fixes one case of bug 20711; bugfix on 0.2.8.2-alpha.
+    - Clarify the message logged when a remote relay is unexpectedly
+      missing an ORPort or DirPort: users were confusing this with a
+      local port. Fixes another case of bug 20711; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (guards):
+    - Don't warn about a missing guard state on timeout-measurement
+      circuits: they aren't supposed to be using guards. Fixes an
+      instance of bug 21007; bugfix on 0.3.0.1-alpha.
+    - Silence a BUG() warning when attempting to use a guard whose
+      descriptor we don't know, and make this scenario less likely to
+      happen. Fixes bug 21415; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Pass correct buffer length when encoding legacy ESTABLISH_INTRO
+      cells. Previously, we were using sizeof() on a pointer, instead of
+      the real destination buffer. Fortunately, that value was only used
+      to double-check that there was enough room--which was already
+      enforced elsewhere. Fixes bug 21553; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix Raspbian build issues related to missing socket errno in
+      test_util.c. Fixes bug 21116; bugfix on 0.2.8.2. Patch
+      by "hein".
+    - Rename "make fuzz" to "make test-fuzz-corpora", since it doesn't
+      actually fuzz anything. Fixes bug 21447; bugfix on 0.3.0.3-alpha.
+    - Use bash in src/test/test-network.sh. This ensures we reliably
+      call chutney's newer tools/test-network.sh when available. Fixes
+      bug 21562; bugfix on 0.2.9.1-alpha.
+
+  o Documentation:
+    - Small fixes to the fuzzing documentation. Closes ticket 21472.
+
+
+Changes in version 0.2.9.10 - 2017-03-01
+  Tor 0.2.9.10 backports a security fix from later Tor release.  It also
+  includes fixes for some major issues affecting directory authorities,
+  LibreSSL compatibility, and IPv6 correctness.
+
+  The Tor 0.2.9.x release series is now marked as a long-term-support
+  series.  We intend to backport security fixes to 0.2.9.x until at
+  least January of 2020.
+
+  o Major bugfixes (directory authority, 0.3.0.3-alpha):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (IPv6 Exits, backport from 0.3.0.3-alpha):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing, also in 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (directory authorities, also in 0.3.0.4-rc):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (portability, compilation, backport from 0.3.0.3-alpha):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor bugfixes (code correctness, also in 0.3.0.4-rc):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (tor-resolve, backport from 0.3.0.3-alpha):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+
+Changes in version 0.3.0.3-alpha - 2017-02-03
+  Tor 0.3.0.3-alpha fixes a few significant bugs introduced over the
+  0.3.0.x development series, including some that could cause
+  authorities to behave badly. There is also a fix for a longstanding
+  bug that could prevent IPv6 exits from working. Tor 0.3.0.3-alpha also
+  includes some smaller features and bugfixes.
+
+  The Tor 0.3.0.x release series is now in patch-freeze: no additional
+  features will be considered for inclusion in 0.3.0.x. We suspect that
+  some bugs will probably remain, however, and we encourage people to
+  test this release.
+
+  o Major bugfixes (directory authority):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+    - When deciding whether we have just found a router to be reachable,
+      do not penalize it for not having performed an Ed25519 link
+      handshake if it does not claim to support an Ed25519 handshake.
+      Previously, we would treat such relays as non-running. Fixes bug
+      21107; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (entry guards):
+    - Stop trying to build circuits through entry guards for which we
+      have no descriptor. Also, stop crashing in the case that we *do*
+      accidentally try to build a circuit in such a state. Fixes bug
+      21242; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (IPv6 Exits):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Minor feature (client):
+    - Enable IPv6 traffic on the SocksPort by default. To disable this,
+      a user will have to specify "NoIPv6Traffic". Closes ticket 21269.
+
+  o Minor feature (fallback scripts):
+    - Add a check_existing mode to updateFallbackDirs.py, which checks
+      if fallbacks in the hard-coded list are working. Closes ticket
+      20174. Patch by haxxpop.
+
+  o Minor features (ciphersuite selection):
+    - Clients now advertise a list of ciphersuites closer to the ones
+      preferred by Firefox. Closes part of ticket 15426.
+    - Allow relays to accept a wider range of ciphersuites, including
+      chacha20-poly1305 and AES-CCM. Closes the other part of 15426.
+
+  o Minor features (controller, configuration):
+    - Each of the *Port options, such as SocksPort, ORPort, ControlPort,
+      and so on, now comes with a __*Port variant that will not be saved
+      to the torrc file by the controller's SAVECONF command. This
+      change allows TorBrowser to set up a single-use domain socket for
+      each time it launches Tor. Closes ticket 20956.
+    - The GETCONF command can now query options that may only be
+      meaningful in context-sensitive lists. This allows the controller
+      to query the mixed SocksPort/__SocksPort style options introduced
+      in feature 20956. Implements ticket 21300.
+
+  o Minor features (portability, compilation):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor features (relay):
+    - We now allow separation of exit and relay traffic to different
+      source IP addresses, using the OutboundBindAddressExit and
+      OutboundBindAddressOR options respectively. Closes ticket 17975.
+      Written by Michael Sonntag.
+
+  o Minor bugfix (logging):
+    - Don't recommend the use of Tor2web in non-anonymous mode.
+      Recommending Tor2web is a bad idea because the client loses all
+      anonymity. Tor2web should only be used in specific cases by users
+      who *know* and understand the issues. Fixes bug 21294; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (client):
+    - Always recover from failures in extend_info_from_node(), in an
+      attempt to prevent any recurrence of bug 21242. Fixes bug 21372;
+      bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (client, entry guards):
+    - Fix a bug warning (with backtrace) when we fail a channel that
+      circuits to fallback directories on it. Fixes bug 21128; bugfix
+      on 0.3.0.1-alpha.
+    - Fix a spurious bug warning (with backtrace) when removing an
+      expired entry guard. Fixes bug 21129; bugfix on 0.3.0.1-alpha.
+    - Fix a bug of the new guard algorithm where tor could stall for up
+      to 10 minutes before retrying a guard after a long period of no
+      network. Fixes bug 21052; bugfix on 0.3.0.1-alpha.
+    - Do not try to build circuits until we have descriptors for our
+      primary entry guards. Related to fix for bug 21242.
+
+  o Minor bugfixes (configure, autoconf):
+    - Rename the configure option --enable-expensive-hardening to
+      --enable-fragile-hardening. Expensive hardening makes the tor
+      daemon abort when some kinds of issues are detected. Thus, it
+      makes tor more at risk of remote crashes but safer against RCE or
+      heartbleed bug category. We now try to explain this issue in a
+      message from the configure script. Fixes bug 21290; bugfix
+      on 0.2.5.4-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the (deprecated) DROPGUARDS controller command. Fixes bug
+      20824; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Clean up the code for expiring intro points with no associated
+      circuits. It was causing, rarely, a service with some expiring
+      introduction points to not open enough additional introduction
+      points. Fixes part of bug 21302; bugfix on 0.2.7.2-alpha.
+    - Stop setting the torrc option HiddenServiceStatistics to "0" just
+      because we're not a bridge or relay. Instead, we preserve whatever
+      value the user set (or didn't set). Fixes bug 21150; bugfix
+      on 0.2.6.2-alpha.
+    - Resolve two possible underflows which could lead to creating and
+      closing a lot of introduction point circuits in a non-stop loop.
+      Fixes bug 21302; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Use "OpenBSD" compiler macro instead of "OPENBSD" or "__OpenBSD__".
+      It is supported by OpenBSD itself, and also by most OpenBSD
+      variants (such as Bitrig). Fixes bug 20980; bugfix
+      on 0.1.2.1-alpha.
+    - When mapping a file of length greater than SIZE_MAX, do not
+      silently truncate its contents. This issue could occur on 32 bit
+      systems with large file support and files which are larger than 4
+      GB. Fixes bug 21134; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+  o Minor bugfixes (Windows services):
+    - Be sure to initialize the monotonic time subsystem before using
+      it, even when running as an NT service. Fixes bug 21356; bugfix
+      on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.2-alpha - 2017-01-23
+  Tor 0.3.0.2-alpha fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  Tor 0.3.0.2-alpha also improves how exit relays and clients handle DNS
+  time-to-live values, makes directory authorities enforce the 1-to-1
+  mapping of relay RSA identity keys to ED25519 identity keys, fixes a
+  client-side onion service reachability bug, does better at selecting
+  the set of fallback directories, and more.
+
+  o Major bugfixes (security, also in 0.2.9.9):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided."  This hardening option, like
+      others, can turn survivable bugs into crashes--and having it on by
+      default made a (relatively harmless) integer overflow bug into a
+      denial-of-service bug. Fixes bug 21278 (TROVE-2017-001); bugfix on
+      0.2.9.1-alpha.
+
+  o Major features (security):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major features (directory authority, security):
+    - The default for AuthDirPinKeys is now 1: directory authorities
+      will reject relays where the RSA identity key matches a previously
+      seen value, but the Ed25519 key has changed. Closes ticket 18319.
+
+  o Major bugfixes (client, guard, crash):
+    - In circuit_get_global_origin_list(), return the actual list of
+      origin circuits. The previous version of this code returned the
+      list of all the circuits, and could have caused strange bugs,
+      including possible crashes. Fixes bug 21118; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (client, onion service, also in 0.2.9.9):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Major bugfixes (DNS):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Minor features (controller):
+    - Add "GETINFO sr/current" and "GETINFO sr/previous" keys, to expose
+      shared-random values to the controller. Closes ticket 19925.
+
+  o Minor features (entry guards):
+    - Add UseEntryGuards to TEST_OPTIONS_DEFAULT_VALUES in order to not
+      break regression tests.
+    - Require UseEntryGuards when UseBridges is set, in order to make
+      sure bridges aren't bypassed. Resolves ticket 20502.
+
+  o Minor features (fallback directories):
+    - Select 200 fallback directories for each release. Closes
+      ticket 20881.
+    - Allow 3 fallback relays per operator, which is safe now that we
+      are choosing 200 fallback relays. Closes ticket 20912.
+    - Exclude relays affected by bug 20499 from the fallback list.
+      Exclude relays from the fallback list if they are running versions
+      known to be affected by bug 20499, or if in our tests they deliver
+      a stale consensus (i.e. one that expired more than 24 hours ago).
+      Closes ticket 20539.
+    - Reduce the minimum fallback bandwidth to 1 MByte/s. Part of
+      ticket 18828.
+    - Require fallback directories to have the same address and port for
+      7 days (now that we have enough relays with this stability).
+      Relays whose OnionOO stability timer is reset on restart by bug
+      18050 should upgrade to Tor 0.2.8.7 or later, which has a fix for
+      this issue. Closes ticket 20880; maintains short-term fix
+      in 0.2.8.2-alpha.
+    - Require fallbacks to have flags for 90% of the time (weighted
+      decaying average), rather than 95%. This allows at least 73% of
+      clients to bootstrap in the first 5 seconds without contacting an
+      authority. Part of ticket 18828.
+    - Annotate updateFallbackDirs.py with the bandwidth and consensus
+      weight for each candidate fallback. Closes ticket 20878.
+    - Make it easier to change the output sort order of fallbacks.
+      Closes ticket 20822.
+    - Display the relay fingerprint when downloading consensuses from
+      fallbacks. Closes ticket 20908.
+
+  o Minor features (geoip, also in 0.2.9.9):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (next-gen onion service directories):
+    - Remove the "EnableOnionServicesV3" consensus parameter that we
+      introduced in 0.3.0.1-alpha: relays are now always willing to act
+      as v3 onion service directories. Resolves ticket 19899.
+
+  o Minor features (linting):
+    - Enhance the changes file linter to warn on Tor versions that are
+      prefixed with "tor-". Closes ticket 21096.
+
+  o Minor features (logging):
+    - In several places, describe unset ed25519 keys as "<unset>",
+      rather than the scary "AAAAAAAA...AAA". Closes ticket 21037.
+
+  o Minor bugfix (control protocol):
+    - The reply to a "GETINFO config/names" request via the control
+      protocol now spells the type "Dependent" correctly. This is a
+      breaking change in the control protocol. (The field seems to be
+      ignored by the most common known controllers.) Fixes bug 18146;
+      bugfix on 0.1.1.4-alpha.
+
+  o Minor bugfixes (bug resilience):
+    - Fix an unreachable size_t overflow in base64_decode(). Fixes bug
+      19222; bugfix on 0.2.0.9-alpha. Found by Guido Vranken; fixed by
+      Hans Jerry Illikainen.
+
+  o Minor bugfixes (build):
+    - Replace obsolete Autoconf macros with their modern equivalent and
+      prevent similar issues in the future. Fixes bug 20990; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (client, guards):
+    - Fix bug where Tor would think that there are circuits waiting for
+      better guards even though those circuits have been freed. Fixes
+      bug 21142; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (config):
+    - Don't assert on startup when trying to get the options list and
+      LearnCircuitBuildTimeout is set to 0: we are currently parsing the
+      options so of course they aren't ready yet. Fixes bug 21062;
+      bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (controller):
+    - Make the GETINFO interface for inquiring about entry guards
+      support the new guards backend. Fixes bug 20823; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (dead code):
+    - Remove a redundant check for PidFile changes at runtime in
+      options_transition_allowed(): this check is already performed
+      regardless of whether the sandbox is active. Fixes bug 21123;
+      bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (documentation):
+    - Update the tor manual page to document every option that can not
+      be changed while tor is running. Fixes bug 21122.
+
+  o Minor bugfixes (fallback directories):
+    - Stop failing when a relay has no uptime data in
+      updateFallbackDirs.py. Fixes bug 20945; bugfix on 0.2.8.1-alpha.
+    - Avoid checking fallback candidates' DirPorts if they are down in
+      OnionOO. When a relay operator has multiple relays, this
+      prioritizes relays that are up over relays that are down. Fixes
+      bug 20926; bugfix on 0.2.8.3-alpha.
+    - Stop failing when OUTPUT_COMMENTS is True in updateFallbackDirs.py.
+      Fixes bug 20877; bugfix on 0.2.8.3-alpha.
+
+  o Minor bugfixes (guards, bootstrapping):
+    - When connecting to a directory guard during bootstrap, do not mark
+      the guard as successful until we receive a good-looking directory
+      response from it. Fixes bug 20974; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Fix the config reload pruning of old vs new services so it
+      actually works when both ephemeral and non-ephemeral services are
+      configured. Fixes bug 21054; bugfix on 0.3.0.1-alpha.
+    - Allow the number of introduction points to be as low as 0, rather
+      than as low as 3. Fixes bug 21033; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (IPv6):
+    - Make IPv6-using clients try harder to find an IPv6 directory
+      server. Fixes bug 20999; bugfix on 0.2.8.2-alpha.
+    - When IPv6 addresses have not been downloaded yet (microdesc
+      consensus documents don't list relay IPv6 addresses), use hard-
+      coded addresses for authorities, fallbacks, and configured
+      bridges. Now IPv6-only clients can use microdescriptors. Fixes bug
+      20996; bugfix on b167e82 from 19608 in 0.2.8.5-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a memory leak when configuring hidden services. Fixes bug
+      20987; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (portability, also in 0.2.9.9):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Honor DataDirectoryGroupReadable when tor is a relay. Previously,
+      initializing the keys would reset the DataDirectory to 0700
+      instead of 0750 even if DataDirectoryGroupReadable was set to 1.
+      Fixes bug 19953; bugfix on 0.0.2pre16. Patch by "redfish".
+
+  o Minor bugfixes (testing):
+    - Remove undefined behavior from the backtrace generator by removing
+      its signal handler. Fixes bug 21026; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Allow the unit tests to pass even when DNS lookups of bogus
+      addresses do not fail as expected. Fixes bug 20862 and 20863;
+      bugfix on unit tests introduced in 0.2.8.1-alpha
+      through 0.2.9.4-alpha.
+
+  o Code simplification and refactoring:
+    - Refactor code to manipulate global_origin_circuit_list into
+      separate functions. Closes ticket 20921.
+
+  o Documentation (formatting):
+    - Clean up formatting of tor.1 man page and HTML doc, where <pre>
+      blocks were incorrectly appearing. Closes ticket 20885.
+
+  o Documentation (man page):
+    - Clarify many options in tor.1 and add some min/max values for
+      HiddenService options. Closes ticket 21058.
+
+
+Changes in version 0.2.9.9 - 2017-01-23
+  Tor 0.2.9.9 fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  This release also resolves a client-side onion service reachability
+  bug, and resolves a pair of small portability issues.
+
+  o Major bugfixes (security):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes -- and having
+      it on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Major bugfixes (client, onion service):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.1-alpha - 2016-12-19
+  Tor 0.3.0.1-alpha is the first alpha release in the 0.3.0 development
+  series. It strengthens Tor's link and circuit handshakes by
+  identifying relays by their Ed25519 keys, improves the algorithm that
+  clients use to choose and maintain their list of guards, and includes
+  additional backend support for the next-generation hidden service
+  design. It also contains numerous other small features and
+  improvements to security, correctness, and performance.
+
+  Below are the changes since 0.2.9.8.
+
+  o Major features (guard selection algorithm):
+    - Tor's guard selection algorithm has been redesigned from the
+      ground up, to better support unreliable networks and restrictive
+      sets of entry nodes, and to better resist guard-capture attacks by
+      hostile local networks. Implements proposal 271; closes
+      ticket 19877.
+
+  o Major features (next-generation hidden services):
+    - Relays can now handle v3 ESTABLISH_INTRO cells as specified by
+      prop224 aka "Next Generation Hidden Services". Service and clients
+      don't use this functionality yet. Closes ticket 19043. Based on
+      initial code by Alec Heifetz.
+    - Relays now support the HSDir version 3 protocol, so that they can
+      can store and serve v3 descriptors. This is part of the next-
+      generation onion service work detailed in proposal 224. Closes
+      ticket 17238.
+
+  o Major features (protocol, ed25519 identity keys):
+    - Relays now use Ed25519 to prove their Ed25519 identities and to
+      one another, and to clients. This algorithm is faster and more
+      secure than the RSA-based handshake we've been doing until now.
+      Implements the second big part of proposal 220; Closes
+      ticket 15055.
+    - Clients now support including Ed25519 identity keys in the EXTEND2
+      cells they generate. By default, this is controlled by a consensus
+      parameter, currently disabled. You can turn this feature on for
+      testing by setting ExtendByEd25519ID in your configuration. This
+      might make your traffic appear different than the traffic
+      generated by other users, however. Implements part of ticket
+      15056; part of proposal 220.
+    - Relays now understand requests to extend to other relays by their
+      Ed25519 identity keys. When an Ed25519 identity key is included in
+      an EXTEND2 cell, the relay will only extend the circuit if the
+      other relay can prove ownership of that identity. Implements part
+      of ticket 15056; part of proposal 220.
+
+  o Major bugfixes (scheduler):
+    - Actually compare circuit policies in ewma_cmp_cmux(). This bug
+      caused the channel scheduler to behave more or less randomly,
+      rather than preferring channels with higher-priority circuits.
+      Fixes bug 20459; bugfix on 0.2.6.2-alpha.
+
+  o Minor features (controller):
+    - When HSFETCH arguments cannot be parsed, say "Invalid argument"
+      rather than "unrecognized." Closes ticket 20389; patch from
+      Ivan Markin.
+
+  o Minor features (diagnostic, directory client):
+    - Warn when we find an unexpected inconsistency in directory
+      download status objects. Prevents some negative consequences of
+      bug 20593.
+
+  o Minor features (directory authority):
+    - Add a new authority-only AuthDirTestEd25519LinkKeys option (on by
+      default) to control whether authorities should try to probe relays
+      by their Ed25519 link keys. This option will go away in a few
+      releases--unless we encounter major trouble in our ed25519 link
+      protocol rollout, in which case it will serve as a safety option.
+
+  o Minor features (directory cache):
+    - Relays and bridges will now refuse to serve the consensus they
+      have if they know it is too old for a client to use. Closes
+      ticket 20511.
+
+  o Minor features (ed25519 link handshake):
+    - Advertise support for the ed25519 link handshake using the
+      subprotocol-versions mechanism, so that clients can tell which
+      relays can identity themselves by Ed25519 ID. Closes ticket 20552.
+
+  o Minor features (fingerprinting resistance, authentication):
+    - Extend the length of RSA keys used for TLS link authentication to
+      2048 bits. (These weren't used for forward secrecy; for forward
+      secrecy, we used P256.) Closes ticket 13752.
+
+  o Minor features (infrastructure):
+    - Implement smartlist_add_strdup() function. Replaces the use of
+      smartlist_add(sl, tor_strdup(str)). Closes ticket 20048.
+
+  o Minor bugfixes (client):
+    - When clients that use bridges start up with a cached consensus on
+      disk, they were ignoring it and downloading a new one. Now they
+      use the cached one. Fixes bug 20269; bugfix on 0.2.3.12-alpha.
+
+  o Minor bugfixes (configuration):
+    - Accept non-space whitespace characters after the severity level in
+      the `Log` option. Fixes bug 19965; bugfix on 0.2.1.1-alpha.
+    - Support "TByte" and "TBytes" units in options given in bytes.
+      "TB", "terabyte(s)", "TBit(s)" and "terabit(s)" were already
+      supported. Fixes bug 20622; bugfix on 0.2.0.14-alpha.
+
+  o Minor bugfixes (consensus weight):
+    - Add new consensus method that initializes bw weights to 1 instead
+      of 0. This prevents a zero weight from making it all the way to
+      the end (happens in small testing networks) and causing an error.
+      Fixes bug 14881; bugfix on 0.2.2.17-alpha.
+
+  o Minor bugfixes (descriptors):
+    - Correctly recognise downloaded full descriptors as valid, even
+      when using microdescriptors as circuits. This affects clients with
+      FetchUselessDescriptors set, and may affect directory authorities.
+      Fixes bug 20839; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (directory system):
+    - Download all consensus flavors, descriptors, and authority
+      certificates when FetchUselessDescriptors is set, regardless of
+      whether tor is a directory cache or not. Fixes bug 20667; bugfix
+      on all recent tor versions.
+    - Bridges and relays now use microdescriptors (like clients do)
+      rather than old-style router descriptors. Now bridges will blend
+      in with clients in terms of the circuits they build. Fixes bug
+      6769; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (ed25519 certificates):
+    - Correctly interpret ed25519 certificates that would expire some
+      time after 19 Jan 2038. Fixes bug 20027; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop ignoring misconfigured hidden services. Instead, refuse to
+      start tor until the misconfigurations have been corrected. Fixes
+      bug 20559; bugfix on multiple commits in 0.2.7.1-alpha
+      and earlier.
+
+  o Minor bugfixes (memory leak at exit):
+    - Fix a small harmless memory leak at exit of the previously unused
+      RSA->Ed identity cross-certificate. Fixes bug 17779; bugfix
+      on 0.2.7.2-alpha.
+
+  o Minor bugfixes (util):
+    - When finishing writing a file to disk, if we were about to replace
+      the file with the temporary file created before and we fail to
+      replace it, remove the temporary file so it doesn't stay on disk.
+      Fixes bug 20646; bugfix on 0.2.0.7-alpha. Patch by fk.
+
+  o Minor bugfixes (Windows):
+    - Check for getpagesize before using it to mmap files. This fixes
+      compilation in some MinGW environments. Fixes bug 20530; bugfix on
+      0.1.2.1-alpha. Reported by "ice".
+
+  o Code simplification and refactoring:
+    - Abolish all global guard context in entrynodes.c; replace with new
+      guard_selection_t structure as preparation for proposal 271.
+      Closes ticket 19858.
+    - Introduce rend_service_is_ephemeral() that tells if given onion
+      service is ephemeral. Replace unclear NULL-checkings for service
+      directory with this function. Closes ticket 20526.
+    - Extract magic numbers in circuituse.c into defined variables.
+    - Refactor circuit_is_available_for_use to remove unnecessary check.
+    - Refactor circuit_predict_and_launch_new for readability and
+      testability. Closes ticket 18873.
+    - Refactor large if statement in purpose_needs_anonymity to use
+      switch statement instead. Closes part of ticket 20077.
+    - Refactor the hashing API to return negative values for errors, as
+      is done as throughout the codebase. Closes ticket 20717.
+    - Remove data structures that were used to index or_connection
+      objects by their RSA identity digests. These structures are fully
+      redundant with the similar structures used in the
+      channel abstraction.
+    - Remove duplicate code in the channel_write_*cell() functions.
+      Closes ticket 13827; patch from Pingl.
+    - Remove redundant behavior of is_sensitive_dir_purpose, refactor to
+      use only purpose_needs_anonymity. Closes part of ticket 20077.
+    - The code to generate and parse EXTEND and EXTEND2 cells has been
+      replaced with code automatically generated by the
+      "trunnel" utility.
+
+  o Documentation:
+    - Include the "TBits" unit in Tor's man page. Fixes part of bug
+      20622; bugfix on 0.2.5.1-alpha.
+    - Change '1' to 'weight_scale' in consensus bw weights calculation
+      comments, as that is reality. Closes ticket 20273. Patch
+      from pastly.
+    - Correct the value for AuthDirGuardBWGuarantee in the manpage, from
+      250 KBytes to 2 MBytes. Fixes bug 20435; bugfix
+      on 0.2.5.6-alpha.
+    - Stop the man page from incorrectly stating that HiddenServiceDir
+      must already exist. Fixes 20486.
+    - Clarify that when ClientRejectInternalAddresses is enabled (which
+      is the default), multicast DNS hostnames for machines on the local
+      network (of the form *.local) are also rejected. Closes
+      ticket 17070.
+
+  o Removed features:
+    - The AuthDirMaxServersPerAuthAddr option no longer exists: The same
+      limit for relays running on a single IP applies to authority IP
+      addresses as well as to non-authority IP addresses. Closes
+      ticket 20960.
+    - The UseDirectoryGuards torrc option no longer exists: all users
+      that use entry guards will also use directory guards. Related to
+      proposal 271; implements part of ticket 20831.
+
+  o Testing:
+    - New unit tests for tor_htonll(). Closes ticket 19563. Patch
+      from "overcaffeinated".
+    - Perform the coding style checks when running the tests and fail
+      when coding style violations are found. Closes ticket 5500.
+    - Add tests for networkstatus_compute_bw_weights_v10.
+    - Add unit tests circuit_predict_and_launch_new.
+    - Extract dummy_origin_circuit_new so it can be used by other
+      test functions.
+
+
+Changes in version 0.2.8.12 - 2016-12-19
+  Tor 0.2.8.12 backports a fix for a medium-severity issue (bug 21018
+  below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  It also includes an updated list of fallback directories, backported
+  from 0.2.9.
+
+  Now that the Tor 0.2.9 series is stable, only major bugfixes will be
+  backported to 0.2.8 in the future.
+
+  o Major bugfixes (parsing, security, backported from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list, backported from 0.2.9.8):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (geoip, backported from 0.2.9.7-rc):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.8 - 2016-12-19
+  Tor 0.2.9.8 is the first stable release of the Tor 0.2.9 series.
+
+  The Tor 0.2.9 series makes mandatory a number of security features
+  that were formerly optional. It includes support for a new shared-
+  randomness protocol that will form the basis for next generation
+  hidden services, includes a single-hop hidden service mode for
+  optimizing .onion services that don't actually want to be hidden,
+  tries harder not to overload the directory authorities with excessive
+  downloads, and supports a better protocol versioning scheme for
+  improved compatibility with other implementations of the Tor protocol.
+
+  And of course, there are numerous other bugfixes and improvements.
+
+  This release also includes a fix for a medium-severity issue (bug
+  21018 below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  Below are the changes since 0.2.9.7-rc. For a list of all changes
+  since 0.2.8, see the ReleaseNotes file.
+
+  o Major bugfixes (parsing, security):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+
+Changes in version 0.2.9.7-rc - 2016-12-12
+  Tor 0.2.9.7-rc fixes a few small bugs remaining in Tor 0.2.9.6-rc,
+  including a few that had prevented tests from passing on
+  some platforms.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (build):
+    - The current Git revision when building from a local repository is
+      now detected correctly when using git worktrees. Fixes bug 20492;
+      bugfix on 0.2.3.9-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When computing old Tor protocol line version in protover, we were
+      looking at 0.2.7.5 twice instead of a specific case for
+      0.2.9.1-alpha. Fixes bug 20810; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (download scheduling):
+    - Resolve a "bug" warning when considering a download schedule whose
+      delay had approached INT_MAX. Fixes 20875; bugfix on 0.2.9.5-alpha.
+
+  o Minor bugfixes (logging):
+    - Downgrade a harmless log message about the
+      pending_entry_connections list from "warn" to "info". Mitigates
+      bug 19926.
+
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak when receiving AF_UNIX connections on a
+      SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
+    - When moving a signed descriptor object from a source to an
+      existing destination, free the allocated memory inside that
+      destination object. Fixes bug 20715; bugfix on 0.2.8.3-alpha.
+
+  o Minor bugfixes (memory leak, use-after-free, linux seccomp2 sandbox):
+    - Fix a memory leak and use-after-free error when removing entries
+      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
+      0.2.5.5-alpha. Patch from "cypherpunks".
+
+  o Minor bugfixes (portability):
+    - Use the correct spelling of MAC_OS_X_VERSION_10_12 on configure.ac
+      Fixes bug 20935; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (unit tests):
+    - Stop expecting NetBSD unit tests to report success for ipfw. Part
+      of a fix for bug 19960; bugfix on 0.2.9.5-alpha.
+    - Fix tolerances in unit tests for monotonic time comparisons
+      between nanoseconds and microseconds. Previously, we accepted a 10
+      us difference only, which is not realistic on every platform's
+      clock_gettime(). Fixes bug 19974; bugfix on 0.2.9.1-alpha.
+    - Remove a double-free in the single onion service unit test. Stop
+      ignoring a return value. Make future changes less error-prone.
+      Fixes bug 20864; bugfix on 0.2.9.6-rc.
+
+
+Changes in version 0.2.8.11 - 2016-12-08
+  Tor 0.2.8.11 backports fixes for additional portability issues that
+  could prevent Tor from building correctly on OSX Sierra, or with
+  OpenSSL 1.1. Affected users should upgrade; others can safely stay
+  with 0.2.8.10.
+
+  o Minor bugfixes (portability):
+    - Avoid compilation errors when building on OSX Sierra. Sierra began
+      to support the getentropy() and clock_gettime() APIs, but created
+      a few problems in doing so. Tor 0.2.9 has a more thorough set of
+      workarounds; in 0.2.8, we are just using the /dev/urandom and mach
+      monotonic time interfaces. Fixes bug 20865. Bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+
+
+Changes in version 0.2.8.10 - 2016-12-02
+  Tor 0.2.8.10 backports a fix for a bug that would sometimes make clients
+  unusable after they left standby mode. It also backports fixes for
+  a few portability issues and a small but problematic memory leak.
+
+  o Major bugfixes (client reliability, backport from 0.2.9.5-alpha):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client performance, backport from 0.2.9.5-alpha):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.6-rc):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.2.9.5-alpha):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+Changes in version 0.2.9.6-rc - 2016-12-02
+  Tor 0.2.9.6-rc fixes a few remaining bugs found in the previous alpha
+  version. We hope that it will be ready to become stable soon, and we
+  encourage everyone to test this release. If no showstopper bugs are
+  found here, the next 0.2.9 release will be stable.
+
+  o Major bugfixes (relay, resolver, logging):
+    - For relays that don't know their own address, avoid attempting a
+      local hostname resolve for each descriptor we download. This
+      will cut down on the number of "Success: chose address 'x.x.x.x'"
+      log lines, and also avoid confusing clock jumps if the resolver
+      is slow. Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (client, fascistfirewall):
+    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
+      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop ignoring the anonymity status of saved keys for hidden
+      services and single onion services when first starting tor.
+      Instead, refuse to start tor if any hidden service key has been
+      used in a different hidden service anonymity mode. Fixes bug
+      20638; bugfix on 17178 in 0.2.9.3-alpha; reported by ahf.
+
+  o Minor bugfixes (portability):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+    - Run correctly when built on Windows build environments that
+      require _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web):
+    - Stop complaining about long-term one-hop circuits deliberately
+      created by single onion services and Tor2web. These log messages
+      are intended to diagnose issue 8387, which relates to circuits
+      hanging around forever for no reason. Fixes bug 20613; bugfix on
+      0.2.9.1-alpha. Reported by "pastly".
+
+  o Minor bugfixes (unit tests):
+    - Stop spurious failures in the local interface address discovery
+      unit tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by
+      Neel Chauhan.
+
+  o Documentation:
+    - Correct the minimum bandwidth value in torrc.sample, and queue a
+      corresponding change for torrc.minimal. Closes ticket 20085.
+
+
+Changes in version 0.2.9.5-alpha - 2016-11-08
+  Tor 0.2.9.5-alpha fixes numerous bugs discovered in the previous alpha
+  version. We believe one or two probably remain, and we encourage
+  everyone to test this release.
+
+  o Major bugfixes (client performance):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client reliability):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (download scheduling):
+    - When using an exponential backoff schedule, do not give up on
+      downloading just because we have failed a bunch of times. Since
+      each delay is longer than the last, retrying indefinitely won't
+      hurt. Fixes bug 20536; bugfix on 0.2.9.1-alpha.
+    - If a consensus expires while we are waiting for certificates to
+      download, stop waiting for certificates.
+    - If we stop waiting for certificates less than a minute after we
+      started downloading them, do not consider the certificate download
+      failure a separate failure. Fixes bug 20533; bugfix
+      on 0.2.0.9-alpha.
+    - Remove the maximum delay on exponential-backoff scheduling. Since
+      we now allow an infinite number of failures (see ticket 20536), we
+      must now allow the time to grow longer on each failure. Fixes part
+      of bug 20534; bugfix on 0.2.9.1-alpha.
+    - Make our initial download delays closer to those from 0.2.8. Fixes
+      another part of bug 20534; bugfix on 0.2.9.1-alpha.
+    - When determining when to download a directory object, handle times
+      after 2038 if the operating system supports them. (Someday this
+      will be important!) Fixes bug 20587; bugfix on 0.2.8.1-alpha.
+    - When using exponential backoff in test networks, use a lower
+      exponent, so the delays do not vary as much. This helps test
+      networks bootstrap consistently. Fixes bug 20597; bugfix on 20499.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (client directory scheduling):
+    - Treat "relay too busy to answer request" as a failed request and a
+      reason to back off on our retry frequency. This is safe now that
+      exponential backoffs retry indefinitely, and avoids a bug where we
+      would reset our download schedule erroneously. Fixes bug 20593;
+      bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (client, logging):
+    - Remove a BUG warning in circuit_pick_extend_handshake(). Instead,
+      assume all nodes support EXTEND2. Use ntor whenever a key is
+      available. Fixes bug 20472; bugfix on 0.2.9.3-alpha.
+    - On DNSPort, stop logging a BUG warning on a failed hostname
+      lookup. Fixes bug 19869; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - When configuring hidden services, check every hidden service
+      directory's permissions. Previously, we only checked the last
+      hidden service. Fixes bug 20529; bugfix the work to fix 13942
+      in 0.2.6.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
+      removed the ECDH ciphers which caused the tests to fail on
+      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay bootstrap):
+    - Ensure relays don't make multiple connections during bootstrap.
+      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+    - Avoid a small memory leak when informing worker threads about
+      rotated onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
+    - Do not try to parallelize workers more than 16x without the user
+      explicitly configuring us to do so, even if we do detect more than
+      16 CPU cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Start correctly when creating a single onion service in a
+      directory that did not previously exist. Fixes bug 20484; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid a unit test failure on systems with over 16 detectable CPU
+      cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Documentation:
+    - Clarify that setting HiddenServiceNonAnonymousMode requires you to
+      also set "SOCKSPort 0". Fixes bug 20487; bugfix on 0.2.9.3-alpha.
+    - Module-level documentation for several more modules. Closes
+      tickets 19287 and 19290.
 
 
 Changes in version 0.2.8.9 - 2016-10-17
@@ -123,7 +7094,7 @@ Changes in version 0.2.9.4-alpha - 2016-10-17
   o Minor bugfixes (getpass):
     - Defensively fix a non-triggerable heap corruption at do_getpass()
       to protect ourselves from mistakes in the future. Fixes bug
-      #19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
+      19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
       by nherring.
 
   o Minor bugfixes (hidden service):
@@ -194,7 +7165,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
 
   o Major bugfixes (circuit building):
     - Hidden service client-to-intro-point and service-to-rendezvous-
-      point cicruits use the TAP key supplied by the protocol, to avoid
+      point circuits use the TAP key supplied by the protocol, to avoid
       epistemic attacks. Fixes bug 19163; bugfix on 0.2.4.18-rc.
 
   o Major bugfixes (compilation, OpenBSD):
@@ -216,8 +7187,9 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
       OpenSSL 0.9.7 or later since 2009. Closes ticket 19998.
 
   o Minor feature (fallback directories):
-    - Remove broken entries from the hard-coded fallback directory list.
-      Closes ticket 20190; patch by teor.
+    - Remove 8 fallbacks that are no longer suitable, leaving 81 of the
+      100 fallbacks originally introduced in Tor 0.2.8.2-alpha in March
+      2016. Closes ticket 20190; patch by teor.
 
   o Minor features (geoip, also in 0.2.8.8):
     - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
@@ -312,7 +7284,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
     - Document the default PathsNeededToBuildCircuits value that's used
       by clients when the directory authorities don't set
       min_paths_for_circs_pct. Fixes bug 20117; bugfix on 02c320916e02
-      in tor-0.2.4.10-alpha. Patch by teor, reported by Jesse V.
+      in 0.2.4.10-alpha. Patch by teor, reported by Jesse V.
     - Fix manual for the User option: it takes a username, not a UID.
       Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have
       a manpage!).
@@ -351,12 +7323,12 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
 
   o Minor bugfixes (options):
     - Check the consistency of UseEntryGuards and EntryNodes more
-      reliably. Fixes bug 20074; bugfix on tor- 0.2.4.12-alpha. Patch
+      reliably. Fixes bug 20074; bugfix on 0.2.4.12-alpha. Patch
       by teor.
     - Stop changing the configured value of UseEntryGuards on
       authorities and Tor2web clients. Fixes bug 20074; bugfix on
-      commits 51fc6799 in tor-0.1.1.16-rc and acda1735 in tor-0.2.4.3-
-      alpha. Patch by teor.
+      commits 51fc6799 in 0.1.1.16-rc and acda1735 in 0.2.4.3-alpha.
+      Patch by teor.
 
   o Minor bugfixes (Tor2web):
     - Prevent Tor2web clients running hidden services, these services
@@ -395,8 +7367,9 @@ Changes in version 0.2.8.8 - 2016-09-23
       this one.
 
   o Minor feature (fallback directories):
-    - Remove broken fallbacks from the hard-coded fallback directory
-      list. Closes ticket 20190; patch by teor.
+    - Remove 8 fallbacks that are no longer suitable, leaving 81 of the
+      100 fallbacks originally introduced in Tor 0.2.8.2-alpha in March
+      2016. Closes ticket 20190; patch by teor.
 
   o Minor features (geoip):
     - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
@@ -478,7 +7451,7 @@ Changes in version 0.2.9.2-alpha - 2016-08-24
   o Deprecated features:
     - A number of DNS-cache-related sub-options for client ports are now
       deprecated for security reasons, and may be removed in a future
-      version of Tor. (We believe that client-side DNS cacheing is a bad
+      version of Tor. (We believe that client-side DNS caching is a bad
       idea for anonymity, and you should not turn it on.) The options
       are: CacheDNS, CacheIPv4DNS, CacheIPv6DNS, UseDNSCache,
       UseIPv4Cache, and UseIPv6Cache.
@@ -850,8 +7823,9 @@ Changes in version 0.2.8.6 - 2016-08-02
       is signed. Fixes bug 19682; bugfix on 0.2.8.1-alpha.
 
   o Minor bugfixes (fallback directories):
-    - Remove a fallback that was on the hardcoded list, then opted-out.
-      Fixes bug 19782; update to fallback list from 0.2.8.2-alpha.
+    - Remove 1 fallback that was on the hardcoded list, then opted-out,
+      leaving 89 of the 100 fallbacks originally introduced in Tor
+      0.2.8.2-alpha in March 2016. Closes ticket 19782; patch by teor.
 
   o Minor bugfixes (Linux seccomp2 sandbox):
     - Allow more syscalls when running with "Sandbox 1" enabled:
@@ -875,7 +7849,7 @@ Changes in version 0.2.8.5-rc - 2016-07-07
   o Major bugfixes (heartbeat):
     - Fix a regression that would crash Tor when the periodic
       "heartbeat" log messages were disabled. Fixes bug 19454; bugfix on
-      tor-0.2.8.1-alpha. Reported by "kubaku".
+      0.2.8.1-alpha. Reported by "kubaku".
 
   o Minor features (build):
     - Tor now again builds with the recent OpenSSL 1.1 development
@@ -923,8 +7897,9 @@ Changes in version 0.2.8.5-rc - 2016-07-07
     - Update fallback whitelist and blacklist based on relay operator
       emails. Blacklist unsuitable (non-working, over-volatile)
       fallbacks. Resolves ticket 19071. Patch by teor.
-    - Update hard-coded fallback list to remove unsuitable fallbacks.
-      Resolves ticket 19071. Patch by teor.
+    - Remove 10 unsuitable fallbacks, leaving 90 of the 100 fallbacks
+      originally introduced in Tor 0.2.8.2-alpha in March 2016. Closes
+      ticket 19071; patch by teor.
 
 
 Changes in version 0.2.8.4-rc - 2016-06-15
@@ -1021,7 +7996,7 @@ Changes in version 0.2.8.3-alpha - 2016-05-26
 
   o Major bugfixes (testing):
     - Fix a bug that would block 'make test-network-all' on systems where
-      IPv6 packets were lost. Fixes bug 19008; bugfix on tor-0.2.7.3-rc.
+      IPv6 packets were lost. Fixes bug 19008; bugfix on 0.2.7.3-rc.
     - Avoid "WSANOTINITIALISED" warnings in the unit tests. Fixes bug 18668;
       bugfix on 0.2.8.1-alpha.
 
@@ -1034,10 +8009,12 @@ Changes in version 0.2.8.3-alpha - 2016-05-26
     - Give each fallback the same weight for client selection; restrict
       fallbacks to one per operator; report fallback directory detail
       changes when rebuilding list; add new fallback directory mirrors
-      to the whitelist; update fallback directories based on the latest
-      OnionOO data; and any other minor simplifications and fixes.
-      Closes tasks 17158, 17905, 18749, bug 18689, and fixes part of bug
-      18812 on 0.2.8.1-alpha; patch by "teor".
+      to the whitelist; and many other minor simplifications and fixes.
+      Closes tasks 17905, 18749, bug 18689, and fixes part of bug 18812 on
+      0.2.8.1-alpha; patch by "teor".
+    - Replace the 21 fallbacks generated in January 2016 and included in
+      Tor 0.2.8.1-alpha, with a list of 100 fallbacks generated in March
+      2016. Closes task 17158; patch by "teor".
 
   o Minor features (geoip):
     - Update geoip and geoip6 to the May 4 2016 Maxmind GeoLite2
@@ -1307,7 +8284,7 @@ Changes in version 0.2.8.2-alpha - 2016-03-28
       testing versions of the static libraries. Fixes bug 18490; bugfix
       on 0.2.7.1-alpha.
     - Avoid spurious failures from configure files related to calling
-      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18625; bugfix on
+      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18626; bugfix on
       0.2.0.1-alpha. Patch from "cypherpunks".
     - Silence spurious clang-scan warnings in the ed25519_donna code by
       explicitly initializing some objects. Fixes bug 18384; bugfix on
@@ -1481,10 +8458,11 @@ Changes in version 0.2.8.1-alpha - 2016-02-04
       should reduces failures due to fallback churn. Implements ticket
       4483. Patch by "teor". Implements IPv4 portions of proposal 210 by
       "mikeperry" and "teor".
-    - Include a trial list of default fallback directories, based on an
-      opt-in survey of suitable relays. Doing this should make clients
-      bootstrap more quickly and reliably, and reduce the load on the
-      directory authorities. Closes ticket 15775. Patch by "teor".
+    - Include a trial list of 21 default fallback directories, generated
+      in January 2016, based on an opt-in survey of suitable relays.
+      Doing this should make clients bootstrap more quickly and reliably,
+      and reduce the load on the directory authorities. Closes ticket
+      15775. Patch by "teor".
       Candidates identified using an OnionOO script by "weasel", "teor",
       "gsathya", and "karsten".
     - Previously only relays that explicitly opened a directory port
@@ -2596,7 +9574,7 @@ Changes in version 0.2.6.8 - 2015-05-21
 
   o Major bugfixes (hidden services, backport from 0.2.7.1-alpha):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
 
@@ -2637,7 +9615,7 @@ Changes in version 0.2.7.1-alpha - 2015-05-12
 
   o Major bugfixes (hidden services):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
 
@@ -2943,7 +9921,7 @@ Changes in version 0.2.6.5-rc - 2015-03-18
   o Major bugfixes (pluggable transports):
     - Initialize the extended OR Port authentication cookie before
       launching pluggable transports. This prevents a race condition
-      that occured when server-side pluggable transports would cache the
+      that occurred when server-side pluggable transports would cache the
       authentication cookie before it has been (re)generated. Fixes bug
       15240; bugfix on 0.2.5.1-alpha.
 
@@ -3686,7 +10664,7 @@ Changes in version 0.2.6.2-alpha - 2014-12-31
       some bugs where we would look at (but fortunately, not reveal)
       uninitialized memory on the stack. Fixes bug 14013; bugfix on all
       versions of Tor.
-    - Clear all memory targetted by tor_addr_{to,from}_sockaddr(), not
+    - Clear all memory targeted by tor_addr_{to,from}_sockaddr(), not
       just the part that's used. This makes it harder for data leak bugs
       to occur in the event of other programming failures. Resolves
       ticket 14041.
@@ -4829,7 +11807,7 @@ Changes in version 0.2.5.5-alpha - 2014-06-18
       directory authority options, remove the documentation for a
       V2-directory fetching option that no longer exists. Resolves
       ticket 11634.
-    - Correct the documenation so that it lists the correct directory
+    - Correct the documentation so that it lists the correct directory
       for the stats files. (They are in a subdirectory called "stats",
       not "status".)
     - In the manpage, move more authority-only options into the
@@ -6114,7 +13092,7 @@ Changes in version 0.2.5.1-alpha - 2013-10-02
       from Arlo Breault.
     - Remove instances of strcpy() from the unit tests. They weren't
       hurting anything, since they were only in the unit tests, but it's
-      embarassing to have strcpy() in the code at all, and some analysis
+      embarrassing to have strcpy() in the code at all, and some analysis
       tools don't like it. Fixes bug 8790; bugfix on 0.2.3.6-alpha and
       0.2.3.8-alpha. Patch from Arlo Breault.
 
@@ -6215,7 +13193,7 @@ Changes in version 0.2.4.16-rc - 2013-08-10
       0.2.4.15-rc. Found by stem integration tests.
 
   o Minor bugfixes:
-    - Fix an invalid memory read that occured when a pluggable
+    - Fix an invalid memory read that occurred when a pluggable
       transport proxy failed its configuration protocol.
       Fixes bug 9288; bugfix on 0.2.4.1-alpha.
     - When evaluating whether to use a connection that we haven't
@@ -6473,7 +13451,7 @@ Changes in version 0.2.4.12-alpha - 2013-04-18
       0.2.0.10-alpha. Reported pseudonymously.
     - Make the format and order of STREAM events for DNS lookups
       consistent among the various ways to launch DNS lookups. Fixes
-      bug 8203; bugfix on 0.2.0.24-rc. Patch by "Desoxy."
+      bug 8203; bugfix on 0.2.0.24-rc. Patch by "Desoxy".
     - Correct our check for which versions of Tor support the EXTEND2
       cell. We had been willing to send it to Tor 0.2.4.7-alpha and
       later, when support was really added in version 0.2.4.8-alpha.
@@ -9006,7 +15984,7 @@ Changes in version 0.2.3.11-alpha - 2012-01-22
       CloseHSServiceRendCircuitsImmediatelyOnTimeout option. Fixes the
       remaining part of bug 1297; bugfix on 0.2.2.2-alpha.
     - Make sure we never mark the wrong rendezvous circuit as having
-      had its introduction cell acknowleged by the introduction-point
+      had its introduction cell acknowledged by the introduction-point
       relay. Previously, when we received an INTRODUCE_ACK cell on a
       client-side hidden-service introduction circuit, we might have
       marked a rendezvous circuit other than the one we specified in
@@ -10313,7 +17291,7 @@ Changes in version 0.2.3.3-alpha - 2011-09-01
       raised by bug 3898.
     - The "--quiet" and "--hush" options now apply not only to Tor's
       behavior before logs are configured, but also to Tor's behavior in
-      the absense of configured logs. Fixes bug 3550; bugfix on
+      the absence of configured logs. Fixes bug 3550; bugfix on
       0.2.0.10-alpha.
 
   o Minor bugfixes (also part of 0.2.2.31-rc):
@@ -12814,7 +19792,7 @@ Changes in version 0.2.2.8-alpha - 2010-01-26
   please upgrade.
 
   o Major bugfixes:
-    - Fix a memory corruption bug on bridges that occured during the
+    - Fix a memory corruption bug on bridges that occurred during the
       inclusion of stats data in extra-info descriptors. Also fix the
       interface for geoip_get_bridge_stats* to prevent similar bugs in
       the future. Diagnosis by Tas, patch by Karsten and Sebastian.
@@ -13228,7 +20206,7 @@ Changes in version 0.2.2.2-alpha - 2009-09-21
       to EDGE and find out if the build-time data in the .tor/state gets
       reset without loss of Tor usability. You should also see a notice
       log message telling you that Tor has reset its timeout.
-    - Directory authorities can now vote on arbitary integer values as
+    - Directory authorities can now vote on arbitrary integer values as
       part of the consensus process. This is designed to help set
       network-wide parameters. Implements proposal 167.
     - Tor now reads the "circwindow" parameter out of the consensus,
@@ -16256,7 +23234,7 @@ Changes in version 0.2.0.9-alpha - 2007-10-24
     - Distinguish between detached signatures for the wrong period, and
       detached signatures for a divergent vote.
     - Fix a small memory leak when computing a consensus.
-    - When there's no concensus, we were forming a vote every 30
+    - When there's no consensus, we were forming a vote every 30
       minutes, but writing the "valid-after" line in our vote based
       on our configured V3AuthVotingInterval: so unless the intervals
       matched up, we immediately rejected our own vote because it didn't
@@ -19643,7 +26621,7 @@ Changes in version 0.1.0.9-rc - 2005-06-09
       KeepalivePeriod, ClientOnly, NoPublish, HttpProxy, HttpsProxy,
       HttpProxyAuthenticator
     - Stop warning about sigpipes in the logs. We're going to
-      pretend that getting these occassionally is normal and fine.
+      pretend that getting these occasionally is normal and fine.
     - Resolve OS X installer bugs: stop claiming to be 0.0.9.2 in
       certain
       installer screens; and don't put stuff into StartupItems unless
@@ -20108,7 +27086,7 @@ Changes in version 0.1.0.1-rc - 2005-03-28
 
 Changes in version 0.0.9.6 - 2005-03-24
   o Bugfixes on 0.0.9.x (crashes and asserts):
-    - Add new end stream reasons to maintainance branch. Fix bug where
+    - Add new end stream reasons to maintenance branch. Fix bug where
       reason (8) could trigger an assert. Prevent bug from recurring.
     - Apparently win32 stat wants paths to not end with a slash.
     - Fix assert triggers in assert_cpath_layer_ok(), where we were
diff --git a/Doxyfile.in b/Doxyfile.in
index a39348f2cb..4cf9c30ded 100644
--- a/Doxyfile.in
+++ b/Doxyfile.in
@@ -446,12 +446,6 @@ MAX_INITIALIZER_LINES  = 30
 
 SHOW_USED_FILES        = YES
 
-# If the sources in your project are distributed over multiple directories 
-# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy 
-# in the documentation. The default is NO.
-
-SHOW_DIRECTORIES       = NO
-
 # Set the SHOW_FILES tag to NO to disable the generation of the Files page.
 # This will remove the Files entry from the Quick Index and from the 
 # Folder Tree View (if specified). The default is YES.
@@ -760,12 +754,6 @@ HTML_FOOTER            =
 
 HTML_STYLESHEET        = 
 
-# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, 
-# files or namespaces will be aligned in HTML using tables. If set to 
-# NO a bullet list will be used.
-
-HTML_ALIGN_MEMBERS     = YES
-
 # If the GENERATE_HTMLHELP tag is set to YES, additional index files 
 # will be generated that can be used as input for tools like the 
 # Microsoft HTML help workshop to generate a compiled HTML help file (.chm) 
@@ -1047,18 +1035,6 @@ GENERATE_XML           = NO
 
 XML_OUTPUT             = xml
 
-# The XML_SCHEMA tag can be used to specify an XML schema, 
-# which can be used by a validating XML parser to check the 
-# syntax of the XML files.
-
-XML_SCHEMA             = 
-
-# The XML_DTD tag can be used to specify an XML DTD, 
-# which can be used by a validating XML parser to check the 
-# syntax of the XML files.
-
-XML_DTD                = 
-
 # If the XML_PROGRAMLISTING tag is set to YES Doxygen will 
 # dump the program listings (including syntax highlighting 
 # and cross-referencing information) to the XML output. Note that 
@@ -1264,7 +1240,7 @@ HAVE_DOT               = NO
 # DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory 
 # containing the font.
 
-DOT_FONTNAME           = FreeSans
+DOT_FONTNAME           =
 
 # By default doxygen will tell dot to use the output directory to look for the 
 # FreeSans.ttf font (which doxygen will put there itself). If you specify a 
diff --git a/LICENSE b/LICENSE
index 56d1002251..3d0f8c121b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -13,7 +13,7 @@ Tor is distributed under this license:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-Copyright (c) 2007-2016, The Tor Project, Inc.
+Copyright (c) 2007-2017, The Tor Project, Inc.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
diff --git a/Makefile.am b/Makefile.am
index 48d3df2654..191620c062 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 # Copyright (c) 2001-2004, Roger Dingledine
 # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-# Copyright (c) 2007-2015, The Tor Project, Inc.
+# Copyright (c) 2007-2017, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 ACLOCAL_AMFLAGS = -I m4
@@ -9,13 +9,14 @@ noinst_LIBRARIES=
 EXTRA_DIST=
 noinst_HEADERS=
 bin_PROGRAMS=
+EXTRA_PROGRAMS=
 CLEANFILES=
 TESTS=
 noinst_PROGRAMS=
 DISTCLEANFILES=
 bin_SCRIPTS=
 AM_CPPFLAGS=
-AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@
+AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@
 SHELL=@SHELL@
 
 if COVERAGE_ENABLED
@@ -24,17 +25,27 @@ else
 TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT)
 endif
 
+if USE_RUST
+## this MUST be $(), otherwise am__DEPENDENCIES will not track it
+rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \
+	$(TOR_RUST_EXTRA_LIBS)
+else
+rust_ldadd=
+endif
+
 include src/include.am
 include doc/include.am
 include contrib/include.am
 
 EXTRA_DIST+= \
 	ChangeLog					\
+	CONTRIBUTING					\
 	INSTALL						\
 	LICENSE						\
 	Makefile.nmake					\
 	README						\
-	ReleaseNotes
+	ReleaseNotes					\
+	scripts/maint/checkSpace.pl
 
 ## This tells etags how to find mockable function definitions.
 AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
@@ -42,16 +53,22 @@ AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
 if COVERAGE_ENABLED
 TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage
 if DISABLE_ASSERTS_IN_UNIT_TESTS
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
 else
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE @TOR_MODULES_ALL_ENABLED@
 endif
 TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1
 else
 TEST_CFLAGS=
-TEST_CPPFLAGS=-DTOR_UNIT_TESTS
+TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
+TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
+
+if LIBFUZZER_ENABLED
+TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div
+# not "edge"
+endif
 
 TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
 TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
@@ -82,6 +99,8 @@ doxygen:
 test: all
 	$(top_builddir)/src/test/test
 
+check-local: check-spaces check-changes
+
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
 		echo '$$CHUTNEY_PATH was not set.'; \
@@ -101,18 +120,20 @@ test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert
 
 # Run all available tests using automake's test-driver
 # only run IPv6 tests if we can ping6 ::1 (localhost)
+# only run IPv6 tests if we can ping ::1 (localhost)
 # some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
 # only run mixed tests if we have a tor-stable binary
-# Try both the BSD and the Linux ping6 syntax, because they're incompatible
+# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
+# because they're incompatible
 test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
 	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
 	rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
 	@flavors="$(TEST_CHUTNEY_FLAVORS)"; \
-	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1; then \
-		echo "ping6 ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+	if ping6 -q -c 1 -o ::1 >/dev/null 2>&1 || ping6 -q -c 1 -W 1 ::1 >/dev/null 2>&1 || ping -6 -c 1 -W 1 ::1 >/dev/null 2>&1; then \
+		echo "ping6 ::1 or ping ::1 succeeded, running IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
 		flavors="$$flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
 	else \
-		echo "ping6 ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
+		echo "ping6 ::1 and ping ::1 failed, skipping IPv6 flavors: $(TEST_CHUTNEY_FLAVORS_IPV6)."; \
 		skip_flavors="$$skip_flavors $(TEST_CHUTNEY_FLAVORS_IPV6)"; \
 	fi; \
 	if command -v tor-stable >/dev/null 2>&1; then \
@@ -127,6 +148,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/
 	done; \
 	for f in $$flavors; do \
 		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
+		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \
 	done; \
 	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
 	! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
@@ -184,11 +206,14 @@ coverage-html-full: all
 # Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
 # tinytest*.[ch]
 check-spaces:
-	$(top_srcdir)/scripts/maint/checkSpace.pl -C \
+if USE_PERL
+	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
 		$(top_srcdir)/src/common/*.[ch] \
 		$(top_srcdir)/src/or/*.[ch] \
 		$(top_srcdir)/src/test/*.[ch] \
+		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
+endif
 
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
@@ -197,16 +222,42 @@ check-logs:
 	$(top_srcdir)/scripts/maint/checkLogs.pl \
 		$(top_srcdir)/src/*/*.[ch] | sort -n
 
+.PHONY: check-typos
+check-typos:
+	@if test -x "`which misspell 2>&1;true`"; then \
+		echo "Checking for Typos ..."; \
+		(misspell \
+			$(top_srcdir)/src/[^e]*/*.[ch] \
+			$(top_srcdir)/doc \
+			$(top_srcdir)/contrib \
+			$(top_srcdir)/scripts \
+			$(top_srcdir)/README \
+			$(top_srcdir)/ChangeLog \
+			$(top_srcdir)/INSTALL \
+			$(top_srcdir)/ReleaseNotes \
+			$(top_srcdir)/LICENSE); \
+	else \
+		echo "Tor can use misspell to check for typos."; \
+		echo "It seems that you don't have misspell installed."; \
+		echo "You can install the latest version of misspell here: https://github.com/client9/misspell#install"; \
+	fi
+
 .PHONY: check-changes
 check-changes:
+if USEPYTHON
 	@if test -d "$(top_srcdir)/changes"; then \
-		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes/*; \
+		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
 		fi
+endif
 
 .PHONY: update-versions
 update-versions:
 	$(PERL) $(top_builddir)/scripts/maint/updateVersions.pl
 
+.PHONY: callgraph
+callgraph:
+	$(top_builddir)/scripts/maint/run_calltool.sh
+
 version:
 	@echo "Tor @VERSION@"
 	@if test -d "$(top_srcdir)/.git" && test -x "`which git 2>&1;true`"; then \
@@ -220,6 +271,14 @@ mostlyclean-local:
 	rm -rf $(top_builddir)/doc/doxygen
 	rm -rf $(TEST_NETWORK_ALL_LOG_DIR)
 
+clean-local:
+	rm -rf $(top_builddir)/src/rust/target
+	rm -rf $(top_builddir)/src/rust/.cargo/registry
+
+if USE_RUST
+distclean-local: distclean-rust
+endif
+
 # This relies on some internal details of how automake implements
 # distcheck.  We check two directories because automake-1.15 changed
 # from $(distdir)/_build to $(distdir)/_build/sub.
diff --git a/README b/README
index d246a6930e..9e0f4b8241 100644
--- a/README
+++ b/README
@@ -27,3 +27,6 @@ Frequently Asked Questions:
 
 To get started working on Tor development:
         See the doc/HACKING directory.
+
+Release timeline:
+         https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/CoreTorReleases
diff --git a/ReleaseNotes b/ReleaseNotes
index af61a4d739..8a38289c2c 100644
--- a/ReleaseNotes
+++ b/ReleaseNotes
@@ -1,6 +1,5834 @@
-This document summarizes new features and bugfixes in each stable release
-of Tor. If you want to see more detailed descriptions of the changes in
-each development snapshot, see the ChangeLog file.
+This document summarizes new features and bugfixes in each stable
+release of Tor. If you want to see more detailed descriptions of the
+changes in each development snapshot, see the ChangeLog file.
+
+Changes in version 0.3.3.7 - 2018-06-12
+  Tor 0.3.3.7 backports several changes from the 0.3.4.x series, including
+  fixes for bugs affecting compatibility and stability.
+
+  o Directory authority changes:
+    - Add an IPv6 address for the "dannenberg" directory authority.
+      Closes ticket 26343.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26351.
+
+  o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha):
+    - Work around a change in OpenSSL 1.1.1 where return values that
+      would previously indicate "no password" now indicate an empty
+      password. Without this workaround, Tor instances running with
+      OpenSSL 1.1.1 would accept descriptors that other Tor instances
+      would reject. Fixes bug 26116; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (compilation, backport from 0.3.4.2-alpha):
+    - Silence unused-const-variable warnings in zstd.h with some GCC
+      versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.4.2-alpha):
+    - Improve accuracy of the BUILDTIMEOUT_SET control port event's
+      TIMEOUT_RATE and CLOSE_RATE fields. (We were previously
+      miscounting the total number of circuits for these field values.)
+      Fixes bug 26121; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (hardening, backport from 0.3.4.2-alpha):
+    - Prevent a possible out-of-bounds smartlist read in
+      protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (path selection, backport from 0.3.4.1-alpha):
+    - Only select relays when they have the descriptors we prefer to use
+      for them. This change fixes a bug where we could select a relay
+      because it had _some_ descriptor, but reject it later with a
+      nonfatal assertion error because it didn't have the exact one we
+      wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha.
+
+
+Changes in version 0.3.3.6 - 2018-05-22
+  Tor 0.3.3.6 is the first stable release in the 0.3.3 series. It
+  backports several important fixes from the 0.3.4.1-alpha.
+
+  The Tor 0.3.3 series includes controller support and other
+  improvements for v3 onion services, official support for embedding Tor
+  within other applications, and our first non-trivial module written in
+  the Rust programming language. (Rust is still not enabled by default
+  when building Tor.) And as usual, there are numerous other smaller
+  bugfixes, features, and improvements.
+
+  Below are the changes since 0.3.2.10. For a list of only the changes
+  since 0.3.3.5-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - When built with Rust, Tor now depends on version 0.2.39 of the
+      libc crate. Closes tickets 25310 and 25664.
+
+  o Major features (embedding):
+    - There is now a documented stable API for programs that need to
+      embed Tor. See tor_api.h for full documentation and known bugs.
+      Closes ticket 23684.
+    - Tor now has support for restarting in the same process.
+      Controllers that run Tor using the "tor_api.h" interface can now
+      restart Tor after Tor has exited. This support is incomplete,
+      however: we fixed crash bugs that prevented it from working at
+      all, but many bugs probably remain, including a possibility of
+      security issues. Implements ticket 24581.
+
+  o Major features (IPv6, directory documents):
+    - Add consensus method 27, which adds IPv6 ORPorts to the microdesc
+      consensus. This information makes it easier for IPv6 clients to
+      bootstrap and choose reachable entry guards. Implements
+      ticket 23826.
+    - Add consensus method 28, which removes IPv6 ORPorts from
+      microdescriptors. Now that the consensus contains IPv6 ORPorts,
+      they are redundant in microdescs. This change will be used by Tor
+      clients on 0.2.8.x and later. (That is to say, with all Tor
+      clients that have IPv6 bootstrap and guard support.) Implements
+      ticket 23828.
+    - Expand the documentation for AuthDirHasIPv6Connectivity when it is
+      set by different numbers of authorities. Fixes 23870
+      on 0.2.4.1-alpha.
+
+  o Major features (onion service v3, control port):
+    - The control port now supports commands and events for v3 onion
+      services. It is now possible to create ephemeral v3 services using
+      ADD_ONION. Additionally, several events (HS_DESC, HS_DESC_CONTENT,
+      CIRC and CIRC_MINOR) and commands (GETINFO, HSPOST, ADD_ONION and
+      DEL_ONION) have been extended to support v3 onion services. Closes
+      ticket 20699; implements proposal 284.
+
+  o Major features (onion services):
+    - Provide torrc options to pin the second and third hops of onion
+      service circuits to a list of nodes. The option HSLayer2Guards
+      pins the second hop, and the option HSLayer3Guards pins the third
+      hop. These options are for use in conjunction with experiments
+      with "vanguards" for preventing guard enumeration attacks. Closes
+      ticket 13837.
+    - When v3 onion service clients send introduce cells, they now
+      include the IPv6 address of the rendezvous point, if it has one.
+      Current v3 onion services running 0.3.2 ignore IPv6 addresses, but
+      in future Tor versions, IPv6-only v3 single onion services will be
+      able to use IPv6 addresses to connect directly to the rendezvous
+      point. Closes ticket 23577. Patch by Neel Chauhan.
+
+  o Major features (relay):
+    - Implement an option, ReducedExitPolicy, to allow an Tor exit relay
+      operator to use a more reasonable ("reduced") exit policy, rather
+      than the default one. If you want to run an exit node without
+      thinking too hard about which ports to allow, this one is for you.
+      Closes ticket 13605. Patch from Neel Chauhan.
+
+  o Major features (rust, portability, experimental):
+    - Tor now ships with an optional implementation of one of its
+      smaller modules (protover.c) in the Rust programming language. To
+      try it out, install a Rust build environment, and configure Tor
+      with "--enable-rust --enable-cargo-online-mode". This should not
+      cause any user-visible changes, but should help us gain more
+      experience with Rust, and plan future Rust integration work.
+      Implementation by Chelsea Komlo. Closes ticket 22840.
+
+  o Major bugfixes (directory authorities, security, backport from 0.3.4.1-alpha):
+    - When directory authorities read a zero-byte bandwidth file, they
+      would previously log a warning with the contents of an
+      uninitialised buffer. They now log a warning about the empty file
+      instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha.
+
+  o Major bugfixes (security, directory authority, denial-of-service):
+    - Fix a bug that could have allowed an attacker to force a directory
+      authority to use up all its RAM by passing it a maliciously
+      crafted protocol versions string. Fixes bug 25517; bugfix on
+      0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005.
+
+  o Major bugfixes (crash, backport from 0.3.4.1-alpha):
+    - Avoid a rare assertion failure in the circuit build timeout code
+      if we fail to allow any circuits to actually complete. Fixes bug
+      25733; bugfix on 0.2.2.2-alpha.
+
+  o Major bugfixes (netflow padding):
+    - Stop adding unneeded channel padding right after we finish
+      flushing to a connection that has been trying to flush for many
+      seconds. Instead, treat all partial or complete flushes as
+      activity on the channel, which will defer the time until we need
+      to add padding. This fix should resolve confusing and scary log
+      messages like "Channel padding timeout scheduled 221453ms in the
+      past." Fixes bug 22212; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (networking):
+    - Tor will no longer reject IPv6 address strings from Tor Browser
+      when they are passed as hostnames in SOCKS5 requests. Fixes bug
+      25036, bugfix on Tor 0.3.1.2.
+
+  o Major bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Correctly detect when onion services get disabled after HUP. Fixes
+      bug 25761; bugfix on 0.3.2.1.
+
+  o Major bugfixes (performance, load balancing):
+    - Directory authorities no longer vote in favor of the Guard flag
+      for relays without directory support. Starting in Tor
+      0.3.0.1-alpha, clients have been avoiding using such relays in the
+      Guard position, leading to increasingly broken load balancing for
+      the 5%-or-so of Guards that don't advertise directory support.
+      Fixes bug 22310; bugfix on 0.3.0.6.
+
+  o Major bugfixes (relay):
+    - If we have failed to connect to a relay and received a connection
+      refused, timeout, or similar error (at the TCP level), do not try
+      that same address/port again for 60 seconds after the failure has
+      occurred. Fixes bug 24767; bugfix on 0.0.6.
+
+  o Major bugfixes (relay, denial of service, backport from 0.3.4.1-alpha):
+    - Impose a limit on circuit cell queue size. The limit can be
+      controlled by a consensus parameter. Fixes bug 25226; bugfix
+      on 0.2.4.14-alpha.
+
+  o Minor features (cleanup):
+    - Tor now deletes the CookieAuthFile and ExtORPortCookieAuthFile
+      when it stops. Closes ticket 23271.
+
+  o Minor features (compatibility, backport from 0.3.4.1-alpha):
+    - Avoid some compilation warnings with recent versions of LibreSSL.
+      Closes ticket 26006.
+
+  o Minor features (config options):
+    - Change the way the default value for MaxMemInQueues is calculated.
+      We now use 40% of the hardware RAM if the system has 8 GB RAM or
+      more. Otherwise we use the former value of 75%. Closes
+      ticket 24782.
+
+  o Minor features (continuous integration):
+    - Update the Travis CI configuration to use the stable Rust channel,
+      now that we have decided to require that. Closes ticket 25714.
+
+  o Minor features (continuous integration, backport from 0.3.4.1-alpha):
+    - Our .travis.yml configuration now includes support for testing the
+      results of "make distcheck". (It's not uncommon for "make check"
+      to pass but "make distcheck" to fail.) Closes ticket 25814.
+    - Our Travis CI configuration now integrates with the Coveralls
+      coverage analysis tool. Closes ticket 25818.
+
+  o Minor features (defensive programming):
+    - Most of the functions in Tor that free objects have been replaced
+      with macros that free the objects and set the corresponding
+      pointers to NULL. This change should help prevent a large class of
+      dangling pointer bugs. Closes ticket 24337.
+    - Where possible, the tor_free() macro now only evaluates its input
+      once. Part of ticket 24337.
+    - Check that microdesc ed25519 ids are non-zero in
+      node_get_ed25519_id() before returning them. Implements ticket
+      24001, patch by "aruna1234".
+
+  o Minor features (directory authority):
+    - When directory authorities are unable to add signatures to a
+      pending consensus, log the reason why. Closes ticket 24849.
+
+  o Minor features (embedding):
+    - Tor can now start with a preauthenticated control connection
+      created by the process that launched it. This feature is meant for
+      use by programs that want to launch and manage a Tor process
+      without allowing other programs to manage it as well. For more
+      information, see the __OwningControllerFD option documented in
+      control-spec.txt. Closes ticket 23900.
+    - On most errors that would cause Tor to exit, it now tries to
+      return from the tor_main() function, rather than calling the
+      system exit() function. Most users won't notice a difference here,
+      but it should be significant for programs that run Tor inside a
+      separate thread: they should now be able to survive Tor's exit
+      conditions rather than having Tor shut down the entire process.
+      Closes ticket 23848.
+    - Applications that want to embed Tor can now tell Tor not to
+      register any of its own POSIX signal handlers, using the
+      __DisableSignalHandlers option. Closes ticket 24588.
+
+  o Minor features (fallback directory list):
+    - Avoid selecting fallbacks that change their IP addresses too
+      often. Select more fallbacks by ignoring the Guard flag, and
+      allowing lower cutoffs for the Running and V2Dir flags. Also allow
+      a lower bandwidth, and a higher number of fallbacks per operator
+      (5% of the list). Implements ticket 24785.
+    - Update the fallback whitelist and blacklist based on opt-ins and
+      relay changes. Closes tickets 22321, 24678, 22527, 24135,
+      and 24695.
+
+  o Minor features (fallback directory mirror configuration):
+    - Add a nickname to each fallback in a C comment. This makes it
+      easier for operators to find their relays, and allows stem to use
+      nicknames to identify fallbacks. Implements ticket 24600.
+    - Add a type and version header to the fallback directory mirror
+      file. Also add a delimiter to the end of each fallback entry. This
+      helps external parsers like stem and Relay Search. Implements
+      ticket 24725.
+    - Add an extrainfo cache flag for each fallback in a C comment. This
+      allows stem to use fallbacks to fetch extra-info documents, rather
+      than using authorities. Implements ticket 22759.
+    - Add the generateFallbackDirLine.py script for automatically
+      generating fallback directory mirror lines from relay fingerprints.
+      No more typos! Add the lookupFallbackDirContact.py script for
+      automatically looking up operator contact info from relay
+      fingerprints. Implements ticket 24706, patch by teor and atagar.
+    - Reject any fallback directory mirror that serves an expired
+      consensus. Implements ticket 20942, patch by "minik".
+    - Remove commas and equals signs from external string inputs to the
+      fallback list. This avoids format confusion attacks. Implements
+      ticket 24726.
+    - Remove the "weight=10" line from fallback directory mirror
+      entries. Ticket 24681 will maintain the current fallback weights
+      by changing Tor's default fallback weight to 10. Implements
+      ticket 24679.
+    - Stop logging excessive information about fallback netblocks.
+      Implements ticket 24791.
+
+  o Minor features (forward-compatibility):
+    - If a relay supports some link authentication protocol that we do
+      not recognize, then include that relay's ed25519 key when telling
+      other relays to extend to it. Previously, we treated future
+      versions as if they were too old to support ed25519 link
+      authentication. Closes ticket 20895.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country
+      database. Closes ticket 26104.
+
+  o Minor features (heartbeat):
+    - Add onion service information to our heartbeat logs, displaying
+      stats about the activity of configured onion services. Closes
+      ticket 24896.
+
+  o Minor features (instrumentation, development):
+    - Add the MainloopStats option to allow developers to get
+      instrumentation information from the main event loop via the
+      heartbeat messages. We hope to use this to improve Tor's behavior
+      when it's trying to sleep. Closes ticket 24605.
+
+  o Minor features (IPv6):
+    - Make IPv6-only clients wait for microdescs for relays, even if we
+      were previously using descriptors (or were using them as a bridge)
+      and have a cached descriptor for them. Implements ticket 23827.
+    - When a consensus has IPv6 ORPorts, make IPv6-only clients use
+      them, rather than waiting to download microdescriptors. Implements
+      ticket 23827.
+
+  o Minor features (log messages):
+    - Improve log message in the out-of-memory handler to include
+      information about memory usage from the different compression
+      backends. Closes ticket 25372.
+    - Improve a warning message that happens when we fail to re-parse an
+      old router because of an expired certificate. Closes ticket 20020.
+    - Make the log more quantitative when we hit MaxMemInQueues
+      threshold exposing some values. Closes ticket 24501.
+
+  o Minor features (logging):
+    - Clarify the log messages produced when getrandom() or a related
+      entropy-generation mechanism gives an error. Closes ticket 25120.
+    - Added support for the Android logging subsystem. Closes
+      ticket 24362.
+
+  o Minor features (performance):
+    - Support predictive circuit building for onion service circuits
+      with multiple layers of guards. Closes ticket 23101.
+    - Use stdatomic.h where available, rather than mutexes, to implement
+      atomic_counter_t. Closes ticket 23953.
+
+  o Minor features (performance, 32-bit):
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when calculating the timestamp in milliseconds for channel padding
+      computations. Implements ticket 24613.
+    - Improve performance on 32-bit systems by avoiding 64-bit division
+      when timestamping cells and buffer chunks for OOM calculations.
+      Implements ticket 24374.
+
+  o Minor features (performance, OSX, iOS):
+    - Use the mach_approximate_time() function (when available) to
+      implement coarse monotonic time. Having a coarse time function
+      should avoid a large number of system calls, and improve
+      performance slightly, especially under load. Closes ticket 24427.
+
+  o Minor features (performance, windows):
+    - Improve performance on Windows Vista and Windows 7 by adjusting
+      TCP send window size according to the recommendation from
+      SIO_IDEAL_SEND_BACKLOG_QUERY. Closes ticket 22798. Patch
+      from Vort.
+
+  o Minor features (sandbox):
+    - Explicitly permit the poll() system call when the Linux
+      seccomp2-based sandbox is enabled: apparently, some versions of
+      libc use poll() when calling getpwnam(). Closes ticket 25313.
+
+  o Minor features (storage, configuration):
+    - Users can store cached directory documents somewhere other than
+      the DataDirectory by using the CacheDirectory option. Similarly,
+      the storage location for relay's keys can be overridden with the
+      KeyDirectory option. Closes ticket 22703.
+
+  o Minor features (testing):
+    - Add a "make test-rust" target to run the rust tests only. Closes
+      ticket 25071.
+
+  o Minor features (testing, debugging, embedding):
+    - For development purposes, Tor now has a mode in which it runs for
+      a few seconds, then stops, and starts again without exiting the
+      process. This mode is meant to help us debug various issues with
+      ticket 23847. To use this feature, compile with
+      --enable-restart-debugging, and set the TOR_DEBUG_RESTART
+      environment variable. This is expected to crash a lot, and is
+      really meant for developers only. It will likely be removed in a
+      future release. Implements ticket 24583.
+
+  o Minor bugfixes (build, rust):
+    - Fix output of autoconf checks to display success messages for Rust
+      dependencies and a suitable rustc compiler version. Fixes bug
+      24612; bugfix on 0.3.1.3-alpha.
+    - Don't pass the --quiet option to cargo: it seems to suppress some
+      errors, which is not what we want to do when building. Fixes bug
+      24518; bugfix on 0.3.1.7.
+    - Build correctly when building from outside Tor's source tree with
+      the TOR_RUST_DEPENDENCIES option set. Fixes bug 22768; bugfix
+      on 0.3.1.7.
+
+  o Minor bugfixes (C correctness):
+    - Fix a very unlikely (impossible, we believe) null pointer
+      dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by
+      Coverity; this is CID 1430932.
+
+  o Minor bugfixes (channel, client):
+    - Better identify client connection when reporting to the geoip
+      client cache. Fixes bug 24904; bugfix on 0.3.1.7.
+
+  o Minor bugfixes (circuit, cannibalization):
+    - Don't cannibalize preemptively-built circuits if we no longer
+      recognize their first hop. This situation can happen if our Guard
+      relay went off the consensus after the circuit was created. Fixes
+      bug 24469; bugfix on 0.0.6.
+
+  o Minor bugfixes (client, backport from 0.3.4.1-alpha):
+    - Don't consider Tor running as a client if the ControlPort is open,
+      but no actual client ports are open. Fixes bug 26062; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a C99 compliance issue in our configuration script that caused
+      compilation issues when compiling Tor with certain versions of
+      xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the correct operation of the RESOLVE command, which had
+      been broken since we added the ability to enable/disable DNS on
+      specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha.
+    - Avoid a (nonfatal) assertion failure when extending a one-hop
+      circuit from the controller to become a multihop circuit. Fixes
+      bug 24903; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness):
+    - Remove a nonworking, unnecessary check to see whether a circuit
+      hop's identity digest was set when the circuit failed. Fixes bug
+      24927; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha):
+    - Upon receiving a malformed connected cell, stop processing the
+      cell immediately. Previously we would mark the connection for
+      close, but continue processing the cell as if the connection were
+      open. Fixes bug 26072; bugfix on 0.2.4.7-alpha.
+
+  o Minor bugfixes (directory authorities, IPv6):
+    - When creating a routerstatus (vote) from a routerinfo (descriptor),
+      set the IPv6 address to the unspecified IPv6 address, and
+      explicitly initialize the port to zero. Fixes bug 24488; bugfix
+      on 0.2.4.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document that the PerConnBW{Rate,Burst} options will fall back to
+      their corresponding consensus parameters only if those parameters
+      are set. Previously we had claimed that these values would always
+      be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha.
+
+  o Minor bugfixes (documentation, backport from 0.3.4.1-alpha):
+    - Stop saying in the manual that clients cache ipv4 dns answers from
+      exit relays. We haven't used them since 0.2.6.3-alpha, and in
+      ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but
+      we forgot to say so in the man page. Fixes bug 26052; bugfix
+      on 0.3.2.6-alpha.
+
+  o Minor bugfixes (exit relay DNS retries):
+    - Re-attempt timed-out DNS queries 3 times before failure, since our
+      timeout is 5 seconds for them, but clients wait 10-15. Also allow
+      slightly more timeouts per resolver when an exit has multiple
+      resolvers configured. Fixes bug 21394; bugfix on 0.3.1.9.
+
+  o Minor bugfixes (fallback directory mirrors):
+    - Make updateFallbackDirs.py search harder for python. (Some OSs
+      don't put it in /usr/bin.) Fixes bug 24708; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hibernation, bandwidth accounting, shutdown):
+    - When hibernating, close connections normally and allow them to
+      flush. Fixes bug 23571; bugfix on 0.2.4.7-alpha. Also fixes
+      bug 7267.
+    - Do not attempt to launch self-reachability tests when entering
+      hibernation. Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+    - Resolve several bugs related to descriptor fetching on bridge
+      clients with bandwidth accounting enabled. (This combination is
+      not recommended!) Fixes a case of bug 12062; bugfix
+      on 0.2.0.3-alpha.
+    - When hibernating, do not attempt to launch DNS checks. Fixes a
+      case of bug 12062; bugfix on 0.1.2.2-alpha.
+    - When hibernating, do not try to upload or download descriptors.
+      Fixes a case of bug 12062; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (IPv6, bridges):
+    - Tor now always sets IPv6 preferences for bridges. Fixes bug 24573;
+      bugfix on 0.2.8.2-alpha.
+    - Tor now sets IPv6 address in the routerstatus as well as in the
+      router descriptors when updating addresses for a bridge. Closes
+      ticket 24572; bugfix on 0.2.4.5-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - When running with the sandbox enabled, reload configuration files
+      correctly even when %include was used. Previously we would crash.
+      Fixes bug 22605; bugfix on 0.3.1. Patch from Daniel Pinto.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha):
+    - Allow the nanosleep() system call, which glibc uses to implement
+      sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Fix a (mostly harmless) race condition when invoking
+      LOG_PROTOCOL_WARN message from a subthread while the torrc options
+      are changing. Fixes bug 23954; bugfix on 0.1.1.9-alpha.
+
+  o Minor bugfixes (man page, SocksPort):
+    - Remove dead code from the old "SocksSocket" option, and rename
+      SocksSocketsGroupWritable to UnixSocksGroupWritable. The old
+      option still works, but is deprecated. Fixes bug 24343; bugfix
+      on 0.2.6.3.
+
+  o Minor bugfixes (memory leaks):
+    - Avoid possible at-exit memory leaks related to use of Libevent's
+      event_base_once() function. (This function tends to leak memory if
+      the event_base is closed before the event fires.) Fixes bug 24584;
+      bugfix on 0.2.8.1-alpha.
+    - Fix a harmless memory leak in tor-resolve. Fixes bug 24582; bugfix
+      on 0.2.1.1-alpha.
+
+  o Minor bugfixes (network IPv6 test):
+    - Tor's test scripts now check if "ping -6 ::1" works when the user
+      runs "make test-network-all". Fixes bug 24677; bugfix on
+      0.2.9.3-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (networking):
+    - string_is_valid_hostname() will not consider IP strings to be
+      valid hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5.
+
+  o Minor bugfixes (onion service v3):
+    - Avoid an assertion failure when the next onion service descriptor
+      rotation type is out of sync with the consensus's valid-after
+      time. Instead, log a warning message with extra information, so we
+      can better hunt down the cause of this assertion. Fixes bug 25306;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.4.1-alpha):
+    - Fix a memory leak when a v3 onion service is configured and gets a
+      SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha.
+    - When parsing the descriptor signature, look for the token plus an
+      extra white-space at the end. This is more correct but also will
+      allow us to support new fields that might start with "signature".
+      Fixes bug 26069; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - If we are configured to offer a single onion service, don't log
+      long-term established one hop rendezvous points in the heartbeat.
+      Fixes bug 25116; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (performance):
+    - Reduce the number of circuits that will be opened at once during
+      the circuit build timeout phase. This is done by increasing the
+      idle timeout to 3 minutes, and lowering the maximum number of
+      concurrent learning circuits to 10. Fixes bug 24769; bugfix
+      on 0.3.1.1-alpha.
+    - Avoid calling protocol_list_supports_protocol() from inside tight
+      loops when running with cached routerinfo_t objects. Instead,
+      summarize the relevant protocols as flags in the routerinfo_t, as
+      we do for routerstatus_t objects. This change simplifies our code
+      a little, and saves a large amount of short-term memory allocation
+      operations. Fixes bug 25008; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (performance, timeouts):
+    - Consider circuits for timeout as soon as they complete a hop. This
+      is more accurate than applying the timeout in
+      circuit_expire_building() because that function is only called
+      once per second, which is now too slow for typical timeouts on the
+      current network. Fixes bug 23114; bugfix on 0.2.2.2-alpha.
+    - Use onion service circuits (and other circuits longer than 3 hops)
+      to calculate a circuit build timeout. Previously, Tor only
+      calculated its build timeout based on circuits that planned to be
+      exactly 3 hops long. With this change, we include measurements
+      from all circuits at the point where they complete their third
+      hop. Fixes bug 23100; bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha):
+    - Avoid a crash when running with DirPort set but ORPort tuned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (Rust FFI):
+    - Fix a minor memory leak which would happen whenever the C code
+      would call the Rust implementation of
+      protover_get_supported_protocols(). This was due to the C version
+      returning a static string, whereas the Rust version newly allocated
+      a CString to pass across the FFI boundary. Consequently, the C
+      code was not expecting to need to free() what it was given. Fixes
+      bug 25127; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (spelling):
+    - Use the "misspell" tool to detect and fix typos throughout the
+      source code. Fixes bug 23650; bugfix on various versions of Tor.
+      Patch from Deepesh Pathak.
+
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      onion service introduction point creation finishing within 5
+      seconds of real clock time. Fixes bug 25450; bugfix
+      on 0.3.1.3-alpha.
+    - Give out Exit flags in bootstrapping networks. Fixes bug 24137;
+      bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (unit test, monotonic time):
+    - Increase a constant (1msec to 10msec) in the monotonic time test
+      that makes sure the nsec/usec/msec times read are synchronized.
+      This change was needed to accommodate slow systems like armel or
+      when the clock_gettime() is not a VDSO on the running kernel.
+      Fixes bug 25113; bugfix on 0.2.9.1.
+
+  o Code simplification and refactoring:
+    - Move the list of default directory authorities to its own file.
+      Closes ticket 24854. Patch by "beastr0".
+    - Remove the old (deterministic) directory retry logic entirely:
+      We've used exponential backoff exclusively for some time. Closes
+      ticket 23814.
+    - Remove the unused nodelist_recompute_all_hsdir_indices(). Closes
+      ticket 25108.
+    - Remove a series of counters used to track circuit extend attempts
+      and connection status but that in reality we aren't using for
+      anything other than stats logged by a SIGUSR1 signal. Closes
+      ticket 25163.
+    - Remove /usr/athena from search path in configure.ac. Closes
+      ticket 24363.
+    - Remove duplicate code in node_has_curve25519_onion_key() and
+      node_get_curve25519_onion_key(), and add a check for a zero
+      microdesc curve25519 onion key. Closes ticket 23966, patch by
+      "aruna1234" and teor.
+    - Rewrite channel_rsa_id_group_set_badness to reduce temporary
+      memory allocations with large numbers of OR connections (e.g.
+      relays). Closes ticket 24119.
+    - Separate the function that deletes ephemeral files when Tor
+      stops gracefully.
+    - Small changes to Tor's buf_t API to make it suitable for use as a
+      general-purpose safe string constructor. Closes ticket 22342.
+    - Switch -Wnormalized=id to -Wnormalized=nfkc in configure.ac to
+      avoid source code identifier confusion. Closes ticket 24467.
+    - The tor_git_revision[] constant no longer needs to be redeclared
+      by everything that links against the rest of Tor. Done as part of
+      ticket 23845, to simplify our external API.
+    - We make extend_info_from_node() use node_get_curve25519_onion_key()
+      introduced in ticket 23577 to access the curve25519 public keys
+      rather than accessing it directly. Closes ticket 23760. Patch by
+      Neel Chauhan.
+    - Add a function to log channels' scheduler state changes to aid
+      debugging efforts. Closes ticket 24531.
+
+  o Documentation:
+    - Improved the documentation of AccountingStart parameter. Closes
+      ticket 23635.
+    - Update the documentation for "Log" to include the current list of
+      logging domains. Closes ticket 25378.
+    - Add documentation on how to build tor with Rust dependencies
+      without having to be online. Closes ticket 22907; bugfix
+      on 0.3.0.3-alpha.
+    - Clarify the behavior of RelayBandwidth{Rate,Burst} with client
+      traffic. Closes ticket 24318.
+    - Document that OutboundBindAddress doesn't apply to DNS requests.
+      Closes ticket 22145. Patch from Aruna Maurya.
+
+  o Code simplification and refactoring (channels):
+    - Remove the incoming and outgoing channel queues. These were never
+      used, but still took up a step in our fast path.
+    - The majority of the channel unit tests have been rewritten and the
+      code coverage has now been raised to 83.6% for channel.c. Closes
+      ticket 23709.
+    - Remove other dead code from the channel subsystem: All together,
+      this cleanup has removed more than 1500 lines of code overall and
+      adding very little except for unit test.
+
+  o Code simplification and refactoring (circuit rendezvous):
+    - Split the client-side rendezvous circuit lookup into two
+      functions: one that returns only established circuits and another
+      that returns all kinds of circuits. Closes ticket 23459.
+
+  o Code simplification and refactoring (controller):
+    - Make most of the variables in networkstatus_getinfo_by_purpose()
+      const. Implements ticket 24489.
+
+  o Documentation (backport from 0.3.4.1-alpha):
+    - Correct an IPv6 error in the documentation for ExitPolicy. Closes
+      ticket 25857. Patch from "CTassisF".
+
+  o Documentation (man page):
+    - The HiddenServiceVersion torrc option accepts only one number:
+      either version 2 or 3. Closes ticket 25026; bugfix
+      on 0.3.2.2-alpha.
+
+  o Documentation (manpage, denial of service):
+    - Provide more detail about the denial-of-service options, by
+      listing each mitigation and explaining how they relate. Closes
+      ticket 25248.
+
+
+Changes in version 0.3.1.10 - 2018-03-03
+  Tor 0.3.1.10 backports a number of bugfixes, including important fixes for
+  security issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.3.1.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  Please note: according to our release calendar, Tor 0.3.1 will no
+  longer be supported after 1 July 2018. If you will be running Tor
+  after that date, you should make sure to plan to upgrade to the latest
+  stable version, or downgrade to 0.2.9 (which will receive long-term
+  support).
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Minor features (linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor feature (relay statistics, backport from 0.3.2.6-alpha):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (fallback directory mirrors, backport from 0.3.2.9):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection, backport from 0.3.2.9):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Major bugfixes (bootstrapping, backport from 0.3.2.5-alpha):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.7-rc):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox, backport from 0.3.2.5-alpha):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (directory cache, backport from 0.3.2.5-alpha):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (entry guards, backport from 0.3.2.3-alpha):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (memory usage, backport from 0.3.2.8-rc):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer, backport from 0.3.2.5-alpha):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (path selection, backport from 0.3.2.4-alpha):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (portability, msvc, backport from 0.3.2.9):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cyptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay, partial backport):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+
+Changes in version 0.2.9.15 - 2018-03-03
+  Tor 0.2.9.15 backports important security and stability bugfixes from
+  later Tor releases.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  All directory authorities should upgrade to one of the versions
+  released today. Relays running 0.2.9.x may wish to update to one of
+  the versions released today, for the DoS mitigations.
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major features (denial-of-service mitigation):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+
+  o Major bugfixes (onion services, retry behavior):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+
+  o Minor feature (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (compatibility, OpenSSL):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor bugfix (channel connection):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfix (directory authority):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (OSX):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (portability, msvc):
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Make the internal channel_is_client() function look at what sort
+      of connection handshake the other side used, rather than whether
+      the other side ever sent a create_fast cell to us. Backports part
+      of the fixes from bugs 22805 and 24898.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+
+Changes in version 0.3.2.10 - 2018-03-03
+  Tor 0.3.2.10 is the second stable release in the 0.3.2 series. It
+  backports a number of bugfixes, including important fixes for security
+  issues.
+
+  It includes an important security fix for a remote crash attack
+  against directory authorities, tracked as TROVE-2018-001.
+
+  Additionally, it backports a fix for a bug whose severity we have
+  upgraded: Bug 24700, which was fixed in 0.3.3.2-alpha, can be remotely
+  triggered in order to crash relays with a use-after-free pattern. As
+  such, we are now tracking that bug as TROVE-2018-002 and
+  CVE-2018-0491, and backporting it to earlier releases. This bug
+  affected versions 0.3.2.1-alpha through 0.3.2.9, as well as version
+  0.3.3.1-alpha.
+
+  This release also backports our new system for improved resistance to
+  denial-of-service attacks against relays.
+
+  This release also fixes several minor bugs and annoyances from
+  earlier releases.
+
+  Relays running 0.3.2.x SHOULD upgrade to one of the versions released
+  today, for the fix to TROVE-2018-002. Directory authorities should
+  also upgrade. (Relays on earlier versions might want to update too for
+  the DoS mitigations.)
+
+  o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha):
+    - Fix a protocol-list handling bug that could be used to remotely crash
+      directory authorities with a null-pointer exception. Fixes bug 25074;
+      bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and
+      CVE-2018-0490.
+
+  o Major bugfixes (scheduler, KIST, denial-of-service, backport from 0.3.3.2-alpha):
+    - Avoid adding the same channel twice in the KIST scheduler pending
+      list, which could lead to remote denial-of-service use-after-free
+      attacks against relays. Fixes bug 24700; bugfix on 0.3.2.1-alpha.
+
+  o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha):
+    - Give relays some defenses against the recent network overload. We
+      start with three defenses (default parameters in parentheses).
+      First: if a single client address makes too many concurrent
+      connections (>100), hang up on further connections. Second: if a
+      single client address makes circuits too quickly (more than 3 per
+      second, with an allowed burst of 90) while also having too many
+      connections open (3), refuse new create cells for the next while
+      (1-2 hours). Third: if a client asks to establish a rendezvous
+      point to you directly, ignore the request. These defenses can be
+      manually controlled by new torrc options, but relays will also
+      take guidance from consensus parameters, so there's no need to
+      configure anything manually. Implements ticket 24902.
+
+  o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha):
+    - Fix an "off by 2" error in counting rendezvous failures on the
+      onion service side. While we thought we would stop the rendezvous
+      attempt after one failed circuit, we were actually making three
+      circuit attempts before giving up. Now switch to a default of 2,
+      and allow the consensus parameter "hs_service_max_rdv_failures" to
+      override. Fixes bug 24895; bugfix on 0.0.6.
+    - New-style (v3) onion services now obey the "max rendezvous circuit
+      attempts" logic. Previously they would make as many rendezvous
+      circuit attempts as they could fit in the MAX_REND_TIMEOUT second
+      window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha.
+
+  o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha):
+    - Add Link protocol version 5 to the supported protocols list. Fixes
+      bug 25070; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (relay, backport from 0.3.3.1-alpha):
+    - Fix a set of false positives where relays would consider
+      connections to other relays as being client-only connections (and
+      thus e.g. deserving different link padding schemes) if those
+      relays fell out of the consensus briefly. Now we look only at the
+      initial handshake and whether the connection authenticated as a
+      relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (scheduler, consensus, backport from 0.3.3.2-alpha):
+    - The scheduler subsystem was failing to promptly notice changes in
+      consensus parameters, making it harder to switch schedulers
+      network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha.
+
+  o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha):
+    - Make our OOM handler aware of the geoip client history cache so it
+      doesn't fill up the memory. This check is important for IPv6 and
+      our DoS mitigation subsystem. Closes ticket 25122.
+
+  o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha):
+    - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released.
+      Previous versions of Tor would not have worked with OpenSSL 1.1.1,
+      since they neither disabled TLS 1.3 nor enabled any of the
+      ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites.
+      Closes ticket 24978.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (logging, diagnostic, backport from 0.3.3.2-alpha):
+    - When logging a failure to create an onion service's descriptor,
+      also log what the problem with the descriptor was. Diagnostic
+      for ticket 24972.
+
+  o Minor bugfix (channel connection, backport from 0.3.3.2-alpha):
+    - Use the actual observed address of an incoming relay connection,
+      not the canonical address of the relay from its descriptor, when
+      making decisions about how to handle the incoming connection.
+      Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera".
+
+  o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha):
+    - Fix a possible crash on malformed consensus. If a consensus had
+      contained an unparseable protocol line, it could have made clients
+      and relays crash with a null-pointer exception. To exploit this
+      issue, however, an attacker would need to be able to subvert the
+      directory authority system. Fixes bug 25251; bugfix on
+      0.2.9.4-alpha. Also tracked as TROVE-2018-004.
+
+  o Minor bugfix (directory authority, backport from 0.3.3.2-alpha):
+    - Directory authorities, when refusing a descriptor from a rejected
+      relay, now explicitly tell the relay (in its logs) to set a valid
+      ContactInfo address and contact the bad-relays@ mailing list.
+      Fixes bug 25170; bugfix on 0.2.9.1.
+
+  o Minor bugfixes (build, rust, backport from 0.3.3.1-alpha):
+    - When building with Rust on OSX, link against libresolv, to work
+      around the issue at https://github.com/rust-lang/rust/issues/46797.
+      Fixes bug 24652; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.3.3.2-alpha):
+    - Remove a BUG() statement when a client fetches an onion descriptor
+      that has a lower revision counter than the one in its cache. This
+      can happen in normal circumstances due to HSDir desync. Fixes bug
+      24976; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.3.3.2-alpha):
+    - Don't treat inability to store a cached consensus object as a bug:
+      it can happen normally when we are out of disk space. Fixes bug
+      24859; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha):
+    - Improve the performance of our consensus-diff application code
+      when Tor is built with the --enable-fragile-hardening option set.
+      Fixes bug 24826; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (OSX, backport from 0.3.3.1-alpha):
+    - Don't exit the Tor process if setrlimit() fails to change the file
+      limit (which can happen sometimes on some versions of OSX). Fixes
+      bug 21074; bugfix on 0.0.9pre5.
+
+  o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha):
+    - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on
+      0.2.9.4-alpha.
+    - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249;
+      bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.3.1-alpha):
+    - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug
+      25005; bugfix on 0.3.2.7-rc.
+
+  o Minor bugfixes (v3 onion services, backport from 0.3.3.2-alpha):
+    - Look at the "HSRend" protocol version, not the "HSDir" protocol
+      version, when deciding whether a consensus entry can support the
+      v3 onion service protocol as a rendezvous point. Fixes bug 25105;
+      bugfix on 0.3.2.1-alpha.
+
+  o Code simplification and refactoring (backport from 0.3.3.3-alpha):
+    - Update the "rust dependencies" submodule to be a project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
+  o Documentation (backport from 0.3.3.1-alpha)
+    - Document that operators who run more than one relay or bridge are
+      expected to set MyFamily and ContactInfo correctly. Closes
+      ticket 24526.
+
+
+Changes in version 0.3.2.9 - 2018-01-09
+  Tor 0.3.2.9 is the first stable release in the 0.3.2 series.
+
+  The 0.3.2 series includes our long-anticipated new onion service
+  design, with numerous security features. (For more information, see
+  our blog post at https://blog.torproject.org/fall-harvest.) We also
+  have a new circuit scheduler algorithm for improved performance on
+  relays everywhere (see https://blog.torproject.org/kist-and-tell),
+  along with many smaller features and bugfixes.
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.1.7. For a list of all
+  changes since 0.3.2.8-rc, see the ChangeLog file.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Major features (next-generation onion services):
+    - Tor now supports the next-generation onion services protocol for
+      clients and services! As part of this release, the core of
+      proposal 224 has been implemented and is available for
+      experimentation and testing by our users. This newer version of
+      onion services ("v3") features many improvements over the legacy
+      system, including:
+
+      a) Better crypto (replaced SHA1/DH/RSA1024
+      with SHA3/ed25519/curve25519)
+
+      b) Improved directory protocol, leaking much less information to
+      directory servers.
+
+      c) Improved directory protocol, with smaller surface for
+      targeted attacks.
+
+      d) Better onion address security against impersonation.
+
+      e) More extensible introduction/rendezvous protocol.
+
+      f) A cleaner and more modular codebase.
+
+      You can identify a next-generation onion address by its length:
+      they are 56 characters long, as in
+      "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad.onion".
+
+      In the future, we will release more options and features for v3
+      onion services, but we first need a testing period, so that the
+      current codebase matures and becomes more robust. Planned features
+      include: offline keys, advanced client authorization, improved
+      guard algorithms, and statistics. For full details, see
+      proposal 224.
+
+      Legacy ("v2") onion services will still work for the foreseeable
+      future, and will remain the default until this new codebase gets
+      tested and hardened. Service operators who want to experiment with
+      the new system can use the 'HiddenServiceVersion 3' torrc
+      directive along with the regular onion service configuration
+      options. For more information, see our blog post at
+      "https://blog.torproject.org/fall-harvest". Enjoy!
+
+  o Major feature (scheduler, channel):
+    - Tor now uses new schedulers to decide which circuits should
+      deliver cells first, in order to improve congestion at relays. The
+      first type is called "KIST" ("Kernel Informed Socket Transport"),
+      and is only available on Linux-like systems: it uses feedback from
+      the kernel to prevent the kernel's TCP buffers from growing too
+      full. The second new scheduler type is called "KISTLite": it
+      behaves the same as KIST, but runs on systems without kernel
+      support for inspecting TCP implementation details. The old
+      scheduler is still available, under the name "Vanilla". To change
+      the default scheduler preference order, use the new "Schedulers"
+      option. (The default preference order is "KIST,KISTLite,Vanilla".)
+
+      Matt Traudt implemented KIST, based on research by Rob Jansen,
+      John Geddes, Christ Wacek, Micah Sherr, and Paul Syverson. For
+      more information, see the design paper at
+      http://www.robgjansen.com/publications/kist-sec2014.pdf and the
+      followup implementation paper at https://arxiv.org/abs/1709.01044.
+      Closes ticket 12541. For more information, see our blog post at
+      "https://blog.torproject.org/kist-and-tell".
+
+  o Major bugfixes (security, general):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+
+  o Major bugfixes (security, directory authority):
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+
+  o Major bugfixes (security, onion service v2):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, relay):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (bootstrapping):
+    - Fetch descriptors aggressively whenever we lack enough to build
+      circuits, regardless of how many descriptors we are missing.
+      Previously, we would delay launching the fetch when we had fewer
+      than 15 missing descriptors, even if some of those descriptors
+      were blocking circuits from building. Fixes bug 23985; bugfix on
+      0.1.1.11-alpha. The effects of this bug became worse in
+      0.3.0.3-alpha, when we began treating missing descriptors from our
+      primary guards as a reason to delay circuits.
+    - Don't try fetching microdescriptors from relays that have failed
+      to deliver them in the past. Fixes bug 23817; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit prediction):
+    - Fix circuit prediction logic so that a client doesn't treat a port
+      as being "handled" by a circuit if that circuit already has
+      isolation settings on it. This change should make Tor clients more
+      responsive by improving their chances of having a pre-created
+      circuit ready for use when a request arrives. Fixes bug 18859;
+      bugfix on 0.2.3.3-alpha.
+
+  o Major bugfixes (exit relays, DNS):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (relay, crash, assertion failure):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Major bugfixes (usability, control port):
+    - Report trusted clock skew indications as bootstrap errors, so
+      controllers can more easily alert users when their clocks are
+      wrong. Fixes bug 23506; bugfix on 0.1.2.6-alpha.
+
+  o Minor features (bridge):
+    - Bridge relays can now set the BridgeDistribution config option to
+      add a "bridge-distribution-request" line to their bridge
+      descriptor, which tells BridgeDB how they'd like their bridge
+      address to be given out. (Note that as of Oct 2017, BridgeDB does
+      not yet implement this feature.) As a side benefit, this feature
+      provides a way to distinguish bridge descriptors from non-bridge
+      descriptors. Implements tickets 18329.
+    - When handling the USERADDR command on an ExtOrPort, warn when the
+      transports provides a USERADDR with no port. In a future version,
+      USERADDR commands of this format may be rejected. Detects problems
+      related to ticket 23080.
+
+  o Minor features (bug detection):
+    - Log a warning message with a stack trace for any attempt to call
+      get_options() during option validation. This pattern has caused
+      subtle bugs in the past. Closes ticket 22281.
+
+  o Minor features (build, compilation):
+    - The "check-changes" feature is now part of the "make check" tests;
+      we'll use it to try to prevent misformed changes files from
+      accumulating. Closes ticket 23564.
+    - Tor builds should now fail if there are any mismatches between the
+      C type representing a configuration variable and the C type the
+      data-driven parser uses to store a value there. Previously, we
+      needed to check these by hand, which sometimes led to mistakes.
+      Closes ticket 23643.
+
+  o Minor features (client):
+    - You can now use Tor as a tunneled HTTP proxy: use the new
+      HTTPTunnelPort option to open a port that accepts HTTP CONNECT
+      requests. Closes ticket 22407.
+    - Add an extra check to make sure that we always use the newer guard
+      selection code for picking our guards. Closes ticket 22779.
+    - When downloading (micro)descriptors, don't split the list into
+      multiple requests unless we want at least 32 descriptors.
+      Previously, we split at 4, not 32, which led to significant
+      overhead in HTTP request size and degradation in compression
+      performance. Closes ticket 23220.
+    - Improve log messages when missing descriptors for primary guards.
+      Resolves ticket 23670.
+
+  o Minor features (command line):
+    - Add a new commandline option, --key-expiration, which prints when
+      the current signing key is going to expire. Implements ticket
+      17639; patch by Isis Lovecruft.
+
+  o Minor features (control port):
+    - If an application tries to use the control port as an HTTP proxy,
+      respond with a meaningful "This is the Tor control port" message,
+      and log the event. Closes ticket 1667. Patch from Ravi
+      Chandra Padmala.
+    - Provide better error message for GETINFO desc/(id|name) when not
+      fetching router descriptors. Closes ticket 5847. Patch by
+      Kevin Butler.
+    - Add GETINFO "{desc,md}/download-enabled", to inform the controller
+      whether Tor will try to download router descriptors and
+      microdescriptors respectively. Closes ticket 22684.
+    - Added new GETINFO targets "ip-to-country/{ipv4,ipv6}-available",
+      so controllers can tell whether the geoip databases are loaded.
+      Closes ticket 23237.
+    - Adds a timestamp field to the CIRC_BW and STREAM_BW bandwidth
+      events. Closes ticket 19254. Patch by "DonnchaC".
+
+  o Minor features (development support):
+    - Developers can now generate a call-graph for Tor using the
+      "calltool" python program, which post-processes object dumps. It
+      should work okay on many Linux and OSX platforms, and might work
+      elsewhere too. To run it, install calltool from
+      https://gitweb.torproject.org/user/nickm/calltool.git and run
+      "make callgraph". Closes ticket 19307.
+
+  o Minor features (directory authority):
+    - Make the "Exit" flag assignment only depend on whether the exit
+      policy allows connections to ports 80 and 443. Previously relays
+      would get the Exit flag if they allowed connections to one of
+      these ports and also port 6667. Resolves ticket 23637.
+
+  o Minor features (ed25519):
+    - Add validation function to checks for torsion components in
+      ed25519 public keys, used by prop224 client-side code. Closes
+      ticket 22006. Math help by Ian Goldberg.
+
+  o Minor features (exit relay, DNS):
+    - Improve the clarity and safety of the log message from evdns when
+      receiving an apparently spoofed DNS reply. Closes ticket 3056.
+
+  o Minor features (fallback directory mirrors):
+    - The fallback directory list has been re-generated based on the
+      current status of the network. Tor uses fallback directories to
+      bootstrap when it doesn't yet have up-to-date directory
+      information. Closes ticket 24801.
+    - Make the default DirAuthorityFallbackRate 0.1, so that clients
+      prefer to bootstrap from fallback directory mirrors. This is a
+      follow-up to 24679, which removed weights from the default
+      fallbacks. Implements ticket 24681.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 5 2018 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (integration, hardening):
+    - Add a new NoExec option to prevent Tor from running other
+      programs. When this option is set to 1, Tor will never try to run
+      another program, regardless of the settings of
+      PortForwardingHelper, ClientTransportPlugin, or
+      ServerTransportPlugin. Once NoExec is set, it cannot be disabled
+      without restarting Tor. Closes ticket 22976.
+
+  o Minor features (linux seccomp2 sandbox):
+    - Update the sandbox rules so that they should now work correctly
+      with Glibc 2.26. Closes ticket 24315.
+
+  o Minor features (logging):
+    - Provide better warnings when the getrandom() syscall fails. Closes
+      ticket 24500.
+    - Downgrade a pair of log messages that could occur when an exit's
+      resolver gave us an unusual (but not forbidden) response. Closes
+      ticket 24097.
+    - Improve the message we log when re-enabling circuit build timeouts
+      after having received a consensus. Closes ticket 20963.
+    - Log more circuit information whenever we are about to try to
+      package a relay cell on a circuit with a nonexistent n_chan.
+      Attempt to diagnose ticket 8185.
+    - Improve info-level log identification of particular circuits, to
+      help with debugging. Closes ticket 23645.
+    - Improve the warning message for specifying a relay by nickname.
+      The previous message implied that nickname registration was still
+      part of the Tor network design, which it isn't. Closes
+      ticket 20488.
+    - If the sandbox filter fails to load, suggest to the user that
+      their kernel might not support seccomp2. Closes ticket 23090.
+
+  o Minor features (onion service, circuit, logging):
+    - Improve logging of many callsite in the circuit subsystem to print
+      the circuit identifier(s).
+    - Log when we cleanup an intro point from a service so we know when
+      and for what reason it happened. Closes ticket 23604.
+
+  o Minor features (portability):
+    - Tor now compiles correctly on arm64 with libseccomp-dev installed.
+      (It doesn't yet work with the sandbox enabled.) Closes
+      ticket 24424.
+    - Check at configure time whether uint8_t is the same type as
+      unsigned char. Lots of existing code already makes this
+      assumption, and there could be strict aliasing issues if the
+      assumption is violated. Closes ticket 22410.
+
+  o Minor features (relay):
+    - When choosing which circuits can be expired as unused, consider
+      circuits from clients even if those clients used regular CREATE
+      cells to make them; and do not consider circuits from relays even
+      if they were made with CREATE_FAST. Part of ticket 22805.
+    - Reject attempts to use relative file paths when RunAsDaemon is
+      set. Previously, Tor would accept these, but the directory-
+      changing step of RunAsDaemon would give strange and/or confusing
+      results. Closes ticket 22731.
+
+  o Minor features (relay statistics):
+    - Change relay bandwidth reporting stats interval from 4 hours to 24
+      hours in order to reduce the efficiency of guard discovery
+      attacks. Fixes ticket 23856.
+
+  o Minor features (reverted deprecations):
+    - The ClientDNSRejectInternalAddresses flag can once again be set in
+      non-testing Tor networks, so long as they do not use the default
+      directory authorities. This change also removes the deprecation of
+      this flag from 0.2.9.2-alpha. Closes ticket 21031.
+
+  o Minor features (robustness):
+    - Change several fatal assertions when flushing buffers into non-
+      fatal assertions, to prevent any recurrence of 23690.
+
+  o Minor features (startup, safety):
+    - When configured to write a PID file, Tor now exits if it is unable
+      to do so. Previously, it would warn and continue. Closes
+      ticket 20119.
+
+  o Minor features (static analysis):
+    - The BUG() macro has been changed slightly so that Coverity no
+      longer complains about dead code if the bug is impossible. Closes
+      ticket 23054.
+
+  o Minor features (testing):
+    - Our fuzzing tests now test the encrypted portions of v3 onion
+      service descriptors. Implements more of 21509.
+    - Add a unit test to make sure that our own generated platform
+      string will be accepted by directory authorities. Closes
+      ticket 22109.
+    - The default chutney network tests now include tests for the v3
+      onion service design. Make sure you have the latest version of
+      chutney if you want to run these. Closes ticket 22437.
+    - Add a unit test to verify that we can parse a hardcoded v2 onion
+      service descriptor. Closes ticket 15554.
+
+  o Minor bugfixes (address selection):
+    - When the fascist_firewall_choose_address_ functions don't find a
+      reachable address, set the returned address to the null address
+      and port. This is a precautionary measure, because some callers do
+      not check the return value. Fixes bug 24736; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (bootstrapping):
+    - When warning about state file clock skew, report the correct
+      direction for the detected skew. Fixes bug 23606; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (bridge clients, bootstrap):
+    - Retry directory downloads when we get our first bridge descriptor
+      during bootstrap or while reconnecting to the network. Keep
+      retrying every time we get a bridge descriptor, until we have a
+      reachable bridge. Fixes part of bug 24367; bugfix on 0.2.0.3-alpha.
+    - Stop delaying bridge descriptor fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when we
+      have at least one reachable bridge. Fixes part of bug 24367;
+      bugfix on 0.2.0.3-alpha.
+    - Stop delaying directory fetches when we have cached bridge
+      descriptors. Instead, only delay bridge descriptor fetches when
+      all our bridges are definitely unreachable. Fixes part of bug
+      24367; bugfix on 0.2.0.3-alpha.
+
+  o Minor bugfixes (bridge):
+    - Overwrite the bridge address earlier in the process of retrieving
+      its descriptor, to make sure we reach it on the configured
+      address. Fixes bug 20532; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (build, compilation):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+    - When searching for OpenSSL, don't accept any OpenSSL library that
+      lacks TLSv1_1_method(): Tor doesn't build with those versions.
+      Additionally, look in /usr/local/opt/openssl, if it's present.
+      These changes together repair the default build on OSX systems
+      with Homebrew installed. Fixes bug 23602; bugfix on 0.2.7.2-alpha.
+    - Fix a signed/unsigned comparison warning introduced by our fix to
+      TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16.
+    - Fix a memory leak warning in one of the libevent-related
+      configuration tests that could occur when manually specifying
+      -fsanitize=address. Fixes bug 24279; bugfix on 0.3.0.2-alpha.
+      Found and patched by Alex Xu.
+    - Fix unused-variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (certificate handling):
+    - Fix a time handling bug in Tor certificates set to expire after
+      the year 2106. Fixes bug 23055; bugfix on 0.3.0.1-alpha. Found by
+      Coverity as CID 1415728.
+
+  o Minor bugfixes (client):
+    - By default, do not enable storage of client-side DNS values. These
+      values were unused by default previously, but they should not have
+      been cached at all. Fixes bug 24050; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (client, usability):
+    - Refrain from needlessly rejecting SOCKS5-with-hostnames and
+      SOCKS4a requests that contain IP address strings, even when
+      SafeSocks in enabled, as this prevents user from connecting to
+      known IP addresses without relying on DNS for resolving. SafeSocks
+      still rejects SOCKS connections that connect to IP addresses when
+      those addresses are _not_ encoded as hostnames. Fixes bug 22461;
+      bugfix on Tor 0.2.6.2-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Call htons() in extend_cell_format() for encoding a 16-bit value.
+      Previously we used ntohs(), which happens to behave the same on
+      all the platforms we support, but which isn't really correct.
+      Fixes bug 23106; bugfix on 0.2.4.8-alpha.
+    - For defense-in-depth, make the controller's write_escaped_data()
+      function robust to extremely long inputs. Fixes bug 19281; bugfix
+      on 0.1.1.1-alpha. Reported by Guido Vranken.
+    - Fix several places in our codebase where a C compiler would be
+      likely to eliminate a check, based on assuming that undefined
+      behavior had not happened elsewhere in the code. These cases are
+      usually a sign of redundant checking or dubious arithmetic. Found
+      by Georg Koppen using the "STACK" tool from Wang, Zeldovich,
+      Kaashoek, and Solar-Lezama. Fixes bug 24423; bugfix on various
+      Tor versions.
+
+  o Minor bugfixes (compression):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (consensus expiry):
+    - Check for adequate directory information correctly. Previously, Tor
+      would reconsider whether it had sufficient directory information
+      every 2 minutes. Fixes bug 23091; bugfix on 0.2.0.19-alpha.
+
+  o Minor bugfixes (control port, linux seccomp2 sandbox):
+    - Avoid a crash when attempting to use the seccomp2 sandbox together
+      with the OwningControllerProcess feature. Fixes bug 24198; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (control port, onion services):
+    - Report "FAILED" instead of "UPLOAD_FAILED" "FAILED" for the
+      HS_DESC event when a service is not able to upload a descriptor.
+      Fixes bug 24230; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (directory cache):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+    - When a consensus diff calculation is only partially successful,
+      only record the successful parts as having succeeded. Partial
+      success can happen if (for example) one compression method fails
+      but the others succeed. Previously we misrecorded all the
+      calculations as having succeeded, which would later cause a
+      nonfatal assertion failure. Fixes bug 24086; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (directory client):
+    - On failure to download directory information, delay retry attempts
+      by a random amount based on the "decorrelated jitter" algorithm.
+      Our previous delay algorithm tended to produce extra-long delays
+      too easily. Fixes bug 23816; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Directory servers now include a "Date:" http header for response
+      codes other than 200. Clients starting with a skewed clock and a
+      recent consensus were getting "304 Not modified" responses from
+      directory authorities, so without the Date header, the client
+      would never hear about a wrong clock. Fixes bug 23499; bugfix
+      on 0.0.8rc1.
+    - Make clients wait for 6 seconds before trying to download a
+      consensus from an authority. Fixes bug 17750; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Document better how to read gcov, and what our gcov postprocessing
+      scripts do. Fixes bug 23739; bugfix on 0.2.9.1-alpha.
+    - Fix manpage to not refer to the obsolete (and misspelled)
+      UseEntryGuardsAsDirectoryGuards parameter in the description of
+      NumDirectoryGuards. Fixes bug 23611; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (DoS-resistance):
+    - If future code asks if there are any running bridges, without
+      checking if bridges are enabled, log a BUG warning rather than
+      crashing. Fixes bug 23524; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (entry guards):
+    - Tor now updates its guard state when it reads a consensus
+      regardless of whether it's missing descriptors. That makes tor use
+      its primary guards to fetch descriptors in some edge cases where
+      it would previously have used fallback directories. Fixes bug
+      23862; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (format strictness):
+    - Restrict several data formats to decimal. Previously, the
+      BuildTimeHistogram entries in the state file, the "bw=" entries in
+      the bandwidth authority file, and the process IDs passed to the
+      __OwningControllerProcess option could all be specified in hex or
+      octal as well as in decimal. This was not an intentional feature.
+      Fixes bug 22802; bugfixes on 0.2.2.1-alpha, 0.2.2.2-alpha,
+      and 0.2.2.28-beta.
+
+  o Minor bugfixes (heartbeat):
+    - If we fail to write a heartbeat message, schedule a retry for the
+      minimum heartbeat interval number of seconds in the future. Fixes
+      bug 19476; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Suppress a log notice when relay descriptors arrive. We already
+      have a bootstrap progress for this so no need to log notice
+      everytime tor receives relay descriptors. Microdescriptors behave
+      the same. Fixes bug 23861; bugfix on 0.2.8.2-alpha.
+    - Remove duplicate log messages regarding opening non-local
+      SocksPorts upon parsing config and opening listeners at startup.
+      Fixes bug 4019; bugfix on 0.2.3.3-alpha.
+    - Use a more comprehensible log message when telling the user
+      they've excluded every running exit node. Fixes bug 7890; bugfix
+      on 0.2.2.25-alpha.
+    - When logging the number of descriptors we intend to download per
+      directory request, do not log a number higher than then the number
+      of descriptors we're fetching in total. Fixes bug 19648; bugfix
+      on 0.1.1.8-alpha.
+    - When warning about a directory owned by the wrong user, log the
+      actual name of the user owning the directory. Previously, we'd log
+      the name of the process owner twice. Fixes bug 23487; bugfix
+      on 0.2.9.1-alpha.
+    - Fix some messages on unexpected errors from the seccomp2 library.
+      Fixes bug 22750; bugfix on 0.2.5.1-alpha. Patch from "cypherpunks".
+    - The tor specification says hop counts are 1-based, so fix two log
+      messages that mistakenly logged 0-based hop counts. Fixes bug
+      18982; bugfix on 0.2.6.2-alpha and 0.2.4.5-alpha. Patch by teor.
+      Credit to Xiaofan Li for reporting this issue.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (memory safety, defensive programming):
+    - Clear the target address when node_get_prim_orport() returns
+      early. Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (memory usage):
+    - When queuing DESTROY cells on a channel, only queue the circuit-id
+      and reason fields: not the entire 514-byte cell. This fix should
+      help mitigate any bugs or attacks that fill up these queues, and
+      free more RAM for other uses. Fixes bug 24666; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (network layer):
+    - When closing a connection via close_connection_immediately(), we
+      mark it as "not blocked on bandwidth", to prevent later calls from
+      trying to unblock it, and give it permission to read. This fixes a
+      backtrace warning that can happen on relays under various
+      circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc.
+
+  o Minor bugfixes (onion services):
+    - The introduction circuit was being timed out too quickly while
+      waiting for the rendezvous circuit to complete. Keep the intro
+      circuit around longer instead of timing out and reopening new ones
+      constantly. Fixes bug 23681; bugfix on 0.2.4.8-alpha.
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+    - When handling multiple SOCKS request for the same .onion address,
+      only fetch the service descriptor once.
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+    - When reloading configured onion services, copy all information
+      from the old service object. Previously, some data was omitted,
+      causing delays in descriptor upload, and other bugs. Fixes bug
+      23790; bugfix on 0.2.1.9-alpha.
+
+  o Minor bugfixes (path selection):
+    - When selecting relays by bandwidth, avoid a rounding error that
+      could sometimes cause load to be imbalanced incorrectly.
+      Previously, we would always round upwards; now, we round towards
+      the nearest integer. This had the biggest effect when a relay's
+      weight adjustments should have given it weight 0, but it got
+      weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha.
+    - When calculating the fraction of nodes that have descriptors, and
+      all nodes in the network have zero bandwidths, count the number of
+      nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha.
+    - Actually log the total bandwidth in compute_weighted_bandwidths().
+      Fixes bug 24170; bugfix on 0.2.4.3-alpha.
+
+  o Minor bugfixes (portability):
+    - Stop using the PATH_MAX variable, which is not defined on GNU
+      Hurd. Fixes bug 23098; bugfix on 0.3.1.1-alpha.
+    - Fix a bug in the bit-counting parts of our timing-wheel code on
+      MSVC. (Note that MSVC is still not a supported build platform, due
+      to cryptographic timing channel risks.) Fixes bug 24633; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - When uploading our descriptor for the first time after startup,
+      report the reason for uploading as "Tor just started" rather than
+      leaving it blank. Fixes bug 22885; bugfix on 0.2.3.4-alpha.
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix a spurious fuzzing-only use of an uninitialized value. Found
+      by Brian Carpenter. Fixes bug 24082; bugfix on 0.3.0.3-alpha.
+    - Test that IPv6-only clients can use microdescriptors when running
+      "make test-network-all". Requires chutney master 61c28b9 or later.
+      Closes ticket 24109.
+    - Prevent scripts/test/coverage from attempting to move gcov output
+      to the root directory. Fixes bug 23741; bugfix on 0.2.5.1-alpha.
+    - Capture and detect several "Result does not fit" warnings in unit
+      tests on platforms with 32-bit time_t. Fixes bug 21800; bugfix
+      on 0.2.9.3-alpha.
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+    - Fix a bug in our fuzzing mock replacement for crypto_pk_checksig(),
+      to correctly handle cases where a caller gives it an RSA key of
+      under 160 bits. (This is not actually a bug in Tor itself, but
+      rather in our fuzzing code.) Fixes bug 24247; bugfix on
+      0.3.0.3-alpha. Found by OSS-Fuzz as issue 4177.
+    - Fix a broken unit test for the OutboundAddress option: the parsing
+      function was never returning an error on failure. Fixes bug 23366;
+      bugfix on 0.3.0.3-alpha.
+    - Fix a signed-integer overflow in the unit tests for
+      dir/download_status_random_backoff, which was untriggered until we
+      fixed bug 17750. Fixes bug 22924; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (usability, control port):
+    - Stop making an unnecessary routerlist check in NETINFO clock skew
+      detection; this was preventing clients from reporting NETINFO clock
+      skew to controllers. Fixes bug 23532; bugfix on 0.2.4.4-alpha.
+
+  o Code simplification and refactoring:
+    - Remove various ways of testing circuits and connections for
+      "clientness"; instead, favor channel_is_client(). Part of
+      ticket 22805.
+    - Extract the code for handling newly-open channels into a separate
+      function from the general code to handle channel state
+      transitions. This change simplifies our callgraph, reducing the
+      size of the largest strongly connected component by roughly a
+      factor of two. Closes ticket 22608.
+    - Remove dead code for largely unused statistics on the number of
+      times we've attempted various public key operations. Fixes bug
+      19871; bugfix on 0.1.2.4-alpha. Fix by Isis Lovecruft.
+    - Remove several now-obsolete functions for asking about old
+      variants directory authority status. Closes ticket 22311; patch
+      from "huyvq".
+    - Remove some of the code that once supported "Named" and "Unnamed"
+      routers. Authorities no longer vote for these flags. Closes
+      ticket 22215.
+    - Rename the obsolete malleable hybrid_encrypt functions used in TAP
+      and old hidden services, to indicate that they aren't suitable for
+      new protocols or formats. Closes ticket 23026.
+    - Replace our STRUCT_OFFSET() macro with offsetof(). Closes ticket
+      22521. Patch from Neel Chauhan.
+    - Split the enormous circuit_send_next_onion_skin() function into
+      multiple subfunctions. Closes ticket 22804.
+    - Split the portions of the buffer.c module that handle particular
+      protocols into separate modules. Part of ticket 23149.
+    - Use our test macros more consistently, to produce more useful
+      error messages when our unit tests fail. Add coccinelle patches to
+      allow us to re-check for test macro uses. Closes ticket 22497.
+
+  o Deprecated features:
+    - The ReachableDirAddresses and ClientPreferIPv6DirPort options are
+      now deprecated; they do not apply to relays, and they have had no
+      effect on clients since 0.2.8.x. Closes ticket 19704.
+    - Deprecate HTTPProxy/HTTPProxyAuthenticator config options. They
+      only applies to direct unencrypted HTTP connections to your
+      directory server, which your Tor probably isn't using. Closes
+      ticket 20575.
+
+  o Documentation:
+    - Add notes in man page regarding OS support for the various
+      scheduler types. Attempt to use less jargon in the scheduler
+      section. Closes ticket 24254.
+    - Clarify that the Address option is entirely about setting an
+      advertised IPv4 address. Closes ticket 18891.
+    - Clarify the manpage's use of the term "address" to clarify what
+      kind of address is intended. Closes ticket 21405.
+    - Document that onion service subdomains are allowed, and ignored.
+      Closes ticket 18736.
+    - Clarify in the manual that "Sandbox 1" is only supported on Linux
+      kernels. Closes ticket 22677.
+    - Document all values of PublishServerDescriptor in the manpage.
+      Closes ticket 15645.
+    - Improve the documentation for the directory port part of the
+      DirAuthority line. Closes ticket 20152.
+    - Restore documentation for the authorities' "approved-routers"
+      file. Closes ticket 21148.
+
+  o Removed features:
+    - The AllowDotExit option has been removed as unsafe. It has been
+      deprecated since 0.2.9.2-alpha. Closes ticket 23426.
+    - The ClientDNSRejectInternalAddresses flag can no longer be set on
+      non-testing networks. It has been deprecated since 0.2.9.2-alpha.
+      Closes ticket 21031.
+    - The controller API no longer includes an AUTHDIR_NEWDESCS event:
+      nobody was using it any longer. Closes ticket 22377.
+
+
+Changes in version 0.3.1.9 - 2017-12-01:
+  Tor 0.3.1.9 backports important security and stability fixes from the
+  0.3.2 development series. All Tor users should upgrade to this
+  release, or to another of the releases coming out today.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (bridge):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (onion service, backport from 0.3.2.5-alpha):
+    - Rename the consensus parameter "hsdir-interval" to "hsdir_interval"
+      so it matches dir-spec.txt. Fixes bug 24262; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+
+Changes in version 0.3.0.13 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+    - When running as a relay, make sure that we never choose ourselves
+      as a guard. Fixes part of bug 21534; bugfix on 0.3.0.1-alpha. This
+      issue is also tracked as TROVE-2017-012 and CVE-2017-8822.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.9.14 - 2017-12-01
+  Tor 0.3.0.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  o Major bugfixes (exit relays, DNS, backport from 0.3.2.4-alpha):
+    - Fix an issue causing DNS to fail on high-bandwidth exit nodes,
+      making them nearly unusable. Fixes bugs 21394 and 18580; bugfix on
+      0.1.2.2-alpha, which introduced eventdns. Thanks to Dhalgren for
+      identifying and finding a workaround to this bug and to Moritz,
+      Arthur Edelstein, and Roger for helping to track it down and
+      analyze it.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - Fix a denial of service issue where an attacker could crash a
+      directory authority using a malformed router descriptor. Fixes bug
+      24245; bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2017-010
+      and CVE-2017-8820.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (security, windows, backport from 0.3.1.1-alpha):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor bugfix (relay address resolution, backport from 0.3.2.1-alpha):
+    - Avoid unnecessary calls to directory_fetches_from_authorities() on
+      relays, to prevent spurious address resolutions and descriptor
+      rebuilds. This is a mitigation for bug 21789. Fixes bug 23470;
+      bugfix on in 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.1-alpha):
+    - Fix unused variable warnings in donna's Curve25519 SSE2 code.
+      Fixes bug 22895; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (logging, relay shutdown, annoyance, backport from 0.3.2.2-alpha):
+    - When a circuit is marked for close, do not attempt to package any
+      cells for channels on that circuit. Previously, we would detect
+      this condition lower in the call stack, when we noticed that the
+      circuit had no attached channel, and log an annoying message.
+      Fixes bug 8185; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, crash, backport from 0.3.2.4-alpha):
+    - Avoid a crash when transitioning from client mode to bridge mode.
+      Previously, we would launch the worker threads whenever our
+      "public server" mode changed, but not when our "server" mode
+      changed. Fixes bug 23693; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.8.17 - 2017-12-01
+  Tor 0.2.8.17 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please upgrade with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, onion service v2, backport from 0.3.2.6-alpha):
+    - Fix a use-after-free error that could crash v2 Tor onion services
+      when they failed to open circuits while expiring introduction
+      points. Fixes bug 24313; bugfix on 0.2.7.2-alpha. This issue is
+      also tracked as TROVE-2017-013 and CVE-2017-8823.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path through
+      ourselves, even in the case where we have somehow lost the version of
+      our descriptor appearing in the consensus. Fixes part of bug 21534;
+      bugfix on 0.2.0.1-alpha. This issue is also tracked as TROVE-2017-012
+      and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (directory authority, backport from 0.3.2.6-alpha):
+    - Add an IPv6 address for the "bastet" directory authority. Closes
+      ticket 24394.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (testing, backport from 0.3.1.6-rc):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+
+
+Changes in version 0.2.5.16 - 2017-12-01
+  Tor 0.2.5.13 backports important security and stability bugfixes from
+  later Tor releases. All Tor users should upgrade to this release, or
+  to another of the releases coming out today.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Major bugfixes (security, backport from 0.3.2.6-alpha):
+    - Fix a denial of service bug where an attacker could use a
+      malformed directory object to cause a Tor instance to pause while
+      OpenSSL would try to read a passphrase from the terminal. (Tor
+      instances run without a terminal, which is the case for most Tor
+      packages, are not impacted.) Fixes bug 24246; bugfix on every
+      version of Tor. Also tracked as TROVE-2017-011 and CVE-2017-8821.
+      Found by OSS-Fuzz as testcase 6360145429790720.
+    - When checking for replays in the INTRODUCE1 cell data for a
+      (legacy) onion service, correctly detect replays in the RSA-
+      encrypted part of the cell. We were previously checking for
+      replays on the entire cell, but those can be circumvented due to
+      the malleability of Tor's legacy hybrid encryption. This fix helps
+      prevent a traffic confirmation attack. Fixes bug 24244; bugfix on
+      0.2.4.1-alpha. This issue is also tracked as TROVE-2017-009
+      and CVE-2017-8819.
+
+  o Major bugfixes (security, relay, backport from 0.3.2.6-alpha):
+    - When running as a relay, make sure that we never build a path
+      through ourselves, even in the case where we have somehow lost the
+      version of our descriptor appearing in the consensus. Fixes part
+      of bug 21534; bugfix on 0.2.0.1-alpha. This issue is also tracked
+      as TROVE-2017-012 and CVE-2017-8822.
+
+  o Minor features (bridge, backport from 0.3.1.9):
+    - Bridges now include notice in their descriptors that they are
+      bridges, and notice of their distribution status, based on their
+      publication settings. Implements ticket 18329. For more fine-
+      grained control of how a bridge is distributed, upgrade to 0.3.2.x
+      or later.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.5.15 - 2017-10-25
+  Tor 0.2.5.15 backports a collection of bugfixes from later Tor release
+  series. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.5 series will no longer be supported after 1 May
+  2018. If you need a release with long-term support, please upgrade to
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.8.16 - 2017-10-25
+  Tor 0.2.8.16 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.2.8 series will no longer be supported after 1 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.13 - 2017-10-25
+  Tor 0.2.9.13 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.0.12 - 2017-10-25
+  Tor 0.3.0.12 backports a collection of bugfixes from later Tor release
+  series, including a bugfix for a crash issue that had affected relays
+  under memory pressure. It also adds a new directory authority, Bastet.
+
+  Note: the Tor 0.3.0 series will no longer be supported after 26 Jan
+  2018. If you need a release with long-term support, please stick with
+  the 0.2.9 series. Otherwise, please upgrade to 0.3.1 or later.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.5-alpha):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (Windows service, backport from 0.3.1.6-rc):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+
+Changes in version 0.3.1.8 - 2017-10-25
+  Tor 0.3.1.8 is the second stable release in the 0.3.1 series.
+  It includes several bugfixes, including a bugfix for a crash issue
+  that had affected relays under memory pressure. It also adds
+  a new directory authority, Bastet.
+
+  o Directory authority changes:
+    - Add "Bastet" as a ninth directory authority to the default list.
+      Closes ticket 23910.
+    - The directory authority "Longclaw" has changed its IP address.
+      Closes ticket 23592.
+
+  o Major bugfixes (relay, crash, assertion failure, backport from 0.3.2.2-alpha):
+    - Fix a timing-based assertion failure that could occur when the
+      circuit out-of-memory handler freed a connection's output buffer.
+      Fixes bug 23690; bugfix on 0.2.6.1-alpha.
+
+  o Minor features (directory authorities, backport from 0.3.2.2-alpha):
+    - Remove longclaw's IPv6 address, as it will soon change. Authority
+      IPv6 addresses were originally added in 0.2.8.1-alpha. This leaves
+      3/8 directory authorities with IPv6 addresses, but there are also
+      52 fallback directory mirrors with IPv6 addresses. Resolves 19760.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.2.2-alpha):
+    - Fix a compilation warning when building with zstd support on
+      32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha. Found
+      and fixed by Andreas Stieger.
+
+  o Minor bugfixes (compression, backport from 0.3.2.2-alpha):
+    - Handle a pathological case when decompressing Zstandard data when
+      the output buffer size is zero. Fixes bug 23551; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.2.1-alpha):
+    - Remove the length limit on HTTP status lines that authorities can
+      send in their replies. Fixes bug 23499; bugfix on 0.3.1.6-rc.
+
+  o Minor bugfixes (hidden service, relay, backport from 0.3.2.2-alpha):
+    - Avoid a possible double close of a circuit by the intro point on
+      error of sending the INTRO_ESTABLISHED cell. Fixes bug 23610;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory safety, backport from 0.3.2.3-alpha):
+    - Clear the address when node_get_prim_orport() returns early.
+      Fixes bug 23874; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.2.2-alpha):
+    - Fix additional channelpadding unit test failures by using mocked
+      time instead of actual time for all tests. Fixes bug 23608; bugfix
+      on 0.3.1.1-alpha.
+
+
+Changes in version 0.2.8.15 - 2017-09-18
+  Tor 0.2.8.15 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Note that Tor 0.2.8.x will no longer be supported after 1 Jan
+  2018.  We suggest that you upgrade to the latest stable release if
+  possible.  If you can't, we recommend that you upgrade at least to
+  0.2.9, which will be supported until 2020.
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.2.9.12 - 2017-09-18
+  Tor 0.2.9.12 backports a collection of bugfixes from later
+  Tor series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Major features (security, backport from 0.3.0.2-alpha):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (crash, directory connections, backport from 0.3.0.5-rc):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (DNS, backport from 0.3.0.2-alpha):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (code style, backport from 0.3.1.3-alpha):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.1-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt support
+      on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions the mingw C header files, avoid
+      float-conversion warnings when calling the C functions isfinite(),
+      isnan(), and signbit(). Fixes bug 22801; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for running tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an
+      empty body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (defensive programming, undefined behavior, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (relay, backport from 0.3.0.5-rc):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Build features (backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+
+Changes in version 0.3.0.11 - 2017-09-18
+  Tor 0.3.0.11 backports a collection of bugfixes from Tor the 0.3.1
+  series.
+
+  Most significantly, it includes a fix for TROVE-2017-008, a
+  security bug that affects hidden services running with the
+  SafeLogging option disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  o Minor features (code style, backport from 0.3.1.7):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features:
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.3.1.7):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (controller, backport from 0.3.1.7):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+
+  o Minor bugfixes (file limits, osx, backport from 0.3.1.5-alpha):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (logging, relay, backport from 0.3.1.6-rc):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+
+Changes in version 0.3.1.7 - 2017-09-18
+  Tor 0.3.1.7 is the first stable release in the 0.3.1 series.
+
+  With the 0.3.1 series, Tor now serves and downloads directory
+  information in more compact formats, to save on bandwidth overhead. It
+  also contains a new padding system to resist netflow-based traffic
+  analysis, and experimental support for building parts of Tor in Rust
+  (though no parts of Tor are in Rust yet). There are also numerous
+  small features, bugfixes on earlier release series, and groundwork for
+  the hidden services revamp of 0.3.2.
+
+  This release also includes a fix for TROVE-2017-008, a security bug
+  that affects hidden services running with the SafeLogging option
+  disabled. For more information, see
+  https://trac.torproject.org/projects/tor/ticket/23490
+
+  Per our stable release policy, we plan to support each stable release
+  series for at least the next nine months, or for three months after
+  the first stable release of the next series: whichever is longer. If
+  you need a release with long-term support, we recommend that you stay
+  with the 0.2.9 series.
+
+  Below is a list of the changes since 0.3.0. For a list of all
+  changes since 0.3.1.6-rc, see the ChangeLog file.
+
+  o New dependencies:
+    - To build with zstd and lzma support, Tor now requires the
+      pkg-config tool at build time.
+
+  o Major bugfixes (security, hidden services, loggging):
+    - Fix a bug where we could log uninitialized stack when a certain
+      hidden service error occurred while SafeLogging was disabled.
+      Fixes bug #23490; bugfix on 0.2.7.2-alpha.
+      This is also tracked as TROVE-2017-008 and CVE-2017-0380.
+
+  o Major features (build system, continuous integration):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major features (directory protocol):
+    - Tor relays and authorities can now serve clients an abbreviated
+      version of the consensus document, containing only the changes
+      since an older consensus document that the client holds. Clients
+      now request these documents when available. When both client and
+      server use this new protocol, they will use far less bandwidth (up
+      to 94% less) to keep the client's consensus up-to-date. Implements
+      proposal 140; closes ticket 13339. Based on work by Daniel Martí.
+    - Tor can now compress directory traffic with lzma or with zstd
+      compression algorithms, which can deliver better bandwidth
+      performance. Because lzma is computationally expensive, it's only
+      used for documents that can be compressed once and served many
+      times. Support for these algorithms requires that tor is built
+      with the libzstd and/or liblzma libraries available. Implements
+      proposal 278; closes ticket 21662.
+    - Relays now perform the more expensive compression operations, and
+      consensus diff generation, in worker threads. This separation
+      avoids delaying the main thread when a new consensus arrives.
+
+  o Major features (experimental):
+    - Tor can now build modules written in Rust. To turn this on, pass
+      the "--enable-rust" flag to the configure script. It's not time to
+      get excited yet: currently, there is no actual Rust functionality
+      beyond some simple glue code, and a notice at startup to tell you
+      that Rust is running. Still, we hope that programmers and
+      packagers will try building Tor with Rust support, so that we can
+      find issues and solve portability problems. Closes ticket 22106.
+
+  o Major features (traffic analysis resistance):
+    - Connections between clients and relays now send a padding cell in
+      each direction every 1.5 to 9.5 seconds (tunable via consensus
+      parameters). This padding will not resist specialized
+      eavesdroppers, but it should be enough to make many ISPs' routine
+      network flow logging less useful in traffic analysis against
+      Tor users.
+
+      Padding is negotiated using Tor's link protocol, so both relays
+      and clients must upgrade for this to take effect. Clients may
+      still send padding despite the relay's version by setting
+      ConnectionPadding 1 in torrc, and may disable padding by setting
+      ConnectionPadding 0 in torrc. Padding may be minimized for mobile
+      users with the torrc option ReducedConnectionPadding. Implements
+      Proposal 251 and Section 2 of Proposal 254; closes ticket 16861.
+    - Relays will publish 24 hour totals of padding and non-padding cell
+      counts to their extra-info descriptors, unless PaddingStatistics 0
+      is set in torrc. These 24 hour totals are also rounded to
+      multiples of 10000.
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (path selection, security):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (connection usage):
+    - We use NETINFO cells to try to determine if both relays involved
+      in a connection will agree on the canonical status of that
+      connection. We prefer the connections where this is the case for
+      extend cells, and try to close connections where relays disagree
+      on their canonical status early. Also, we now prefer the oldest
+      valid connection for extend cells. These two changes should reduce
+      the number of long-term connections that are kept open between
+      relays. Fixes bug 17604; bugfix on 0.2.5.5-alpha.
+    - Relays now log hourly statistics (look for
+      "channel_check_for_duplicates" lines) on the total number of
+      connections to other relays. If the number of connections per
+      relay is unexpectedly large, this log message is at notice level.
+      Otherwise it is at info.
+
+  o Major bugfixes (entry guards):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (linux TPROXY support):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xx" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Major bugfixes (relay, link handshake):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (security, windows):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (bridge authority):
+    - Add "fingerprint" lines to the networkstatus-bridges file produced
+      by bridge authorities. Closes ticket 22207.
+
+  o Minor features (code style):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (config options):
+    - Allow "%include" directives in torrc configuration files. These
+      directives import the settings from other files, or from all the
+      files in a directory. Closes ticket 1922. Code by Daniel Pinto.
+    - Make SAVECONF return an error when overwriting a torrc that has
+      includes. Using SAVECONF with the FORCE option will allow it to
+      overwrite torrc even if includes are used. Related to ticket 1922.
+    - Add "GETINFO config-can-saveconf" to tell controllers if SAVECONF
+      will work without the FORCE option. Related to ticket 1922.
+
+  o Minor features (controller):
+    - Warn the first time that a controller requests data in the long-
+      deprecated 'GETINFO network-status' format. Closes ticket 21703.
+
+  o Minor features (defaults):
+    - The default value for UseCreateFast is now 0: clients which
+      haven't yet received a consensus document will now use a proper
+      ntor handshake to talk to their directory servers whenever they
+      can. Closes ticket 21407.
+    - Onion key rotation and expiry intervals are now defined as a
+      network consensus parameter, per proposal 274. The default
+      lifetime of an onion key is increased from 7 to 28 days. Old onion
+      keys will expire after 7 days by default. This change will make
+      consensus diffs much smaller, and save significant bandwidth.
+      Closes ticket 21641.
+
+  o Minor features (defensive programming):
+    - Create a pair of consensus parameters, nf_pad_tor2web and
+      nf_pad_single_onion, to disable netflow padding in the consensus
+      for non-anonymous connections in case the overhead is high. Closes
+      ticket 17857.
+
+  o Minor features (diagnostic):
+    - Add a stack trace to the bug warnings that can be logged when
+      trying to send an outgoing relay cell with n_chan == 0. Diagnostic
+      attempt for bug 23105.
+    - Add logging messages to try to diagnose a rare bug that seems to
+      generate RSA->Ed25519 cross-certificates dated in the 1970s. We
+      think this is happening because of incorrect system clocks, but
+      we'd like to know for certain. Diagnostic for bug 22466.
+    - Avoid an assertion failure, and log a better error message, when
+      unable to remove a file from the consensus cache on Windows.
+      Attempts to mitigate and diagnose bug 22752.
+
+  o Minor features (directory authority):
+    - Improve the message that authorities report to relays that present
+      RSA/Ed25519 keypairs that conflict with previously pinned keys.
+      Closes ticket 22348.
+
+  o Minor features (directory cache, consensus diff):
+    - Add a new MaxConsensusAgeForDiffs option to allow directory cache
+      operators with low-resource environments to adjust the number of
+      consensuses they'll store and generate diffs from. Most cache
+      operators should leave it unchanged. Helps to work around
+      bug 22883.
+
+  o Minor features (fallback directory list):
+    - Update the fallback directory mirror whitelist and blacklist based
+      on operator emails. Closes task 21121.
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the September 6 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (hidden services, logging):
+    - Log a message when a hidden service descriptor has fewer
+      introduction points than specified in
+      HiddenServiceNumIntroductionPoints. Closes tickets 21598.
+    - Log a message when a hidden service reaches its introduction point
+      circuit limit, and when that limit is reset. Follow up to ticket
+      21594; closes ticket 21622.
+    - Warn user if multiple entries in EntryNodes and at least one
+      HiddenService are used together. Pinning EntryNodes along with a
+      hidden service can be possibly harmful; for instance see ticket
+      14917 or 21155. Closes ticket 21155.
+
+  o Minor features (linux seccomp2 sandbox):
+    - We now have a document storage backend compatible with the Linux
+      seccomp2 sandbox. This backend is used for consensus documents and
+      diffs between them; in the long term, we'd like to use it for
+      unparseable directory material too. Closes ticket 21645
+    - Increase the maximum allowed size passed to mprotect(PROT_WRITE)
+      from 1MB to 16MB. This was necessary with the glibc allocator in
+      order to allow worker threads to allocate more memory -- which in
+      turn is necessary because of our new use of worker threads for
+      compression. Closes ticket 22096.
+
+  o Minor features (logging):
+    - Log files are no longer created world-readable by default.
+      (Previously, most distributors would store the logs in a non-
+      world-readable location to prevent inappropriate access. This
+      change is an extra precaution.) Closes ticket 21729; patch
+      from toralf.
+
+  o Minor features (performance):
+    - Our Keccak (SHA-3) implementation now accesses memory more
+      efficiently, especially on little-endian systems. Closes
+      ticket 21737.
+    - Add an O(1) implementation of channel_find_by_global_id(), to
+      speed some controller functions.
+
+  o Minor features (relay, configuration):
+    - The MyFamily option may now be repeated as many times as desired,
+      for relays that want to configure large families. Closes ticket
+      4998; patch by Daniel Pinto.
+
+  o Minor features (relay, performance):
+    - Always start relays with at least two worker threads, to prevent
+      priority inversion on slow tasks. Part of the fix for bug 22883.
+    - Allow background work to be queued with different priorities, so
+      that a big pile of slow low-priority jobs will not starve out
+      higher priority jobs. This lays the groundwork for a fix for
+      bug 22883.
+
+  o Minor features (safety):
+    - Add an explicit check to extrainfo_parse_entry_from_string() for
+      NULL inputs. We don't believe this can actually happen, but it may
+      help silence a warning from the Clang analyzer. Closes
+      ticket 21496.
+
+  o Minor features (testing):
+    - Add more tests for compression backend initialization. Closes
+      ticket 22286.
+    - Add a "--disable-memory-sentinels" feature to help with fuzzing.
+      When Tor is compiled with this option, we disable a number of
+      redundant memory-safety failsafes that are intended to stop bugs
+      from becoming security issues. This makes it easier to hunt for
+      bugs that would be security issues without the failsafes turned
+      on. Closes ticket 21439.
+    - Add a general event-tracing instrumentation support to Tor. This
+      subsystem will enable developers and researchers to add fine-
+      grained instrumentation to their Tor instances, for use when
+      examining Tor network performance issues. There are no trace
+      events yet, and event-tracing is off by default unless enabled at
+      compile time. Implements ticket 13802.
+    - Improve our version parsing tests: add tests for typical version
+      components, add tests for invalid versions, including numeric
+      range and non-numeric prefixes. Unit tests 21278, 21450, and
+      21507. Partially implements 21470.
+
+  o Minor bugfixes (bandwidth accounting):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (code correctness):
+    - Accurately identify client connections by their lack of peer
+      authentication. This means that we bail out earlier if asked to
+      extend to a client. Follow-up to 21407. Fixes bug 21406; bugfix
+      on 0.2.4.23.
+
+  o Minor bugfixes (compilation warnings):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug
+      22915; bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Avoid compiler warnings in the unit tests for calling tor_sscanf()
+      with wide string outputs. Fixes bug 15582; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (compression):
+    - When spooling compressed data to an output buffer, don't try to
+      spool more data when there is no more data to spool and we are not
+      trying to flush the input. Previously, we would sometimes launch
+      compression requests with nothing to do, which interferes with our
+      22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha.
+
+  o Minor bugfixes (configuration):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection lifespan):
+    - Allow more control over how long TLS connections are kept open:
+      unify CircuitIdleTimeout and PredictedPortsRelevanceTime into a
+      single option called CircuitsAvailableTimeout. Also, allow the
+      consensus to control the default values for both this preference
+      and the lifespan of relay-to-relay connections. Fixes bug 17592;
+      bugfix on 0.2.5.5-alpha.
+    - Increase the initial circuit build timeout testing frequency, to
+      help ensure that ReducedConnectionPadding clients finish learning
+      a timeout before their orconn would expire. The initial testing
+      rate was set back in the days of TAP and before the Tor Browser
+      updater, when we had to be much more careful about new clients
+      making lots of circuits. With this change, a circuit build timeout
+      is learned in about 15-20 minutes, instead of 100-120 minutes.
+
+  o Minor bugfixes (controller):
+    - Do not crash when receiving a HSPOST command with an empty body.
+      Fixes part of bug 22644; bugfix on 0.2.7.1-alpha.
+    - Do not crash when receiving a POSTDESCRIPTOR command with an empty
+      body. Fixes part of bug 22644; bugfix on 0.2.0.1-alpha.
+    - GETINFO onions/current and onions/detached no longer respond with
+      551 on empty lists. Fixes bug 21329; bugfix on 0.2.7.1-alpha.
+    - Trigger HS descriptor events on the control port when the client
+      fails to pick a hidden service directory for a hidden service.
+      This can happen if all the hidden service directories are in
+      ExcludeNodes, or they have all been queried within the last 15
+      minutes. Fixes bug 22042; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (coverity build support):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (defensive programming):
+    - Detect and break out of infinite loops in our compression code. We
+      don't think that any such loops exist now, but it's best to be
+      safe. Closes ticket 22672.
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+  o Minor bugfixes (directory authority):
+    - When a directory authority rejects a descriptor or extrainfo with
+      a given digest, mark that digest as undownloadable, so that we do
+      not attempt to download it again over and over. We previously
+      tried to avoid downloading such descriptors by other means, but we
+      didn't notice if we accidentally downloaded one anyway. This
+      behavior became problematic in 0.2.7.2-alpha, when authorities
+      began pinning Ed25519 keys. Fixes bug 22349; bugfix
+      on 0.2.1.19-alpha.
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+    - Prevent the shared randomness subsystem from asserting when
+      initialized by a bridge authority with an incomplete configuration
+      file. Fixes bug 21586; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (error reporting, windows):
+    - When formatting Windows error messages, use the English format to
+      avoid codepage issues. Fixes bug 22520; bugfix on 0.1.2.8-alpha.
+      Patch from "Vort".
+
+  o Minor bugfixes (exit-side DNS):
+    - Fix an untriggerable assertion that checked the output of a
+      libevent DNS error, so that the assertion actually behaves as
+      expected. Fixes bug 22244; bugfix on 0.2.0.20-rc. Found by Andrey
+      Karpov using PVS-Studio.
+
+  o Minor bugfixes (fallback directories):
+    - Make the usage example in updateFallbackDirs.py actually work, and
+      explain what it does. Fixes bug 22270; bugfix on 0.3.0.3-alpha.
+    - Decrease the guard flag average required to be a fallback. This
+      allows us to keep relays that have their guard flag removed when
+      they restart. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+    - Decrease the minimum number of fallbacks to 100. Fixes bug 20913;
+      bugfix on 0.2.8.1-alpha.
+    - Make sure fallback directory mirrors have the same address, port,
+      and relay identity key for at least 30 days before they are
+      selected. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (file limits, osx):
+    - When setting the maximum number of connections allowed by the OS,
+      always allow some extra file descriptors for other files. Fixes
+      bug 22797; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Increase the number of circuits that a service is allowed to
+      open over a specific period of time. The value was lower than it
+      should be (8 vs 12) in the normal case of 3 introduction points.
+      Fixes bug 22159; bugfix on 0.3.0.5-rc.
+    - Fix a BUG warning during HSv3 descriptor decoding that could be
+      cause by a specially crafted descriptor. Fixes bug 23233; bugfix
+      on 0.3.0.1-alpha. Bug found by "haxxpop".
+    - Stop printing a cryptic warning when a hidden service gets a
+      request to connect to a virtual port that it hasn't configured.
+      Fixes bug 16706; bugfix on 0.2.6.3-alpha.
+    - Simplify hidden service descriptor creation by using an existing
+      flag to check if an introduction point is established. Fixes bug
+      21599; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (link handshake):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (logging):
+    - When decompressing, do not warn if we fail to decompress using a
+      compression method that we merely guessed. Fixes part of bug
+      22670; bugfix on 0.1.1.14-alpha.
+    - When decompressing, treat mismatch between content-encoding and
+      actual compression type as a protocol warning. Fixes part of bug
+      22670; bugfix on 0.1.1.9-alpha.
+    - Downgrade "assigned_to_cpuworker failed" message to info-level
+      severity. In every case that can reach it, either a better warning
+      has already been logged, or no warning is warranted. Fixes bug
+      22356; bugfix on 0.2.6.3-alpha.
+    - Log a better message when a directory authority replies to an
+      upload with an unexpected status code. Fixes bug 11121; bugfix
+      on 0.1.0.1-rc.
+    - Downgrade a log statement about unexpected relay cells from "bug"
+      to "protocol warning", because there is at least one use case
+      where it can be triggered by a buggy tor implementation. Fixes bug
+      21293; bugfix on 0.1.1.14-alpha.
+
+  o Minor bugfixes (logging, relay):
+    - Remove a forgotten debugging message when an introduction point
+      successfully establishes a hidden service prop224 circuit with
+      a client.
+    - Change three other log_warn() for an introduction point to
+      protocol warnings, because they can be failure from the network
+      and are not relevant to the operator. Fixes bug 23078; bugfix on
+      0.3.0.1-alpha and 0.3.0.2-alpha.
+
+  o Minor bugfixes (relay):
+    - Inform the geoip and rephist modules about all requests, even on
+      relays that are only fetching microdescriptors. Fixes a bug
+      related to 21585; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a small memory leak at exit from the backtrace handler code.
+      Fixes bug 21788; bugfix on 0.2.5.2-alpha. Patch from Daniel Pinto.
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+    - Fix a small memory leak when validating a configuration that uses
+      two or more AF_UNIX sockets for the same port type. Fixes bug
+      23053; bugfix on 0.2.6.3-alpha. This is CID 1415725.
+
+  o Minor bugfixes (process behavior):
+    - When exiting because of an error, always exit with a nonzero exit
+      status. Previously, we would fail to report an error in our exit
+      status in cases related to __OwningControllerProcess failure,
+      lockfile contention, and Ed25519 key initialization. Fixes bug
+      22720; bugfix on versions 0.2.1.6-alpha, 0.2.2.28-beta, and
+      0.2.7.2-alpha respectively. Reported by "f55jwk4f"; patch
+      from "huyvq".
+
+  o Minor bugfixes (robustness, error handling):
+    - Improve our handling of the cases where OpenSSL encounters a
+      memory error while encoding keys and certificates. We haven't
+      observed these errors in the wild, but if they do happen, we now
+      detect and respond better. Fixes bug 19418; bugfix on all versions
+      of Tor. Reported by Guido Vranken.
+
+  o Minor bugfixes (testing):
+    - Fix an undersized buffer in test-memwipe.c. Fixes bug 23291;
+      bugfix on 0.2.7.2-alpha. Found and patched by Ties Stuij.
+    - Use unbuffered I/O for utility functions around the
+      process_handle_t type. This fixes unit test failures reported on
+      OpenBSD and FreeBSD. Fixes bug 21654; bugfix on 0.2.3.1-alpha.
+    - Make display of captured unit test log messages consistent. Fixes
+      bug 21510; bugfix on 0.2.9.3-alpha.
+    - Make test-network.sh always call chutney's test-network.sh.
+      Previously, this only worked on systems which had bash installed,
+      due to some bash-specific code in the script. Fixes bug 19699;
+      bugfix on 0.3.0.4-rc. Follow-up to ticket 21581.
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+    - The unit tests now pass on systems where localhost is misconfigured
+      to some IPv4 address other than 127.0.0.1. Fixes bug 6298; bugfix
+      on 0.0.9pre2.
+
+  o Minor bugfixes (voting consistency):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (Windows service):
+    - When running as a Windows service, set the ID of the main thread
+      correctly. Failure to do so made us fail to send log messages to
+      the controller in 0.2.1.16-rc, slowed down controller event
+      delivery in 0.2.7.3-rc and later, and crash with an assertion
+      failure in 0.3.1.1-alpha. Fixes bug 23081; bugfix on 0.2.1.6-alpha.
+      Patch and diagnosis from "Vort".
+
+  o Minor bugfixes (windows, relay):
+    - Resolve "Failure from drain_fd: No error" warnings on Windows
+      relays. Fixes bug 21540; bugfix on 0.2.6.3-alpha.
+
+  o Code simplification and refactoring:
+    - Break up the 630-line function connection_dir_client_reached_eof()
+      into a dozen smaller functions. This change should help
+      maintainability and readability of the client directory code.
+    - Isolate our use of the openssl headers so that they are only
+      included from our crypto wrapper modules, and from tests that
+      examine those modules' internals. Closes ticket 21841.
+    - Simplify our API to launch directory requests, making it more
+      extensible and less error-prone. Now it's easier to add extra
+      headers to directory requests. Closes ticket 21646.
+    - Our base64 decoding functions no longer overestimate the output
+      space that they need when parsing unpadded inputs. Closes
+      ticket 17868.
+    - Remove unused "ROUTER_ADDED_NOTIFY_GENERATOR" internal value.
+      Resolves ticket 22213.
+    - The logic that directory caches use to spool request to clients,
+      serving them one part at a time so as not to allocate too much
+      memory, has been refactored for consistency. Previously there was
+      a separate spooling implementation per type of spoolable data. Now
+      there is one common spooling implementation, with extensible data
+      types. Closes ticket 21651.
+    - Tor's compression module now supports multiple backends. Part of
+      the implementation for proposal 278; closes ticket 21663.
+
+  o Documentation:
+    - Add a manpage description for the key-pinning-journal file. Closes
+      ticket 22347.
+    - Correctly note that bandwidth accounting values are stored in the
+      state file, and the bw_accounting file is now obsolete. Closes
+      ticket 16082.
+    - Document more of the files in the Tor data directory, including
+      cached-extrainfo, secret_onion_key{,_ntor}.old, hidserv-stats,
+      approved-routers, sr-random, and diff-cache. Found while fixing
+      ticket 22347.
+    - Clarify the manpage for the (deprecated) torify script. Closes
+      ticket 6892.
+    - Clarify the behavior of the KeepAliveIsolateSOCKSAuth sub-option.
+      Closes ticket 21873.
+    - Correct documentation about the default DataDirectory value.
+      Closes ticket 21151.
+    - Document the default behavior of NumEntryGuards and
+      NumDirectoryGuards correctly. Fixes bug 21715; bugfix
+      on 0.3.0.1-alpha.
+    - Document key=value pluggable transport arguments for Bridge lines
+      in torrc. Fixes bug 20341; bugfix on 0.2.5.1-alpha.
+    - Note that bandwidth-limiting options don't affect TCP headers or
+      DNS. Closes ticket 17170.
+
+  o Removed features (configuration options, all in ticket 22060):
+    - These configuration options are now marked Obsolete, and no longer
+      have any effect: AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ExcludeSingleHopRelays, FastFirstHopPK,
+      TLSECGroup, WarnUnsafeSocks. They were first marked as deprecated
+      in 0.2.9.2-alpha and have now been removed. The previous default
+      behavior is now always chosen; the previous (less secure) non-
+      default behavior is now unavailable.
+    - CloseHSClientCircuitsImmediatelyOnTimeout and
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout were deprecated in
+      0.2.9.2-alpha and now have been removed. HS circuits never close
+      on circuit build timeout; they have a longer timeout period.
+    - {Control,DNS,Dir,Socks,Trans,NATD,OR}ListenAddress were deprecated
+      in 0.2.9.2-alpha and now have been removed. Use the ORPort option
+      (and others) to configure listen-only and advertise-only addresses.
+
+  o Removed features (tools):
+    - We've removed the tor-checkkey tool from src/tools. Long ago, we
+      used it to help people detect RSA keys that were generated by
+      versions of Debian affected by CVE-2008-0166. But those keys have
+      been out of circulation for ages, and this tool is no longer
+      required. Closes ticket 21842.
+
+
+Changes in version 0.3.0.10 - 2017-08-02
+   Tor 0.3.0.10 backports a collection of small-to-medium bugfixes
+   from the current Tor alpha series. OpenBSD users and TPROXY users
+   should upgrade; others are probably okay sticking with 0.3.0.9.
+
+  o Major features (build system, continuous integration, backport from 0.3.1.5-alpha):
+    - Tor's repository now includes a Travis Continuous Integration (CI)
+      configuration file (.travis.yml). This is meant to help new
+      developers and contributors who fork Tor to a Github repository be
+      better able to test their changes, and understand what we expect
+      to pass. To use this new build feature, you must fork Tor to your
+      Github account, then go into the "Integrations" menu in the
+      repository settings for your fork and enable Travis, then push
+      your changes. Closes ticket 22636.
+
+  o Major bugfixes (linux TPROXY support, backport from 0.3.1.1-alpha):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Major bugfixes (openbsd, denial-of-service, backport from 0.3.1.5-alpha):
+    - Avoid an assertion failure bug affecting our implementation of
+      inet_pton(AF_INET6) on certain OpenBSD systems whose strtol()
+      handling of "0xbar" differs from what we had expected. Fixes bug
+      22789; bugfix on 0.2.3.8-alpha. Also tracked as TROVE-2017-007.
+
+  o Minor features (backport from 0.3.1.5-alpha):
+    - Update geoip and geoip6 to the July 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (bandwidth accounting, backport from 0.3.1.2-alpha):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (compilation warnings, backport from 0.3.1.5-alpha):
+    - Suppress -Wdouble-promotion warnings with clang 4.0. Fixes bug 22915;
+      bugfix on 0.2.8.1-alpha.
+    - Fix warnings when building with libscrypt and openssl scrypt
+      support on Clang. Fixes bug 22916; bugfix on 0.2.7.2-alpha.
+    - When building with certain versions of the mingw C header files,
+      avoid float-conversion warnings when calling the C functions
+      isfinite(), isnan(), and signbit(). Fixes bug 22801; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
+    - Backport a fix for an "unused variable" warning that appeared
+      in some versions of mingw. Fixes bug 22838; bugfix on
+      0.2.8.1-alpha.
+
+  o Minor bugfixes (coverity build support, backport from 0.3.1.5-alpha):
+    - Avoid Coverity build warnings related to our BUG() macro. By
+      default, Coverity treats BUG() as the Linux kernel does: an
+      instant abort(). We need to override that so our BUG() macro
+      doesn't prevent Coverity from analyzing functions that use it.
+      Fixes bug 23030; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.1.1-alpha):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.5-alpha):
+    - Avoid a sandbox failure when trying to re-bind to a socket and
+      mark it as IPv6-only. Fixes bug 20247; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (unit tests, backport from 0.3.1.5-alpha)
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.0.9 - 2017-06-29
+  Tor 0.3.0.9 fixes a path selection bug that would allow a client
+  to use a guard that was in the same network family as a chosen exit
+  relay. This is a security regression; all clients running earlier
+  versions of 0.3.0.x or 0.3.1.x should upgrade to 0.3.0.9 or
+  0.3.1.4-alpha.
+
+  This release also backports several other bugfixes from the 0.3.1.x
+  series.
+
+  o Major bugfixes (path selection, security, backport from 0.3.1.4-alpha):
+    - When choosing which guard to use for a circuit, avoid the exit's
+      family along with the exit itself. Previously, the new guard
+      selection logic avoided the exit, but did not consider its family.
+      Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked as TROVE-2017-
+      006 and CVE-2017-0377.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.1-alpha):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (entry guards, backport from 0.3.1.4-alpha):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (voting consistency, backport from 0.3.1.1-alpha):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (linux seccomp2 sandbox, backport from 0.3.1.4-alpha):
+    - Permit the fchmod system call, to avoid crashing on startup when
+      starting with the seccomp2 sandbox and an unexpected set of
+      permissions on the data directory or its contents. Fixes bug
+      22516; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (defensive programming, backport from 0.3.1.4-alpha):
+    - Fix a memset() off the end of an array when packing cells. This
+      bug should be harmless in practice, since the corrupted bytes are
+      still in the same structure, and are always padding bytes,
+      ignored, or immediately overwritten, depending on compiler
+      behavior. Nevertheless, because the memset()'s purpose is to make
+      sure that any other cell-handling bugs can't expose bytes to the
+      network, we need to fix it. Fixes bug 22737; bugfix on
+      0.2.4.11-alpha. Fixes CID 1401591.
+
+
+Changes in version 0.3.0.8 - 2017-06-08
+  Tor 0.3.0.8 fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.0.8 also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management, backport from 0.3.1.3-alpha):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha):
+    - Stop rejecting v3 hidden service descriptors because their size
+      did not match an old padding rule. Fixes bug 22447; bugfix on
+      tor-0.3.0.1-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (configuration, backport from 0.3.1.1-alpha):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (link handshake, backport from 0.3.1.3-alpha):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport from
+    0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.2.9.11 - 2017-06-08
+  Tor 0.2.9.11 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005. (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  Tor 0.2.9.11 also backports fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (future-proofing, backport from 0.3.0.7):
+    - Tor no longer refuses to download microdescriptors or descriptors if
+      they are listed as "published in the future".  This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor features (directory authorities, backport from 0.3.0.4-rc)
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port, backport from 0.3.0.6):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.0.7):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport
+    from 0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+Changes in version 0.2.8.14 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.7.8 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.2.6.12 - 2017-06-08
+  Tor 0.2.6.12 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.5.14 - 2017-06-08
+  Tor 0.2.5.14 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.4.29 - 2017-06-08
+  Tor 0.2.4.29 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.3.0.7 - 2017-05-15
+  Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions
+  of Tor 0.3.0.x, where an attacker could cause a Tor relay process
+  to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade;
+  clients are not affected.
+
+  o Major bugfixes (hidden service directory, security):
+    - Fix an assertion failure in the hidden service directory code, which
+      could be used by an attacker to remotely cause a Tor relay process to
+      exit. Relays running earlier versions of Tor 0.3.0.x should upgrade.
+      should upgrade. This security issue is tracked as TROVE-2017-002.
+      Fixes bug 22246; bugfix on 0.3.0.1-alpha.
+
+  o Minor features:
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (future-proofing):
+    - Tor no longer refuses to download microdescriptors or descriptors
+      if they are listed as "published in the future". This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.0.6 - 2017-04-26
+  Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series.
+
+  With the 0.3.0 series, clients and relays now use Ed25519 keys to
+  authenticate their link connections to relays, rather than the old
+  RSA1024 keys that they used before. (Circuit crypto has been
+  Curve25519-authenticated since 0.2.4.8-alpha.) We have also replaced
+  the guard selection and replacement algorithm to behave more robustly
+  in the presence of unreliable networks, and to resist guard-
+  capture attacks.
+
+  This series also includes numerous other small features and bugfixes,
+  along with more groundwork for the upcoming hidden-services revamp.
+
+  Per our stable release policy, we plan to support the Tor 0.3.0
+  release series for at least the next nine months, or for three months
+  after the first stable release of the 0.3.1 series: whichever is
+  longer. If you need a release with long-term support, we recommend
+  that you stay with the 0.2.9 series.
+
+  Below are the changes since 0.2.9.10. For a list of only the changes
+  since 0.3.0.5-rc, see the ChangeLog file.
+
+  o Major features (directory authority, security):
+    - The default for AuthDirPinKeys is now 1: directory authorities
+      will reject relays where the RSA identity key matches a previously
+      seen value, but the Ed25519 key has changed. Closes ticket 18319.
+
+  o Major features (guard selection algorithm):
+    - Tor's guard selection algorithm has been redesigned from the
+      ground up, to better support unreliable networks and restrictive
+      sets of entry nodes, and to better resist guard-capture attacks by
+      hostile local networks. Implements proposal 271; closes
+      ticket 19877.
+
+  o Major features (next-generation hidden services):
+    - Relays can now handle v3 ESTABLISH_INTRO cells as specified by
+      prop224 aka "Next Generation Hidden Services". Service and clients
+      don't use this functionality yet. Closes ticket 19043. Based on
+      initial code by Alec Heifetz.
+    - Relays now support the HSDir version 3 protocol, so that they can
+      can store and serve v3 descriptors. This is part of the next-
+      generation onion service work detailed in proposal 224. Closes
+      ticket 17238.
+
+  o Major features (protocol, ed25519 identity keys):
+    - Clients now support including Ed25519 identity keys in the EXTEND2
+      cells they generate. By default, this is controlled by a consensus
+      parameter, currently disabled. You can turn this feature on for
+      testing by setting ExtendByEd25519ID in your configuration. This
+      might make your traffic appear different than the traffic
+      generated by other users, however. Implements part of ticket
+      15056; part of proposal 220.
+    - Relays now understand requests to extend to other relays by their
+      Ed25519 identity keys. When an Ed25519 identity key is included in
+      an EXTEND2 cell, the relay will only extend the circuit if the
+      other relay can prove ownership of that identity. Implements part
+      of ticket 15056; part of proposal 220.
+    - Relays now use Ed25519 to prove their Ed25519 identities and to
+      one another, and to clients. This algorithm is faster and more
+      secure than the RSA-based handshake we've been doing until now.
+      Implements the second big part of proposal 220; Closes
+      ticket 15055.
+
+  o Major features (security):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (client, onion service, also in 0.2.9.9):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Major bugfixes (crash, directory connections):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (directory authority):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (DNS):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (IPv6 Exits):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+    - When parsing a malformed content-length field from an HTTP
+      message, do not read off the end of the buffer. This bug was a
+      potential remote denial-of-service attack against Tor clients and
+      relays. A workaround was released in October 2016, to prevent this
+      bug from crashing Tor. This is a fix for the underlying issue,
+      which should no longer matter (if you applied the earlier patch).
+      Fixes bug 20894; bugfix on 0.2.0.16-alpha. Bug found by fuzzing
+      using AFL (http://lcamtuf.coredump.cx/afl/).
+
+  o Major bugfixes (scheduler):
+    - Actually compare circuit policies in ewma_cmp_cmux(). This bug
+      caused the channel scheduler to behave more or less randomly,
+      rather than preferring channels with higher-priority circuits.
+      Fixes bug 20459; bugfix on 0.2.6.2-alpha.
+
+  o Major bugfixes (security, also in 0.2.9.9):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes--and having it
+      on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Minor feature (client):
+    - Enable IPv6 traffic on the SocksPort by default. To disable this,
+      a user will have to specify "NoIPv6Traffic". Closes ticket 21269.
+
+  o Minor feature (fallback scripts):
+    - Add a check_existing mode to updateFallbackDirs.py, which checks
+      if fallbacks in the hard-coded list are working. Closes ticket
+      20174. Patch by haxxpop.
+
+  o Minor feature (protocol versioning):
+    - Add new protocol version for proposal 224. HSIntro now advertises
+      version "3-4" and HSDir version "1-2". Fixes ticket 20656.
+
+  o Minor features (ciphersuite selection):
+    - Allow relays to accept a wider range of ciphersuites, including
+      chacha20-poly1305 and AES-CCM. Closes the other part of 15426.
+    - Clients now advertise a list of ciphersuites closer to the ones
+      preferred by Firefox. Closes part of ticket 15426.
+
+  o Minor features (controller):
+    - Add "GETINFO sr/current" and "GETINFO sr/previous" keys, to expose
+      shared-random values to the controller. Closes ticket 19925.
+    - When HSFETCH arguments cannot be parsed, say "Invalid argument"
+      rather than "unrecognized." Closes ticket 20389; patch from
+      Ivan Markin.
+
+  o Minor features (controller, configuration):
+    - Each of the *Port options, such as SocksPort, ORPort, ControlPort,
+      and so on, now comes with a __*Port variant that will not be saved
+      to the torrc file by the controller's SAVECONF command. This
+      change allows TorBrowser to set up a single-use domain socket for
+      each time it launches Tor. Closes ticket 20956.
+    - The GETCONF command can now query options that may only be
+      meaningful in context-sensitive lists. This allows the controller
+      to query the mixed SocksPort/__SocksPort style options introduced
+      in feature 20956. Implements ticket 21300.
+
+  o Minor features (diagnostic, directory client):
+    - Warn when we find an unexpected inconsistency in directory
+      download status objects. Prevents some negative consequences of
+      bug 20593.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (directory authority):
+    - Add a new authority-only AuthDirTestEd25519LinkKeys option (on by
+      default) to control whether authorities should try to probe relays
+      by their Ed25519 link keys. This option will go away in a few
+      releases--unless we encounter major trouble in our ed25519 link
+      protocol rollout, in which case it will serve as a safety option.
+
+  o Minor features (directory cache):
+    - Relays and bridges will now refuse to serve the consensus they
+      have if they know it is too old for a client to use. Closes
+      ticket 20511.
+
+  o Minor features (ed25519 link handshake):
+    - Advertise support for the ed25519 link handshake using the
+      subprotocol-versions mechanism, so that clients can tell which
+      relays can identity themselves by Ed25519 ID. Closes ticket 20552.
+
+  o Minor features (entry guards):
+    - Add UseEntryGuards to TEST_OPTIONS_DEFAULT_VALUES in order to not
+      break regression tests.
+    - Require UseEntryGuards when UseBridges is set, in order to make
+      sure bridges aren't bypassed. Resolves ticket 20502.
+
+  o Minor features (fallback directories):
+    - Allow 3 fallback relays per operator, which is safe now that we
+      are choosing 200 fallback relays. Closes ticket 20912.
+    - Annotate updateFallbackDirs.py with the bandwidth and consensus
+      weight for each candidate fallback. Closes ticket 20878.
+    - Display the relay fingerprint when downloading consensuses from
+      fallbacks. Closes ticket 20908.
+    - Exclude relays affected by bug 20499 from the fallback list.
+      Exclude relays from the fallback list if they are running versions
+      known to be affected by bug 20499, or if in our tests they deliver
+      a stale consensus (i.e. one that expired more than 24 hours ago).
+      Closes ticket 20539.
+    - Make it easier to change the output sort order of fallbacks.
+      Closes ticket 20822.
+    - Reduce the minimum fallback bandwidth to 1 MByte/s. Part of
+      ticket 18828.
+    - Require fallback directories to have the same address and port for
+      7 days (now that we have enough relays with this stability).
+      Relays whose OnionOO stability timer is reset on restart by bug
+      18050 should upgrade to Tor 0.2.8.7 or later, which has a fix for
+      this issue. Closes ticket 20880; maintains short-term fix
+      in 0.2.8.2-alpha.
+    - Require fallbacks to have flags for 90% of the time (weighted
+      decaying average), rather than 95%. This allows at least 73% of
+      clients to bootstrap in the first 5 seconds without contacting an
+      authority. Part of ticket 18828.
+    - Select 200 fallback directories for each release. Closes
+      ticket 20881.
+
+  o Minor features (fingerprinting resistance, authentication):
+    - Extend the length of RSA keys used for TLS link authentication to
+      2048 bits. (These weren't used for forward secrecy; for forward
+      secrecy, we used P256.) Closes ticket 13752.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (geoip, also in 0.2.9.9):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (infrastructure):
+    - Implement smartlist_add_strdup() function. Replaces the use of
+      smartlist_add(sl, tor_strdup(str)). Closes ticket 20048.
+
+  o Minor features (linting):
+    - Enhance the changes file linter to warn on Tor versions that are
+      prefixed with "tor-". Closes ticket 21096.
+
+  o Minor features (logging):
+    - In several places, describe unset ed25519 keys as "<unset>",
+      rather than the scary "AAAAAAAA...AAA". Closes ticket 21037.
+
+  o Minor features (portability, compilation):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor features (relay):
+    - We now allow separation of exit and relay traffic to different
+      source IP addresses, using the OutboundBindAddressExit and
+      OutboundBindAddressOR options respectively. Closes ticket 17975.
+      Written by Michael Sonntag.
+
+  o Minor features (reliability, crash):
+    - Try better to detect problems in buffers where they might grow (or
+      think they have grown) over 2 GB in size. Diagnostic for
+      bug 21369.
+
+  o Minor features (testing):
+    - During 'make test-network-all', if tor logs any warnings, ask
+      chutney to output them. Requires a recent version of chutney with
+      the 21572 patch. Implements 21570.
+
+  o Minor bugfix (control protocol):
+    - The reply to a "GETINFO config/names" request via the control
+      protocol now spells the type "Dependent" correctly. This is a
+      breaking change in the control protocol. (The field seems to be
+      ignored by the most common known controllers.) Fixes bug 18146;
+      bugfix on 0.1.1.4-alpha.
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfix (logging):
+    - Don't recommend the use of Tor2web in non-anonymous mode.
+      Recommending Tor2web is a bad idea because the client loses all
+      anonymity. Tor2web should only be used in specific cases by users
+      who *know* and understand the issues. Fixes bug 21294; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (bug resilience):
+    - Fix an unreachable size_t overflow in base64_decode(). Fixes bug
+      19222; bugfix on 0.2.0.9-alpha. Found by Guido Vranken; fixed by
+      Hans Jerry Illikainen.
+
+  o Minor bugfixes (build):
+    - Replace obsolete Autoconf macros with their modern equivalent and
+      prevent similar issues in the future. Fixes bug 20990; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (certificate expiration time):
+    - Avoid using link certificates that don't become valid till some
+      time in the future. Fixes bug 21420; bugfix on 0.2.4.11-alpha
+
+  o Minor bugfixes (client):
+    - Always recover from failures in extend_info_from_node(), in an
+      attempt to prevent any recurrence of bug 21242. Fixes bug 21372;
+      bugfix on 0.2.3.1-alpha.
+    - When clients that use bridges start up with a cached consensus on
+      disk, they were ignoring it and downloading a new one. Now they
+      use the cached one. Fixes bug 20269; bugfix on 0.2.3.12-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (config):
+    - Don't assert on startup when trying to get the options list and
+      LearnCircuitBuildTimeout is set to 0: we are currently parsing the
+      options so of course they aren't ready yet. Fixes bug 21062;
+      bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (configuration):
+    - Accept non-space whitespace characters after the severity level in
+      the `Log` option. Fixes bug 19965; bugfix on 0.2.1.1-alpha.
+    - Support "TByte" and "TBytes" units in options given in bytes.
+      "TB", "terabyte(s)", "TBit(s)" and "terabit(s)" were already
+      supported. Fixes bug 20622; bugfix on 0.2.0.14-alpha.
+
+  o Minor bugfixes (configure, autoconf):
+    - Rename the configure option --enable-expensive-hardening to
+      --enable-fragile-hardening. Expensive hardening makes the tor
+      daemon abort when some kinds of issues are detected. Thus, it
+      makes tor more at risk of remote crashes but safer against RCE or
+      heartbleed bug category. We now try to explain this issue in a
+      message from the configure script. Fixes bug 21290; bugfix
+      on 0.2.5.4-alpha.
+
+  o Minor bugfixes (consensus weight):
+    - Add new consensus method that initializes bw weights to 1 instead
+      of 0. This prevents a zero weight from making it all the way to
+      the end (happens in small testing networks) and causing an error.
+      Fixes bug 14881; bugfix on 0.2.2.17-alpha.
+
+  o Minor bugfixes (crash prevention):
+    - Fix an (currently untriggerable, but potentially dangerous) crash
+      bug when base32-encoding inputs whose sizes are not a multiple of
+      5. Fixes bug 21894; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (dead code):
+    - Remove a redundant check for PidFile changes at runtime in
+      options_transition_allowed(): this check is already performed
+      regardless of whether the sandbox is active. Fixes bug 21123;
+      bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (descriptors):
+    - Correctly recognise downloaded full descriptors as valid, even
+      when using microdescriptors as circuits. This affects clients with
+      FetchUselessDescriptors set, and may affect directory authorities.
+      Fixes bug 20839; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Allow relays to use directory mirrors without a DirPort: these
+      relays need to be contacted over their ORPorts using a begindir
+      connection. Fixes one case of bug 20711; bugfix on 0.2.8.2-alpha.
+    - Clarify the message logged when a remote relay is unexpectedly
+      missing an ORPort or DirPort: users were confusing this with a
+      local port. Fixes another case of bug 20711; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (directory system):
+    - Bridges and relays now use microdescriptors (like clients do)
+      rather than old-style router descriptors. Now bridges will blend
+      in with clients in terms of the circuits they build. Fixes bug
+      6769; bugfix on 0.2.3.2-alpha.
+    - Download all consensus flavors, descriptors, and authority
+      certificates when FetchUselessDescriptors is set, regardless of
+      whether tor is a directory cache or not. Fixes bug 20667; bugfix
+      on all recent tor versions.
+
+  o Minor bugfixes (documentation):
+    - Update the tor manual page to document every option that can not
+      be changed while tor is running. Fixes bug 21122.
+
+  o Minor bugfixes (ed25519 certificates):
+    - Correctly interpret ed25519 certificates that would expire some
+      time after 19 Jan 2038. Fixes bug 20027; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (fallback directories):
+    - Avoid checking fallback candidates' DirPorts if they are down in
+      OnionOO. When a relay operator has multiple relays, this
+      prioritizes relays that are up over relays that are down. Fixes
+      bug 20926; bugfix on 0.2.8.3-alpha.
+    - Stop failing when OUTPUT_COMMENTS is True in updateFallbackDirs.py.
+      Fixes bug 20877; bugfix on 0.2.8.3-alpha.
+    - Stop failing when a relay has no uptime data in
+      updateFallbackDirs.py. Fixes bug 20945; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Clean up the code for expiring intro points with no associated
+      circuits. It was causing, rarely, a service with some expiring
+      introduction points to not open enough additional introduction
+      points. Fixes part of bug 21302; bugfix on 0.2.7.2-alpha.
+    - Resolve two possible underflows which could lead to creating and
+      closing a lot of introduction point circuits in a non-stop loop.
+      Fixes bug 21302; bugfix on 0.2.7.2-alpha.
+    - Stop setting the torrc option HiddenServiceStatistics to "0" just
+      because we're not a bridge or relay. Instead, we preserve whatever
+      value the user set (or didn't set). Fixes bug 21150; bugfix
+      on 0.2.6.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Make hidden services check for failed intro point connections,
+      even when they have exceeded their intro point creation limit.
+      Fixes bug 21596; bugfix on 0.2.7.2-alpha. Reported by Alec Muffett.
+    - Make hidden services with 8 to 10 introduction points check for
+      failed circuits immediately after startup. Previously, they would
+      wait for 5 minutes before performing their first checks. Fixes bug
+      21594; bugfix on 0.2.3.9-alpha. Reported by Alec Muffett.
+    - Stop ignoring misconfigured hidden services. Instead, refuse to
+      start tor until the misconfigurations have been corrected. Fixes
+      bug 20559; bugfix on multiple commits in 0.2.7.1-alpha
+      and earlier.
+
+  o Minor bugfixes (IPv6):
+    - Make IPv6-using clients try harder to find an IPv6 directory
+      server. Fixes bug 20999; bugfix on 0.2.8.2-alpha.
+    - When IPv6 addresses have not been downloaded yet (microdesc
+      consensus documents don't list relay IPv6 addresses), use hard-
+      coded addresses for authorities, fallbacks, and configured
+      bridges. Now IPv6-only clients can use microdescriptors. Fixes bug
+      20996; bugfix on b167e82 from 19608 in 0.2.8.5-alpha.
+
+  o Minor bugfixes (memory leak at exit):
+    - Fix a small harmless memory leak at exit of the previously unused
+      RSA->Ed identity cross-certificate. Fixes bug 17779; bugfix
+      on 0.2.7.2-alpha.
+
+  o Minor bugfixes (onion services):
+    - Allow the number of introduction points to be as low as 0, rather
+      than as low as 3. Fixes bug 21033; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Use "OpenBSD" compiler macro instead of "OPENBSD" or "__OpenBSD__".
+      It is supported by OpenBSD itself, and also by most OpenBSD
+      variants (such as Bitrig). Fixes bug 20980; bugfix
+      on 0.1.2.1-alpha.
+
+  o Minor bugfixes (portability, also in 0.2.9.9):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+    - Honor DataDirectoryGroupReadable when tor is a relay. Previously,
+      initializing the keys would reset the DataDirectory to 0700
+      instead of 0750 even if DataDirectoryGroupReadable was set to 1.
+      Fixes bug 19953; bugfix on 0.0.2pre16. Patch by "redfish".
+
+  o Minor bugfixes (testing):
+    - Fix Raspbian build issues related to missing socket errno in
+      test_util.c. Fixes bug 21116; bugfix on 0.2.8.2. Patch by "hein".
+    - Remove undefined behavior from the backtrace generator by removing
+      its signal handler. Fixes bug 21026; bugfix on 0.2.5.2-alpha.
+    - Use bash in src/test/test-network.sh. This ensures we reliably
+      call chutney's newer tools/test-network.sh when available. Fixes
+      bug 21562; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+  o Minor bugfixes (unit tests):
+    - Allow the unit tests to pass even when DNS lookups of bogus
+      addresses do not fail as expected. Fixes bug 20862 and 20863;
+      bugfix on unit tests introduced in 0.2.8.1-alpha
+      through 0.2.9.4-alpha.
+
+  o Minor bugfixes (util):
+    - When finishing writing a file to disk, if we were about to replace
+      the file with the temporary file created before and we fail to
+      replace it, remove the temporary file so it doesn't stay on disk.
+      Fixes bug 20646; bugfix on 0.2.0.7-alpha. Patch by fk.
+
+  o Minor bugfixes (Windows services):
+    - Be sure to initialize the monotonic time subsystem before using
+      it, even when running as an NT service. Fixes bug 21356; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (Windows):
+    - Check for getpagesize before using it to mmap files. This fixes
+      compilation in some MinGW environments. Fixes bug 20530; bugfix on
+      0.1.2.1-alpha. Reported by "ice".
+
+  o Code simplification and refactoring:
+    - Abolish all global guard context in entrynodes.c; replace with new
+      guard_selection_t structure as preparation for proposal 271.
+      Closes ticket 19858.
+    - Extract magic numbers in circuituse.c into defined variables.
+    - Introduce rend_service_is_ephemeral() that tells if given onion
+      service is ephemeral. Replace unclear NULL-checkings for service
+      directory with this function. Closes ticket 20526.
+    - Refactor circuit_is_available_for_use to remove unnecessary check.
+    - Refactor circuit_predict_and_launch_new for readability and
+      testability. Closes ticket 18873.
+    - Refactor code to manipulate global_origin_circuit_list into
+      separate functions. Closes ticket 20921.
+    - Refactor large if statement in purpose_needs_anonymity to use
+      switch statement instead. Closes part of ticket 20077.
+    - Refactor the hashing API to return negative values for errors, as
+      is done as throughout the codebase. Closes ticket 20717.
+    - Remove data structures that were used to index or_connection
+      objects by their RSA identity digests. These structures are fully
+      redundant with the similar structures used in the
+      channel abstraction.
+    - Remove duplicate code in the channel_write_*cell() functions.
+      Closes ticket 13827; patch from Pingl.
+    - Remove redundant behavior of is_sensitive_dir_purpose, refactor to
+      use only purpose_needs_anonymity. Closes part of ticket 20077.
+    - The code to generate and parse EXTEND and EXTEND2 cells has been
+      replaced with code automatically generated by the
+      "trunnel" utility.
+
+  o Documentation (formatting):
+    - Clean up formatting of tor.1 man page and HTML doc, where <pre>
+      blocks were incorrectly appearing. Closes ticket 20885.
+
+  o Documentation (man page):
+    - Clarify many options in tor.1 and add some min/max values for
+      HiddenService options. Closes ticket 21058.
+
+  o Documentation:
+    - Change '1' to 'weight_scale' in consensus bw weights calculation
+      comments, as that is reality. Closes ticket 20273. Patch
+      from pastly.
+    - Clarify that when ClientRejectInternalAddresses is enabled (which
+      is the default), multicast DNS hostnames for machines on the local
+      network (of the form *.local) are also rejected. Closes
+      ticket 17070.
+    - Correct the value for AuthDirGuardBWGuarantee in the manpage, from
+      250 KBytes to 2 MBytes. Fixes bug 20435; bugfix on 0.2.5.6-alpha.
+    - Include the "TBits" unit in Tor's man page. Fixes part of bug
+      20622; bugfix on 0.2.5.1-alpha.
+    - Small fixes to the fuzzing documentation. Closes ticket 21472.
+    - Stop the man page from incorrectly stating that HiddenServiceDir
+      must already exist. Fixes 20486.
+    - Update the description of the directory server options in the
+      manual page, to clarify that a relay no longer needs to set
+      DirPort in order to be a directory cache. Closes ticket 21720.
+
+  o Removed features:
+    - The AuthDirMaxServersPerAuthAddr option no longer exists: The same
+      limit for relays running on a single IP applies to authority IP
+      addresses as well as to non-authority IP addresses. Closes
+      ticket 20960.
+    - The UseDirectoryGuards torrc option no longer exists: all users
+      that use entry guards will also use directory guards. Related to
+      proposal 271; implements part of ticket 20831.
+
+  o Testing:
+    - Add tests for networkstatus_compute_bw_weights_v10.
+    - Add unit tests circuit_predict_and_launch_new.
+    - Extract dummy_origin_circuit_new so it can be used by other
+      test functions.
+    - New unit tests for tor_htonll(). Closes ticket 19563. Patch
+      from "overcaffeinated".
+    - Perform the coding style checks when running the tests and fail
+      when coding style violations are found. Closes ticket 5500.
+
+
+Changes in version 0.2.8.13 - 2017-03-03
+  Tor 0.2.8.13 backports a security fix from later Tor
+  releases.  Anybody running Tor 0.2.8.12 or earlier should upgrade to this
+  this release, if for some reason they cannot upgrade to a later
+  release series, and if they build Tor with the --enable-expensive-hardening
+  option.
+
+  Note that support for Tor 0.2.8.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.8.x series after 1 Jan 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.7.7 - 2017-03-03
+  Tor 0.2.7.7 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.7.6 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.7.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.7.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.6.11 - 2017-03-03
+  Tor 0.2.6.11 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.6.10 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.6.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.6.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+
+Changes in version 0.2.5.13 - 2017-03-03
+  Tor 0.2.5.13 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.5.13 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.5.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.5.x series after 1 May 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (crypto error-handling, backport from 0.2.7.2-alpha):
+    - Check for failures from crypto_early_init, and refuse to continue.
+      A previous typo meant that we could keep going with an
+      uninitialized crypto library, and would have OpenSSL initialize
+      its own PRNG. Fixes bug 16360; bugfix on 0.2.5.2-alpha, introduced
+      when implementing ticket 4900. Patch by "teor".
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.4.28 - 2017-03-03
+  Tor 0.2.4.28 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.4.27 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.4.x is ending soon: we will not issue
+  any fixes for the Tor 0.2.4.x series after 1 August 2017.  If you need
+  a Tor release series with long-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (DoS-resistance, backport from 0.2.7.1-alpha):
+    - Make it harder for attackers to overload hidden services with
+      introductions, by blocking multiple introduction requests on the
+      same circuit. Resolves ticket 15515.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.9.10 - 2017-03-01
+  Tor 0.2.9.10 backports a security fix from later Tor release.  It also
+  includes fixes for some major issues affecting directory authorities,
+  LibreSSL compatibility, and IPv6 correctness.
+
+  The Tor 0.2.9.x release series is now marked as a long-term-support
+  series.  We intend to backport security fixes to 0.2.9.x until at
+  least January of 2020.
+
+  o Major bugfixes (directory authority, 0.3.0.3-alpha):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (IPv6 Exits, backport from 0.3.0.3-alpha):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing, also in 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (directory authorities, also in 0.3.0.4-rc):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (portability, compilation, backport from 0.3.0.3-alpha):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor bugfixes (code correctness, also in 0.3.0.4-rc):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (tor-resolve, backport from 0.3.0.3-alpha):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+
+Changes in version 0.2.9.9 - 2017-01-23
+  Tor 0.2.9.9 fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  This release also resolves a client-side onion service reachability
+  bug, and resolves a pair of small portability issues.
+
+  o Major bugfixes (security):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes -- and having
+      it on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Major bugfixes (client, onion service):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.2.8.12 - 2016-12-19
+  Tor 0.2.8.12 backports a fix for a medium-severity issue (bug 21018
+  below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  It also includes an updated list of fallback directories, backported
+  from 0.2.9.
+
+  Now that the Tor 0.2.9 series is stable, only major bugfixes will be
+  backported to 0.2.8 in the future.
+
+  o Major bugfixes (parsing, security, backported from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list, backported from 0.2.9.8):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (geoip, backported from 0.2.9.7-rc):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.8 - 2016-12-19
+  Tor 0.2.9.8 is the first stable release of the Tor 0.2.9 series.
+
+  The Tor 0.2.9 series makes mandatory a number of security features
+  that were formerly optional. It includes support for a new shared-
+  randomness protocol that will form the basis for next generation
+  hidden services, includes a single-hop hidden service mode for
+  optimizing .onion services that don't actually want to be hidden,
+  tries harder not to overload the directory authorities with excessive
+  downloads, and supports a better protocol versioning scheme for
+  improved compatibility with other implementations of the Tor protocol.
+
+  And of course, there are numerous other bugfixes and improvements.
+
+  This release also includes a fix for a medium-severity issue (bug
+  21018 below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  Below are listed the changes since Tor 0.2.8.11.  For a list of
+  changes since 0.2.9.7-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - When building with OpenSSL, Tor now requires version 1.0.1 or
+      later. OpenSSL 1.0.0 and earlier are no longer supported by the
+      OpenSSL team, and should not be used. Closes ticket 20303.
+    - Tor now requires Libevent version 2.0.10-stable or later. Older
+      versions of Libevent have less efficient backends for several
+      platforms, and lack the DNS code that we use for our server-side
+      DNS support. This implements ticket 19554.
+    - Tor now requires zlib version 1.2 or later, for security,
+      efficiency, and (eventually) gzip support. (Back when we started,
+      zlib 1.1 and zlib 1.0 were still found in the wild. 1.2 was
+      released in 2003. We recommend the latest version.)
+
+  o Deprecated features:
+    - A number of DNS-cache-related sub-options for client ports are now
+      deprecated for security reasons, and may be removed in a future
+      version of Tor. (We believe that client-side DNS caching is a bad
+      idea for anonymity, and you should not turn it on.) The options
+      are: CacheDNS, CacheIPv4DNS, CacheIPv6DNS, UseDNSCache,
+      UseIPv4Cache, and UseIPv6Cache.
+    - A number of options are deprecated for security reasons, and may
+      be removed in a future version of Tor. The options are:
+      AllowDotExit, AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ClientDNSRejectInternalAddresses,
+      CloseHSClientCircuitsImmediatelyOnTimeout,
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout,
+      ExcludeSingleHopRelays, FastFirstHopPK, TLSECGroup,
+      UseNTorHandshake, and WarnUnsafeSocks.
+    - The *ListenAddress options are now deprecated as unnecessary: the
+      corresponding *Port options should be used instead. These options
+      may someday be removed. The affected options are:
+      ControlListenAddress, DNSListenAddress, DirListenAddress,
+      NATDListenAddress, ORListenAddress, SocksListenAddress,
+      and TransListenAddress.
+
+  o Major bugfixes (parsing, security, new since 0.2.9.7-rc):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major features (build, hardening):
+    - Tor now builds with -ftrapv by default on compilers that support
+      it. This option detects signed integer overflow (which C forbids),
+      and turns it into a hard-failure. We do not apply this option to
+      code that needs to run in constant time to avoid side-channels;
+      instead, we use -fwrapv in that code. Closes ticket 17983.
+    - When --enable-expensive-hardening is selected, stop applying the
+      clang/gcc sanitizers to code that needs to run in constant time.
+      Although we are aware of no introduced side-channels, we are not
+      able to prove that there are none. Related to ticket 17983.
+
+  o Major features (circuit building, security):
+    - Authorities, relays, and clients now require ntor keys in all
+      descriptors, for all hops (except for rare hidden service protocol
+      cases), for all circuits, and for all other roles. Part of
+      ticket 19163.
+    - Authorities, relays, and clients only use ntor, except for
+      rare cases in the hidden service protocol. Part of ticket 19163.
+
+  o Major features (compilation):
+    - Our big list of extra GCC warnings is now enabled by default when
+      building with GCC (or with anything like Clang that claims to be
+      GCC-compatible). To make all warnings into fatal compilation
+      errors, pass --enable-fatal-warnings to configure. Closes
+      ticket 19044.
+    - Use the Autoconf macro AC_USE_SYSTEM_EXTENSIONS to automatically
+      turn on C and POSIX extensions. (Previously, we attempted to do
+      this on an ad hoc basis.) Closes ticket 19139.
+
+  o Major features (directory authorities, hidden services):
+    - Directory authorities can now perform the shared randomness
+      protocol specified by proposal 250. Using this protocol, directory
+      authorities generate a global fresh random value every day. In the
+      future, this value will be used by hidden services to select
+      HSDirs. This release implements the directory authority feature;
+      the hidden service side will be implemented in the future as part
+      of proposal 224. Resolves ticket 16943; implements proposal 250.
+
+  o Major features (downloading, random exponential backoff):
+    - When we fail to download an object from a directory service, wait
+      for an (exponentially increasing) randomized amount of time before
+      retrying, rather than a fixed interval as we did before. This
+      prevents a group of Tor instances from becoming too synchronized,
+      or a single Tor instance from becoming too predictable, in its
+      download schedule. Closes ticket 15942.
+
+  o Major features (resource management):
+    - Tor can now notice it is about to run out of sockets, and
+      preemptively close connections of lower priority. (This feature is
+      off by default for now, since the current prioritizing method is
+      yet not mature enough. You can enable it by setting
+      "DisableOOSCheck 0", but watch out: it might close some sockets
+      you would rather have it keep.) Closes ticket 18640.
+
+  o Major features (single-hop "hidden" services):
+    - Add experimental HiddenServiceSingleHopMode and
+      HiddenServiceNonAnonymousMode options. When both are set to 1,
+      every hidden service on that Tor instance becomes a non-anonymous
+      Single Onion Service. Single Onions make one-hop (direct)
+      connections to their introduction and rendezvous points. One-hop
+      circuits make Single Onion servers easily locatable, but clients
+      remain location-anonymous. This is compatible with the existing
+      hidden service implementation, and works on the current Tor
+      network without any changes to older relays or clients. Implements
+      proposal 260, completes ticket 17178. Patch by teor and asn.
+
+  o Major features (subprotocol versions):
+    - Tor directory authorities now vote on a set of recommended
+      "subprotocol versions", and on a set of required subprotocol
+      versions. Clients and relays that lack support for a _required_
+      subprotocol version will not start; those that lack support for a
+      _recommended_ subprotocol version will warn the user to upgrade.
+      This change allows compatible implementations of the Tor protocol(s)
+      to exist without pretending to be 100% bug-compatible with
+      particular releases of Tor itself. Closes ticket 19958; implements
+      part of proposal 264.
+
+  o Major bugfixes (circuit building):
+    - Hidden service client-to-intro-point and service-to-rendezvous-
+      point circuits use the TAP key supplied by the protocol, to avoid
+      epistemic attacks. Fixes bug 19163; bugfix on 0.2.4.18-rc.
+
+  o Major bugfixes (download scheduling):
+    - Avoid resetting download status for consensuses hourly, since we
+      already have another, smarter retry mechanism. Fixes bug 8625;
+      bugfix on 0.2.0.9-alpha.
+    - If a consensus expires while we are waiting for certificates to
+      download, stop waiting for certificates.
+    - If we stop waiting for certificates less than a minute after we
+      started downloading them, do not consider the certificate download
+      failure a separate failure. Fixes bug 20533; bugfix
+      on 0.2.0.9-alpha.
+    - When using exponential backoff in test networks, use a lower
+      exponent, so the delays do not vary as much. This helps test
+      networks bootstrap consistently. Fixes bug 20597; bugfix on 20499.
+
+  o Major bugfixes (exit policies):
+    - Avoid disclosing exit outbound bind addresses, configured port
+      bind addresses, and local interface addresses in relay descriptors
+      by default under ExitPolicyRejectPrivate. Instead, only reject
+      these (otherwise unlisted) addresses if
+      ExitPolicyRejectLocalInterfaces is set. Fixes bug 18456; bugfix on
+      0.2.7.2-alpha. Patch by teor.
+
+  o Major bugfixes (hidden services):
+    - Allow Tor clients with appropriate controllers to work with
+      FetchHidServDescriptors set to 0. Previously, this option also
+      disabled descriptor cache lookup, thus breaking hidden services
+      entirely. Fixes bug 18704; bugfix on 0.2.0.20-rc. Patch by "twim".
+    - Clients now require hidden services to include the TAP keys for
+      their intro points in the hidden service descriptor. This prevents
+      an inadvertent upgrade to ntor, which a malicious hidden service
+      could use to distinguish clients by consensus version. Fixes bug
+      20012; bugfix on 0.2.4.8-alpha. Patch by teor.
+
+  o Major bugfixes (relay, resolver, logging):
+    - For relays that don't know their own address, avoid attempting a
+      local hostname resolve for each descriptor we download. This
+      will cut down on the number of "Success: chose address 'x.x.x.x'"
+      log lines, and also avoid confusing clock jumps if the resolver
+      is slow. Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
+
+  o Minor features (port flags):
+    - Add new flags to the *Port options to give finer control over which
+      requests are allowed. The flags are NoDNSRequest, NoOnionTraffic,
+      and the synthetic flag OnionTrafficOnly, which is equivalent to
+      NoDNSRequest, NoIPv4Traffic, and NoIPv6Traffic. Closes enhancement
+      18693; patch by "teor".
+
+  o Minor features (build, hardening):
+    - Detect and work around a libclang_rt problem that would prevent
+      clang from finding __mulodi4() on some 32-bit platforms, and thus
+      keep -ftrapv from linking on those systems. Closes ticket 19079.
+    - When building on a system without runtime support for the runtime
+      hardening options, try to log a useful warning at configuration
+      time, rather than an incomprehensible warning at link time. If
+      expensive hardening was requested, this warning becomes an error.
+      Closes ticket 18895.
+
+  o Minor features (client, directory):
+    - Since authorities now omit all routers that lack the Running and
+      Valid flags, we assume that any relay listed in the consensus must
+      have those flags. Closes ticket 20001; implements part of
+      proposal 272.
+
+  o Minor features (code safety):
+    - In our integer-parsing functions, ensure that the maximum value we
+      allow is no smaller than the minimum value. Closes ticket 19063;
+      patch from "U+039b".
+
+  o Minor features (compilation, portability):
+    - Compile correctly on MacOS 10.12 (aka "Sierra"). Closes
+      ticket 20241.
+
+  o Minor features (config):
+    - Warn users when descriptor and port addresses are inconsistent.
+      Mitigates bug 13953; patch by teor.
+
+  o Minor features (controller):
+    - Allow controllers to configure basic client authorization on
+      hidden services when they create them with the ADD_ONION controller
+      command. Implements ticket 15588. Patch by "special".
+    - Fire a STATUS_SERVER controller event whenever the hibernation
+      status changes between "awake"/"soft"/"hard". Closes ticket 18685.
+    - Implement new GETINFO queries for all downloads that use
+      download_status_t to schedule retries. This allows controllers to
+      examine the schedule for pending downloads. Closes ticket 19323.
+
+  o Minor features (development tools, etags):
+    - Teach the "make tags" Makefile target how to correctly find
+      "MOCK_IMPL" function definitions. Patch from nherring; closes
+      ticket 16869.
+
+  o Minor features (directory authority):
+    - After voting, if the authorities decide that a relay is not
+      "Valid", they no longer include it in the consensus at all. Closes
+      ticket 20002; implements part of proposal 272.
+    - Directory authorities now only give the Guard flag to a relay if
+      they are also giving it the Stable flag. This change allows us to
+      simplify path selection for clients. It should have minimal effect
+      in practice, since >99% of Guards already have the Stable flag.
+      Implements ticket 18624.
+    - Directory authorities now write their v3-status-votes file out to
+      disk earlier in the consensus process, so we have a record of the
+      votes even if we abort the consensus process. Resolves
+      ticket 19036.
+
+  o Minor features (fallback directory list, new since 0.2.9.7-rc):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (hidden service):
+    - Stop being so strict about the payload length of "rendezvous1"
+      cells. We used to be locked in to the "TAP" handshake length, and
+      now we can handle better handshakes like "ntor". Resolves
+      ticket 18998.
+
+  o Minor features (infrastructure, time):
+    - Tor now includes an improved timer backend, so that we can
+      efficiently support tens or hundreds of thousands of concurrent
+      timers, as will be needed for some of our planned anti-traffic-
+      analysis work. This code is based on William Ahern's "timeout.c"
+      project, which implements a "tickless hierarchical timing wheel".
+      Closes ticket 18365.
+    - Tor now uses the operating system's monotonic timers (where
+      available) for internal fine-grained timing. Previously we would
+      look at the system clock, and then attempt to compensate for the
+      clock running backwards. Closes ticket 18908.
+
+  o Minor features (logging):
+    - Add a set of macros to check nonfatal assertions, for internal
+      use. Migrating more of our checks to these should help us avoid
+      needless crash bugs. Closes ticket 18613.
+    - Provide a more useful warning message when configured with an
+      invalid Nickname. Closes ticket 18300; patch from "icanhasaccount".
+    - When dumping unparseable router descriptors, optionally store them
+      in separate files, named by digest, up to a configurable size
+      limit. You can change the size limit by setting the
+      MaxUnparseableDescSizeToLog option, and disable this feature by
+      setting that option to 0. Closes ticket 18322.
+
+  o Minor features (performance):
+    - Change the "optimistic data" extension from "off by default" to
+      "on by default". The default was ordinarily overridden by a
+      consensus option, but when clients were bootstrapping for the
+      first time, they would not have a consensus to get the option
+      from. Changing this default saves a round-trip during startup.
+      Closes ticket 18815.
+
+  o Minor features (relay, usability):
+    - When the directory authorities refuse a bad relay's descriptor,
+      encourage the relay operator to contact us. Many relay operators
+      won't notice this line in their logs, but it's a win if even a few
+      learn why we don't like what their relay was doing. Resolves
+      ticket 18760.
+
+  o Minor features (security, TLS):
+    - Servers no longer support clients that lack AES ciphersuites.
+      (3DES is no longer considered an acceptable cipher.) We believe
+      that no such Tor clients currently exist, since Tor has required
+      OpenSSL 0.9.7 or later since 2009. Closes ticket 19998.
+
+  o Minor features (testing):
+    - Disable memory protections on OpenBSD when performing our unit
+      tests for memwipe(). The test deliberately invokes undefined
+      behavior, and the OpenBSD protections interfere with this. Patch
+      from "rubiate". Closes ticket 20066.
+    - Move the test-network.sh script to chutney, and modify tor's test-
+      network.sh to call the (newer) chutney version when available.
+      Resolves ticket 19116. Patch by teor.
+    - Use the lcov convention for marking lines as unreachable, so that
+      we don't count them when we're generating test coverage data.
+      Update our coverage tools to understand this convention. Closes
+      ticket 16792.
+    - Our link-handshake unit tests now check that when invalid
+      handshakes fail, they fail with the error messages we expected.
+    - Our unit testing code that captures log messages no longer
+      prevents them from being written out if the user asked for them
+      (by passing --debug or --info or --notice or --warn to the "test"
+      binary). This change prevents us from missing unexpected log
+      messages simply because we were looking for others. Related to
+      ticket 19999.
+    - The unit tests now log all warning messages with the "BUG" flag.
+      Previously, they only logged errors by default. This change will
+      help us make our testing code more correct, and make sure that we
+      only hit this code when we mean to. In the meantime, however,
+      there will be more warnings in the unit test logs than before.
+      This is preparatory work for ticket 19999.
+    - The unit tests now treat any failure of a "tor_assert_nonfatal()"
+      assertion as a test failure.
+    - We've done significant work to make the unit tests run faster.
+
+  o Minor features (testing, ipv6):
+    - Add the hs-ipv6 chutney target to make test-network-all's IPv6
+      tests. Remove bridges+hs, as it's somewhat redundant. This
+      requires a recent chutney version that supports IPv6 clients,
+      relays, and authorities. Closes ticket 20069; patch by teor.
+    - Add the single-onion and single-onion-ipv6 chutney targets to
+      "make test-network-all". This requires a recent chutney version
+      with the single onion network flavors (git c72a652 or later).
+      Closes ticket 20072; patch by teor.
+
+  o Minor features (Tor2web):
+    - Make Tor2web clients respect ReachableAddresses. This feature was
+      inadvertently enabled in 0.2.8.6, then removed by bugfix 19973 on
+      0.2.8.7. Implements feature 20034. Patch by teor.
+
+  o Minor features (unix domain sockets):
+    - When configuring a unix domain socket for a SocksPort,
+      ControlPort, or Hidden service, you can now wrap the address in
+      quotes, using C-style escapes inside the quotes. This allows unix
+      domain socket paths to contain spaces. Resolves ticket 18753.
+
+  o Minor features (user interface):
+    - Tor now supports the ability to declare options deprecated, so
+      that we can recommend that people stop using them. Previously, this
+      was done in an ad-hoc way. There is a new --list-deprecated-options
+      command-line option to list all of the deprecated options. Closes
+      ticket 19820.
+
+  o Minor features (virtual addresses):
+    - Increase the maximum number of bits for the IPv6 virtual network
+      prefix from 16 to 104. In this way, the condition for address
+      allocation is less restrictive. Closes ticket 20151; feature
+      on 0.2.4.7-alpha.
+
+  o Minor bug fixes (circuits):
+    - Use the CircuitBuildTimeout option whenever
+      LearnCircuitBuildTimeout is disabled. Previously, we would respect
+      the option when a user disabled it, but not when it was disabled
+      because some other option was set. Fixes bug 20073; bugfix on
+      0.2.4.12-alpha. Patch by teor.
+
+  o Minor bugfixes (build):
+    - The current Git revision when building from a local repository is
+      now detected correctly when using git worktrees. Fixes bug 20492;
+      bugfix on 0.2.3.9-alpha.
+
+  o Minor bugfixes (relay address discovery):
+    - Stop reordering IP addresses returned by the OS. This makes it
+      more likely that Tor will guess the same relay IP address every
+      time. Fixes issue 20163; bugfix on 0.2.7.1-alpha, ticket 17027.
+      Reported by René Mayrhofer, patch by "cypherpunks".
+
+  o Minor bugfixes (memory allocation):
+    - Change how we allocate memory for large chunks on buffers, to
+      avoid a (currently impossible) integer overflow, and to waste less
+      space when allocating unusually large chunks. Fixes bug 20081;
+      bugfix on 0.2.0.16-alpha. Issue identified by Guido Vranken.
+
+  o Minor bugfixes (bootstrap):
+    - Remember the directory server we fetched the consensus or previous
+      certificates from, and use it to fetch future authority
+      certificates. This change improves bootstrapping performance.
+      Fixes bug 18963; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (circuits):
+    - Make sure extend_info_from_router() is only called on servers.
+      Fixes bug 19639; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (client, fascistfirewall):
+    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
+      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, unix domain sockets):
+    - Disable IsolateClientAddr when using AF_UNIX backed SocksPorts as
+      the client address is meaningless. Fixes bug 20261; bugfix
+      on 0.2.6.3-alpha.
+
+  o Minor bugfixes (code style):
+    - Fix an integer signedness conversion issue in the case conversion
+      tables. Fixes bug 19168; bugfix on 0.2.1.11-alpha.
+
+  o Minor bugfixes (compilation):
+    - Build correctly on versions of libevent2 without support for
+      evutil_secure_rng_add_bytes(). Fixes bug 19904; bugfix
+      on 0.2.5.4-alpha.
+    - When building with Clang, use a full set of GCC warnings.
+      (Previously, we included only a subset, because of the way we
+      detected them.) Fixes bug 19216; bugfix on 0.2.0.1-alpha.
+    - Detect Libevent2 functions correctly on systems that provide
+      libevent2, but where libevent1 is linked with -levent. Fixes bug
+      19904; bugfix on 0.2.2.24-alpha. Patch from Rubiate.
+    - Run correctly when built on Windows build environments that
+      require _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
+
+  o Minor bugfixes (configuration):
+    - When parsing quoted configuration values from the torrc file,
+      handle Windows line endings correctly. Fixes bug 19167; bugfix on
+      0.2.0.16-alpha. Patch from "Pingl".
+
+  o Minor bugfixes (directory authority):
+    - Authorities now sort the "package" lines in their votes, for ease
+      of debugging. (They are already sorted in consensus documents.)
+      Fixes bug 18840; bugfix on 0.2.6.3-alpha.
+    - Die with a more useful error when the operator forgets to place
+      the authority_signing_key file into the keys directory. This
+      avoids an uninformative assert & traceback about having an invalid
+      key. Fixes bug 20065; bugfix on 0.2.0.1-alpha.
+    - When allowing private addresses, mark Exits that only exit to
+      private locations as such. Fixes bug 20064; bugfix
+      on 0.2.2.9-alpha.
+    - When parsing a detached signature, make sure we use the length of
+      the digest algorithm instead of a hardcoded DIGEST256_LEN in
+      order to avoid comparing bytes out-of-bounds with a smaller digest
+      length such as SHA1. Fixes bug 19066; bugfix on 0.2.2.6-alpha.
+
+  o Minor bugfixes (getpass):
+    - Defensively fix a non-triggerable heap corruption at do_getpass()
+      to protect ourselves from mistakes in the future. Fixes bug
+      19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
+      by nherring.
+
+  o Minor bugfixes (guard selection):
+    - Don't mark guards as unreachable if connection_connect() fails.
+      That function fails for local reasons, so it shouldn't reveal
+      anything about the status of the guard. Fixes bug 14334; bugfix
+      on 0.2.3.10-alpha.
+    - Use a single entry guard even if the NumEntryGuards consensus
+      parameter is not provided. Fixes bug 17688; bugfix
+      on 0.2.5.6-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Increase the minimum number of internal circuits we preemptively
+      build from 2 to 3, so a circuit is available when a client
+      connects to another onion service. Fixes bug 13239; bugfix
+      on 0.1.0.1-rc.
+    - Allow hidden services to run on IPv6 addresses even when the
+      IPv6Exit option is not set. Fixes bug 18357; bugfix
+      on 0.2.4.7-alpha.
+    - Stop logging intro point details to the client log on certain
+      error conditions. Fixed as part of bug 20012; bugfix on
+      0.2.4.8-alpha. Patch by teor.
+    - When deleting an ephemeral hidden service, close its intro points
+      even if they are not completely open. Fixes bug 18604; bugfix
+      on 0.2.7.1-alpha.
+    - When configuring hidden services, check every hidden service
+      directory's permissions. Previously, we only checked the last
+      hidden service. Fixes bug 20529; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (IPv6, testing):
+    - Check for IPv6 correctly on Linux when running test networks.
+      Fixes bug 19905; bugfix on 0.2.7.3-rc; patch by teor.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Add permission to run the sched_yield() and sigaltstack() system
+      calls, in order to support versions of Tor compiled with asan or
+      ubsan code that use these calls. Now "sandbox 1" and
+      "--enable-expensive-hardening" should be compatible on more
+      systems. Fixes bug 20063; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Downgrade a harmless log message about the
+      pending_entry_connections list from "warn" to "info". Mitigates
+      bug 19926.
+    - Log a more accurate message when we fail to dump a microdescriptor.
+      Fixes bug 17758; bugfix on 0.2.2.8-alpha. Patch from Daniel Pinto.
+    - When logging a directory ownership mismatch, log the owning
+      username correctly. Fixes bug 19578; bugfix on 0.2.2.29-beta.
+    - When we are unable to remove the bw_accounting file, do not warn
+      if the reason we couldn't remove it was that it didn't exist.
+      Fixes bug 19964; bugfix on 0.2.5.4-alpha. Patch from pastly.
+
+  o Minor bugfixes (memory leak):
+    - Fix a series of slow memory leaks related to parsing torrc files
+      and options. Fixes bug 19466; bugfix on 0.2.1.6-alpha.
+    - Avoid a small memory leak when informing worker threads about
+      rotated onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
+    - Fix a small memory leak when receiving AF_UNIX connections on a
+      SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
+    - When moving a signed descriptor object from a source to an
+      existing destination, free the allocated memory inside that
+      destination object. Fixes bug 20715; bugfix on 0.2.8.3-alpha.
+    - Fix a memory leak and use-after-free error when removing entries
+      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
+      0.2.5.5-alpha. Patch from "cypherpunks".
+    - Fix a small, uncommon memory leak that could occur when reading a
+      truncated ed25519 key file. Fixes bug 18956; bugfix
+      on 0.2.6.1-alpha.
+
+  o Minor bugfixes (option parsing):
+    - Count unix sockets when counting client listeners (SOCKS, Trans,
+      NATD, and DNS). This has no user-visible behavior changes: these
+      options are set once, and never read. Required for correct
+      behavior in ticket 17178. Fixes bug 19677; bugfix on
+      0.2.6.3-alpha. Patch by teor.
+
+  o Minor bugfixes (options):
+    - Check the consistency of UseEntryGuards and EntryNodes more
+      reliably. Fixes bug 20074; bugfix on 0.2.4.12-alpha. Patch
+      by teor.
+    - Stop changing the configured value of UseEntryGuards on
+      authorities and Tor2web clients. Fixes bug 20074; bugfix on
+      commits 51fc6799 in 0.1.1.16-rc and acda1735 in 0.2.4.3-alpha.
+      Patch by teor.
+
+  o Minor bugfixes (relay):
+    - Ensure relays don't make multiple connections during bootstrap.
+      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
+    - Do not try to parallelize workers more than 16x without the user
+      explicitly configuring us to do so, even if we do detect more than
+      16 CPU cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (testing):
+    - The test-stem and test-network makefile targets now depend only on
+      the tor binary that they are testing. Previously, they depended on
+      "make all". Fixes bug 18240; bugfix on 0.2.8.2-alpha. Based on a
+      patch from "cypherpunks".
+    - Allow clients to retry HSDirs much faster in test networks. Fixes
+      bug 19702; bugfix on 0.2.7.1-alpha. Patch by teor.
+    - Avoid a unit test failure on systems with over 16 detectable CPU
+      cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+    - Let backtrace tests work correctly under AddressSanitizer:
+      disable ASAN's detection of segmentation faults while running
+      test_bt.sh, so that we can make sure that our own backtrace
+      generation code works. Fixes bug 18934; bugfix
+      on 0.2.5.2-alpha. Patch from "cypherpunks".
+    - Fix the test-network-all target on out-of-tree builds by using the
+      correct path to the test driver script. Fixes bug 19421; bugfix
+      on 0.2.7.3-rc.
+    - Stop spurious failures in the local interface address discovery
+      unit tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by
+      Neel Chauhan.
+    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
+      removed the ECDH ciphers which caused the tests to fail on
+      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
+    - The tor_tls_server_info_callback unit test no longer crashes when
+      debug-level logging is turned on. Fixes bug 20041; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (time):
+    - Improve overflow checks in tv_udiff and tv_mdiff. Fixes bug 19483;
+      bugfix on all released tor versions.
+    - When computing the difference between two times in milliseconds,
+      we now round to the nearest millisecond correctly. Previously, we
+      could sometimes round in the wrong direction. Fixes bug 19428;
+      bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (Tor2web):
+    - Prevent Tor2web clients from running hidden services: these services
+      are not anonymous due to the one-hop client paths. Fixes bug
+      19678. Patch by teor.
+
+  o Minor bugfixes (user interface):
+    - Display a more accurate number of suppressed messages in the log
+      rate-limiter. Previously, there was a potential integer overflow
+      in the counter. Now, if the number of messages hits a maximum, the
+      rate-limiter doesn't count any further. Fixes bug 19435; bugfix
+      on 0.2.4.11-alpha.
+    - Fix a typo in the passphrase prompt for the ed25519 identity key.
+      Fixes bug 19503; bugfix on 0.2.7.2-alpha.
+
+  o Code simplification and refactoring:
+    - Remove redundant declarations of the MIN macro. Closes
+      ticket 18889.
+    - Rename tor_dup_addr() to tor_addr_to_str_dup() to avoid confusion.
+      Closes ticket 18462; patch from "icanhasaccount".
+    - Split the 600-line directory_handle_command_get function into
+      separate functions for different URL types. Closes ticket 16698.
+
+  o Documentation:
+    - Add module-level internal documentation for 36 C files that
+      previously didn't have a high-level overview. Closes ticket 20385.
+    - Correct the IPv6 syntax in our documentation for the
+      VirtualAddrNetworkIPv6 torrc option. Closes ticket 19743.
+    - Correct the minimum bandwidth value in torrc.sample, and queue a
+      corresponding change for torrc.minimal. Closes ticket 20085.
+    - Fix spelling of "--enable-tor2web-mode" in the manpage. Closes
+      ticket 19153. Patch from "U+039b".
+    - Module-level documentation for several more modules. Closes
+      tickets 19287 and 19290.
+    - Document the --passphrase-fd option in the tor manpage. Fixes bug
+      19504; bugfix on 0.2.7.3-rc.
+    - Document the default PathsNeededToBuildCircuits value that's used
+      by clients when the directory authorities don't set
+      min_paths_for_circs_pct. Fixes bug 20117; bugfix on 0.2.4.10-alpha.
+      Patch by teor, reported by Jesse V.
+    - Fix manual for the User option: it takes a username, not a UID.
+      Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have
+      a manpage!).
+    - Fix the description of the --passphrase-fd option in the
+      tor-gencert manpage. The option is used to pass the number of a
+      file descriptor to read the passphrase from, not to read the file
+      descriptor from. Fixes bug 19505; bugfix on 0.2.0.20-alpha.
+
+  o Removed code:
+    - We no longer include the (dead, deprecated) bufferevent code in
+      Tor. Closes ticket 19450. Based on a patch from "U+039b".
+
+  o Removed features:
+    - Remove support for "GET /tor/bytes.txt" DirPort request, and
+      "GETINFO dir-usage" controller request, which were only available
+      via a compile-time option in Tor anyway. Feature was added in
+      0.2.2.1-alpha. Resolves ticket 19035.
+    - There is no longer a compile-time option to disable support for
+      TransPort. (If you don't want TransPort, just don't use it.) Patch
+      from "U+039b". Closes ticket 19449.
+
+  o Testing:
+    - Run more workqueue tests as part of "make check". These had
+      previously been implemented, but you needed to know special
+      command-line options to enable them.
+    - We now have unit tests for our code to reject zlib "compression
+      bombs". (Fortunately, the code works fine.)
+
+
+Changes in version 0.2.8.11 - 2016-12-08
+  Tor 0.2.8.11 backports fixes for additional portability issues that
+  could prevent Tor from building correctly on OSX Sierra, or with
+  OpenSSL 1.1. Affected users should upgrade; others can safely stay
+  with 0.2.8.10.
+
+  o Minor bugfixes (portability):
+    - Avoid compilation errors when building on OSX Sierra. Sierra began
+      to support the getentropy() and clock_gettime() APIs, but created
+      a few problems in doing so. Tor 0.2.9 has a more thorough set of
+      workarounds; in 0.2.8, we are just using the /dev/urandom and mach
+      monotonic time interfaces. Fixes bug 20865. Bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+
+
+Changes in version 0.2.8.10 - 2016-12-02
+  Tor 0.2.8.10 backports a fix for a bug that would sometimes make clients
+  unusable after they left standby mode. It also backports fixes for
+  a few portability issues and a small but problematic memory leak.
+
+  o Major bugfixes (client reliability, backport from 0.2.9.5-alpha):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client performance, backport from 0.2.9.5-alpha):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.6-rc):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.2.9.5-alpha):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
 
 Changes in version 0.2.8.9 - 2016-10-17
   Tor 0.2.8.9 backports a fix for a security hole in previous versions
@@ -449,7 +6277,7 @@ Changes in version 0.2.8.6 - 2016-08-02
 
   o Minor bugfixes (build):
     - Avoid spurious failures from configure files related to calling
-      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18625; bugfix on
+      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18626; bugfix on
       0.2.0.1-alpha. Patch from "cypherpunks".
     - Do not link the unit tests against both the testing and non-
       testing versions of the static libraries. Fixes bug 18490; bugfix
@@ -991,7 +6819,7 @@ Changes in version 0.2.7.5 - 2015-11-20
 
   o Major bugfixes (hidden services):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
     - When cannibalizing a circuit for an introduction point, always
@@ -1606,7 +7434,7 @@ Changes in version 0.2.6.8 - 2015-05-21
 
   o Major bugfixes (hidden services, backport from 0.2.7.1-alpha):
     - Revert commit that made directory authorities assign the HSDir
-      flag to relay without a DirPort; this was bad because such relays
+      flag to relays without a DirPort; this was bad because such relays
       can't handle BEGIN_DIR cells. Fixes bug 15850; bugfix
       on 0.2.6.3-alpha.
 
@@ -1905,7 +7733,7 @@ Changes in version 0.2.6.6 - 2015-03-24
   o Major bugfixes (pluggable transports):
     - Initialize the extended OR Port authentication cookie before
       launching pluggable transports. This prevents a race condition
-      that occured when server-side pluggable transports would cache the
+      that occurred when server-side pluggable transports would cache the
       authentication cookie before it has been (re)generated. Fixes bug
       15240; bugfix on 0.2.5.1-alpha.
 
@@ -2378,7 +8206,7 @@ Changes in version 0.2.6.6 - 2015-03-24
       some bugs where we would look at (but fortunately, not reveal)
       uninitialized memory on the stack. Fixes bug 14013; bugfix on all
       versions of Tor.
-    - Clear all memory targetted by tor_addr_{to,from}_sockaddr(), not
+    - Clear all memory targeted by tor_addr_{to,from}_sockaddr(), not
       just the part that's used. This makes it harder for data leak bugs
       to occur in the event of other programming failures. Resolves
       ticket 14041.
@@ -3555,7 +9383,7 @@ Changes in version 0.2.5.10 - 2014-10-24
       from Arlo Breault.
     - Remove instances of strcpy() from the unit tests. They weren't
       hurting anything, since they were only in the unit tests, but it's
-      embarassing to have strcpy() in the code at all, and some analysis
+      embarrassing to have strcpy() in the code at all, and some analysis
       tools don't like it. Fixes bug 8790; bugfix on 0.2.3.6-alpha and
       0.2.3.8-alpha. Patch from Arlo Breault.
     - Remove is_internal_IP() function. Resolves ticket 4645.
@@ -3579,7 +9407,7 @@ Changes in version 0.2.5.10 - 2014-10-24
       directory authority options, remove the documentation for a
       V2-directory fetching option that no longer exists. Resolves
       ticket 11634.
-    - Correct the documenation so that it lists the correct directory
+    - Correct the documentation so that it lists the correct directory
       for the stats files. (They are in a subdirectory called "stats",
       not "status".)
     - In the manpage, move more authority-only options into the
@@ -5722,7 +11550,7 @@ Changes in version 0.2.3.25 - 2012-11-19
       bugfix on 0.2.0.3-alpha.
     - The "--quiet" and "--hush" options now apply not only to Tor's
       behavior before logs are configured, but also to Tor's behavior in
-      the absense of configured logs. Fixes bug 3550; bugfix on
+      the absence of configured logs. Fixes bug 3550; bugfix on
       0.2.0.10-alpha.
     - Change the AllowDotExit rules so they should actually work.
       We now enforce AllowDotExit only immediately after receiving an
@@ -6835,7 +12663,7 @@ Changes in version 0.2.2.32 - 2011-08-27
       algorithms for signatures and resource selection. Newer formats
       are signed with SHA256, with a possibility for moving to a better
       hash algorithm in the future.
-    - Directory authorities can now vote on arbitary integer values as
+    - Directory authorities can now vote on arbitrary integer values as
       part of the consensus process. This is designed to help set
       network-wide parameters. Implements proposal 167.
 
@@ -12024,7 +17852,7 @@ Changes in version 0.1.0.10 - 2005-06-14
     - Use correct errno on win32 if libevent fails.
     - Check and warn about known-bad/slow libevent versions.
     - Stop warning about sigpipes in the logs. We're going to
-      pretend that getting these occassionally is normal and fine.
+      pretend that getting these occasionally is normal and fine.
 
   o New contrib scripts:
     - New experimental script tor/contrib/exitlist: a simple python
@@ -12143,7 +17971,7 @@ Changes in version 0.0.9.7 - 2005-04-01
 
 Changes in version 0.0.9.6 - 2005-03-24
   o Bugfixes on 0.0.9.x (crashes and asserts):
-    - Add new end stream reasons to maintainance branch. Fix bug where
+    - Add new end stream reasons to maintenance branch. Fix bug where
       reason (8) could trigger an assert.  Prevent bug from recurring.
     - Apparently win32 stat wants paths to not end with a slash.
     - Fix assert triggers in assert_cpath_layer_ok(), where we were
diff --git a/acinclude.m4 b/acinclude.m4
index ab12317139..49d4f14471 100644
--- a/acinclude.m4
+++ b/acinclude.m4
@@ -2,7 +2,7 @@ dnl Helper macros for Tor configure.ac
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
 dnl Copyright (c) 2007-2008, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2015, The Tor Project, Inc.
+dnl Copyright (c) 2007-2017, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_DEFUN([TOR_EXTEND_CODEPATH],
@@ -51,12 +51,12 @@ AC_DEFUN([TOR_TRY_COMPILE_WITH_CFLAGS], [
   AC_CACHE_CHECK([whether the compiler accepts $1], VAR, [
     tor_saved_CFLAGS="$CFLAGS"
     CFLAGS="$CFLAGS -pedantic -Werror $1"
-    AC_TRY_COMPILE([], [return 0;],
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                    [AS_VAR_SET(VAR,yes)],
                    [AS_VAR_SET(VAR,no)])
     if test x$2 != x; then
       AS_VAR_PUSHDEF([can_link],[tor_can_link_$1])
-      AC_TRY_LINK([], [return 0;],
+      AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                   [AS_VAR_SET(can_link,yes)],
                   [AS_VAR_SET(can_link,no)])
       AS_VAR_POPDEF([can_link])
@@ -93,7 +93,7 @@ AC_DEFUN([TOR_CHECK_LDFLAGS], [
     AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>], [fputs("", stdout)])],
                   [AS_VAR_SET(VAR,yes)],
                   [AS_VAR_SET(VAR,no)],
-	          [AC_TRY_LINK([], [return 0;],
+                  [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                                    [AS_VAR_SET(VAR,yes)],
                                    [AS_VAR_SET(VAR,no)])])
     CFLAGS="$tor_saved_CFLAGS"
@@ -113,21 +113,21 @@ if test x$2 = xdevpkg; then
   h=" headers for"
 fi
 if test -f /etc/debian_version && test x"$tor_$1_$2_debian" != x; then
-  AC_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
+  AC_MSG_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
   if test x"$tor_$1_$2_debian" != x"$tor_$1_devpkg_debian"; then 
-    AC_WARN([   You will probably need $tor_$1_devpkg_debian too.])
+    AC_MSG_WARN([   You will probably need $tor_$1_devpkg_debian too.])
   fi 
 fi
 if test -f /etc/fedora-release && test x"$tor_$1_$2_redhat" != x; then
-  AC_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
+  AC_MSG_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
   if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-    AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+    AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
   fi 
 else
   if test -f /etc/redhat-release && test x"$tor_$1_$2_redhat" != x; then
-    AC_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
+    AC_MSG_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
     if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-      AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+      AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
     fi 
   fi
 fi
@@ -245,7 +245,10 @@ if test "$cross_compiling" != yes; then
        LDFLAGS="$tor_tryextra $orig_LDFLAGS"
      fi
      AC_RUN_IFELSE([AC_LANG_PROGRAM([$5], [$6])],
-                   [runnable=yes], [runnable=no])
+                   [runnable=yes], [runnable=no],
+                   [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
+                                   [runnable=yes],
+                                   [runnable=no])])
      if test "$runnable" = yes; then
         tor_cv_library_$1_linker_option=$tor_tryextra
         break
diff --git a/autogen.sh b/autogen.sh
index 8c43a9798a..276dd4047c 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,12 +1,12 @@
 #!/bin/sh
 
 if [ -x "`which autoreconf 2>/dev/null`" ] ; then
-  opt="-if"
+  opt="-i -f -W all,error"
 
   for i in $@; do
     case "$i" in
       -v)
-        opt=$opt"v"
+        opt="${opt} -v"
         ;;
     esac
   done
diff --git a/changes/27286 b/changes/27286
new file mode 100644
index 0000000000..5f5f7a4ae7
--- /dev/null
+++ b/changes/27286
@@ -0,0 +1,4 @@
+  o Minor features (directory authorities):
+    - Authorities no longer vote to make the subprotocol version "LinkAuth=1"
+      a requirement: it is unsupportable with NSS, and hasn't been needed
+      since Tor 0.3.0.1-alpha. Closes ticket 27286.
diff --git a/changes/bug20424_029_minimal b/changes/bug20424_029_minimal
new file mode 100644
index 0000000000..eb7886233e
--- /dev/null
+++ b/changes/bug20424_029_minimal
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - When compiling with --enable-openbsd-malloc or --enable-tcmalloc, tell
+      the compiler not to include the system malloc implementation. Fixes bug
+      20424; bugfix on 0.2.0.20-rc.
diff --git a/changes/bug21394.2 b/changes/bug21394.2
deleted file mode 100644
index b580d2a786..0000000000
--- a/changes/bug21394.2
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfix (Exit node DNS retries):
-    - Re-attempt timed-out DNS queries 3 times before failure, since our
-      timeout is 5 seconds for them, but clients wait 10-15. Also allow
-      slightly more timeouts per resolver before giving up on it in the
-      case where an exit has multiple resolvers configured. Fixes bug 21394;
-      bugfix on 0.3.1.9.
-
diff --git a/changes/bug23681 b/changes/bug23681
deleted file mode 100644
index e317f36d50..0000000000
--- a/changes/bug23681
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hidden service client):
-    - The introduction circuit was being timed out too quickly while waiting
-      for the rendezvous circuit to complete. Keep the intro circuit around
-      longer instead of timing out and reopening new ones constantly. Fixes
-      bug 23681; bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug23790 b/changes/bug23790
deleted file mode 100644
index 4aaf616e4d..0000000000
--- a/changes/bug23790
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (hidden service v2):
-    - When reloading tor (HUP) configured with hidden service(s), some
-      information weren't copy to the new service object. One problem with this
-      was that tor would wait at least the RendPostPeriod time before uploading
-      the descriptor if the reload happened before the descriptor needed to be
-      published. Fixes bug 23790; bugfix on 0.2.1.9-alpha.
diff --git a/changes/bug24891 b/changes/bug24891
new file mode 100644
index 0000000000..403b2b1123
--- /dev/null
+++ b/changes/bug24891
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - Refrain from compiling unit testing related object files
+      when --disable-unittests is set to configure script.
+      Fixes bug 24891; bugfix on 0.2.5.1-alpha.
diff --git a/changes/bug24903 b/changes/bug24903
deleted file mode 100644
index 01c9b53f23..0000000000
--- a/changes/bug24903
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (controller, reliability):
-    - Avoid a (nonfatal) assertion failure when extending a one-hop circuit
-      from the controller to become a multihop circuit. Fixes bug 24903;
-      bugfix on 0.2.5.2-alpha.
-
diff --git a/changes/bug24977 b/changes/bug24977
new file mode 100644
index 0000000000..f8127a2a73
--- /dev/null
+++ b/changes/bug24977
@@ -0,0 +1,5 @@
+  o Minor bugfixes (onion services):
+    - Recompute some consensus information after clock skews or when we
+      transition from a non-live consensus to a live consensus. We do this to
+      avoid having an outdated state which could impact next-generation onion
+      services. Fixes bug 24977; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug25113 b/changes/bug25113
deleted file mode 100644
index 4a020b784d..0000000000
--- a/changes/bug25113
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (unit test, monotonic time):
-    - Bump a gap of 1msec to 10msec used in the monotonic time test that makes
-      sure the nsec/usec/msec time read are synchronized. This change was
-      needed to accommodate slow system like armel or when the clock_gettime()
-      is not a VDSO on the running kernel. Fixes bug 25113; bugfix on 0.2.9.1.
diff --git a/changes/bug25116 b/changes/bug25116
deleted file mode 100644
index b3e73feeaa..0000000000
--- a/changes/bug25116
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden service, heartbeat):
-    - Don't log in the heartbeat any long term established one hop rendezvous
-      points if tor is a single onion service. Fixes bug 25116; bugfix on
-      0.2.9.6-rc;
diff --git a/changes/bug25440 b/changes/bug25440
new file mode 100644
index 0000000000..f8d9dd4fab
--- /dev/null
+++ b/changes/bug25440
@@ -0,0 +1,5 @@
+  o Minor bugfixes (linux seccomp2 sandbox):
+    - Fix a bug in out sandboxing rules for the openat() syscall.
+      Previously, no openat() call would be permitted, which would break
+      filesystem operations on recent glibc versions. Fixes bug 25440;
+      bugfix on 0.2.9.15. Diagnosis and patch from Daniel Pinto.
diff --git a/changes/bug25505 b/changes/bug25505
new file mode 100644
index 0000000000..101c7d5246
--- /dev/null
+++ b/changes/bug25505
@@ -0,0 +1,3 @@
+  o Minor bugfixes (netflow padding):
+    - Ensure circuitmux queues are empty before scheduling or sending padding.
+      Fixes bug 25505; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug25686_diagnostic b/changes/bug25686_diagnostic
new file mode 100644
index 0000000000..96323145d8
--- /dev/null
+++ b/changes/bug25686_diagnostic
@@ -0,0 +1,4 @@
+  o Minor features (relay, diagnostic):
+    - Add several checks to detect whether Tor relays are uploading their
+      descriptors without specifying why they regenerated. Diagnostic for
+      ticket 25686.
diff --git a/changes/bug25787 b/changes/bug25787
new file mode 100644
index 0000000000..3041e8a603
--- /dev/null
+++ b/changes/bug25787
@@ -0,0 +1,7 @@
+  o Minor bugfixes (testing):
+    - Instead of trying to  read the geoip configuration files from within the
+      unit tests, instead create our own ersatz files with just enough
+      geoip data in the format we expect. Trying to read from the source
+      directory created problems on Windows with mingw, where the
+      build system's paths are not the same as the platform's paths.
+      Fixes bug 25787; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug26158 b/changes/bug26158
new file mode 100644
index 0000000000..0d74cf1167
--- /dev/null
+++ b/changes/bug26158
@@ -0,0 +1,5 @@
+  o Minor bugfixes (relay):
+     - Relays now correctly block attempts to re-extend to the previous
+       relay by Ed25519 identity. Previously they would warn in this case,
+       but not actually reject the attempt. Fixes bug 26158; bugfix on
+       0.3.0.1-alpha.
diff --git a/changes/bug26196 b/changes/bug26196
new file mode 100644
index 0000000000..e63f09a2d6
--- /dev/null
+++ b/changes/bug26196
@@ -0,0 +1,7 @@
+  o Minor bugfixes (testing, compilation):
+    - Refrain from trying to get an item from empty smartlist in
+      test_bridges_clear_bridge_list. Set DEBUG_SMARTLIST in unit
+      tests to catch improper smartlist usage. Furthermore,
+      enable DEBUG_SMARTLIST globally when build is configured
+      with fragile hardening. Fixes bug 26196; bugfix on
+      0.3.4.1-alpha.
diff --git a/changes/bug26214 b/changes/bug26214
new file mode 100644
index 0000000000..4277b9c6ec
--- /dev/null
+++ b/changes/bug26214
@@ -0,0 +1,3 @@
+  o Minor bugfixes (correctness, flow control):
+    - Upon receiving a stream-level SENDME cell, verify that our window has
+      not grown too large. Fixes bug 26214; bugfix on svn r54 (pre-0.0.1)
diff --git a/changes/bug26245 b/changes/bug26245
new file mode 100644
index 0000000000..7a14cea0bc
--- /dev/null
+++ b/changes/bug26245
@@ -0,0 +1,3 @@
+  o Minor features (compilation):
+    o The --enable-fatal-warnings flag now affects Rust code as well.
+      Closes ticket 26245.
diff --git a/changes/bug26258_033 b/changes/bug26258_033
new file mode 100644
index 0000000000..ceca383335
--- /dev/null
+++ b/changes/bug26258_033
@@ -0,0 +1,4 @@
+  o Major bugfixes (rust, testing):
+    - Fix a bug where a failure in the rust unit tests would not actually
+      cause the build to fail. Fixes bug 26258; bugfix on 0.3.3.4-alpha.
+
diff --git a/changes/bug26269 b/changes/bug26269
new file mode 100644
index 0000000000..73dcdbf5c5
--- /dev/null
+++ b/changes/bug26269
@@ -0,0 +1,5 @@
+  o Minor bugfixes (compilation):
+    - Fix a compilation warning on some versions of GCC when
+      building code that calls routerinfo_get_my_routerinfo() twice,
+      assuming that the second call will succeed if the first one did.
+      Fixes bug 26269; bugfix on 0.2.8.2-alpha.
diff --git a/changes/bug26402 b/changes/bug26402
new file mode 100644
index 0000000000..b21283a2d2
--- /dev/null
+++ b/changes/bug26402
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - When linking the libtor_testing.a library, only include the dirauth
+      object files once.  Previously, they were getting added twice.
+      Fixes bug 26402; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug26415 b/changes/bug26415
new file mode 100644
index 0000000000..497fbb7365
--- /dev/null
+++ b/changes/bug26415
@@ -0,0 +1,3 @@
+  o Minor bugfixes (testing):
+    - Fix compilation of the doctests in the Rust crypto crate.  Fixes
+      bug 26415; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug26435 b/changes/bug26435
new file mode 100644
index 0000000000..f66c503dd5
--- /dev/null
+++ b/changes/bug26435
@@ -0,0 +1,5 @@
+  o Major bugfixes (directory authority):
+    - Fix a memory leak where directory authorities would leak a chunk
+      of memory for every router descriptor every time they considered
+      voting. This bug was taking down directory authorities due to
+      out-of-memory issues. Fixes bug 26435; bugfix on 0.3.3.6.
diff --git a/changes/bug26470 b/changes/bug26470
new file mode 100644
index 0000000000..854ec7ea72
--- /dev/null
+++ b/changes/bug26470
@@ -0,0 +1,4 @@
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to
+      an HTTP request we couldn't handle. Fixes bug 26470; bugfix on
+      0.3.2.1-alpha.
diff --git a/changes/bug26485 b/changes/bug26485
new file mode 100644
index 0000000000..5a40b7a78e
--- /dev/null
+++ b/changes/bug26485
@@ -0,0 +1,4 @@
+  o Minor bugfixes (directory authority):
+    - When voting for recommended versions, make sure that all of the
+      versions are well-formed and parsable. Fixes bug 26485; bugfix on
+      0.1.1.6-alpha.
diff --git a/changes/bug26497 b/changes/bug26497
new file mode 100644
index 0000000000..d0c05ff3e4
--- /dev/null
+++ b/changes/bug26497
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Stop setting $CARGO_HOME. cargo will use the user's $CARGO_HOME, or
+      $HOME/.cargo by default. Fixes bug 26497; bugfix on 0.3.1.5-alpha.
diff --git a/changes/bug26497-backport b/changes/bug26497-backport
new file mode 100644
index 0000000000..1d86e01bf3
--- /dev/null
+++ b/changes/bug26497-backport
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Backport test_rust.sh from master.
+      Fixes bug 26497; bugfix on 0.3.1.5-alpha.
diff --git a/changes/bug26497-cd b/changes/bug26497-cd
new file mode 100644
index 0000000000..37bf1bc956
--- /dev/null
+++ b/changes/bug26497-cd
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - cd to ${abs_top_builddir}/src/rust before running cargo in
+      src/test/test_rust.sh. This makes the working directory consistent
+      between builds and tests. Fixes bug 26497; bugfix on 0.3.3.2-alpha.
diff --git a/changes/bug26523 b/changes/bug26523
new file mode 100644
index 0000000000..a739d240e9
--- /dev/null
+++ b/changes/bug26523
@@ -0,0 +1,5 @@
+  o Minor bugfixes (hidden service, control port):
+    - The HSPOST command wasn't parsing properly the HSADDRESS= parameter and
+      thus not using it. It now handles it correctly. Fixes bug 26523; bugfix on
+      0.3.3.1-alpha. Patch by "akwizgran".
+
diff --git a/changes/bug26535.029 b/changes/bug26535.029
new file mode 100644
index 0000000000..111b539f17
--- /dev/null
+++ b/changes/bug26535.029
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing, compatibility):
+    - When running the ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this.  Fixes bug 26535; bugfix on
+      0.2.5.5-alpha.
diff --git a/changes/bug26535.032 b/changes/bug26535.032
new file mode 100644
index 0000000000..395d08d816
--- /dev/null
+++ b/changes/bug26535.032
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing, compatibility):
+    - When running the hs_ntor_ref.py test, make sure only to pass strings
+      (rather than "bytes" objects) to the Python subprocess module.
+      Python 3 on Windows seems to require this.  Fixes bug 26535; bugfix on
+      0.3.1.1-alpha.
diff --git a/changes/bug26568 b/changes/bug26568
new file mode 100644
index 0000000000..0c4c05d043
--- /dev/null
+++ b/changes/bug26568
@@ -0,0 +1,3 @@
+  o Minor bugfixes (controller):
+    - Report the port correctly when a port is configured to bind to "auto".
+      Fixes bug 26568; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug26627 b/changes/bug26627
new file mode 100644
index 0000000000..d28bd05d53
--- /dev/null
+++ b/changes/bug26627
@@ -0,0 +1,7 @@
+  o Minor bugfixes (v3 onion services):
+    - Stop sending ed25519 link specifiers in v3 onion service introduce
+      cells, when the rendezvous point doesn't support ed25519 link
+      authentication. Fixes bug 26627; bugfix on 0.3.2.4-alpha.
+    - Stop putting ed25519 link specifiers in v3 onion service descriptors,
+      when the intro point doesn't support ed25519 link authentication.
+      Fixes bug 26627; bugfix on 0.3.2.4-alpha.
diff --git a/changes/bug26779 b/changes/bug26779
new file mode 100644
index 0000000000..fb7f6160ea
--- /dev/null
+++ b/changes/bug26779
@@ -0,0 +1,4 @@
+  o Minor features (bug workaround):
+    - Compile correctly on systems that provide the C11 stdatomic.h header,
+      but where C11 atomic functions don't actually compile.
+      Closes ticket 26779; workaround for Debian issue 903709.
diff --git a/changes/bug26785 b/changes/bug26785
new file mode 100644
index 0000000000..e6392fcbdd
--- /dev/null
+++ b/changes/bug26785
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation, portability):
+    - Don't try to use a pragma to temporarily disable
+      -Wunused-const-variable if the compiler doesn't support it.
+      Fixes bug 26785; bugfix on 0.3.2.11.
diff --git a/changes/bug26787 b/changes/bug26787
new file mode 100644
index 0000000000..b32e519a93
--- /dev/null
+++ b/changes/bug26787
@@ -0,0 +1,3 @@
+  o Minor bugfixes (testing):
+    - Disable core dumps in test_bt.sh, to avoid failures in "make
+      distcheck". Fixes bug 26787; bugfix on 0.2.5.2-alpha.
diff --git a/changes/bug26789 b/changes/bug26789
new file mode 100644
index 0000000000..9b3520543d
--- /dev/null
+++ b/changes/bug26789
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+     - Update build system so that tor builds again with
+       --disable-unittests after recent refactoring efforts.
+       Fixes bug 26789; bugfix on 0.3.4.3-alpha.
diff --git a/changes/bug26830 b/changes/bug26830
new file mode 100644
index 0000000000..c002f19530
--- /dev/null
+++ b/changes/bug26830
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - Skip an unreliable key generation test on Windows, until the underlying
+      issue in bug 26076 is resolved. Fixes bug 26830; bugfix on 0.2.7.3-rc.
diff --git a/changes/bug26853 b/changes/bug26853
new file mode 100644
index 0000000000..6ee47789b9
--- /dev/null
+++ b/changes/bug26853
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - Skip an unreliable key expiration test on Windows, until the underlying
+      issue in bug 26076 is resolved. Fixes bug 26853; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug26873 b/changes/bug26873
new file mode 100644
index 0000000000..565f8bf0b1
--- /dev/null
+++ b/changes/bug26873
@@ -0,0 +1,4 @@
+  o Minor bugfixes (portability):
+    - Fix compilation of the unit tests on GNU/Hurd, which does not
+      define PATH_MAX. Fixes bug 26873; bugfix on 0.3.3.1-alpha.
+      Patch from "paulusASol".
diff --git a/changes/bug26876 b/changes/bug26876
new file mode 100644
index 0000000000..b661104236
--- /dev/null
+++ b/changes/bug26876
@@ -0,0 +1,4 @@
+  o Minor bugfixes (portability):
+    - Work around two different bugs in the OS X 10.10 and later SDKs that
+      would prevent us from successfully targeting earlier versions of OS X.
+      Fixes bug 26876; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug26896 b/changes/bug26896
new file mode 100644
index 0000000000..9762dc7fac
--- /dev/null
+++ b/changes/bug26896
@@ -0,0 +1,3 @@
+  o Minor bugfixes (directory connection shutdown):
+    - Avoid a double-close when shutting down a stalled directory connection.
+      Fixes bug 26896; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug26924 b/changes/bug26924
new file mode 100644
index 0000000000..882db56b40
--- /dev/null
+++ b/changes/bug26924
@@ -0,0 +1,4 @@
+  o Minor bugfixes (single onion services, Tor2web):
+    - Log a protocol warning when single onion services or Tor2web clients
+      fail to authenticate direct connections to relays.
+      Fixes bug 26924; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug26927 b/changes/bug26927
new file mode 100644
index 0000000000..cd035bba8e
--- /dev/null
+++ b/changes/bug26927
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging):
+    - Improve the log message when connection initiators fail to authenticate
+      direct connections to relays.
+      Fixes bug 26927; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug26932 b/changes/bug26932
new file mode 100644
index 0000000000..7d9481dcd3
--- /dev/null
+++ b/changes/bug26932
@@ -0,0 +1,3 @@
+  o Minor bugfixes (onion services):
+    - Fix bug that causes services to not ever rotate their descriptors if they
+      were getting SIGHUPed often. Fixes bug 26932; bugfix on 0.3.2.1-alpha.
\ No newline at end of file
diff --git a/changes/bug26948 b/changes/bug26948
new file mode 100644
index 0000000000..0f0728843f
--- /dev/null
+++ b/changes/bug26948
@@ -0,0 +1,4 @@
+  o Minor bugfixes (in-process restart):
+    - Always call tor_free_all() when leaving tor_run_main(). When we
+      did not, restarting tor in-process would cause an assertion failure.
+      Fixes bug 26948; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug26979 b/changes/bug26979
new file mode 100644
index 0000000000..e615207b74
--- /dev/null
+++ b/changes/bug26979
@@ -0,0 +1,4 @@
+  o Minor bugfixes (appveyor ci):
+    - Improve Appveyor CI IRC logging. Generate correct branches and URLs for
+      pull requests and tags. Use unambiguous short commits.
+      Fixes bug 26979; bugfix on master.
diff --git a/changes/bug26994 b/changes/bug26994
new file mode 100644
index 0000000000..664894a7fc
--- /dev/null
+++ b/changes/bug26994
@@ -0,0 +1,3 @@
+  o Minor bugfixes (portability):
+    - Avoid a compilation error in test_bwmgt.c on Solaris 10.
+      Fixes bug 26994; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27003 b/changes/bug27003
new file mode 100644
index 0000000000..4f2045afc7
--- /dev/null
+++ b/changes/bug27003
@@ -0,0 +1,6 @@
+  o Major bugfixes (event scheduler):
+    - When we enable a periodic event, schedule it in the event loop
+      rather than running it immediately. Previously, we would re-run
+      periodic events immediately in the middle of (for example)
+      changing our options, with unpredictable effects.  Fixes bug
+      27003; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27081 b/changes/bug27081
new file mode 100644
index 0000000000..74e0efbd29
--- /dev/null
+++ b/changes/bug27081
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation, windows):
+    - Don't link or search for pthreads when building for Windows, even if we
+      are using build environment (like mingw) that provides a pthreads
+      library. Fixes bug 27081; bugfix on 0.1.0.1-rc.
diff --git a/changes/bug27088 b/changes/bug27088
new file mode 100644
index 0000000000..d4d3b292c5
--- /dev/null
+++ b/changes/bug27088
@@ -0,0 +1,5 @@
+  o Minor bugfixes (continuous integration):
+    - Pass the module flags to distcheck configure, and
+      log the flags before running configure. (Backported
+      to 0.2.9 and later as a precaution.)
+      Fixes bug 27088; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27090 b/changes/bug27090
new file mode 100644
index 0000000000..3d119a9c30
--- /dev/null
+++ b/changes/bug27090
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - Build with zstd on macOS.
+      Fixes bug 27090; bugfix on 0.3.1.5-alpha.
diff --git a/changes/bug27093 b/changes/bug27093
new file mode 100644
index 0000000000..6c097f1196
--- /dev/null
+++ b/changes/bug27093
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Consistently use ../../.. as a fallback for $abs_top_srcdir in
+      test_rust.sh. Fixes bug 27093; bugfix on 0.3.4.3-alpha.
diff --git a/changes/bug27139 b/changes/bug27139
new file mode 100644
index 0000000000..0d1e3b4329
--- /dev/null
+++ b/changes/bug27139
@@ -0,0 +1,14 @@
+  o Minor bugfixes (32-bit OSX and iOS, timing):
+    - Fix an integer overflow bug in our optimized 32-bit millisecond-
+      difference algorithm for 32-bit Apple platforms. Previously, it
+      would overflow when calculating the difference between two times
+      more than 47 days apart.  Fixes part of bug 27139; bugfix on
+      0.3.4.1-alpha.
+    - Improve the precision of our 32-bit millisecond difference
+      algorithm for 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+    - Relax the tolerance on the mainloop/update_time_jumps test
+      when running on 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+
+
diff --git a/changes/bug27164 b/changes/bug27164
new file mode 100644
index 0000000000..d04d2f28f4
--- /dev/null
+++ b/changes/bug27164
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - Protover parsing was ignoring a 2nd hyphen and everything after it,
+      accepting entries like "Link=1-5-foo". Fixes bug 27164; bugfix on
+      0.3.3.1-alpha.
diff --git a/changes/bug27177 b/changes/bug27177
new file mode 100644
index 0000000000..b03bbc96ea
--- /dev/null
+++ b/changes/bug27177
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - Protover parsing was accepting the presence of whitespace in version
+      strings, which the C implementation would choke on, e.g. "Desc=1\t,2".
+      Fixes bug 27177; bugfix on 0.3.3.5-rc.
diff --git a/changes/bug27185 b/changes/bug27185
new file mode 100644
index 0000000000..79221b3df4
--- /dev/null
+++ b/changes/bug27185
@@ -0,0 +1,3 @@
+  o Minor bugfixes (Windows, compilation):
+    - Silence a compilation warning on MSVC 2017 and clang-cl.
+      Fixes bug 27185; bugfix on 0.2.2.2-alpha.
diff --git a/changes/bug27199 b/changes/bug27199
new file mode 100644
index 0000000000..f9d2a422f9
--- /dev/null
+++ b/changes/bug27199
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Abort on panic in all build profiles, instead of potentially unwinding
+      into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27206 b/changes/bug27206
new file mode 100644
index 0000000000..c0fbbed702
--- /dev/null
+++ b/changes/bug27206
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - protover_all_supported() would attempt to allocate up to 16GB on some
+      inputs, leading to a potential memory DoS. Fixes bug 27206; bugfix on
+      0.3.3.5-rc.
diff --git a/changes/bug27226 b/changes/bug27226
new file mode 100644
index 0000000000..9030773cd5
--- /dev/null
+++ b/changes/bug27226
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing, openssl compatibility):
+    - Our "tortls/cert_matches_key" unit test no longer relies on OpenSSL
+      internals.  Previously, it relied on unsupported OpenSSL behavior in
+      a way that caused it to crash with OpenSSL 1.0.2p. Fixes bug 27226;
+      bugfix on 0.2.5.1-alpha.
diff --git a/changes/bug27236 b/changes/bug27236
new file mode 100644
index 0000000000..76d792f4c9
--- /dev/null
+++ b/changes/bug27236
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing, bootstrap):
+    - When calculating bootstrap progress, check exit policies and the exit
+      flag. Previously, Tor would only check the exit flag, which caused
+      race conditions in small and fast networks like chutney.
+      Fixes bug 27236; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug27295 b/changes/bug27295
new file mode 100644
index 0000000000..c5a364877a
--- /dev/null
+++ b/changes/bug27295
@@ -0,0 +1,3 @@
+  o Minor bugfixes (testing, chutney):
+    - Before running make test-network-all, delete old logs and test result
+      files, to avoid spurious failures. Fixes bug 27295; bugfix on 0.2.7.3-rc.
diff --git a/changes/bug27335 b/changes/bug27335
new file mode 100644
index 0000000000..dcc55a945a
--- /dev/null
+++ b/changes/bug27335
@@ -0,0 +1,4 @@
+  o Minor bugfixes (hidden service v3):
+    - In case the hidden service directory can't be created or has wrong
+      permissions, do not BUG() on it which lead to a non fatal stacktrace.
+      Fixes bug 27335; bugfix on 0.3.2.1.
diff --git a/changes/bug27344 b/changes/bug27344
new file mode 100644
index 0000000000..9f66855586
--- /dev/null
+++ b/changes/bug27344
@@ -0,0 +1,4 @@
+  o Minor features (compatibility):
+    - Tell OpenSSL to maintain backward compatibility with previous
+      RSA1024/DH1024 users in Tor. With OpenSSL 1.1.1-pre6, these ciphers
+      are disabled by default. Closes ticket 27344.
diff --git a/changes/bug27345 b/changes/bug27345
new file mode 100644
index 0000000000..d98f4afbcc
--- /dev/null
+++ b/changes/bug27345
@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing):
+    - When running make test-network-all, use the mixed+hs-v2 network.
+      (A previous fix to chutney removed v3 onion services from the
+      mixed+hs-v23 network, so seeing "mixed+hs-v23" in tests is
+      confusing.) Fixes bug 27345; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug27418 b/changes/bug27418
new file mode 100644
index 0000000000..1d99497dc4
--- /dev/null
+++ b/changes/bug27418
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - When we use echo in Travis, don't pass a --flag as the first argument.
+      Fixes bug 27418; bugfix on 0.3.4.7-rc.
diff --git a/changes/bug27453 b/changes/bug27453
new file mode 100644
index 0000000000..4501346d2c
--- /dev/null
+++ b/changes/bug27453
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - When a Travis build fails, and showing a log fails, keep trying to
+      show the other logs. Fixes bug 27453; bugfix on 0.3.4.7-rc.
diff --git a/changes/bug27460 b/changes/bug27460
new file mode 100644
index 0000000000..53c4c7daf3
--- /dev/null
+++ b/changes/bug27460
@@ -0,0 +1,6 @@
+  o Minor bugfixes (continuous integration):
+    - Disable gcc hardening in Appveyor Windows 64-bit builds. As of
+      August 29, 2018, Appveyor images come with gcc 8.2.0 by default.
+      64-bit Windows executables compiled with gcc 8.2.0 and
+      tor's --enable-gcc-hardening crash.
+      Fixes bug 27460; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27461 b/changes/bug27461
new file mode 100644
index 0000000000..3571ee816a
--- /dev/null
+++ b/changes/bug27461
@@ -0,0 +1,5 @@
+  o Minor bugfixes (compilation):
+    - Stop calling SetProcessDEPPolicy() on 64-bit Windows. It is not
+      supported, and always fails. Some compilers warn about the function
+      pointer cast on 64-bit Windows.
+      Fixes bug 27461; bugfix on 0.2.2.23-alpha.
diff --git a/changes/bug27463 b/changes/bug27463
new file mode 100644
index 0000000000..073acdd997
--- /dev/null
+++ b/changes/bug27463
@@ -0,0 +1,3 @@
+  o Minor bugfixes (onion services):
+    - Silence a spurious compiler warning in rend_client_send_introduction().
+      Fixes bug 27463; bugfix on 0.1.1.2-alpha.
diff --git a/changes/bug27464 b/changes/bug27464
new file mode 100644
index 0000000000..9dedd06f41
--- /dev/null
+++ b/changes/bug27464
@@ -0,0 +1,3 @@
+  o Minor bugfixes (continuous integration):
+    - Stop reinstalling identical packages in our Windows CI.
+      Fixes bug 27464; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27465 b/changes/bug27465
new file mode 100644
index 0000000000..743b35130f
--- /dev/null
+++ b/changes/bug27465
@@ -0,0 +1,5 @@
+  o Minor bugfixes (compilation):
+    - Silence a spurious compiler warning on the GetAdaptersAddresses
+      function pointer cast. This issue is already fixed by 26481 in
+      0.3.5 and later, by removing the lookup and cast.
+      Fixes bug 27465; bugfix on 0.2.3.11-alpha.
diff --git a/changes/bug27649 b/changes/bug27649
new file mode 100644
index 0000000000..55bfc3a842
--- /dev/null
+++ b/changes/bug27649
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - The protover rewrite in 24031 allowed repeated votes from the same
+      voter for the same protocol version to be counted multiple times in
+      protover_compute_vote(). Fixes bug 27649; bugfix on 0.3.3.5-rc.
diff --git a/changes/bug27687 b/changes/bug27687
new file mode 100644
index 0000000000..8b7903b63e
--- /dev/null
+++ b/changes/bug27687
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - protover parsed and accepted unknown protocol names containing invalid
+      characters outside the range [A-Za-z0-9-]. Fixes bug 27687; bugfix on
+      0.3.3.1-alpha.
diff --git a/changes/bug27708 b/changes/bug27708
new file mode 100644
index 0000000000..d283b19515
--- /dev/null
+++ b/changes/bug27708
@@ -0,0 +1,4 @@
+  o Major bugfixes (restart-in-process):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27740 b/changes/bug27740
new file mode 100644
index 0000000000..76a17b7dda
--- /dev/null
+++ b/changes/bug27740
@@ -0,0 +1,4 @@
+  o Minor bugfixes (rust):
+    - Return a string that can be safely freed by C code, not one created by
+      the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix
+      on 0.3.3.1-alpha.
diff --git a/changes/bug27741 b/changes/bug27741
new file mode 100644
index 0000000000..531e264b63
--- /dev/null
+++ b/changes/bug27741
@@ -0,0 +1,5 @@
+  o Minor bugfixes (rust, directory authority):
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
diff --git a/changes/bug27750 b/changes/bug27750
new file mode 100644
index 0000000000..c234788b1c
--- /dev/null
+++ b/changes/bug27750
@@ -0,0 +1,6 @@
+  o Minor bugfixes (connection, relay):
+    - Avoid a wrong BUG() stacktrace in case a closing connection is being held
+      open because the write side is rate limited but not the read side. Now,
+      the connection read side is simply shutdown instead of kept open until tor
+      is able to flush the connection and then fully close it. Fixes bug 27750;
+      bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27781 b/changes/bug27781
new file mode 100644
index 0000000000..44d838af8a
--- /dev/null
+++ b/changes/bug27781
@@ -0,0 +1,4 @@
+  o Major bugfixes (compilation):
+    - Fix compilation on arm (and other less-used CPUs)
+      when compiling with OpenSSL before 1.1. Fixes bug 27781;
+      bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27800 b/changes/bug27800
new file mode 100644
index 0000000000..63d5dbc681
--- /dev/null
+++ b/changes/bug27800
@@ -0,0 +1,4 @@
+  o Minor bugfixes (directory authority):
+    - Log additional info when we get a relay that shares an ed25519
+      ID with a different relay, instead making a BUG() warning.
+      Fixes bug 27800; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug27804 b/changes/bug27804
new file mode 100644
index 0000000000..fa7fec0bc5
--- /dev/null
+++ b/changes/bug27804
@@ -0,0 +1,3 @@
+  o Minor bugfixes (rust):
+    - Fix a potential null dereference in protover_all_supported().
+      Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28399 b/changes/bug28399
new file mode 100644
index 0000000000..9096db70b0
--- /dev/null
+++ b/changes/bug28399
@@ -0,0 +1,4 @@
+  o Minor bugfixes (continuous integration, Windows):
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI.
+      Fixes bug 28399; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28419 b/changes/bug28419
new file mode 100644
index 0000000000..52ceb0a2a7
--- /dev/null
+++ b/changes/bug28419
@@ -0,0 +1,3 @@
+  o Minor bugfixes (memory leaks):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha.  Patch from Martin Kepplinger.
\ No newline at end of file
diff --git a/changes/bug28454 b/changes/bug28454
new file mode 100644
index 0000000000..ca46ae2777
--- /dev/null
+++ b/changes/bug28454
@@ -0,0 +1,4 @@
+  o Minor bugfixes (continuous integration, Windows):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not
+      come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28554 b/changes/bug28554
new file mode 100644
index 0000000000..9a0b281406
--- /dev/null
+++ b/changes/bug28554
@@ -0,0 +1,3 @@
+  o Minor bugfixes (unit tests, guard selection):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug28619 b/changes/bug28619
new file mode 100644
index 0000000000..86be8cb2fb
--- /dev/null
+++ b/changes/bug28619
@@ -0,0 +1,6 @@
+  o Minor bugfixes (hidden service v3):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close any
+      rendezvous circuits in order to let the existing client connections
+      finish by themselves or closed by the application. The HS v2 is doing
+      that already so now we have the same behavior for all versions. Fixes
+      bug 28619; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug29017 b/changes/bug29017
new file mode 100644
index 0000000000..5c4a53c43f
--- /dev/null
+++ b/changes/bug29017
@@ -0,0 +1,4 @@
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017;
+      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug29244 b/changes/bug29244
new file mode 100644
index 0000000000..6206a95463
--- /dev/null
+++ b/changes/bug29244
@@ -0,0 +1,4 @@
+  o Minor bugfixes (build, compatibility):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again.
+      Fixes bug 29244; bugfix on 0.3.3.4-alpha.
diff --git a/changes/feature26372_029 b/changes/feature26372_029
new file mode 100644
index 0000000000..150ac30555
--- /dev/null
+++ b/changes/feature26372_029
@@ -0,0 +1,4 @@
+  o Minor features (compilation):
+
+    - When building Tor, prefer to use Python 3 over Python 2, and more
+      recent (contemplated) versions over older ones. Closes ticket 26372.
diff --git a/changes/geoip-2018-07-03 b/changes/geoip-2018-07-03
new file mode 100644
index 0000000000..e921d63c99
--- /dev/null
+++ b/changes/geoip-2018-07-03
@@ -0,0 +1,4 @@
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the July 3 2018 Maxmind GeoLite2
+      Country database. Closes ticket 26674.
+
diff --git a/changes/geoip-2018-08-07 b/changes/geoip-2018-08-07
new file mode 100644
index 0000000000..9ddbe7b1b2
--- /dev/null
+++ b/changes/geoip-2018-08-07
@@ -0,0 +1,4 @@
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the August 7 2018 Maxmind GeoLite2
+      Country database. Closes ticket 27089.
+
diff --git a/changes/rust_cross b/changes/rust_cross
new file mode 100644
index 0000000000..d490403a28
--- /dev/null
+++ b/changes/rust_cross
@@ -0,0 +1,2 @@
+  o Minor feature (Rust, portability):
+    - Rust cross-compilation is now supported. Closes ticket 25895.
diff --git a/changes/task26771 b/changes/task26771
new file mode 100644
index 0000000000..fd700900f7
--- /dev/null
+++ b/changes/task26771
@@ -0,0 +1,4 @@
+  o Directory authority changes:
+    - The "Bifroest" bridge authority has been retired; the new bridge
+      authority is "Serge", and it is operated by George from the
+      TorBSD project. Closes ticket 26771.
diff --git a/changes/ticket24629 b/changes/ticket24629
new file mode 100644
index 0000000000..482c0a1a6d
--- /dev/null
+++ b/changes/ticket24629
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Enable macOS builds in our Travis CI configuration.
+      Closes ticket 24629.
diff --git a/changes/ticket24629-backport b/changes/ticket24629-backport
new file mode 100644
index 0000000000..dfbc465634
--- /dev/null
+++ b/changes/ticket24629-backport
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Backport Travis rust distcheck to 0.3.3.
+      Closes ticket 24629.
diff --git a/changes/ticket25573 b/changes/ticket25573
new file mode 100644
index 0000000000..9939601b50
--- /dev/null
+++ b/changes/ticket25573
@@ -0,0 +1,5 @@
+  o Minor features (controller):
+    - For purposes of CIRC_BW-based dropped cell detection, track half-closed
+      stream ids, and allow their ENDs, SENDMEs, DATA and path bias check
+      cells to arrive without counting it as dropped until either the END arrvies,
+      or the windows are empty. Closes ticket 25573.
diff --git a/changes/ticket25947 b/changes/ticket25947
new file mode 100644
index 0000000000..68559a73f8
--- /dev/null
+++ b/changes/ticket25947
@@ -0,0 +1,4 @@
+  o Minor feature (unit tests):
+    - Test complete bandwidth measurements files and test that incomplete lines
+      only give warnings when the end of the header has not been
+      detected. Fixes bug 25947; bugfix on 0.2.2.1-alpha
diff --git a/changes/ticket25960 b/changes/ticket25960
new file mode 100644
index 0000000000..0d1be2119b
--- /dev/null
+++ b/changes/ticket25960
@@ -0,0 +1,5 @@
+  o Minor feature (directory authorities):
+    - Stop warning about incomplete bw lines before the first complete bw line
+      has been found, so that additional header lines can be ignored.
+      Fixes bug 25960; bugfix on 0.2.2.1-alpha
+
diff --git a/changes/ticket26467 b/changes/ticket26467
new file mode 100644
index 0000000000..45883786c2
--- /dev/null
+++ b/changes/ticket26467
@@ -0,0 +1,3 @@
+  o Minor bugfixes (memory, correctness):
+    - Fix a number of small memory leaks identified by coverity. Fixes
+      bug 26467; bugfix on numerous Tor versions.
diff --git a/changes/ticket26560 b/changes/ticket26560
new file mode 100644
index 0000000000..5b4fb1bfe7
--- /dev/null
+++ b/changes/ticket26560
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI.  Closes ticket 26560.
diff --git a/changes/ticket26647 b/changes/ticket26647
new file mode 100644
index 0000000000..1c2e917c6d
--- /dev/null
+++ b/changes/ticket26647
@@ -0,0 +1,4 @@
+  o Minor features (controller):
+    - The control port now exposes the list of HTTPTunnelPorts and
+      ExtOrPorts via GETINFO net/listeners/httptunnel and net/listeners/extor
+      respectively.  Closes ticket 26647.
diff --git a/changes/ticket26913 b/changes/ticket26913
new file mode 100644
index 0000000000..d6555764ec
--- /dev/null
+++ b/changes/ticket26913
@@ -0,0 +1,7 @@
+  o Minor bugfixes (directory permissions):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would always override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix on
+      0.3.3.1-alpha.
diff --git a/changes/ticket26952-cargo b/changes/ticket26952-cargo
new file mode 100644
index 0000000000..e1efdfcd74
--- /dev/null
+++ b/changes/ticket26952-cargo
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration, rust):
+    - Use cargo cache in our Travis CI configuration.
+      Closes ticket 26952.
diff --git a/changes/ticket26952-ccache b/changes/ticket26952-ccache
new file mode 100644
index 0000000000..edc115e9de
--- /dev/null
+++ b/changes/ticket26952-ccache
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Use ccache in our Travis CI configuration.
+      Closes ticket 26952.
diff --git a/changes/ticket27087 b/changes/ticket27087
new file mode 100644
index 0000000000..b8af70aaa0
--- /dev/null
+++ b/changes/ticket27087
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Run asciidoc during Travis CI.
+      Implements ticket 27087.
diff --git a/changes/ticket27252-032 b/changes/ticket27252-032
new file mode 100644
index 0000000000..4752aedcf6
--- /dev/null
+++ b/changes/ticket27252-032
@@ -0,0 +1,5 @@
+  o Minor features (continuous integration):
+    - Only run one online rust build in Travis, to reduce network errors.
+      Skip offline rust builds on Travis for Linux gcc, because they're
+      redundant.
+      Implements ticket 27252.
diff --git a/changes/ticket27252-034 b/changes/ticket27252-034
new file mode 100644
index 0000000000..620ad83efe
--- /dev/null
+++ b/changes/ticket27252-034
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Don't do a distcheck with --disable-module-dirauth in Travis.
+      Implements ticket 27252.
diff --git a/changes/ticket27275 b/changes/ticket27275
new file mode 100644
index 0000000000..d1332e9540
--- /dev/null
+++ b/changes/ticket27275
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Only post Appveyor IRC notifications when the build fails.
+      Implements ticket 27275.
diff --git a/changes/ticket27410 b/changes/ticket27410
new file mode 100644
index 0000000000..a21fdde58e
--- /dev/null
+++ b/changes/ticket27410
@@ -0,0 +1,5 @@
+  o Minor bugfixes (hidden service v3):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first one
+      leaving the other hanging and let to time out by themselves. Fixes bug
+      27410; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27430 b/changes/ticket27430
new file mode 100644
index 0000000000..4e016e91e7
--- /dev/null
+++ b/changes/ticket27430
@@ -0,0 +1,4 @@
+  o Minor features (continuous integration):
+    - Show config.log and test-suite.log after failed Appveyor builds.
+      Also upload the zipped full logs as a build artifact.
+      Implements ticket 27430.
diff --git a/changes/ticket27449 b/changes/ticket27449
new file mode 100644
index 0000000000..2a0984c09c
--- /dev/null
+++ b/changes/ticket27449
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - Log the compiler path and version during Appveyor builds.
+      Implements ticket 27449.
diff --git a/changes/ticket27550 b/changes/ticket27550
new file mode 100644
index 0000000000..87f9b5cbe9
--- /dev/null
+++ b/changes/ticket27550
@@ -0,0 +1,5 @@
+  o Minor bugfixes (hidden service v3):
+    - Don't warn so loudly when tor is unable to decode a descriptor. This can
+      now happen as a normal use case if a client gets a descriptor with
+      client authorization but the client is not authorized. Fixes bug 27550;
+      bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket27797 b/changes/ticket27797
new file mode 100644
index 0000000000..f07e35f84c
--- /dev/null
+++ b/changes/ticket27797
@@ -0,0 +1,5 @@
+  o Minor bugfixes (node, hidden service v3):
+    - When selecting a v3 rendezvous point, not only look at the protover but
+      also if the curve25519 onion key is present. That way we avoid picking a
+      node that supports the v3 rendezvous but for which we don't have the
+      descriptor yet for the key. Fixes bug 27797; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27810 b/changes/ticket27810
new file mode 100644
index 0000000000..119f781cb9
--- /dev/null
+++ b/changes/ticket27810
@@ -0,0 +1,5 @@
+  o Minor bugfixes (test, hidden service v3):
+    - Make the the hs_service tests uses the same time source when creating
+      the introduction point and testing it. This helps make test work on very
+      slow system like ARM or Travis. Fixes bug 27810; bugfix on
+      0.3.2.1-alpha.
diff --git a/changes/ticket27849 b/changes/ticket27849
new file mode 100644
index 0000000000..c2babeffc7
--- /dev/null
+++ b/changes/ticket27849
@@ -0,0 +1,5 @@
+  o Major bugfixes (mainloop, bootstrap):
+    - Make sure Tor bootstraps and works properly if only the ControlPort is
+      set. Prior to this fix, Tor would only bootstrap with at least a client
+      port being set (Socks, Trans, NATD, DNS or HTTPTunnel port). Fixes bug
+      27849; bugfix on 0.3.4.1-alpha.
diff --git a/changes/ticket28089 b/changes/ticket28089
new file mode 100644
index 0000000000..a900ec18f5
--- /dev/null
+++ b/changes/ticket28089
@@ -0,0 +1,6 @@
+  o Major bugfixes (relay):
+    - When our write bandwidth limit is exhausted, stop writing on the
+      connection. Previously, we had a typo in the code that would make us stop
+      reading leading to relay connections being stuck indefinitely. Fixes bug
+      28089; bugfix on 0.3.4.1-alpha.
+
diff --git a/changes/ticket28459 b/changes/ticket28459
new file mode 100644
index 0000000000..6b5839b52b
--- /dev/null
+++ b/changes/ticket28459
@@ -0,0 +1,4 @@
+  o Minor features (continuous integration, Windows):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
diff --git a/changes/ticket28574 b/changes/ticket28574
new file mode 100644
index 0000000000..562810f511
--- /dev/null
+++ b/changes/ticket28574
@@ -0,0 +1,4 @@
+  o Minor bugfixes (continuous integration, Windows):
+    - Explicitly specify the path to the OpenSSL library and do not download
+      OpenSSL from Pacman, but instead use the library that is already provided
+      by AppVeyor. Fixes bug 28574; bugfix on master.
diff --git a/changes/ticket28912 b/changes/ticket28912
new file mode 100644
index 0000000000..4119b778bc
--- /dev/null
+++ b/changes/ticket28912
@@ -0,0 +1,6 @@
+  o Major bugfixes (relay, directory):
+    - A connection serving directory information wouldn't get reactivated after
+      the first chunk of data was sent (usually 32KB). Tor now always activate
+      the main loop event that goes through these connections as long as at
+      least one connection is still active. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
diff --git a/changes/ticket28973 b/changes/ticket28973
new file mode 100644
index 0000000000..b1d208ee51
--- /dev/null
+++ b/changes/ticket28973
@@ -0,0 +1,6 @@
+  o Minor features (OpenSSL bug workaround):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels.  When this bug
+      is detected, Tor will disable TLS 1.3.  We recommend upgrading to
+      a version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
diff --git a/changes/ticket29168 b/changes/ticket29168
new file mode 100644
index 0000000000..65c5232f65
--- /dev/null
+++ b/changes/ticket29168
@@ -0,0 +1,5 @@
+  o Major bugfixes (cell scheduler, KIST):
+    - Make KIST to always take into account the outbuf length when computing
+      what we can actually put in the outbuf. This could lead to the outbuf
+      being filled up and thus a possible memory DoS vector. TROVE-2019-001.
+      Fixes bug 29168; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket29806 b/changes/ticket29806
new file mode 100644
index 0000000000..6afefd4c04
--- /dev/null
+++ b/changes/ticket29806
@@ -0,0 +1,7 @@
+  o Minor features (bandwidth authority):
+    - Make bandwidth authorities to ignore relays that are reported in the
+      bandwidth file with the key-value "vote=0".
+      This change allows to report the relays that were not measured due
+      some failure and diagnose the reasons without the bandwidth being included in the
+      bandwidth authorities vote.
+      Closes ticket 29806.
diff --git a/changes/ticket29962 b/changes/ticket29962
new file mode 100644
index 0000000000..e36cc0cf9a
--- /dev/null
+++ b/changes/ticket29962
@@ -0,0 +1,3 @@
+  o Minor features (continuous integration):
+    - On Travis Rust builds, cleanup Rust registry and refrain from caching
+      target/ directory to speed up builds. Resolves issue 29962.
diff --git a/changes/tickets_27765_27943 b/changes/tickets_27765_27943
new file mode 100644
index 0000000000..e8468545f1
--- /dev/null
+++ b/changes/tickets_27765_27943
@@ -0,0 +1,5 @@
+  o Minor bugfixes (CI, appveyor):
+    - Only install the necessary mingw packages during our appveyor
+      builds. This change makes the build a little faster, and prevents a
+      conflict with a preinstalled mingw openssl that appveyor now ships.
+      Fixes bugs 27943 and 27765; bugfix on 0.3.4.2-alpha.
diff --git a/config.rust.in b/config.rust.in
new file mode 100644
index 0000000000..4ca5351aec
--- /dev/null
+++ b/config.rust.in
@@ -0,0 +1,22 @@
+# Used by our cargo build.rs script to get variables from autoconf.
+#
+# The "configure" script will generate "config.rust" from "config.rust.in",
+# and then build.rs will read "config.rust".
+
+BUILDDIR=@BUILDDIR@
+TOR_LDFLAGS_zlib=@TOR_LDFLAGS_zlib@
+TOR_LDFLAGS_openssl=@TOR_LDFLAGS_openssl@
+TOR_LDFLAGS_libevent=@TOR_LDFLAGS_libevent@
+TOR_ZLIB_LIBS=@TOR_ZLIB_LIBS@
+TOR_LIB_MATH=@TOR_LIB_MATH@
+TOR_LIBEVENT_LIBS=@TOR_LIBEVENT_LIBS@
+TOR_OPENSSL_LIBS=@TOR_OPENSSL_LIBS@
+TOR_LIB_WS32=@TOR_LIB_WS32@
+TOR_LIB_GDI=@TOR_LIB_GDI@
+TOR_LIB_USERENV=@TOR_LIB_USERENV@
+CURVE25519_LIBS=@CURVE25519_LIBS@
+TOR_SYSTEMD_LIBS=@TOR_SYSTEMD_LIBS@
+TOR_LZMA_LIBS=@TOR_LZMA_LIBS@
+TOR_ZSTD_LIBS=@TOR_ZSTD_LIBS@
+LIBS=@LIBS@
+LDFLAGS=@LDFLAGS@
diff --git a/configure.ac b/configure.ac
index 771769422e..bb807094a5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,17 +1,17 @@
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2015, The Tor Project, Inc.
+dnl Copyright (c) 2007-2017, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_PREREQ([2.63])
-AC_INIT([tor],[0.2.9.17-dev])
+AC_INIT([tor],[0.3.4.11-dev])
 AC_CONFIG_SRCDIR([src/or/main.c])
 AC_CONFIG_MACRO_DIR([m4])
 
 # "foreign" means we don't follow GNU package layout standards
 # "1.11" means we require automake version 1.11 or newer
 # "subdir-objects" means put .o files in the same directory as the .c files
-AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects])
+AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects -Wall -Werror])
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_CONFIG_HEADERS([orconfig.h])
@@ -49,6 +49,20 @@ AC_ARG_ENABLE(asserts-in-tests,
    AS_HELP_STRING(--disable-asserts-in-tests, [disable tor_assert() calls in the unit tests, for branch coverage]))
 AC_ARG_ENABLE(system-torrc,
    AS_HELP_STRING(--disable-system-torrc, [don't look for a system-wide torrc file]))
+AC_ARG_ENABLE(libfuzzer,
+   AS_HELP_STRING(--enable-libfuzzer, [build extra fuzzers based on 'libfuzzer']))
+AC_ARG_ENABLE(oss-fuzz,
+   AS_HELP_STRING(--enable-oss-fuzz, [build extra fuzzers based on 'oss-fuzz' environment]))
+AC_ARG_ENABLE(memory-sentinels,
+   AS_HELP_STRING(--disable-memory-sentinels, [disable code that tries to prevent some kinds of memory access bugs. For fuzzing only.]))
+AC_ARG_ENABLE(rust,
+   AS_HELP_STRING(--enable-rust, [enable rust integration]))
+AC_ARG_ENABLE(cargo-online-mode,
+   AS_HELP_STRING(--enable-cargo-online-mode, [Allow cargo to make network requests to fetch crates. For builds with rust only.]))
+AC_ARG_ENABLE(restart-debugging,
+   AS_HELP_STRING(--enable-restart-debugging, [Build Tor with support for debugging in-process restart. Developers only.]))
+AC_ARG_ENABLE(zstd-advanced-apis,
+   AS_HELP_STRING(--disable-zstd-advanced-apis, [Build without support for zstd's "static-only" APIs.]))
 
 if test "x$enable_coverage" != "xyes" -a "x$enable_asserts_in_tests" = "xno" ; then
     AC_MSG_ERROR([Can't disable assertions outside of coverage build])
@@ -57,6 +71,9 @@ fi
 AM_CONDITIONAL(UNITTESTS_ENABLED, test "x$enable_unittests" != "xno")
 AM_CONDITIONAL(COVERAGE_ENABLED, test "x$enable_coverage" = "xyes")
 AM_CONDITIONAL(DISABLE_ASSERTS_IN_UNIT_TESTS, test "x$enable_asserts_in_tests" = "xno")
+AM_CONDITIONAL(LIBFUZZER_ENABLED, test "x$enable_libfuzzer" = "xyes")
+AM_CONDITIONAL(OSS_FUZZ_ENABLED, test "x$enable_oss_fuzz" = "xyes")
+AM_CONDITIONAL(USE_RUST, test "x$enable_rust" = "xyes")
 
 if test "$enable_static_tor" = "yes"; then
   enable_static_libevent="yes";
@@ -70,6 +87,11 @@ if test "$enable_system_torrc" = "no"; then
             [Defined if we're not going to look for a torrc in SYSCONF])
 fi
 
+if test "$enable_memory_sentinels" = "no"; then
+  AC_DEFINE(DISABLE_MEMORY_SENTINELS, 1,
+           [Defined if we're turning off memory safety code to look for bugs])
+fi
+
 AM_CONDITIONAL(USE_OPENBSD_MALLOC, test "x$enable_openbsd_malloc" = "xyes")
 
 AC_ARG_ENABLE(asciidoc,
@@ -89,7 +111,15 @@ AC_ARG_ENABLE(systemd,
         * ) AC_MSG_ERROR(bad value for --enable-systemd) ;;
       esac], [systemd=auto])
 
+if test "$enable_restart_debugging" = "yes"; then
+  AC_DEFINE(ENABLE_RESTART_DEBUGGING, 1,
+            [Defined if we're building with support for in-process restart debugging.])
+fi
 
+if test "$enable_zstd_advanced_apis" != "no"; then
+   AC_DEFINE(ENABLE_ZSTD_ADVANCED_APIS, 1,
+             [Defined if we're going to try to use zstd's "static-only" APIs.])
+fi
 
 # systemd support
 if test "x$enable_systemd" = "xno"; then
@@ -139,8 +169,15 @@ dnl Others suggest '/gs /safeseh /nxcompat /dynamicbase' for non-gcc on Windows
 AC_ARG_ENABLE(gcc-hardening,
     AS_HELP_STRING(--disable-gcc-hardening, [disable compiler security checks]))
 
+dnl Deprecated --enable-expensive-hardening but keep it for now for backward compat.
 AC_ARG_ENABLE(expensive-hardening,
-    AS_HELP_STRING(--enable-expensive-hardening, [enable more expensive compiler hardening; makes Tor slower]))
+    AS_HELP_STRING(--enable-expensive-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower]))
+AC_ARG_ENABLE(fragile-hardening,
+    AS_HELP_STRING(--enable-fragile-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower]))
+if test "x$enable_expensive_hardening" = "xyes" || test "x$enable_fragile_hardening" = "xyes"; then
+  fragile_hardening="yes"
+  AC_DEFINE(DEBUG_SMARTLIST, 1, [Enable smartlist debugging])
+fi
 
 dnl Linker hardening options
 dnl Currently these options are ELF specific - you can't use this with MacOSX
@@ -170,11 +207,64 @@ AC_ARG_ENABLE(seccomp,
 AC_ARG_ENABLE(libscrypt,
      AS_HELP_STRING(--disable-libscrypt, [do not attempt to use libscrypt]))
 
-dnl check for the correct "ar" when cross-compiling
-AN_MAKEVAR([AR], [AC_PROG_AR])
-AN_PROGRAM([ar], [AC_PROG_AR])
-AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL([AR], [ar], [ar])])
-AC_PROG_AR
+dnl Enable event tracing which are transformed to debug log statement.
+AC_ARG_ENABLE(event-tracing-debug,
+     AS_HELP_STRING(--enable-event-tracing-debug, [build with event tracing to debug log]))
+AM_CONDITIONAL([USE_EVENT_TRACING_DEBUG], [test "x$enable_event_tracing_debug" = "xyes"])
+
+if test x$enable_event_tracing_debug = xyes; then
+  AC_DEFINE([USE_EVENT_TRACING_DEBUG], [1], [Tracing framework to log debug])
+  AC_DEFINE([TOR_EVENT_TRACING_ENABLED], [1], [Compile the event tracing instrumentation])
+fi
+
+dnl Enable Android only features.
+AC_ARG_ENABLE(android,
+     AS_HELP_STRING(--enable-android, [build with Android features enabled]))
+AM_CONDITIONAL([USE_ANDROID], [test "x$enable_android" = "xyes"])
+
+if test "x$enable_android" = "xyes"; then
+  AC_DEFINE([USE_ANDROID], [1], [Compile with Android specific features enabled])
+
+  dnl Check if the Android log library is available.
+  AC_CHECK_HEADERS([android/log.h])
+  AC_SEARCH_LIBS(__android_log_write, [log])
+
+fi
+
+dnl ---
+dnl Tor modules options. These options are namespaced with --disable-module-XXX
+dnl ---
+
+dnl All our modules.
+m4_define(MODULES, dirauth)
+
+dnl Directory Authority module.
+AC_ARG_ENABLE([module-dirauth],
+              AS_HELP_STRING([--disable-module-dirauth],
+                             [Do not build tor with the dirauth module]),
+              [], dnl Action if-given
+              AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
+                        [Compile with Directory Authority feature support]))
+AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"])
+
+dnl Helper variables.
+TOR_MODULES_ALL_ENABLED=
+AC_DEFUN([ADD_MODULE], [
+    MODULE=m4_toupper($1)
+    TOR_MODULES_ALL_ENABLED="${TOR_MODULES_ALL_ENABLED} -DHAVE_MODULE_${MODULE}=1"
+])
+m4_foreach_w([module], MODULES, [ADD_MODULE([module])])
+AC_SUBST(TOR_MODULES_ALL_ENABLED)
+
+dnl check for the correct "ar" when cross-compiling.
+dnl   (AM_PROG_AR was new in automake 1.11.2, which we do not yet require,
+dnl    so kludge up a replacement for the case where it isn't there yet.)
+m4_ifdef([AM_PROG_AR],
+         [AM_PROG_AR],
+         [AN_MAKEVAR([AR], [AC_PROG_AR])
+          AN_PROGRAM([ar], [AC_PROG_AR])
+          AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL([AR], [ar], [:])])
+          AC_PROG_AR])
 
 dnl Check whether the above macro has settled for a simply named tool even
 dnl though we're cross compiling. We must do this before running AC_PROG_CC,
@@ -194,11 +284,11 @@ AC_PROG_CC
 AC_PROG_CPP
 AC_PROG_MAKE_SET
 AC_PROG_RANLIB
+AC_PROG_SED
 
-AC_PATH_PROG([PERL], [perl])
-
-dnl autoconf 2.59 appears not to support AC_PROG_SED
-AC_CHECK_PROG([SED],[sed],[sed],[/bin/false])
+AC_ARG_VAR([PERL], [path to Perl binary])
+AC_CHECK_PROGS([PERL], [perl])
+AM_CONDITIONAL(USE_PERL, [test "x$ac_cv_prog_PERL" != "x"])
 
 dnl check for asciidoc and a2x
 AC_PATH_PROG([ASCIIDOC], [asciidoc], none)
@@ -206,9 +296,6 @@ AC_PATH_PROGS([A2X], [a2x a2x.py], none)
 
 AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue")
 
-AM_CONDITIONAL(USE_FW_HELPER, test "x$natpmp" = "xtrue" || test "x$upnp" = "xtrue")
-AM_CONDITIONAL(NAT_PMP, test "x$natpmp" = "xtrue")
-AM_CONDITIONAL(MINIUPNPC, test "x$upnp" = "xtrue")
 AM_PROG_CC_C_O
 AC_PROG_CC_C99
 
@@ -223,6 +310,13 @@ if test "x$PYTHON" = "x"; then
 fi
 AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"])
 
+dnl List all external rust crates we depend on here. Include the version
+rust_crates=" \
+    digest-0.7.2 \
+    libc-0.2.39 \
+"
+AC_SUBST(rust_crates)
+
 ifdef([AC_C_FLEXIBLE_ARRAY_MEMBER], [
 AC_C_FLEXIBLE_ARRAY_MEMBER
 ], [
@@ -340,6 +434,7 @@ AH_BOTTOM([
 
 
 AM_CONDITIONAL(BUILD_NT_SERVICES, test "x$bwin32" = "xtrue")
+AM_CONDITIONAL(BUILD_LIBTORRUNNER, test "x$bwin32" != "xtrue")
 
 dnl Enable C99 when compiling with MIPSpro
 AC_MSG_CHECKING([for MIPSpro compiler])
@@ -358,6 +453,116 @@ fi
 
 AC_C_BIGENDIAN
 
+AC_ARG_VAR([TOR_RUST_TARGET], [Rust target, must be specified when cross-compiling (HOST != BUILD). example: i686-pc-windows-gnu])
+
+if test "x$enable_rust" = "xyes"; then
+  AC_ARG_VAR([RUSTC], [path to the rustc binary])
+  AC_CHECK_PROG([RUSTC], [rustc], [rustc],[no])
+  if test "x$RUSTC" = "xno"; then
+    AC_MSG_ERROR([rustc unavailable but rust integration requested.])
+  fi
+
+  AC_ARG_VAR([CARGO], [path to the cargo binary])
+  AC_CHECK_PROG([CARGO], [cargo], [cargo],[no])
+  if test "x$CARGO" = "xno"; then
+    AC_MSG_ERROR([cargo unavailable but rust integration requested.])
+  fi
+
+  AC_DEFINE([HAVE_RUST], 1, [have Rust])
+  if test "x$enable_fatal_warnings" = "xyes"; then
+    RUST_WARN=
+  else
+    RUST_WARN=#
+  fi
+  if test "x$enable_cargo_online_mode" = "xyes"; then
+    CARGO_ONLINE=
+    RUST_DL=#
+  else
+    CARGO_ONLINE=--frozen
+    RUST_DL=
+
+    dnl When we're not allowed to touch the network, we need crate dependencies
+    dnl locally available.
+    AC_MSG_CHECKING([rust crate dependencies])
+    AC_ARG_VAR([TOR_RUST_DEPENDENCIES], [path to directory with local crate mirror])
+    if test "x$TOR_RUST_DEPENDENCIES" = "x"; then
+      TOR_RUST_DEPENDENCIES="${srcdir}/src/ext/rust/crates"
+    fi
+    dnl Check whether the path exists before we try to cd into it.
+    if test ! -d "$TOR_RUST_DEPENDENCIES"; then
+      AC_MSG_ERROR([Rust dependency directory $TOR_RUST_DEPENDENCIES does not exist. Specify a dependency directory using the TOR_RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.])
+      ERRORED=1
+    fi
+    dnl Make the path absolute, since we'll be using it from within a
+    dnl subdirectory.
+    TOR_RUST_DEPENDENCIES=$(cd "$TOR_RUST_DEPENDENCIES" ; pwd)
+
+    for dep in $rust_crates; do
+      if test ! -d "$TOR_RUST_DEPENDENCIES"/"$dep"; then
+        AC_MSG_ERROR([Failure to find rust dependency $TOR_RUST_DEPENDENCIES/$dep. Specify a dependency directory using the TOR_RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.])
+        ERRORED=1
+      fi
+    done
+    if test "x$ERRORED" = "x"; then
+      AC_MSG_RESULT([yes])
+    fi
+  fi
+
+  dnl This is a workaround for #46797
+  dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ).  Once the
+  dnl upstream bug is fixed, we can remove this workaround.
+  case "$host_os" in
+      darwin*)
+        TOR_RUST_EXTRA_LIBS="-lresolv"
+	;;
+  esac
+
+  dnl For now both MSVC and MinGW rust libraries will output static libs with
+  dnl the MSVC naming convention.
+  if test "$bwin32" = "true"; then
+    tor_rust_static_name=tor_rust.lib
+  else
+    tor_rust_static_name=libtor_rust.a
+  fi
+
+  AC_CANONICAL_BUILD
+
+  if test -n "$TOR_RUST_TARGET"; then
+    if test "$host" = "$build"; then
+      AC_MSG_ERROR([HOST = BUILD is invalid if TOR_RUST_TARGET is specified, see configure --help for more information.])
+    fi
+    RUST_TARGET_PROP="target = '$TOR_RUST_TARGET'"
+    TOR_RUST_LIB_PATH="src/rust/target/$TOR_RUST_TARGET/release/$tor_rust_static_name"
+  else
+    if test "$host" != "$build"; then
+      AC_MSG_ERROR([TOR_RUST_TARGET must be specified when cross-compiling with Rust enabled.])
+    fi
+    RUST_TARGET_PROP=
+    TOR_RUST_LIB_PATH="src/rust/target/release/$tor_rust_static_name"
+  fi
+
+  AC_SUBST(RUST_TARGET_PROP)
+  AC_SUBST(TOR_RUST_LIB_PATH)
+  AC_SUBST(CARGO_ONLINE)
+  AC_SUBST(RUST_WARN)
+  AC_SUBST(RUST_DL)
+
+  dnl Let's check the rustc version, too
+  AC_MSG_CHECKING([rust version])
+  RUSTC_VERSION=`$RUSTC --version`
+  RUSTC_VERSION_MAJOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 1`
+  RUSTC_VERSION_MINOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 2`
+  if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then
+    AC_MSG_ERROR([rustc version couldn't be identified])
+  fi
+  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 14; then
+    AC_MSG_ERROR([rustc must be at least version 1.14])
+  fi
+  AC_MSG_RESULT([$RUSTC_VERSION])
+fi
+
+AC_SUBST(TOR_RUST_EXTRA_LIBS)
+
 AC_SEARCH_LIBS(socket, [socket network])
 AC_SEARCH_LIBS(gethostbyname, [nsl])
 AC_SEARCH_LIBS(dlopen, [dl])
@@ -390,6 +595,7 @@ AC_CHECK_FUNCS(
 	timingsafe_memcmp \
         flock \
         ftime \
+        get_current_dir_name \
         getaddrinfo \
         getifaddrs \
         getpass \
@@ -406,6 +612,7 @@ AC_CHECK_FUNCS(
         lround \
         memmem \
         memset_s \
+        mmap \
 	pipe \
 	pipe2 \
         prctl \
@@ -429,12 +636,39 @@ AC_CHECK_FUNCS(
 	_vscprintf
 )
 
-# Apple messed up when they added two functions functions in Sierra: they
+# Apple messed up when they added some functions: they
 # forgot to decorate them with appropriate AVAILABLE_MAC_OS_VERSION
-# checks. So we should only probe for those functions if we are sure that we
-# are not targetting OSX 10.11 or earlier.
+# checks.
+
+# We should only probe for these functions if we are sure that we
+# are not targeting OS X 10.9 or earlier.
+AC_MSG_CHECKING([for a pre-Yosemite OS X build target])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+#ifdef __APPLE__
+#  include <AvailabilityMacros.h>
+#  ifndef MAC_OS_X_VERSION_10_10
+#    define MAC_OS_X_VERSION_10_10 101000
+#  endif
+#  if defined(MAC_OS_X_VERSION_MIN_REQUIRED)
+#    if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_10
+#      error "Running on Mac OS X 10.9 or earlier"
+#    endif
+#  endif
+#endif
+]], [[]])],
+   [on_macos_pre_10_10=no ; AC_MSG_RESULT([no])],
+   [on_macos_pre_10_10=yes; AC_MSG_RESULT([yes])])
+
+if test "$on_macos_pre_10_10" = "no"; then
+  AC_CHECK_FUNCS(
+        mach_approximate_time \
+  )
+fi
+
+# We should only probe for these functions if we are sure that we
+# are not targeting OSX 10.11 or earlier.
 AC_MSG_CHECKING([for a pre-Sierra OSX build target])
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #ifdef __APPLE__
 #  include <AvailabilityMacros.h>
 #  ifndef MAC_OS_X_VERSION_10_12
@@ -446,7 +680,7 @@ AC_TRY_COMPILE([
 #    endif
 #  endif
 #endif
-], [],
+]], [[]])],
    [on_macos_pre_10_12=no ; AC_MSG_RESULT([no])],
    [on_macos_pre_10_12=yes; AC_MSG_RESULT([yes])])
 
@@ -474,6 +708,21 @@ fi
 AM_CONDITIONAL(BUILD_READPASSPHRASE_C,
   test "x$ac_cv_func_readpassphrase" = "xno" && test "$bwin32" = "false")
 
+AC_MSG_CHECKING([whether free(NULL) works])
+AC_RUN_IFELSE([AC_LANG_PROGRAM([
+  #include <stdlib.h>
+], [
+char *p = NULL;
+free(p);
+])],
+[free_null_ok=true; AC_MSG_RESULT(yes)],
+[free_null_ok=false; AC_MSG_RESULT(no)],
+[free_null_ok=cross; AC_MSG_RESULT(cross)])
+
+if test "$free_null_ok" = "false"; then
+   AC_MSG_ERROR([Your libc implementation doesn't allow free(NULL), as required by C99.])
+fi
+
 dnl ------------------------------------------------------
 dnl Where do you live, libevent?  And how do we call you?
 
@@ -483,13 +732,16 @@ if test "$bwin32" = "true"; then
   # Some of the cargo-cults recommend -lwsock32 as well, but I don't
   # think it's actually necessary.
   TOR_LIB_GDI=-lgdi32
+  TOR_LIB_USERENV=-luserenv
 else
   TOR_LIB_WS32=
   TOR_LIB_GDI=
+  TOR_LIB_USERENV=
 fi
 AC_SUBST(TOR_LIB_WS32)
 AC_SUBST(TOR_LIB_GDI)
 AC_SUBST(TOR_LIB_IPHLPAPI)
+AC_SUBST(TOR_LIB_USERENV)
 
 tor_libevent_pkg_redhat="libevent"
 tor_libevent_pkg_debian="libevent-dev"
@@ -516,12 +768,13 @@ TOR_SEARCH_LIBRARY(libevent, $trylibeventdir, [-levent $STATIC_LIBEVENT_FLAGS $T
 #include <winsock2.h>
 #endif
 struct event_base;
-struct event_base *event_base_new(void);],
+struct event_base *event_base_new(void);
+void event_base_free(struct event_base *);],
     [
 #ifdef _WIN32
 {WSADATA d; WSAStartup(0x101,&d); }
 #endif
-event_base_new();
+event_base_free(event_base_new());
 ], [--with-libevent-dir], [/opt/libevent])
 
 dnl Determine the incantation needed to link libevent.
@@ -619,11 +872,21 @@ AC_ARG_WITH(ssl-dir,
       fi
   ])
 
-TOR_SEARCH_LIBRARY(openssl, $tryssldir, [-lssl -lcrypto $TOR_LIB_GDI],
-    [#include <openssl/rand.h>],
-    [void RAND_add(const void *buf, int num, double entropy);],
-    [RAND_add((void*)0,0,0);], [],
-    [/usr/local/openssl /usr/lib/openssl /usr/local/ssl /usr/lib/ssl /usr/local /usr/athena /opt/openssl])
+AC_MSG_NOTICE([Now, we'll look for OpenSSL >= 1.0.1])
+TOR_SEARCH_LIBRARY(openssl, $tryssldir, [-lssl -lcrypto $TOR_LIB_GDI $TOR_LIB_WS32],
+    [#include <openssl/ssl.h>
+     char *getenv(const char *);],
+    [struct ssl_cipher_st;
+     unsigned SSL_CIPHER_get_id(const struct ssl_cipher_st *);
+     char *getenv(const char *);],
+    dnl This funny-looking test program calls getenv, so that the compiler
+    dnl will neither make code that call SSL_CIPHER_get_id(NULL) [producing
+    dnl a crash], nor optimize out the call to SSL_CIPHER_get_id().
+    dnl We look for SSL_cipher_get_id() because it is present in
+    dnl OpenSSL >=1.0.1, because it is not deprecated, and because Tor
+    dnl depends on it.
+    [if (getenv("THIS_SHOULDNT_BE_SET_X201803")) SSL_CIPHER_get_id((void *)0);], [],
+    [/usr/local/opt/openssl /usr/local/openssl /usr/lib/openssl /usr/local/ssl /usr/lib/ssl /usr/local /opt/openssl])
 
 dnl XXXX check for OPENSSL_VERSION_NUMBER == SSLeay()
 
@@ -646,16 +909,16 @@ LIBS="$TOR_OPENSSL_LIBS $LIBS"
 LDFLAGS="$TOR_LDFLAGS_openssl $LDFLAGS"
 CPPFLAGS="$TOR_CPPFLAGS_openssl $CPPFLAGS"
 
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
 #if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x1000100fL
 #error "too old"
 #endif
-   ], [],
+   ]], [[]])],
    [ : ],
-   [ AC_ERROR([OpenSSL is too old. We require 1.0.1 or later. You can specify a path to a newer one with --with-openssl-dir.]) ])
+   [ AC_MSG_ERROR([OpenSSL is too old. We require 1.0.1 or later. You can specify a path to a newer one with --with-openssl-dir.]) ])
 
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #if defined(OPENSSL_NO_EC) || defined(OPENSSL_NO_ECDH) || defined(OPENSSL_NO_ECDSA)
@@ -664,9 +927,9 @@ AC_TRY_COMPILE([
 #if !defined(NID_X9_62_prime256v1) || !defined(NID_secp224r1)
 #error "curves unavailable"
 #endif
-   ], [],
+   ]], [[]])],
    [ : ],
-   [ AC_ERROR([OpenSSL is built without full ECC support, including curves P256 and P224. You can specify a path to one with ECC support with --with-openssl-dir.]) ])
+   [ AC_MSG_ERROR([OpenSSL is built without full ECC support, including curves P256 and P224. You can specify a path to one with ECC support with --with-openssl-dir.]) ])
 
 AC_CHECK_MEMBERS([struct ssl_method_st.get_cipher_by_char], , ,
 [#include <openssl/ssl.h>
@@ -691,6 +954,38 @@ AC_CHECK_MEMBERS([SSL.state], , ,
 [#include <openssl/ssl.h>
 ])
 
+AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT()
+#include <openssl/sha.h>
+])
+
+dnl Define the set of checks for KIST scheduler support.
+AC_DEFUN([CHECK_KIST_SUPPORT],[
+  dnl KIST needs struct tcp_info and for certain members to exist.
+  AC_CHECK_MEMBERS(
+    [struct tcp_info.tcpi_unacked, struct tcp_info.tcpi_snd_mss],
+    , ,[[#include <netinet/tcp.h>]])
+  dnl KIST needs SIOCOUTQNSD to exist for an ioctl call.
+  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
+                     #include <linux/sockios.h>
+                     #ifndef SIOCOUTQNSD
+                     #error
+                     #endif
+                     ])], have_siocoutqnsd=yes, have_siocoutqnsd=no)
+  if test "x$have_siocoutqnsd" = "xyes"; then
+    if test "x$ac_cv_member_struct_tcp_info_tcpi_unacked" = "xyes"; then
+      if test "x$ac_cv_member_struct_tcp_info_tcpi_snd_mss" = "xyes"; then
+        have_kist_support=yes
+      fi
+    fi
+  fi
+])
+dnl Now, trigger the check.
+CHECK_KIST_SUPPORT
+AS_IF([test "x$have_kist_support" = "xyes"],
+      [AC_DEFINE(HAVE_KIST_SUPPORT, 1, [Defined if KIST scheduler is supported
+                                        on this system])],
+      [AC_MSG_NOTICE([KIST scheduler can't be used. Missing support.])])
+
 LIBS="$save_LIBS"
 LDFLAGS="$save_LDFLAGS"
 CPPFLAGS="$save_CPPFLAGS"
@@ -721,6 +1016,80 @@ else
 fi
 AC_SUBST(TOR_ZLIB_LIBS)
 
+dnl ------------------------------------------------------
+dnl Where we do we find lzma?
+
+AC_ARG_ENABLE(lzma,
+      AS_HELP_STRING(--enable-lzma, [enable support for the LZMA compression scheme.]),
+      [case "${enableval}" in
+        "yes") lzma=true ;;
+        "no")  lzma=false ;;
+        * ) AC_MSG_ERROR(bad value for --enable-lzma) ;;
+      esac], [lzma=auto])
+
+if test "x$enable_lzma" = "xno"; then
+    have_lzma=no;
+else
+    PKG_CHECK_MODULES([LZMA],
+                      [liblzma],
+                      have_lzma=yes,
+                      have_lzma=no)
+
+    if test "x$have_lzma" = "xno" ; then
+        AC_MSG_WARN([Unable to find liblzma.])
+    fi
+fi
+
+if test "x$have_lzma" = "xyes"; then
+    AC_DEFINE(HAVE_LZMA,1,[Have LZMA])
+    TOR_LZMA_CFLAGS="${LZMA_CFLAGS}"
+    TOR_LZMA_LIBS="${LZMA_LIBS}"
+fi
+AC_SUBST(TOR_LZMA_CFLAGS)
+AC_SUBST(TOR_LZMA_LIBS)
+
+dnl ------------------------------------------------------
+dnl Where we do we find zstd?
+
+AC_ARG_ENABLE(zstd,
+      AS_HELP_STRING(--enable-zstd, [enable support for the Zstandard compression scheme.]),
+      [case "${enableval}" in
+        "yes") zstd=true ;;
+        "no")  zstd=false ;;
+        * ) AC_MSG_ERROR(bad value for --enable-zstd) ;;
+      esac], [zstd=auto])
+
+if test "x$enable_zstd" = "xno"; then
+    have_zstd=no;
+else
+    PKG_CHECK_MODULES([ZSTD],
+                      [libzstd >= 1.1],
+                      have_zstd=yes,
+                      have_zstd=no)
+
+    if test "x$have_zstd" = "xno" ; then
+        AC_MSG_WARN([Unable to find libzstd.])
+    fi
+fi
+
+if test "x$have_zstd" = "xyes"; then
+    AC_DEFINE(HAVE_ZSTD,1,[Have Zstd])
+    TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}"
+    TOR_ZSTD_LIBS="${ZSTD_LIBS}"
+
+    dnl now check for zstd functions
+    save_LIBS="$LIBS"
+    save_CFLAGS="$CFLAGS"
+    LIBS="$LIBS $ZSTD_LIBS"
+    CFLAGS="$CFLAGS $ZSTD_CFLAGS"
+    AC_CHECK_FUNCS(ZSTD_estimateCStreamSize \
+                   ZSTD_estimateDCtxSize)
+    LIBS="$save_LIBS"
+    CFLAGS="$save_CFLAGS"
+fi
+AC_SUBST(TOR_ZSTD_CFLAGS)
+AC_SUBST(TOR_ZSTD_LIBS)
+
 dnl ----------------------------------------------------------------------
 dnl Check if libcap is available for capabilities.
 
@@ -741,13 +1110,14 @@ dnl since sometimes the linker will like an option but not be willing to
 dnl use it with a build of a library.
 
 all_ldflags_for_check="$TOR_LDFLAGS_zlib $TOR_LDFLAGS_openssl $TOR_LDFLAGS_libevent"
-all_libs_for_check="$TOR_ZLIB_LIBS $TOR_LIB_MATH $TOR_LIBEVENT_LIBS $TOR_OPENSSL_LIBS $TOR_SYSTEMD_LIBS $TOR_LIB_WS32 $TOR_LIB_GDI $TOR_CAP_LIBS"
+all_libs_for_check="$TOR_ZLIB_LIBS $TOR_LIB_MATH $TOR_LIBEVENT_LIBS $TOR_OPENSSL_LIBS $TOR_SYSTEMD_LIBS $TOR_LIB_WS32 $TOR_LIB_GDI $TOR_LIB_USERENV $TOR_CAP_LIBS"
 
 CFLAGS_FTRAPV=
 CFLAGS_FWRAPV=
 CFLAGS_ASAN=
 CFLAGS_UBSAN=
 
+
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
 #if !defined(__clang__)
 #error
@@ -771,36 +1141,63 @@ m4_ifdef([AS_VAR_IF],[
     AS_VAR_POPDEF([can_compile])
     TOR_CHECK_CFLAGS(-Wstack-protector)
     TOR_CHECK_CFLAGS(--param ssp-buffer-size=1)
-    if test "$bwin32" = "false"; then
+    if test "$bwin32" = "false" && test "$enable_libfuzzer" != "yes" && test "$enable_oss_fuzz" != "yes"; then
        TOR_CHECK_CFLAGS(-fPIE)
        TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check")
     fi
     TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true)
 fi
 
-if test "x$enable_expensive_hardening" = "xyes"; then
+if test "$fragile_hardening" = "yes"; then
     TOR_TRY_COMPILE_WITH_CFLAGS(-ftrapv, also_link, CFLAGS_FTRAPV="-ftrapv", true)
    if test "$tor_cv_cflags__ftrapv" = "yes" && test "$tor_can_link__ftrapv" != "yes"; then
       AC_MSG_WARN([The compiler supports -ftrapv, but for some reason I was not able to link with -ftrapv. Are you missing run-time support? Run-time hardening will not work as well as it should.])
    fi
 
    if test "$tor_cv_cflags__ftrapv" != "yes"; then
-     AC_MSG_ERROR([You requested expensive hardening, but the compiler does not seem to support -ftrapv.])
+     AC_MSG_ERROR([You requested fragile hardening, but the compiler does not seem to support -ftrapv.])
    fi
 
    TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=address], also_link, CFLAGS_ASAN="-fsanitize=address", true)
     if test "$tor_cv_cflags__fsanitize_address" = "yes" && test "$tor_can_link__fsanitize_address" != "yes"; then
-      AC_MSG_ERROR([The compiler supports -fsanitize=address, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libubsan.so, and with Clang you need libclang_rt.ubsan*])
+      AC_MSG_ERROR([The compiler supports -fsanitize=address, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libubsan.*, and with Clang you need libclang_rt.ubsan*])
     fi
 
    TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=undefined], also_link, CFLAGS_UBSAN="-fsanitize=undefined", true)
     if test "$tor_cv_cflags__fsanitize_address" = "yes" && test "$tor_can_link__fsanitize_address" != "yes"; then
-      AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.so, and with Clang you need libclang_rt.ubsan*])
+      AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*])
     fi
 
 TOR_CHECK_CFLAGS([-fno-omit-frame-pointer])
 fi
 
+dnl Find the correct libraries to add in order to use the sanitizers.
+dnl
+dnl When building Rust, Cargo will run the linker with the -nodefaultlibs
+dnl option, which will prevent the compiler from linking the sanitizer
+dnl libraries it needs.  We need to specify them manually.
+dnl
+dnl What's more, we need to specify them in a linker script rather than
+dnl from build.rs: these options aren't allowed in the cargo:rustc-flags
+dnl variable.
+RUST_LINKER_OPTIONS=""
+if test "x$have_clang" = "xyes"; then
+	if test "x$CFLAGS_ASAN" != "x"; then
+		RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS $CFLAGS_ASAN"
+	fi
+	if test "x$CFLAGS_UBSAN" != "x"; then
+        	RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS $CFLAGS_UBSAN"
+	fi
+else
+	if test "x$CFLAGS_ASAN" != "x"; then
+		RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -lasan"
+	fi
+	if test "x$CFLAGS_UBSAN" != "x"; then
+        	RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -lubsan"
+	fi
+fi
+AC_SUBST(RUST_LINKER_OPTIONS)
+
 CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN"
 CFLAGS_CONSTTIME="$CFLAGS_FWRAPV"
 
@@ -867,7 +1264,7 @@ saved_CFLAGS="$CFLAGS"
 TOR_CHECK_CFLAGS(-fomit-frame-pointer)
 F_OMIT_FRAME_POINTER=''
 if test "$saved_CFLAGS" != "$CFLAGS"; then
-  if test "x$enable_expensive_hardening" != "xyes"; then
+  if test "$fragile_hardening" = "yes"; then
     F_OMIT_FRAME_POINTER='-fomit-frame-pointer'
   fi
 fi
@@ -1036,6 +1433,7 @@ AC_CHECK_HEADERS([assert.h \
                   pwd.h \
                   readpassphrase.h \
                   stdint.h \
+                  stdatomic.h \
                   sys/eventfd.h \
                   sys/file.h \
                   sys/ioctl.h \
@@ -1146,10 +1544,6 @@ if test "x$linux_netfilter_ipv6_ip6_tables" = "x1"; then
 fi
 if test "x$transparent_ok" = "x1"; then
   AC_DEFINE(USE_TRANSPARENT, 1, "Define to enable transparent proxy support")
-  case "$host" in
-    *-*-openbsd* | *-*-bitrig*)
-      AC_DEFINE(OPENBSD, 1, "Define to handle pf on OpenBSD properly") ;;
-  esac
 else
   AC_MSG_NOTICE([Transparent proxy support enabled, but missing headers.])
 fi
@@ -1293,6 +1687,26 @@ AC_CHECK_SIZEOF(socklen_t, , [AC_INCLUDES_DEFAULT()
 
 AC_CHECK_SIZEOF(cell_t)
 
+# Let's see if stdatomic works. (There are some debian clangs that screw it
+# up; see Tor bug #26779 and debian bug 903709.)
+AC_CACHE_CHECK([whether C11 stdatomic.h actually works],
+               tor_cv_stdatomic_works,
+[AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#include <stdatomic.h>
+struct x { atomic_size_t y; };
+void try_atomic_init(struct x *xx)
+{
+  atomic_init(&xx->y, 99);
+  atomic_fetch_add(&xx->y, 1);
+}
+]])], [tor_cv_stdatomic_works=yes], [tor_cv_stdatomic_works=no])])
+
+if test "$tor_cv_stdatomic_works" = "yes"; then
+   AC_DEFINE(STDATOMIC_WORKS, 1, [Set to 1 if we can compile a simple stdatomic example.])
+elif test "$ac_cv_header_stdatomic_h" = "yes"; then
+   AC_MSG_WARN([Your compiler provides the stdatomic.h header, but it doesn't seem to work.  I'll pretend it isn't there. If you are using Clang on Debian, maybe this is because of https://bugs.debian.org/903709 ])
+fi
+
 # Now make sure that NULL can be represented as zero bytes.
 AC_CACHE_CHECK([whether memset(0) sets pointers to NULL], tor_cv_null_is_zero,
 [AC_RUN_IFELSE([AC_LANG_SOURCE(
@@ -1404,6 +1818,24 @@ if test "$tor_cv_sign_extend" != "no"; then
             [Define to 1 iff right-shifting a negative value performs sign-extension])
 fi
 
+# Is uint8_t the same type as unsigned char?
+AC_CACHE_CHECK([whether uint8_t is the same type as unsigned char], tor_cv_uint8_uchar,
+[AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+#include <stdint.h>
+extern uint8_t c;
+unsigned char c;]])],
+       [tor_cv_uint8_uchar=yes],
+       [tor_cv_uint8_uchar=no],
+       [tor_cv_uint8_uchar=cross])])
+
+if test "$tor_cv_uint8_uchar" = "cross"; then
+  AC_MSG_NOTICE([Cross-compiling: we'll assume that uint8_t is the same type as unsigned char])
+fi
+
+if test "$tor_cv_uint8_uchar" = "no"; then
+  AC_MSG_ERROR([We assume that uint8_t is the same type as unsigned char, but your compiler disagrees.])
+fi
+
 # Whether we should use the dmalloc memory allocation debugging library.
 AC_MSG_CHECKING(whether to use dmalloc (debug memory allocation library))
 AC_ARG_WITH(dmalloc,
@@ -1476,9 +1908,9 @@ AC_CHECK_FUNC(gethostbyname_r, [
   AC_MSG_CHECKING([how many arguments gethostbyname_r() wants])
   OLD_CFLAGS=$CFLAGS
   CFLAGS="$CFLAGS $MY_CPPFLAGS $MY_THREAD_CPPFLAGS $MY_CFLAGS"
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
+  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-  ], [[
+  ]], [[
     char *cp1, *cp2;
     struct hostent *h1, *h2;
     int i1, i2;
@@ -1489,27 +1921,27 @@ AC_CHECK_FUNC(gethostbyname_r, [
      [Define this if gethostbyname_r takes 6 arguments])
     AC_MSG_RESULT(6)
   ], [
-    AC_TRY_COMPILE([
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-    ], [
+    ]], [[
       char *cp1, *cp2;
       struct hostent *h1;
       int i1, i2;
       (void)gethostbyname_r(cp1,h1,cp2,i1,&i2);
-    ], [
+    ]])], [
       AC_DEFINE(HAVE_GETHOSTBYNAME_R)
       AC_DEFINE(HAVE_GETHOSTBYNAME_R_5_ARG, 1,
         [Define this if gethostbyname_r takes 5 arguments])
       AC_MSG_RESULT(5)
    ], [
-      AC_TRY_COMPILE([
+      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-     ], [
+     ]], [[
        char *cp1;
        struct hostent *h1;
        struct hostent_data hd;
        (void) gethostbyname_r(cp1,h1,&hd);
-     ], [
+     ]])], [
        AC_DEFINE(HAVE_GETHOSTBYNAME_R)
        AC_DEFINE(HAVE_GETHOSTBYNAME_R_3_ARG, 1,
          [Define this if gethostbyname_r takes 3 arguments])
@@ -1594,6 +2026,12 @@ AC_SUBST(BUILDDIR)
 AH_TEMPLATE([BUILDDIR],[tor's build directory])
 AC_DEFINE_UNQUOTED(BUILDDIR,"$BUILDDIR")
 
+if test "x$SRCDIR" = "x"; then
+  SRCDIR=$(cd "$srcdir"; pwd)
+fi
+AH_TEMPLATE([SRCDIR],[tor's sourcedir directory])
+AC_DEFINE_UNQUOTED(SRCDIR,"$SRCDIR")
+
 if test "x$CONFDIR" = "x"; then
   CONFDIR=`eval echo $sysconfdir/tor`
 fi
@@ -1775,7 +2213,6 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
      -Winvalid-source-encoding
      -Winvalid-token-paste
      -Wknr-promoted-parameter
-     -Wlanguage-extension-token
      -Wlarge-by-value-copy
      -Wliteral-conversion
      -Wliteral-range
@@ -1801,7 +2238,7 @@ if test "x$enable_gcc_warnings_advisory" != "xno"; then
      -Wnon-literal-null-conversion
      -Wnon-pod-varargs
      -Wnonportable-cfstrings
-     -Wnormalized=id
+     -Wnormalized=nfkc
      -Wnull-arithmetic
      -Wnull-character
      -Wnull-conversion
@@ -1913,6 +2350,9 @@ dnl     -Wthread-safety-precise
   if test "$tor_cv_cflags__Woverlength_strings" = "yes"; then
     AC_DEFINE([HAVE_CFLAG_WOVERLENGTH_STRINGS], 1, [True if we have -Woverlength-strings])
   fi
+  if test "$tor_cv_cflags__warn_unused_const_variable_2" = "yes"; then
+    AC_DEFINE([HAVE_CFLAG_WUNUSED_CONST_VARIABLE], 1, [True if we have -Wunused-const-variable])
+  fi
 
   if test "x$enable_fatal_warnings" = "xyes"; then
     # I'd like to use TOR_CHECK_CFLAGS here, but I can't, since the
@@ -1934,6 +2374,8 @@ CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_z
 AC_CONFIG_FILES([
         Doxyfile
         Makefile
+        config.rust
+        link_rust.sh
         contrib/dist/suse/tor.sh
         contrib/operator-tools/tor.logrotate
         contrib/dist/tor.sh
@@ -1941,6 +2383,7 @@ AC_CONFIG_FILES([
         contrib/dist/tor.service
         src/config/torrc.sample
         src/config/torrc.minimal
+        src/rust/.cargo/config
         scripts/maint/checkOptionDocs.pl
         scripts/maint/updateVersions.pl
 ])
@@ -1963,4 +2406,19 @@ if test "x$asciidoc" = "xtrue" && test "$ASCIIDOC" = "none"; then
   done
 fi
 
+if test "$fragile_hardening" = "yes"; then
+  AC_MSG_WARN([
+
+============
+Warning!  Building Tor with --enable-fragile-hardening (also known as
+--enable-expensive-hardening) makes some kinds of attacks harder, but makes
+other kinds of attacks easier. A Tor instance build with this option will be
+somewhat less vulnerable to remote code execution, arithmetic overflow, or
+out-of-bounds read/writes... but at the cost of becoming more vulnerable to
+denial of service attacks. For more information, see
+https://trac.torproject.org/projects/tor/wiki/doc/TorFragileHardening
+============
+  ])
+fi
+
 AC_OUTPUT
diff --git a/contrib/operator-tools/linux-tor-prio.sh b/contrib/operator-tools/linux-tor-prio.sh
index ea9e0ddaa5..30ea5fc659 100644
--- a/contrib/operator-tools/linux-tor-prio.sh
+++ b/contrib/operator-tools/linux-tor-prio.sh
@@ -87,7 +87,7 @@ RATE_UP=5000
 # machine does any other network activity. That is not very fun.
 RATE_UP_TOR=1500
 
-# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor trafic in
+# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor traffic in
 # kbits/sec.
 RATE_UP_TOR_CEIL=5000
 
diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in
index 452720516e..18bc2a62ca 100644
--- a/contrib/win32build/tor-mingw.nsi.in
+++ b/contrib/win32build/tor-mingw.nsi.in
@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.2.9.17-dev"
+!define VERSION "0.3.4.11-dev"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"
diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md
index f1c65850a4..3711f70198 100644
--- a/doc/HACKING/CodingStandards.md
+++ b/doc/HACKING/CodingStandards.md
@@ -4,9 +4,10 @@ Coding conventions for Tor
 tl;dr:
 
    - Run configure with `--enable-fatal-warnings`
-   - Run `make check-spaces` to catch whitespace errors
    - Document your functions
    - Write unit tests
+   - Run `make check` before submitting a patch
+   - Run `make distcheck` if you have made changes to build system components
    - Add a file in `changes` for your branch.
 
 Patch checklist
@@ -22,13 +23,42 @@ preference)
 Did you remember...
 
    - To build your code while configured with `--enable-fatal-warnings`?
-   - To run `make check-spaces` on your code?
    - To run `make check-docs` to see whether all new options are on
      the manpage?
    - To write unit tests, as possible?
+   - To run `make test-full` to test against all unit and integration tests (or
+     `make test-full-online` if you have a working connection to the internet)?
+   - To test that the distribution will actually work via `make distcheck`?
    - To base your code on the appropriate branch?
    - To include a file in the `changes` directory as appropriate?
 
+If you are submitting a major patch or new feature, or want to in the future...
+
+   - Set up Chutney and Stem, see HACKING/WritingTests.md
+   - Run `make test-full` to test against all unit and integration tests.
+
+If you have changed build system components:
+   - Please run `make distcheck`
+   - For example, if you have changed Makefiles, autoconf files, or anything
+     else that affects the build system.
+
+License issues
+==============
+
+Tor is distributed under the license terms in the LICENSE -- in
+brief, the "3-clause BSD license".  If you send us code to
+distribute with Tor, it needs to be code that we can distribute
+under those terms.  Please don't send us patches unless you agree
+to allow this.
+
+Some compatible licenses include:
+
+  - 3-clause BSD
+  - 2-clause BSD
+  - CC0 Public Domain Dedication
+
+
+
 How we use Git branches
 =======================
 
@@ -49,8 +79,17 @@ before it gets merged into maint, but that's rare.
 
 If you're working on a bugfix for a bug that occurs in a particular version,
 base your bugfix branch on the "maint" branch for the first supported series
-that has that bug.  (As of June 2013, we're supporting 0.2.3 and later.) If
-you're working on a new feature, base it on the master branch.
+that has that bug.  (As of June 2013, we're supporting 0.2.3 and later.)
+
+If you're working on a new feature, base it on the master branch. If you're
+working on a new feature and it will take a while to implement and/or you'd
+like to avoid the possibility of unrelated bugs in Tor while you're
+implementing your feature, consider branching off of the latest maint- branch.
+_Never_ branch off a relase- branch. Don't branch off a tag either: they come
+from release branches. Doing so will likely produce a nightmare of merge
+conflicts in the ChangeLog when it comes time to merge your branch into Tor.
+Best advice: don't try to keep an independent branch forked for more than 6
+months and expect it to merge cleanly. Try to merge pieces early and often.
 
 
 How we log changes
@@ -74,17 +113,34 @@ you can use `git describe --contains <sha1 of commit>`.
 If at all possible, try to create this file in the same commit where you are
 making the change.  Please give it a distinctive name that no other branch will
 use for the lifetime of your change. To verify the format of the changes file,
-you can use `make check-changes`.
+you can use `make check-changes`.  This is run automatically as part of
+`make check` -- if it fails, we must fix it before we release.  These
+checks are implemented in `scripts/maint/lintChanges.py`.
+
+Changes file style guide:
+  * Changes files begin with "  o Header (subheading):".  The header
+    should usually be "Minor/Major bugfixes/features". The subheading is a
+    particular area within Tor.  See the ChangeLog for examples.
+
+  * Make everything terse.
+
+  * Write from the user's point of view: describe the user-visible changes
+    right away.
+
+  * Mention configuration options by name.  If they're rare or unusual,
+    remind people what they're for.
+
+  * Describe changes in the present tense and in the imperative: not past.
+
+  * Every bugfix should have a sentence of the form "Fixes bug 1234; bugfix
+    on 0.1.2.3-alpha", describing what bug was fixed and where it came from.
+
+  * "Relays", not "servers", "nodes", or "Tor relays".
 
 When we go to make a release, we will concatenate all the entries
 in changes to make a draft changelog, and clear the directory. We'll
 then edit the draft changelog into a nice readable format.
 
-To make sure that stuff is in the right format, we use
-scripts/maint/lintChanges.py to check the changes files for
-(superficial) validity.  You can run this script on your own changes
-files!
-
 What needs a changes file?
 
    * A not-exhaustive list: Anything that might change user-visible
@@ -93,6 +149,10 @@ What needs a changes file?
    rewrites.  Anything about which somebody might plausibly wonder "when
    did that happen, and/or why did we do that" 6 months down the line.
 
+What does not need a changes file?
+
+   * Bugfixes for code that hasn't shipped in any released version of Tor
+
 Why use changes files instead of Git commit messages?
 
    * Git commit messages are written for developers, not users, and they
@@ -149,6 +209,79 @@ old C functions.  Use `strlcat`, `strlcpy`, or `tor_snprintf/tor_asprintf` inste
 We don't call `memcmp()` directly.  Use `fast_memeq()`, `fast_memneq()`,
 `tor_memeq()`, or `tor_memneq()` for most purposes.
 
+Also see a longer list of functions to avoid in:
+https://people.torproject.org/~nickm/tor-auto/internal/this-not-that.html
+
+Floating point math is hard
+---------------------------
+
+Floating point arithmetic as typically implemented by computers is
+very counterintuitive.  Failure to adequately analyze floating point
+usage can result in surprising behavior and even security
+vulnerabilities!
+
+General advice:
+
+   - Don't use floating point.
+   - If you must use floating point, document how the limits of
+     floating point precision and calculation accuracy affect function
+     outputs.
+   - Try to do as much as possible of your calculations using integers
+     (possibly acting as fixed-point numbers) and convert to floating
+     point for display.
+   - If you must send floating point numbers on the wire, serialize
+     them in a platform-independent way.  Tor avoids exchanging
+     floating-point values, but when it does, it uses ASCII numerals,
+     with a decimal point (".").
+   - Binary fractions behave very differently from decimal fractions.
+     Make sure you understand how these differences affect your
+     calculations.
+   - Every floating point arithmetic operation is an opportunity to
+     lose precision, overflow, underflow, or otherwise produce
+     undesired results.  Addition and subtraction tend to be worse
+     than multiplication and division (due to things like catastrophic
+     cancellation).  Try to arrange your calculations to minimize such
+     effects.
+   - Changing the order of operations changes the results of many
+     floating-point calculations.  Be careful when you simplify
+     calculations!  If the order is significant, document it using a
+     code comment.
+   - Comparing most floating point values for equality is unreliable.
+     Avoid using `==`, instead, use `>=` or `<=`.  If you use an
+     epsilon value, make sure it's appropriate for the ranges in
+     question.
+   - Different environments (including compiler flags and per-thread
+     state on a single platform!) can get different results from the
+     same floating point calculations.  This means you can't use
+     floats in anything that needs to be deterministic, like consensus
+     generation.  This also makes reliable unit tests of
+     floating-point outputs hard to write.
+
+For additional useful advice (and a little bit of background), see
+[What Every Programmer Should Know About Floating-Point
+Arithmetic](http://floating-point-gui.de/).
+
+A list of notable (and surprising) facts about floating point
+arithmetic is at [Floating-point
+complexities](https://randomascii.wordpress.com/2012/04/05/floating-point-complexities/).
+Most of that [series of posts on floating
+point](https://randomascii.wordpress.com/category/floating-point/) is
+helpful.
+
+For more detailed (and math-intensive) background, see [What Every
+Computer Scientist Should Know About Floating-Point
+Arithmetic](https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html).
+
+Other C conventions
+-------------------
+
+The `a ? b : c` trinary operator only goes inside other expressions;
+don't use it as a replacement for if. (You can ignore this inside macro
+definitions when necessary.)
+
+Assignment operators shouldn't nest inside other expressions.  (You can
+ignore this inside macro definitions when necessary.)
+
 Functions not to write
 ----------------------
 
@@ -210,6 +343,64 @@ end-users that they aren't expected to understand the message (perhaps
 with a string like "internal error"). Option (A) is to be preferred to
 option (B).
 
+Assertions In Tor
+-----------------
+
+Assertions should be used for bug-detection only.  Don't use assertions to
+detect bad user inputs, network errors, resource exhaustion, or similar
+issues.
+
+Tor is always built with assertions enabled, so try to only use
+`tor_assert()` for cases where you are absolutely sure that crashing is the
+least bad option.  Many bugs have been caused by use of `tor_assert()` when
+another kind of check would have been safer.
+
+If you're writing an assertion to test for a bug that you _can_ recover from,
+use `tor_assert_nonfatal()` in place of `tor_assert()`.  If you'd like to
+write a conditional that incorporates a nonfatal assertion, use the `BUG()`
+macro, as in:
+
+	if (BUG(ptr == NULL))
+		return -1;
+
+Allocator conventions
+---------------------
+
+By convention, any tor type with a name like `abc_t` should be allocated
+by a function named `abc_new()`.  This function should never return
+NULL.
+
+Also, a type named `abc_t` should be freed by a function named `abc_free_()`.
+Don't call this `abc_free_()` function directly -- instead, wrap it in a
+macro called `abc_free()`, using the `FREE_AND_NULL` macro:
+
+    void abc_free_(abc_t *obj);
+    #define abc_free(obj) FREE_AND_NULL(abc_t, abc_free_, (obj))
+
+This macro will free the underlying `abc_t` object, and will also set
+the object pointer to NULL.
+
+You should define all `abc_free_()` functions to accept NULL inputs:
+
+    void
+    abc_free_(abc_t *obj)
+    {
+      if (!obj)
+        return;
+      tor_free(obj->name);
+      thing_free(obj->thing);
+      tor_free(obj);
+    }
+
+If you need a free function that takes a `void *` argument (for example,
+to use it as a function callback), define it with a name like
+`abc_free_void()`:
+
+    static void
+    abc_free_void_(void *obj)
+    {
+      abc_free_(obj);
+    }
 
 
 Doxygen comment conventions
@@ -243,3 +434,4 @@ the functions that call your function rely on it doing something, then your
 function should mention that it does that something in the documentation.  If
 you rely on a function doing something beyond what is in its documentation,
 then you should watch out, or it might do something else later.
+
diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md
new file mode 100644
index 0000000000..d9496c08f7
--- /dev/null
+++ b/doc/HACKING/CodingStandardsRust.md
@@ -0,0 +1,523 @@
+
+ Rust Coding Standards
+=======================
+
+You MUST follow the standards laid out in `.../doc/HACKING/CodingStandards.md`,
+where applicable.
+
+ Module/Crate Declarations
+---------------------------
+
+Each Tor C module which is being rewritten MUST be in its own crate.
+See the structure of `.../src/rust` for examples.
+
+In your crate, you MUST use `lib.rs` ONLY for pulling in external
+crates (e.g. `extern crate libc;`) and exporting public objects from
+other Rust modules (e.g. `pub use mymodule::foo;`).  For example, if
+you create a crate in `.../src/rust/yourcrate`, your Rust code should
+live in `.../src/rust/yourcrate/yourcode.rs` and the public interface
+to it should be exported in `.../src/rust/yourcrate/lib.rs`.
+
+If your code is to be called from Tor C code, you MUST define a safe
+`ffi.rs`.  See the "Safety" section further down for more details.
+
+For example, in a hypothetical `tor_addition` Rust module:
+
+In `.../src/rust/tor_addition/addition.rs`:
+
+    pub fn get_sum(a: i32, b: i32) -> i32 {
+        a + b
+    }
+
+In `.../src/rust/tor_addition/lib.rs`:
+
+    pub use addition::*;
+
+In `.../src/rust/tor_addition/ffi.rs`:
+
+    #[no_mangle]
+    pub extern "C" fn tor_get_sum(a: c_int, b: c_int) -> c_int {
+        get_sum(a, b)
+    }
+
+If your Rust code must call out to parts of Tor's C code, you must
+declare the functions you are calling in the `external` crate, located
+at `.../src/rust/external`.
+
+<!-- XXX get better examples of how to declare these externs, when/how they -->
+<!-- XXX are unsafe, what they are expected to do —isis                     -->
+
+Modules should strive to be below 500 lines (tests excluded). Single
+responsibility and limited dependencies should be a guiding standard.
+
+If you have any external modules as dependencies (e.g. `extern crate
+libc;`), you MUST declare them in your crate's `lib.rs` and NOT in any
+other module.
+
+ Dependencies and versions
+---------------------------
+
+In general, we use modules from only the Rust standard library
+whenever possible. We will review including external crates on a
+case-by-case basis.
+
+If a crate only contains traits meant for compatibility between Rust
+crates, such as [the digest crate](https://crates.io/crates/digest) or
+[the failure crate](https://crates.io/crates/failure), it is very likely
+permissible to add it as a dependency.  However, a brief review should
+be conducted as to the usefulness of implementing external traits
+(i.e. how widespread is the usage, how many other crates either
+implement the traits or have trait bounds based upon them), as well as
+the stability of the traits (i.e. if the trait is going to change, we'll
+potentially have to re-do all our implementations of it).
+
+For large external libraries, especially which implement features which
+would be labour-intensive to reproduce/maintain ourselves, such as
+cryptographic or mathematical/statistics libraries, only crates which
+have stabilised to 1.0.0 should be considered, however, again, we may
+make exceptions on a case-by-case basis.
+
+Currently, Tor requires that you use the latest stable Rust version. At
+some point in the future, we will freeze on a given stable Rust version,
+to ensure backward compatibility with stable distributions that ship it.
+
+ Updating/Adding Dependencies
+------------------------------
+
+To add/remove/update dependencies, first add your dependencies,
+exactly specifying their versions, into the appropriate *crate-level*
+`Cargo.toml` in `src/rust/` (i.e. *not* `/src/rust/Cargo.toml`, but
+instead the one for your crate).  Also, investigate whether your
+dependency has any optional dependencies which are unnecessary but are
+enabled by default.  If so, you'll likely be able to enable/disable
+them via some feature, e.g.:
+
+```toml
+[dependencies]
+foo = { version = "1.0.0", default-features = false }
+```
+
+Next, run `/scripts/maint/updateRustDependencies.sh`.  Then, go into
+`src/ext/rust` and commit the changes to the `tor-rust-dependencies`
+repo.
+
+ Documentation
+---------------
+
+You MUST include `#[deny(missing_docs)]` in your crate.
+
+For function/method comments, you SHOULD include a one-sentence, "first person"
+description of function behaviour (see requirements for documentation as
+described in `.../src/HACKING/CodingStandards.md`), then an `# Inputs` section
+for inputs or initialisation values, a `# Returns` section for return
+values/types, a `# Warning` section containing warnings for unsafe behaviours or
+panics that could happen.  For publicly accessible
+types/constants/objects/functions/methods, you SHOULD also include an
+`# Examples` section with runnable doctests.
+
+You MUST document your module with _module docstring_ comments,
+i.e. `//!` at the beginning of each line.
+
+ Style
+-------
+
+You SHOULD consider breaking up large literal numbers with `_` when it makes it
+more human readable to do so, e.g. `let x: u64 = 100_000_000_000`.
+
+ Testing
+---------
+
+All code MUST be unittested and integration tested.
+
+Public functions/objects exported from a crate SHOULD include doctests
+describing how the function/object is expected to be used.
+
+Integration tests SHOULD go into a `tests/` directory inside your
+crate.  Unittests SHOULD go into their own module inside the module
+they are testing, e.g. in `.../src/rust/tor_addition/addition.rs` you
+should put:
+
+    #[cfg(test)]
+    mod test {
+        use super::*;
+
+        #[test]
+        fn addition_with_zero() {
+            let sum: i32 = get_sum(5i32, 0i32);
+            assert_eq!(sum, 5);
+        }
+    }
+
+ Benchmarking
+--------------
+
+The external `test` crate can be used for most benchmarking.  However, using
+this crate requires nightly Rust.  Since we may want to switch to a more
+stable Rust compiler eventually, we shouldn't do things which will automatically
+break builds for stable compilers.  Therefore, you MUST feature-gate your
+benchmarks in the following manner.
+
+If you wish to benchmark some of your Rust code, you MUST put the
+following in the `[features]` section of your crate's `Cargo.toml`:
+
+    [features]
+    bench = []
+
+Next, in your crate's `lib.rs` you MUST put:
+
+    #[cfg(all(test, feature = "bench"))]
+    extern crate test;
+
+This ensures that the external crate `test`, which contains utilities
+for basic benchmarks, is only used when running benchmarks via `cargo
+bench --features bench`.
+
+Finally, to write your benchmark code, in
+`.../src/rust/tor_addition/addition.rs` you SHOULD put:
+
+    #[cfg(all(test, features = "bench"))]
+    mod bench {
+        use test::Bencher;
+        use super::*;
+
+        #[bench]
+        fn addition_small_integers(b: &mut Bencher) {
+            b.iter(| | get_sum(5i32, 0i32));
+        }
+    }
+
+ Fuzzing
+---------
+
+If you wish to fuzz parts of your code, please see the
+[`cargo fuzz`](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses
+[libfuzzer-sys](https://github.com/rust-fuzz/libfuzzer-sys).
+
+ Whitespace & Formatting
+-------------------------
+
+You MUST run `rustfmt` (https://github.com/rust-lang-nursery/rustfmt)
+on your code before your code will be merged.  You can install rustfmt
+by doing `cargo install rustfmt-nightly` and then run it with `cargo
+fmt`.
+
+ Safety
+--------
+
+You SHOULD read [the nomicon](https://doc.rust-lang.org/nomicon/) before writing
+Rust FFI code.  It is *highly advised* that you read and write normal Rust code
+before attempting to write FFI or any other unsafe code.
+
+Here are some additional bits of advice and rules:
+
+0. Any behaviours which Rust considers to be undefined are forbidden
+
+   From https://doc.rust-lang.org/reference/behavior-considered-undefined.html:
+
+   > Behavior considered undefined
+   >
+   > The following is a list of behavior which is forbidden in all Rust code,
+   > including within unsafe blocks and unsafe functions. Type checking provides the
+   > guarantee that these issues are never caused by safe code.
+   > 
+   > * Data races
+   > * Dereferencing a null/dangling raw pointer
+   > * Reads of [undef](http://llvm.org/docs/LangRef.html#undefined-values)
+   >   (uninitialized) memory
+   > * Breaking the
+   >   [pointer aliasing rules](http://llvm.org/docs/LangRef.html#pointer-aliasing-rules)
+   >   with raw pointers (a subset of the rules used by C)
+   > * `&mut T` and `&T` follow LLVM’s scoped noalias model, except if the `&T`
+   >   contains an `UnsafeCell<U>`. Unsafe code must not violate these aliasing
+   >   guarantees.
+   > * Mutating non-mutable data (that is, data reached through a shared
+   >   reference or data owned by a `let` binding), unless that data is
+   >   contained within an `UnsafeCell<U>`.
+   > * Invoking undefined behavior via compiler intrinsics:
+   >     - Indexing outside of the bounds of an object with
+   >       `std::ptr::offset` (`offset` intrinsic), with the exception of
+   >       one byte past the end which is permitted.
+   >     - Using `std::ptr::copy_nonoverlapping_memory` (`memcpy32`/`memcpy64`
+   >       intrinsics) on overlapping buffers
+   > * Invalid values in primitive types, even in private fields/locals:
+   >     - Dangling/null references or boxes
+   >     - A value other than `false` (0) or `true` (1) in a `bool`
+   >     - A discriminant in an `enum` not included in the type definition
+   >     - A value in a `char` which is a surrogate or above `char::MAX`
+   >     - Non-UTF-8 byte sequences in a `str`
+   > * Unwinding into Rust from foreign code or unwinding from Rust into foreign
+   >   code. Rust's failure system is not compatible with exception handling in other
+   >   languages. Unwinding must be caught and handled at FFI boundaries.
+
+1. `unwrap()`
+
+   If you call `unwrap()`, anywhere, even in a test, you MUST include
+   an inline comment stating how the unwrap will either 1) never fail,
+   or 2) should fail (i.e. in a unittest).
+
+   You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the
+   potential error with either `expect()` or the eel operator, `?`.
+   For example, consider a function which parses a string into an integer:
+
+        fn parse_port_number(config_string: &str) -> u16 {
+            u16::from_str_radix(config_string, 10).unwrap()
+        }
+
+   There are numerous ways this can fail, and the `unwrap()` will cause the
+   whole program to byte the dust!  Instead, either you SHOULD use `expect()`
+   (or another equivalent function which will return an `Option` or a `Result`)
+   and change the return type to be compatible:
+
+        fn parse_port_number(config_string: &str) -> Option<u16> {
+            u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16")
+        }
+
+   or you SHOULD use `or()` (or another similar method):
+
+        fn parse_port_number(config_string: &str) -> Option<u16> {
+            u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16")
+        }
+
+   Using methods like `or()` can be particularly handy when you must do
+   something afterwards with the data, for example, if we wanted to guarantee
+   that the port is high.  Combining these methods with the eel operator (`?`)
+   makes this even easier:
+
+        fn parse_port_number(config_string: &str) -> Result<u16, Err> {
+            let port = u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16"))?;
+
+            if port > 1024 {
+                return Ok(port);
+            } else {
+                return Err("Low ports not allowed");
+            }
+        }
+
+2. `unsafe`
+
+   If you use `unsafe`, you MUST describe a contract in your
+   documentation which describes how and when the unsafe code may
+   fail, and what expectations are made w.r.t. the interfaces to
+   unsafe code.  This is also REQUIRED for major pieces of FFI between
+   C and Rust.
+
+   When creating an FFI in Rust for C code to call, it is NOT REQUIRED
+   to declare the entire function `unsafe`.  For example, rather than doing:
+
+        #[no_mangle]
+        pub unsafe extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 {
+            for number in &mut numbers {
+                *number += 1;
+            }
+            std::mem::transmute::<[u8; 4], u32>(numbers)
+        }
+
+   You SHOULD instead do:
+
+        #[no_mangle]
+        pub extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 {
+            for index in 0..numbers.len() {
+                numbers[index] += 1;
+            }
+            unsafe {
+                std::mem::transmute::<[u8; 4], u32>(numbers)
+            }
+        }
+
+3. Pass only C-compatible primitive types and bytes over the boundary
+
+   Rust's C-compatible primitive types are integers and floats.
+   These types are declared in the [libc crate](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types).
+   Most Rust objects have different [representations](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types)
+   in C and Rust, so they can't be passed using FFI.
+
+   Tor currently uses the following Rust primitive types from libc for FFI:
+   * defined-size integers: `uint32_t`
+   * native-sized integers: `c_int`
+   * native-sized floats: `c_double`
+   * native-sized raw pointers: `* c_void`, `* c_char`, `** c_char`
+
+   TODO: C smartlist to Stringlist conversion using FFI
+
+   The only non-primitive type which may cross the FFI boundary is
+   bytes, e.g. `&[u8]`.  This SHOULD be done on the Rust side by
+   passing a pointer (`*mut libc::c_char`). The length can be passed
+   explicitly (`libc::size_t`), or the string can be NUL-byte terminated
+   C string.
+
+   One might be tempted to do this via doing
+   `CString::new("blah").unwrap().into_raw()`. This has several problems:
+
+   a) If you do `CString::new("bl\x00ah")` then the unwrap() will fail
+      due to the additional NULL terminator, causing a dangling
+      pointer to be returned (as well as a potential use-after-free).
+
+   b) Returning the raw pointer will cause the CString to run its deallocator,
+      which causes any C code which tries to access the contents to dereference a
+      NULL pointer.
+
+   c) If we were to do `as_raw()` this would result in a potential double-free
+      since the Rust deallocator would run and possibly Tor's deallocator.
+
+   d) Calling `into_raw()` without later using the same pointer in Rust to call
+      `from_raw()` and then deallocate in Rust can result in a
+      [memory leak](https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw).
+
+      [It was determined](https://github.com/rust-lang/rust/pull/41074) that this
+      is safe to do if you use the same allocator in C and Rust and also specify
+      the memory alignment for CString (except that there is no way to specify
+      the alignment for CString).  It is believed that the alignment is always 1,
+      which would mean it's safe to dealloc the resulting `*mut c_char` in Tor's
+      C code.  However, the Rust developers are not willing to guarantee the
+      stability of, or a contract for, this behaviour, citing concerns that this
+      is potentially extremely and subtly unsafe.
+
+4. Perform an allocation on the other side of the boundary
+
+   After crossing the boundary, the other side MUST perform an
+   allocation to copy the data and is therefore responsible for
+   freeing that memory later.
+
+5. No touching other language's enums
+
+   Rust enums should never be touched from C (nor can they be safely
+   `#[repr(C)]`) nor vice versa:
+
+   >  "The chosen size is the default enum size for the target platform's C
+   >  ABI. Note that enum representation in C is implementation defined, so this is
+   >  really a "best guess". In particular, this may be incorrect when the C code
+   >  of interest is compiled with certain flags."
+
+   (from https://gankro.github.io/nomicon/other-reprs.html)
+
+6. Type safety
+
+   Wherever possible and sensical, you SHOULD create new types in a
+   manner which prevents type confusion or misuse.  For example,
+   rather than using an untyped mapping between strings and integers
+   like so:
+
+        use std::collections::HashMap;
+
+        pub fn get_elements_with_over_9000_points(map: &HashMap<String, usize>) -> Vec<String> {
+            ...
+        }
+
+   It would be safer to define a new type, such that some other usage
+   of `HashMap<String, usize>` cannot be confused for this type:
+
+        pub struct DragonBallZPowers(pub HashMap<String, usize>);
+
+        impl DragonBallZPowers {
+            pub fn over_nine_thousand<'a>(&'a self) -> Vec<&'a String> {
+                let mut powerful_enough: Vec<&'a String> = Vec::with_capacity(5);
+
+                for (character, power) in &self.0 {
+                    if *power > 9000 {
+                        powerful_enough.push(character);
+                    }
+                }
+                powerful_enough
+            }
+        }
+
+   Note the following code, which uses Rust's type aliasing, is valid
+   but it does NOT meet the desired type safety goals:
+
+        pub type Power = usize;
+
+        pub fn over_nine_thousand(power: &Power) -> bool {
+            if *power > 9000 {
+                return true;
+            }
+            false
+        }
+
+        // We can still do the following:
+        let his_power: usize = 9001;
+        over_nine_thousand(&his_power);
+
+7. Unsafe mucking around with lifetimes
+
+   Because lifetimes are technically, in type theory terms, a kind, i.e. a
+   family of types, individual lifetimes can be treated as types.  For example,
+   one can arbitrarily extend and shorten lifetime using `std::mem::transmute`:
+
+        struct R<'a>(&'a i32);
+
+        unsafe fn extend_lifetime<'b>(r: R<'b>) -> R<'static> {
+            std::mem::transmute::<R<'b>, R<'static>>(r)
+        }
+
+        unsafe fn shorten_invariant_lifetime<'b, 'c>(r: &'b mut R<'static>) -> &'b mut R<'c> {
+            std::mem::transmute::<&'b mut R<'static>, &'b mut R<'c>>(r)
+        }
+
+   Calling `extend_lifetime()` would cause an `R` passed into it to live forever
+   for the life of the program (the `'static` lifetime).  Similarly,
+   `shorten_invariant_lifetime()` could be used to take something meant to live
+   forever, and cause it to disappear!  This is incredibly unsafe.  If you're
+   going to be mucking around with lifetimes like this, first, you better have
+   an extremely good reason, and second, you may as be honest and explicit about
+   it, and for ferris' sake just use a raw pointer.
+
+   In short, just because lifetimes can be treated like types doesn't mean you
+   should do it.
+
+8. Doing excessively unsafe things when there's a safer alternative
+
+   Similarly to #7, often there are excessively unsafe ways to do a task and a
+   simpler, safer way.  You MUST choose the safer option where possible.
+
+   For example, `std::mem::transmute` can be abused in ways where casting with
+   `as` would be both simpler and safer:
+
+        // Don't do this
+        let ptr = &0;
+        let ptr_num_transmute = unsafe { std::mem::transmute::<&i32, usize>(ptr)};
+
+        // Use an `as` cast instead
+        let ptr_num_cast = ptr as *const i32 as usize;
+
+   In fact, using `std::mem::transmute` for *any* reason is a code smell and as
+   such SHOULD be avoided.
+
+9. Casting integers with `as`
+
+   This is generally fine to do, but it has some behaviours which you should be
+   aware of.  Casting down chops off the high bits, e.g.:
+
+        let x: u32 = 4294967295;
+        println!("{}", x as u16); // prints 65535
+
+   Some cases which you MUST NOT do include:
+
+   * Casting an `u128` down to an `f32` or vice versa (e.g.
+     `u128::MAX as f32` but this isn't only a problem with overflowing
+     as it is also undefined behaviour for `42.0f32 as u128`),
+
+   * Casting between integers and floats when the thing being cast
+     cannot fit into the type it is being casted into, e.g.:
+
+         println!("{}", 42949.0f32 as u8); // prints 197 in debug mode and 0 in release
+         println!("{}", 1.04E+17 as u8);   // prints 0 in both modes
+         println!("{}", (0.0/0.0) as i64); // prints whatever the heck LLVM wants
+
+     Because this behaviour is undefined, it can even produce segfaults in
+     safe Rust code.  For example, the following program built in release
+     mode segfaults:
+
+         #[inline(never)]
+         pub fn trigger_ub(sl: &[u8; 666]) -> &[u8] {
+             // Note that the float is out of the range of `usize`, invoking UB when casting.
+             let idx = 1e99999f64 as usize;
+             &sl[idx..] // The bound check is elided due to `idx` being of an undefined value.
+         }
+
+         fn main() {
+             println!("{}", trigger_ub(&[1; 666])[999999]); // ~ out of bound
+         }
+
+      And in debug mode panics with:
+
+         thread 'main' panicked at 'slice index starts at 140721821254240 but ends at 666', /checkout/src/libcore/slice/mod.rs:754:4
diff --git a/doc/HACKING/Fuzzing.md b/doc/HACKING/Fuzzing.md
new file mode 100644
index 0000000000..2039d6a4c0
--- /dev/null
+++ b/doc/HACKING/Fuzzing.md
@@ -0,0 +1,123 @@
+= Fuzzing Tor
+
+== The simple version (no fuzzing, only tests)
+
+Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place
+where you checked it out.
+
+To run the fuzzing test cases in a deterministic fashion, use:
+      make test-fuzz-corpora
+
+This won't actually fuzz Tor!  It will just run all the fuzz binaries
+on our existing set of testcases for the fuzzer.
+
+
+== Different kinds of fuzzing
+
+Right now we support three different kinds of fuzzer.
+
+First, there's American Fuzzy Lop (AFL), a fuzzer that works by forking
+a target binary and passing it lots of different inputs on stdin.  It's the
+trickiest one to set up, so I'll be describing it more below.
+
+Second, there's libFuzzer, a llvm-based fuzzer that you link in as a library,
+and it runs a target function over and over.  To use this one, you'll need to
+have a reasonably recent clang and libfuzzer installed.  At that point, you
+just build with --enable-expensive-hardening and --enable-libfuzzer.  That
+will produce a set of binaries in src/test/fuzz/lf-fuzz-* .  These programs
+take as input a series of directories full of fuzzing examples.  For more
+information on libfuzzer, see http://llvm.org/docs/LibFuzzer.html
+
+Third, there's Google's OSS-Fuzz infrastructure, which expects to get all of
+its.  For more on this, see https://github.com/google/oss-fuzz and the
+projects/tor subdirectory.  You'll need to mess around with Docker a bit to
+test this one out; it's meant to run on Google's infrastructure.
+
+In all cases, you'll need some starting examples to give the fuzzer when it
+starts out.  There's a set in the "fuzzing-corpora" git repository.  Try
+setting TOR_FUZZ_CORPORA to point to a checkout of that repository
+
+== Writing Tor fuzzers
+
+A tor fuzzing harness should have:
+* a fuzz_init() function to set up any necessary global state.
+* a fuzz_main() function to receive input and pass it to a parser.
+* a fuzz_cleanup() function to clear global state.
+
+Most fuzzing frameworks will produce many invalid inputs - a tor fuzzing
+harness should rejecting invalid inputs without crashing or behaving badly.
+
+But the fuzzing harness should crash if tor fails an assertion, triggers a
+bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect
+"interesting" cases.
+
+
+== Guided Fuzzing with AFL
+
+There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so
+its integrity can't be verified. That said, you really shouldn't fuzz on a
+machine you care about, anyway.
+
+To Build:
+  Get AFL from http://lcamtuf.coredump.cx/afl/ and unpack it
+  cd afl
+  make
+  cd ../tor
+  PATH=$PATH:../afl/ CC="../afl/afl-gcc" ./configure --enable-expensive-hardening
+  AFL_HARDEN=1 make clean fuzzers
+
+To Find The ASAN Memory Limit: (64-bit only)
+
+On 64-bit platforms, afl needs to know how much memory ASAN uses,
+because ASAN tends to allocate a ridiculous amount of virtual memory,
+and then not actually use it.
+
+Read afl/docs/notes_for_asan.txt for more details.
+
+  Download recidivm from http://jwilk.net/software/recidivm
+  Download the signature
+  Check the signature
+  tar xvzf recidivm*.tar.gz
+  cd recidivm*
+  make
+  /path/to/recidivm -v src/test/fuzz/fuzz-http
+  Use the final "ok" figure as the input to -m when calling afl-fuzz
+  (Normally, recidivm would output a figure automatically, but in some cases,
+  the fuzzing harness will hang when the memory limit is too small.)
+
+You could also just say "none" instead of the memory limit below, if you
+don't care about memory limits.
+
+
+To Run:
+  mkdir -p src/test/fuzz/fuzz_http_findings
+  ../afl/afl-fuzz -i ${TOR_FUZZ_CORPORA}/http -o src/test/fuzz/fuzz_http_findings -m <asan-memory-limit> -- src/test/fuzz/fuzz-http
+
+
+AFL has a multi-core mode, check the documentation for details.
+You might find the included fuzz-multi.sh script useful for this.
+
+macOS (OS X) requires slightly more preparation, including:
+* using afl-clang (or afl-clang-fast from the llvm directory)
+* disabling external crash reporting (AFL will guide you through this step)
+
+== Triaging Issues
+
+Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code.
+
+Hangs might be interesting, but they might also be spurious machine slowdowns.
+Check if a hang is reproducible before reporting it. Sometimes, processing
+valid inputs may take a second or so, particularly with the fuzzer and
+sanitizers enabled.
+
+To see what fuzz-http is doing with a test case, call it like this:
+  src/test/fuzz/fuzz-http --debug < /path/to/test.case
+
+(Logging is disabled while fuzzing to increase fuzzing speed.)
+
+== Reporting Issues
+
+Please report any issues discovered using the process in Tor's security issue
+policy:
+
+https://trac.torproject.org/projects/tor/wiki/org/meetings/2016SummerDevMeeting/Notes/SecurityIssuePolicy
diff --git a/doc/HACKING/GettingStarted.md b/doc/HACKING/GettingStarted.md
index 0295adc1ff..0c42404634 100644
--- a/doc/HACKING/GettingStarted.md
+++ b/doc/HACKING/GettingStarted.md
@@ -11,8 +11,9 @@ whole Tor ecosystem.)
 
 
 If you are looking for a more bare-bones, less user-friendly information
-dump of important information, you might like reading doc/HACKING
-instead.  You should probably read it before you write your first patch.
+dump of important information, you might like reading the "torguts"
+documents linked to below. You should probably read it before you write
+your first patch.
 
 
 Required background
diff --git a/doc/HACKING/GettingStartedRust.md b/doc/HACKING/GettingStartedRust.md
new file mode 100644
index 0000000000..aa29c097da
--- /dev/null
+++ b/doc/HACKING/GettingStartedRust.md
@@ -0,0 +1,183 @@
+
+ Hacking on Rust in Tor
+========================
+
+ Getting Started
+-----------------
+
+Please read or review our documentation on Rust coding standards
+(`.../doc/HACKING/CodingStandardsRust.md`) before doing anything.
+
+Please also read
+[the Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). We
+aim to follow the good example set by the Rust community and be
+excellent to one another.  Let's be careful with each other, so we can
+be memory-safe together!
+
+Next, please contact us before rewriting anything!  Rust in Tor is still
+an experiment.  It is an experiment that we very much want to see
+succeed, so we're going slowly and carefully.  For the moment, it's also
+a completely volunteer-driven effort: while many, if not most, of us are
+paid to work on Tor, we are not yet funded to write Rust code for Tor.
+Please be patient with the other people who are working on getting more
+Rust code into Tor, because they are graciously donating their free time
+to contribute to this effort.
+
+ Resources for learning Rust
+-----------------------------
+
+**Beginning resources**
+
+The primary resource for learning Rust is
+[The Book](https://doc.rust-lang.org/book/).  If you'd like to start writing
+Rust immediately, without waiting for anything to install, there is
+[an interactive browser-based playground](https://play.rust-lang.org/).
+
+**Advanced resources**
+
+If you're interested in playing with various Rust compilers and viewing
+a very nicely displayed output of the generated assembly, there is
+[the Godbolt compiler explorer](https://rust.godbolt.org/)
+
+For learning how to write unsafe Rust, read
+[The Rustonomicon](https://doc.rust-lang.org/nomicon/).
+
+For learning everything you ever wanted to know about Rust macros, there
+is
+[The Little Book of Rust Macros](https://danielkeep.github.io/tlborm/book/index.html).
+
+For learning more about FFI and Rust, see Jake Goulding's
+[Rust FFI Omnibus](http://jakegoulding.com/rust-ffi-omnibus/).
+
+ Compiling Tor with Rust enabled
+---------------------------------
+
+You will need to run the `configure` script with the `--enable-rust`
+flag to explicitly build with Rust. Additionally, you will need to
+specify where to fetch Rust dependencies, as we allow for either
+fetching dependencies from Cargo or specifying a local directory.
+
+**Fetch dependencies from Cargo**
+
+    ./configure --enable-rust --enable-cargo-online-mode
+
+**Using a local dependency cache**
+
+You'll need the following Rust dependencies (as of this writing):
+
+    libc==0.2.39
+
+We vendor our Rust dependencies in a separate repo using
+[cargo-vendor](https://github.com/alexcrichton/cargo-vendor).  To use
+them, do:
+
+    git submodule init
+    git submodule update
+
+To specify the local directory containing the dependencies, (assuming
+you are in the top level of the repository) configure tor with:
+
+    TOR_RUST_DEPENDENCIES='path_to_dependencies_directory' ./configure --enable-rust
+
+(Note that TOR_RUST_DEPENDENCIES must be the full path to the directory; it
+cannot be relative.)
+
+Assuming you used the above `git submodule` commands and you're in the
+topmost directory of the repository, this would be:
+
+    TOR_RUST_DEPENDENCIES=`pwd`/src/ext/rust/crates ./configure --enable-rust
+
+
+ Identifying which modules to rewrite
+======================================
+
+The places in the Tor codebase that are good candidates for porting to
+Rust are:
+
+1. loosely coupled to other Tor submodules,
+2. have high test coverage, and
+3. would benefit from being implemented in a memory safe language.
+
+Help in either identifying places such as this, or working to improve
+existing areas of the C codebase by adding regression tests and
+simplifying dependencies, would be really helpful.
+
+Furthermore, as submodules in C are implemented in Rust, this is a good
+opportunity to refactor, add more tests, and split modules into smaller
+areas of responsibility.
+
+A good first step is to build a module-level callgraph to understand how
+interconnected your target module is.
+
+    git clone https://git.torproject.org/user/nickm/calltool.git
+    cd tor
+    CFLAGS=0 ./configure
+    ../calltool/src/main.py module_callgraph
+
+The output will tell you each module name, along with a set of every module that
+the module calls.  Modules which call fewer other modules are better targets.
+
+ Writing your Rust module
+==========================
+
+Strive to change the C API as little as possible.
+
+We are currently targetting Rust stable. (See CodingStandardsRust.md for more
+details.)
+
+It is on our TODO list to try to cultivate good
+standing with various distro maintainers of `rustc` and `cargo`, in
+order to ensure that whatever version we solidify on is readily
+available.
+
+If parts of your Rust code needs to stay in sync with C code (such as
+handling enums across the FFI boundary), annonotate these places in a
+comment structured as follows:
+
+  /// C_RUST_COUPLED: <path_to_file> `<name_of_c_object>`
+
+Where <name_of_c_object> can be an enum, struct, constant, etc.  Then,
+do the same in the C code, to note that rust will need to be changed
+when the C does.
+
+
+ Adding your Rust module to Tor's build system
+-----------------------------------------------
+
+0. Your translation of the C module should live in its own crate(s)
+   in the `.../tor/src/rust/` directory.
+1. Add your crate to `.../tor/src/rust/Cargo.toml`, in the
+   `[workspace.members]` section.
+2. Add your crate's files to src/rust/include.am
+
+If your crate should be available to C (rather than just being included as a
+dependency of other Rust modules):
+0. Declare the crate as a dependency of tor_rust in
+   `src/rust/tor_util/Cargo.toml` and include it in
+   `src/rust/tor_rust/lib.rs`
+
+ How to test your Rust code
+----------------------------
+
+Everything should be tested full stop.  Even non-public functionality.
+
+Be sure to edit `.../tor/src/test/test_rust.sh` to add the name of your
+crate to the `crates` variable! This will ensure that `cargo test` is
+run on your crate.
+
+Configure Tor's build system to build with Rust enabled:
+
+    ./configure --enable-fatal-warnings --enable-rust --enable-cargo-online-mode
+
+Tor's test should be run by doing:
+
+    make check
+
+Tor's integration tests should also pass:
+
+    make test-stem
+
+ Submitting a patch
+=====================
+
+Please follow the instructions in `.../doc/HACKING/GettingStarted.md`.
diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md
index a7f36e6c7e..a0795076e0 100644
--- a/doc/HACKING/HelpfulTools.md
+++ b/doc/HACKING/HelpfulTools.md
@@ -4,6 +4,22 @@ Useful tools
 These aren't strictly necessary for hacking on Tor, but they can help track
 down bugs.
 
+Travis CI
+---------
+It's CI. Looks like this: https://travis-ci.org/torproject/tor.
+
+Runs automatically on Pull Requests sent to torproject/tor. You can set it up
+for your fork to build commits outside of PRs too:
+
+1. sign up for GitHub: https://github.com/join
+2. fork https://github.com/torproject/tor:
+   https://help.github.com/articles/fork-a-repo/
+3. follow https://docs.travis-ci.com/user/getting-started/#To-get-started-with-Travis-CI.
+   skip steps involving `.travis.yml` (we already have one).
+
+Builds should show up on the web at travis-ci.com and on IRC at #tor-ci on
+OFTC. If they don't, ask #tor-dev (also on OFTC).
+
 Jenkins
 -------
 
@@ -111,15 +127,19 @@ Running gcov for unit test coverage
 
 (On OSX, you'll need to start with `--enable-coverage CC=clang`.)
 
-Then, look at the .gcov files in `coverage-output`.  '-' before a line means
-that the compiler generated no code for that line.  '######' means that the
-line was never reached.  Lines with numbers were called that number of times.
-
 If that doesn't work:
 
    * Try configuring Tor with `--disable-gcc-hardening`
    * You might need to run `make clean` after you run `./configure`.
 
+Then, look at the .gcov files in `coverage-output`.  '-' before a line means
+that the compiler generated no code for that line.  '######' means that the
+line was never reached.  Lines with numbers were called that number of times.
+
+For more details about how to read gcov output, see the [Invoking
+gcov](https://gcc.gnu.org/onlinedocs/gcc/Invoking-Gcov.html) chapter
+of the GCC manual.
+
 If you make changes to Tor and want to get another set of coverage results,
 you can run `make reset-gcov` to clear the intermediary gcov output.
 
@@ -128,9 +148,13 @@ a meaningful diff between them, you can run:
 
     ./scripts/test/cov-diff coverage-output1 coverage-output2 | less
 
-In this diff, any lines that were visited at least once will have coverage
-"1".  This lets you inspect what you (probably) really want to know: which
-untested lines were changed?  Are there any new untested lines?
+In this diff, any lines that were visited at least once will have coverage "1",
+and line numbers are deleted.  This lets you inspect what you (probably) really
+want to know: which untested lines were changed?  Are there any new untested
+lines?
+
+If you run ./scripts/test/cov-exclude, it marks excluded unreached
+lines with 'x', and excluded reached lines with '!!!'.
 
 Running integration tests
 -------------------------
@@ -142,6 +166,12 @@ run `make test-network`.
 We also have scripts to run integration tests using Stem.  To try them, set
 `STEM_SOURCE_DIR` to your Stem source directory, and run `test-stem`.
 
+Profiling Tor
+-------------
+
+Ongoing notes about Tor profiling can be found at
+https://pad.riseup.net/p/profiling-tor
+
 Profiling Tor with oprofile
 ---------------------------
 
@@ -168,20 +198,62 @@ Here are some basic instructions
    * `opreport -l that_dir/*`
  - Profit
 
+Profiling Tor with perf
+-----------------------
+
+This works with a running Tor, and requires root.
+
+1. Decide how long you want to profile for. Start with (say) 30 seconds. If that
+   works, try again with longer times.
+
+2. Find the PID of your running tor process.
+
+3. Run `perf record --call-graph dwarf -p <PID> sleep <SECONDS>`
+
+   (You may need to do this as root.)
+
+   You might need to add `-e cpu-clock` as an option to the perf record line
+   above, if you are on an older CPU without access to hardware profiling
+   events, or in a VM, or something.
+
+4. Now you have a perf.data file. Have a look at it with `perf report
+   --no-children --sort symbol,dso` or `perf report --no-children --sort
+   symbol,dso --stdio --header`. How does it look?
+
+5a. Once you have a nice big perf.data file, you can compress it, encrypt it,
+    and send it to your favorite Tor developers.
+
+5b. Or maybe you'd rather not send a nice big perf.data file. Who knows what's
+    in that!? It's kinda scary. To generate a less scary file, you can use `perf
+    report -g > <FILENAME>.out`. Then you can compress that and put it somewhere
+    public.
+
+Profiling Tor with gperftools aka Google-performance-tools
+----------------------------------------------------------
+
+This should work on nearly any unixy system. It doesn't seem to be compatible
+with RunAsDaemon though.
+
+Beforehand, install google-perftools.
+
+1. You need to rebuild Tor, hack the linking steps to add `-lprofiler` to the
+   libs. You can do this by adding `LIBS=-lprofiler` when you call `./configure`.
+
+Now you can run Tor with profiling enabled, and use the pprof utility to look at
+performance! See the gperftools manual for more info, but basically:
+
+2. Run `env CPUPROFILE=/tmp/profile src/or/tor -f <path/torrc>`. The profile file
+   is not written to until Tor finishes execuction.
+
+3. Run `pprof src/or/tor /tm/profile` to start the REPL.
+
 Generating and analyzing a callgraph
 ------------------------------------
 
-1. Run `./scripts/maint/generate_callgraph.sh`.  This will generate a
-   bunch of files in a new ./callgraph directory.
-
-2. Run `./scripts/maint/analyze_callgraph.py callgraph/src/*/*`.  This
-   will do a lot of graph operations and then dump out a new
-   `callgraph.pkl` file, containing data in Python's 'pickle' format.
+0. Build Tor on linux or mac, ideally with -O0 or -fno-inline.
 
-3. Run `./scripts/maint/display_callgraph.py`.  It will display:
-    - the number of functions reachable from each function.
-    - all strongly-connnected components in the Tor callgraph
-    - the largest bottlenecks in the largest SCC in the Tor callgraph.
+1. Clone 'https://gitweb.torproject.org/user/nickm/calltool.git/' .
+   Follow the README in that repository.
 
 Note that currently the callgraph generator can't detect calls that pass
 through function pointers.
diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md
index d53318942f..2d1f3d1c9e 100644
--- a/doc/HACKING/HowToReview.md
+++ b/doc/HACKING/HowToReview.md
@@ -19,6 +19,8 @@ Top-level smell-checks
 
 - Does `make check-spaces` pass?
 
+- Does `make check-changes` pass?
+
 - Does it have a reasonable amount of tests?  Do they pass?  Do they leak
   memory?
 
@@ -32,6 +34,7 @@ Top-level smell-checks
 
 - If this changes Tor's behavior on the wire, is there a design proposal?
 
+- If this changes anything in the code, is there a "changes" file?
 
 
 Let's look at the code!
diff --git a/doc/HACKING/Module.md b/doc/HACKING/Module.md
new file mode 100644
index 0000000000..1028a029d9
--- /dev/null
+++ b/doc/HACKING/Module.md
@@ -0,0 +1,111 @@
+# Modules in Tor #
+
+This document describes the build system and coding standards when writing a
+module in Tor.
+
+## What is a module? ##
+
+In the context of the tor code base, a module is a subsystem that we can
+selectively enable or disable, at `configure` time.
+
+Currently, there is only one module:
+
+  - Directory Authority subsystem (dirauth)
+
+It is located in its own directory in `src/or/dirauth/`. To disable it, one
+need to pass `--disable-module-dirauth` at configure time. All modules are
+currently enabled by default.
+
+## Build System ##
+
+The changes to the build system are pretty straightforward.
+
+1. Locate in the `configure.ac` file this define: `m4_define(MODULES`. It
+   contains a list (white-space separated) of the module in tor. Add yours to
+   the list.
+
+2. Use the `AC_ARG_ENABLE([module-dirauth]` template for your new module. We
+   use the "disable module" approach instead of enabling them one by one. So,
+   by default, tor will build all the modules.
+
+   This will define the `HAVE_MODULE_<name>` statement which can be used in
+   the C code to conditionally compile things for your module. And the
+   `BUILD_MODULE_<name>` is also defined for automake files (e.g: include.am).
+
+3. In the `src/or/include.am` file, locate the `MODULE_DIRAUTH_SOURCES` value.
+   You need to create your own `_SOURCES` variable for your module and then
+   conditionally add the it to `LIBTOR_A_SOURCES` if you should build the
+   module.
+
+   It is then **very** important to add your SOURCES variable to
+   `src_or_libtor_testing_a_SOURCES` so the tests can build it.
+
+4. Do the same for header files, locate `ORHEADERS +=` which always add all
+   headers of all modules so the symbol can be found for the module entry
+   points.
+
+Finally, your module will automatically be included in the
+`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests. They
+always build everything in order to tests everything.
+
+## Coding ##
+
+As mentioned above, a module must be isolated in its own directory (name of
+the module) in `src/or/`.
+
+There are couples of "rules" you want to follow:
+
+* Minimize as much as you can the number of entry points into your module.
+  Less is always better but of course that doesn't work out for every use
+  case. However, it is a good thing to always keep that in mind.
+
+* Do **not** use the `HAVE_MODULE_<name>` define outside of the module code
+  base. Every entry point should have a second definition if the module is
+  disabled. For instance:
+
+  ```
+  #ifdef HAVE_MODULE_DIRAUTH
+
+  int sr_init(int save_to_disk);
+
+  #else /* HAVE_MODULE_DIRAUTH */
+
+  static inline int
+  sr_init(int save_to_disk)
+  {
+    (void) save_to_disk;
+    return 0;
+  }
+
+  #endif /* HAVE_MODULE_DIRAUTH */
+
+  ```
+
+  The main reason for this approach is to avoid having conditional code
+  everywhere in the code base. It should be centralized as much as possible
+  which helps maintainability but also avoids conditional spaghetti code
+  making the code much more difficult to follow/understand.
+
+* It is possible that you end up with code that needs to be used by the rest
+  of the code base but is still part of your module. As a good example, if you
+  look at `src/or/shared_random_client.c`: it contains code needed by the hidden
+  service subsystem but mainly related to the shared random subsystem very
+  specific to the dirauth module.
+
+  This is fine but try to keep it as lean as possible and never use the same
+  filename as the one in the module. For example, this is a bad idea and
+  should never be done:
+
+    - `src/or/shared_random.c`
+    - `src/or/dirauth/shared_random.c`
+
+* When you include headers from the module, **always** use the full module
+  path in your statement. Example:
+
+  `#include "dirauth/dirvote.h"`
+
+  The main reason is that we do **not** add the module include path by default
+  so it needs to be specified. But also, it helps our human brain understand
+  which part comes from a module or not.
+
+  Even **in** the module itself, use the full include path like above.
diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md
index 7595398241..e70416c354 100644
--- a/doc/HACKING/ReleasingTor.md
+++ b/doc/HACKING/ReleasingTor.md
@@ -8,8 +8,15 @@ new Tor release:
 === 0. Preliminaries
 
 1. Get at least three of weasel/arma/Sebastian/Sina to put the new
-   version number in their approved versions list.
+   version number in their approved versions list.  Give them a few
+   days to do this if you can.
 
+2. If this is going to be an important security release, give the packagers
+   some advance warning: See this list of packagers in IV.3 below.
+
+3. Given the release date for Tor, ask the TB team about the likely release
+   date of a TB that contains it.  See note below in "commit, upload,
+   announce".
 
 === I. Make sure it works
 
@@ -18,6 +25,7 @@ new Tor release:
    resolve those.
 
    As applicable, merge the `maint-X` branch into the `release-X` branch.
+   But you've been doing that all along, right?
 
 2. Are all of the jenkins builders happy?  See jenkins.torproject.org.
 
@@ -26,39 +34,37 @@ new Tor release:
 
    What about Coverity Scan?
 
-   Is make check-spaces happy?
+   What about clang scan-build?
 
-   Does 'make distcheck' compain?
+   Does 'make distcheck' complain?
 
-   How about 'make test-stem' and 'make test-network'?
+   How about 'make test-stem' and 'make test-network' and
+   `make test-network-full`?
 
        - Are all those tests still happy with --enable-expensive-hardening ?
 
    Any memory leaks?
 
 
-=== II. Write a changelog.
+=== II. Write a changelog
+
 
+1a. (Alpha release variant)
 
-1. Gather the `changes/*` files into a changelog entry, rewriting many
+   Gather the `changes/*` files into a changelog entry, rewriting many
    of them and reordering to focus on what users and funders would find
    interesting and understandable.
 
-   1. Make sure that everything that wants a bug number has one.
-      Make sure that everything which is a bugfix says what version
-      it was a bugfix on.
-
-   2. Concatenate them.
-
-   3. Sort them by section. Within each section, sort by "version it's
-      a bugfix on", else by numerical ticket order.
+   To do this, first run `./scripts/maint/lintChanges.py changes/*` and
+   fix as many warnings as you can.  Then run `./scripts/maint/sortChanges.py
+   changes/* > changelog.in` to combine headings and sort the entries.
+   After that, it's time to hand-edit and fix the issues that lintChanges
+   can't find:
 
-   4. Clean them up:
+   1. Within each section, sort by "version it's a bugfix on", else by
+      numerical ticket order.
 
-      Standard idioms:
-      `Fixes bug 9999; bugfix on 0.3.3.3-alpha.`
-
-      One space after a period.
+   2. Clean them up:
 
       Make stuff very terse
 
@@ -86,19 +92,32 @@ new Tor release:
       maint-0.2.1), be sure to make the stanzas identical (so people can
       distinguish if these are the same change).
 
-   5. Merge them in.
+   3. Clean everything one last time.
+
+   4. Run `./scripts/maint/format_changelog.py --inplace` to make it prettier
 
-   6. Clean everything one last time.
+1b. (old-stable release variant)
 
-   7. Run `./scripts/maint/format_changelog.py` to make it prettier.
+   For stable releases that backport things from later, we try to compose
+   their releases, we try to make sure that we keep the changelog entries
+   identical to their original versions, with a 'backport from 0.x.y.z'
+   note added to each section.  So in this case, once you have the items
+   from the changes files copied together, don't use them to build a new
+   changelog: instead, look up the corrected versions that were merged
+   into ChangeLog in the master branch, and use those.
 
 2. Compose a short release blurb to highlight the user-facing
    changes. Insert said release blurb into the ChangeLog stanza. If it's
    a stable release, add it to the ReleaseNotes file too. If we're adding
-   to a release-0.2.x branch, manually commit the changelogs to the later
+   to a release-* branch, manually commit the changelogs to the later
    git branches too.
 
-3.  If you're doing the first stable release in a series, you need to
+3. If there are changes that require or suggest operator intervention
+   before or during the update, mail operators (either dirauth or relays
+   list) with a headline that indicates that an action is required or
+   appreciated.
+
+4. If you're doing the first stable release in a series, you need to
    create a ReleaseNotes for the series as a whole.  To get started
    there, copy all of the Changelog entries from the series into a new
    file, and run `./scripts/maint/sortChanges.py` on it.  That will
@@ -111,51 +130,61 @@ new Tor release:
 
 === III. Making the source release.
 
-1. In `maint-0.2.x`, bump the version number in `configure.ac` and run
-   `scripts/maint/updateVersions.pl` to update version numbers in other
-   places, and commit.  Then merge `maint-0.2.x` into `release-0.2.x`.
+1. In `maint-0.?.x`, bump the version number in `configure.ac` and run
+   `perl scripts/maint/updateVersions.pl` to update version numbers in other
+   places, and commit.  Then merge `maint-0.?.x` into `release-0.?.x`.
 
    (NOTE: To bump the version number, edit `configure.ac`, and then run
    either `make`, or `perl scripts/maint/updateVersions.pl`, depending on
    your version.)
 
-2. Make distcheck, put the tarball up somewhere, and tell `#tor` about
-   it. Wait a while to see if anybody has problems building it. Try to
-   get Sebastian or somebody to try building it on Windows.
+   When you merge the maint branch forward to the next maint branch, or into
+   master, merge it with "-s ours" to avoid a needless version bump.
+
+2. Make distcheck, put the tarball up in somewhere (how about your
+   homedir on your homedir on people.torproject.org?) , and tell `#tor`
+   about it. Wait a while to see if anybody has problems building it.
+   (Though jenkins is usually pretty good about catching these things.)
 
 === IV. Commit, upload, announce
 
 1. Sign the tarball, then sign and push the git tag:
 
         gpg -ba <the_tarball>
-        git tag -u <keyid> tor-0.2.x.y-status
-        git push origin tag tor-0.2.x.y-status
+        git tag -u <keyid> tor-0.3.x.y-status
+        git push origin tag tor-0.3.x.y-status
+
+   (You must do this before you update the website: it relies on finding
+   the version by tag.)
 
 2. scp the tarball and its sig to the dist website, i.e.
    `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want
    it to go live, you run "static-update-component dist.torproject.org"
    on dist-master.
 
-   Edit `include/versions.wmi` and `Makefile` to note the new version.
+   In the webwml.git repository, `include/versions.wmi` and `Makefile`
+   to note the new version.
 
    (NOTE: Due to #17805, there can only be one stable version listed at
    once.  Nonetheless, do not call your version "alpha" if it is stable,
    or people will get confused.)
 
-3. Email the packagers (cc'ing tor-assistants) that a new tarball is up.
+3. Email the packagers (cc'ing tor-team) that a new tarball is up.
    The current list of packagers is:
 
        - {weasel,gk,mikeperry} at torproject dot org
        - {blueness} at gentoo dot org
        - {paul} at invizbox dot io
+       - {vincent} at invizbox dot com
        - {lfleischer} at archlinux dot org
        - {Nathan} at freitas dot net
        - {mike} at tig dot as
-       - {tails-rm} at boum dot org (for pre-release announcments)
-
-
-       - {tails-dev} at boum dot org (for at-release announcements)
+       - {tails-rm} at boum dot org
+       - {simon} at sdeziel.info
+       - {yuri} at freebsd.org
+       - {mh+tor} at scrit.ch
 
+   Also, email tor-packagers@lists.torproject.org.
 
 4. Add the version number to Trac.  To do this, go to Trac, log in,
     select "Admin" near the top of the screen, then select "Versions" from
@@ -164,22 +193,30 @@ new Tor release:
     0.2.2.23-alpha" (or whatever the version is), and we select the date as
     the date in the ChangeLog.
 
-5. Wait up to a day or two (for a development release), or until most
-    packages are up (for a stable release), and mail the release blurb and
-    changelog to tor-talk or tor-announce.
+5. Double-check: did the version get recommended in the consensus yet?  Is
+   the website updated?  If not, don't announce until they have the
+   up-to-date versions, or people will get confused.
+
+6. Mail the release blurb and ChangeLog to tor-talk (development release) or
+   tor-announce (stable).
+
+   Post the changelog on the blog as well. You can generate a
+   blog-formatted version of the changelog with the -B option to
+   format-changelog.
 
-   (We might be moving to faster announcements, but don't announce until
-   the website is at least updated.)
+   When you post, include an estimate of when the next TorBrowser
+   releases will come out that include this Tor release.  This will
+   usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it
+   can vary.
 
 
 === V. Aftermath and cleanup
 
-1. If it's a stable release, bump the version number in the `maint-x.y.z`
-    branch to "newversion-dev", and do a `merge -s ours` merge to avoid
-    taking that change into master.  Do a similar `merge -s theirs`
-    merge to get the change (and only that change) into release.  (Some
-    of the build scripts require that maint merge cleanly into release.)
+1. If it's a stable release, bump the version number in the
+    `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours`
+    merge to avoid taking that change into master.
 
 2. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
 
+3. Keep an eye on the blog post, to moderate comments and answer questions.
 
diff --git a/doc/HACKING/Tracing.md b/doc/HACKING/Tracing.md
new file mode 100644
index 0000000000..349aade23a
--- /dev/null
+++ b/doc/HACKING/Tracing.md
@@ -0,0 +1,91 @@
+# Tracing #
+
+This document describes how the event tracing subsystem works in tor so
+developers can add events to the code base but also hook them to an event
+tracing framework.
+
+## Basics ###
+
+Event tracing is separated in two concepts, trace events and a tracer. The
+tracing subsystem can be found in `src/trace`. The `events.h` header file is
+the main file that maps the different tracers to trace events.
+
+### Events ###
+
+A trace event is basically a function from which we can pass any data that
+we want to collect. In addition, we specify a context for the event such as
+a subsystem and an event name.
+
+A trace event in tor has the following standard format:
+
+	tor_trace(subsystem, event\_name, args...)
+
+The `subsystem` parameter is the name of the subsytem the trace event is in.
+For example that could be "scheduler" or "vote" or "hs". The idea is to add
+some context to the event so when we collect them we know where it's coming
+from. The `event_name` is the name of the event which helps a lot with
+adding some semantic to the event. Finally, `args` is any number of
+arguments we want to collect.
+
+Here is an example of a possible tracepoint in main():
+
+	tor_trace(main, init_phase, argc)
+
+The above is a tracepoint in the `main` subsystem with `init_phase` as the
+event name and the `int argc` is passed to the event as well.
+
+How `argc` is collected or used has nothing to do with the instrumentation
+(adding trace events to the code). It is the work of the tracer so this is why
+the trace events and collection framework (tracer) are decoupled. You _can_
+have trace events without a tracer.
+
+### Tracer ###
+
+In `src/trace/events.h`, we map the `tor_trace()` function to the right
+tracer. A tracer support is only enabled at compile time. For instance, the
+file `src/trace/debug.h` contains the mapping of the generic tracing function
+`tor_trace()` to the `log_debug()` function. More specialized function can be
+mapped depending on the tracepoint.
+
+## Build System ##
+
+This section describes how it is integrated into the build system of tor.
+
+By default, every tracing events are disabled in tor that is `tor_trace()`
+is a NOP.
+
+To enable a tracer, there is a configure option on the form of:
+
+	--enable-tracing-<tracer>
+
+We have an option that will send every trace events to a `log_debug()` (as
+mentionned above) which will print you the subsystem and name of the event but
+not the arguments for technical reasons. This is useful if you want to quickly
+see if your trace event is being hit or well written. To do so, use this
+configure option:
+
+	--enable-tracing-debug
+
+## Instrument Tor ##
+
+This is pretty easy. Let's say you want to add a trace event in
+`src/or/rendcache.c`, you only have to add this include statement:
+
+	#include "trace/events.h"
+
+Once done, you can add as many as you want `tor_trace()` that you need.
+Please use the right subsystem (here it would be `hs`) and a unique name that
+tells what the event is for. For example:
+
+	tor_trace(hs, store_desc_as_client, desc, desc_id);
+
+If you look in `src/trace/events.h`, you'll see that if tracing is enabled it
+will be mapped to a function called:
+
+	tor_trace_hs_store_desc_as_client(desc, desc_id)
+
+And the point of all this is for that function to be defined in a new file
+that you might want to add named `src/trace/hs.{c|h}` which would defined how
+to collect the data for the `tor_trace_hs_store_desc_as_client()` function
+like for instance sending it to a `log_debug()` or do more complex operations
+or use a userspace tracer like LTTng (https://lttng.org).
diff --git a/doc/HACKING/WritingTests.md b/doc/HACKING/WritingTests.md
index de80bbdef2..cc393494ec 100644
--- a/doc/HACKING/WritingTests.md
+++ b/doc/HACKING/WritingTests.md
@@ -48,7 +48,7 @@ isolation, you just run `./src/test/test-memwipe`.
 To run tests within the unit test programs, you can specify the name
 of the test.  The string ".." can be used as a wildcard at the end of the
 test name.  For example, to run all the cell format tests, enter
-`./src/test/test cellfmt/..`.  To run
+`./src/test/test cellfmt/..`.
 
 Many tests that need to mess with global state run in forked subprocesses in
 order to keep from contaminating one another.  But when debugging a failing test,
@@ -91,6 +91,9 @@ coverage percentage.
 For a summary of the test coverage for each _function_, run
 `./scripts/test/cov-display -f ${TMPDIR}/*`.
 
+For more details on using gcov, including the helper scripts in
+scripts/test, see HelpfulTools.md.
+
 ### Comparing test coverage
 
 Sometimes it's useful to compare test coverage for a branch you're writing to
@@ -117,7 +120,8 @@ with LCOV_EXCL_START... LCOV_EXCL_STOP.  Note that older versions of
 lcov don't understand these lines.
 
 You can post-process .gcov files to make these lines 'unreached' by
-running ./scripts/test/cov-exclude on them.
+running ./scripts/test/cov-exclude on them.  It marks excluded
+unreached lines with 'x', and excluded reached lines with '!!!'.
 
 Note: you should never do this unless the line is meant to 100%
 unreachable by actual code.
diff --git a/doc/HACKING/android/Simpleperf.md b/doc/HACKING/android/Simpleperf.md
new file mode 100644
index 0000000000..25f39a3d23
--- /dev/null
+++ b/doc/HACKING/android/Simpleperf.md
@@ -0,0 +1,98 @@
+# Using `simpleperf` to collect CPU profiling on Android
+
+This document describes how you can use Android's `simpleperf`
+command-line tool to get CPU profiling information from Tor via the
+Orbot application. The tool is particularly useful for Tor development
+because it is able to profile native applications on the platform
+whereas a lot of the normal tooling for the Android platform is only
+able to collect information from Java-based applications.
+
+## Prerequisites
+
+Before using `simpleperf` there is a couple of steps that must be
+followed. You should make sure you have both a recent installation of
+the Android Software Development Kit (SDK) and Native Development Kit
+(NDK) installed. These can be found on the Android Developers website.
+
+1. Follow the build instructions from the `BUILD` file in the Orbot
+   repository and build an Orbot APK (Android Package) file with
+   debugging enabled. Make sure that when you build the native content of
+   the Orbot application that you run the `make -C external` command with
+   an additional `DEBUG=1` as parameter to ensure that the Orbot build
+   process does not strip the debug symbols from the Tor binary.
+
+2. (Optional) Uninstall and clean-up your old Orbot installation that
+   is most likely downloaded from Google's Play Store or via fdroid:
+
+       $ adb shell pm clear org.torproject.android
+       $ adb uninstall org.torproject.android
+
+3. Install the Android Package you generated in step 1:
+
+       $ adb install /path/to/your/app-fullperm-debug.apk
+
+4. Check on your device that the newly installed Orbot actually works
+   and behaves in the way you expect it to.
+
+## Profiling using `simpleperf`
+
+The `simpleperf` tool can be found in the `simpleperf/` directory in
+the directory where you installed the Android NDK to. In this
+directory there is a set of Python files that will help you deploy the
+tool to a device and collect the measurement data such that you can
+analyze the results on your computer rather than on your phone.
+
+1. Change directory to the location of the `simpleperf` directory.
+2. Open the `app_profiler.config` file and change
+   `app_package_name` to `org.torproject.android`, `apk_file_path` to
+   the path of your Orbot Android Package (APK file).
+3. Optionally change the duration parameter in the `record_options`
+   variable in `app_profiler.config` to the duration which you would like
+   to collect samples in. The value is specified in seconds.
+4. Run the app profiler using `python app_profiler.py`. This helper
+   script will push the `simpleperf` tool to your device, start the
+   profiler, and once it has completed copy the generated `perf.data`
+   file over to your computer with the results.
+
+### Analyzing the results
+
+You can inspect your resulting `perf.data` file via a simple GUI
+program `python report.py` or via the command-line tool `simpleperf
+report`. I've found the GUI tool to be easier to navigate around with
+than the command-line tool.
+
+The `-g` option can be passed to the command line `simpleperf report`
+tool allows you to see the call graph of functions and how much time
+was spend on the call.
+
+## Tips & Tricks
+
+- When you have installed Orbot the first time, you will notice that
+  if you get a shell on the Android device that there is no Tor binary
+  available. This is because Orbot unpacks the Tor binary first time it
+  is executed and places it under the `app_bin/` directory on the
+  device.
+
+  To access binaries, `torrc` files, and other useful information on
+  the device do the following:
+
+      $ adb shell
+      (device):/ $ run-as org.torproject.android
+      (device):/data/data/org.torproject.android $ ls
+      app_bin app_data cache databases files lib shared_prefs
+
+  Descriptors, control authentication cookie, state, and other files can be
+  found in the `app_data` directory. The `torrc` can be found in the `app_bin/`
+  directory.
+
+- You can enable logging in Tor via the syslog (or android) log
+  mechanism with:
+
+      $ adb shell
+      (device):/ $ run-as org.torproject.android
+      (device):/data/data/org.torproject.android $ echo -e "\nLog info syslog" >> app_bin/torrc
+
+  Start Tor the normal way via Orbot and collect the logs from your computer using
+
+      $ adb logcat
+
diff --git a/doc/include.am b/doc/include.am
index 7164a4b2a0..7942188eaf 100644
--- a/doc/include.am
+++ b/doc/include.am
@@ -12,17 +12,11 @@
 # part of the source distribution, so that people without asciidoc can
 # just use the .1 and .html files.
 
-base_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
-all_mans = $(base_mans)
-if USE_FW_HELPER
-install_mans = $(all_mans)
-else
-install_mans = $(base_mans)
-endif
+all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
 
 if USE_ASCIIDOC
-nodist_man1_MANS = $(install_mans:=.1)
-doc_DATA = $(install_mans:=.html)
+nodist_man1_MANS = $(all_mans:=.1)
+doc_DATA = $(all_mans:=.html)
 html_in = $(all_mans:=.html.in)
 man_in = $(all_mans:=.1.in)
 txt_in = $(all_mans:=.1.txt)
@@ -41,10 +35,15 @@ EXTRA_DIST+= doc/asciidoc-helper.sh			\
 	     doc/TUNING						\
 	     doc/HACKING/README.1st.md				\
 	     doc/HACKING/CodingStandards.md 			\
+	     doc/HACKING/CodingStandardsRust.md			\
+	     doc/HACKING/Fuzzing.md				\
 	     doc/HACKING/GettingStarted.md 			\
+	     doc/HACKING/GettingStartedRust.md 			\
 	     doc/HACKING/HelpfulTools.md 			\
 	     doc/HACKING/HowToReview.md  			\
+	     doc/HACKING/Module.md				\
 	     doc/HACKING/ReleasingTor.md                        \
+	     doc/HACKING/Tracing.md				\
 	     doc/HACKING/WritingTests.md
 
 docdir = @docdir@
diff --git a/doc/tor.1.txt b/doc/tor.1.txt
index a7ee7d11ca..3f7a8c39af 100644
--- a/doc/tor.1.txt
+++ b/doc/tor.1.txt
@@ -128,6 +128,16 @@ COMMAND-LINE OPTIONS
     the passphrase, including any trailing newlines.
     Default: read from the terminal.
 
+[[opt-key-expiration]] **--key-expiration** [**purpose**]::
+    The **purpose** specifies which type of key certificate to determine
+    the expiration of.  The only currently recognised **purpose** is
+    "sign". +
+ +
+    Running "tor --key-expiration sign" will attempt to find your signing
+    key certificate and will output, both in the logs as well as to stdout,
+    the signing key certificate's expiration time in ISO-8601 format.
+    For example, the output sent to stdout will be of the form:
+    "signing-cert-expiry: 2017-07-25 08:30:15 UTC"
 
 Other options can be specified on the command-line in the format "--option
 value", in the format "option value", or in a configuration file.  For
@@ -153,6 +163,13 @@ values. To split one configuration entry into multiple lines, use a single
 backslash character (\) before the end of the line.  Comments can be used in
 such multiline entries, but they must start at the beginning of a line.
 
+Configuration options can be imported from files or folders using the %include
+option with the value being a path. If the path is a file, the options from the
+file will be parsed as if they were written where the %include option is. If
+the path is a folder, all files on that folder will be parsed following lexical
+order. Files starting with a dot are ignored. Files on subfolders are ignored.
+The %include option can be used recursively.
+
 By default, an option on the command line overrides an option found in the
 configuration file, and an option in a configuration file overrides one in
 the defaults file.
@@ -176,7 +193,7 @@ forward slash (/) in the configuration file and on the command line.
 GENERAL OPTIONS
 ---------------
 
-[[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     A token bucket limits the average incoming bandwidth usage on this node
     to the specified number of bytes per second, and the average outgoing
     bandwidth usage to that same value.  If you want to run a relay in the
@@ -185,6 +202,9 @@ GENERAL OPTIONS
     course, more is better; we recommend at least 250 KBytes (2 mbits) if
     possible.  (Default: 1 GByte) +
  +
+    Note that this option, and other bandwidth-limiting options, apply to TCP
+    data only: They do not count TCP headers or DNS traffic. +
+ +
     With this option, and in other options that take arguments in bytes,
     KBytes, and so on, other formats are also supported. Notably, "KBytes" can
     also be written as "kilobytes" or "kb"; "MBytes" can be written as
@@ -195,43 +215,48 @@ GENERAL OPTIONS
     To avoid confusion, we recommend writing "bytes" or "bits" explicitly,
     since it's easy to forget that "B" means bytes, not bits.
 
-[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Limit the maximum token bucket size (also known as the burst) to the given
     number of bytes in each direction. (Default: 1 GByte)
 
-[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If set, we will not advertise more than this amount of bandwidth for our
     BandwidthRate. Server operators who want to reduce the number of clients
     who ask to build circuits through them (since this is proportional to
     advertised bandwidth rate) can thus reduce the CPU demands on their server
     without impacting network performance.
 
-[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If not 0, a separate token bucket limits the average incoming bandwidth
     usage for \_relayed traffic_ on this node to the specified number of bytes
     per second, and the average outgoing bandwidth usage to that same value.
     Relayed traffic currently is calculated to include answers to directory
-    requests, but that may change in future versions. (Default: 0)
+    requests, but that may change in future versions. They do not include directory
+    fetches by the relay (from authority or other relays), because that is considered
+    "client" activity.  (Default: 0)
 
-[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If not 0, limit the maximum token bucket size (also known as the burst) for
     \_relayed traffic_ to the given number of bytes in each direction.
-    (Default: 0)
+    They do not include directory fetches by the relay (from authority
+    or other relays), because that is considered "client" activity. (Default: 0)
 
-[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
-    If set, do separate rate limiting for each connection from a non-relay.
-    You should never need to change this value, since a network-wide value is
-    published in the consensus and your relay will use that value. (Default: 0)
+[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If this option is set manually, or via the "perconnbwrate" consensus
+    field, Tor will use it for separate rate limiting for each connection
+    from a non-relay. (Default: 0)
 
-[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
-    If set, do separate rate limiting for each connection from a non-relay.
-    You should never need to change this value, since a network-wide value is
-    published in the consensus and your relay will use that value. (Default: 0)
+[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
+    If this option is set manually, or via the "perconnbwburst" consensus
+    field, Tor will use it for separate rate limiting for each connection
+    from a non-relay. (Default: 0)
 
 [[ClientTransportPlugin]] **ClientTransportPlugin** __transport__ socks4|socks5 __IP__:__PORT__::
 **ClientTransportPlugin** __transport__ exec __path-to-binary__ [options]::
     In its first form, when set along with a corresponding Bridge line, the Tor
-    client forwards its traffic to a SOCKS-speaking proxy on "IP:PORT". It's the
+    client forwards its traffic to a SOCKS-speaking proxy on "IP:PORT".
+    (IPv4 addresses should written as-is; IPv6 addresses should be wrapped in
+    square brackets.) It's the
     duty of that proxy to properly forward the traffic to the bridge. +
  +
     In its second form, when set along with a corresponding Bridge line, the Tor
@@ -248,7 +273,8 @@ GENERAL OPTIONS
 [[ServerTransportListenAddr]] **ServerTransportListenAddr** __transport__ __IP__:__PORT__::
     When this option is set, Tor will suggest __IP__:__PORT__ as the
     listening address of any pluggable transport proxy that tries to
-    launch __transport__.
+    launch __transport__. (IPv4 addresses should written as-is; IPv6
+    addresses should be wrapped in square brackets.)
 
 [[ServerTransportOptions]] **ServerTransportOptions** __transport__ __k=v__ __k=v__ ...::
     When this option is set, Tor will pass the __k=v__ parameters to
@@ -285,7 +311,9 @@ GENERAL OPTIONS
     other than controller connections, and we close (and don't reattempt)
     any outbound
     connections.  Controllers sometimes use this option to avoid using
-    the network until Tor is fully configured. (Default: 0)
+    the network until Tor is fully configured.  Tor will make still certain
+    network-related calls (like DNS lookups) as a part of its configuration
+    process, even if DisableNetwork is set. (Default: 0)
 
 [[ConstrainedSockets]] **ConstrainedSockets** **0**|**1**::
     If set, Tor will tell the kernel to attempt to shrink the buffers for all
@@ -338,17 +366,10 @@ GENERAL OPTIONS
         Unix domain sockets only: Do not insist that the directory
         that holds the socket be read-restricted.
 
-[[ControlListenAddress]] **ControlListenAddress** __IP__[:__PORT__]::
-    Bind the controller listener to this address. If you specify a port, bind
-    to this port rather than the one specified in ControlPort. We strongly
-    recommend that you leave this alone unless you know what you're doing,
-    since giving attackers access to your control listener is really
-    dangerous. This directive can be specified multiple
-    times to bind to multiple addresses/ports.  (Default: 127.0.0.1)
-
 [[ControlSocket]] **ControlSocket** __Path__::
     Like ControlPort, but listens on a Unix domain socket, rather than a TCP
-    socket. '0' disables ControlSocket (Unix and Unix-like systems only.)
+    socket. '0' disables ControlSocket. (Unix and Unix-like systems only.)
+    (Default: 0)
 
 [[ControlSocketsGroupWritable]] **ControlSocketsGroupWritable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read and
@@ -390,14 +411,29 @@ GENERAL OPTIONS
     file readable by the default GID. (Default: 0)
 
 [[DataDirectory]] **DataDirectory** __DIR__::
-    Store working data in DIR (Default: @LOCALSTATEDIR@/lib/tor)
+    Store working data in DIR. Can not be changed while tor is running.
+    (Default: ~/.tor if your home directory is not /; otherwise,
+    @LOCALSTATEDIR@/lib/tor.  On Windows, the default is
+    your ApplicationData folder.)
 
 [[DataDirectoryGroupReadable]] **DataDirectoryGroupReadable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read the
     DataDirectory. If the option is set to 1, make the DataDirectory readable
     by the default GID. (Default: 0)
 
-[[FallbackDir]] **FallbackDir** __address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=__address__:__orport__]::
+[[CacheDirectory]] **CacheDirectory** __DIR__::
+    Store cached directory data in DIR. Can not be changed while tor is
+    running.
+    (Default: uses the value of DataDirectory.)
+
+[[CacheDirectoryGroupReadable]] **CacheDirectoryGroupReadable** **0**|**1**|**auto**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    CacheDirectory. If the option is set to 1, make the CacheDirectory readable
+    by the default GID. If the option is "auto", then we use the
+    setting for DataDirectoryGroupReadable when the CacheDirectory is the
+    same as the DataDirectory, and 0 otherwise. (Default: auto)
+
+[[FallbackDir]] **FallbackDir** __ipv4address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]::
     When we're unable to connect to any directory cache for directory info
     (usually because we don't know about any yet) we try a directory authority.
     Clients also simultaneously try a FallbackDir, to avoid hangs on client
@@ -413,7 +449,7 @@ GENERAL OPTIONS
     FallbackDir line is present, it replaces the hard-coded FallbackDirs,
     regardless of the value of UseDefaultFallbackDirs.) (Default: 1)
 
-[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__::
+[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__::
     Use a nonstandard authoritative directory server at the provided address
     and port, with the specified key fingerprint. This option can be repeated
     many times, for multiple authoritative directory servers. Flags are
@@ -427,13 +463,16 @@ GENERAL OPTIONS
     with probability proportional to that weight (default 1.0). If a
     flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority
     whose v3 long-term signing key has the fingerprint **fp**. Lastly,
-    if an "ipv6=__address__:__orport__" flag is present, then the directory
+    if an "ipv6=**[**__ipv6address__**]**:__orport__" flag is present, then
+    the directory
     authority is listening for IPv6 connections on the indicated IPv6 address
     and OR Port. +
  +
-    Tor will contact the authority at __address__:__port__ (the DirPort) to
-    download directory documents. If an IPv6 address is supplied, Tor will
-    also download directory documents at the IPv6 address on the DirPort. +
+    Tor will contact the authority at __ipv4address__ to
+    download directory documents. The provided __port__ value is a dirport;
+    clients ignore this in favor of the specified "orport=" value.  If an
+    IPv6 ORPort is supplied, Tor will
+    also download directory documents at the IPv6 ORPort. +
  +
     If no **DirAuthority** line is given, Tor will use the default directory
     authorities. NOTE: this option is intended for setting up a private Tor
@@ -448,9 +487,9 @@ GENERAL OPTIONS
     should be 1.0 or less. The default is less than 1, to reduce load on
     authorities. (Default: 0.1)
 
-[[AlternateDirAuthority]] **AlternateDirAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__ +
+[[AlternateDirAuthority]] **AlternateDirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__ +
 
-[[AlternateBridgeAuthority]] **AlternateBridgeAuthority** [__nickname__] [**flags**] __address__:__port__ __ fingerprint__::
+[[AlternateBridgeAuthority]] **AlternateBridgeAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __ fingerprint__::
     These options behave as DirAuthority, but they replace fewer of the
     default directory authorities. Using
     AlternateDirAuthority replaces the default Tor directory authorities, but
@@ -465,7 +504,8 @@ GENERAL OPTIONS
     not supported. We believe that this feature works on modern Gnu/Linux
     distributions, and that it should work on *BSD systems (untested). This
     option requires that you start your Tor as root, and you should use the
-    **User** option to properly reduce Tor's privileges. (Default: 0)
+    **User** option to properly reduce Tor's privileges.
+    Can not be changed while tor is running. (Default: 0)
 
 [[DisableDebuggerAttachment]] **DisableDebuggerAttachment** **0**|**1**::
    If set to 1, Tor will attempt to prevent basic debugging attachment attempts
@@ -505,22 +545,33 @@ GENERAL OPTIONS
     (Default: 1)
 
 [[FetchUselessDescriptors]] **FetchUselessDescriptors** **0**|**1**::
-    If set to 1, Tor will fetch every non-obsolete descriptor from the
-    authorities that it hears about. Otherwise, it will avoid fetching useless
-    descriptors, for example for routers that are not running. This option is
-    useful if you're using the contributed "exitlist" script to enumerate Tor
-    nodes that exit to certain addresses. (Default: 0)
+    If set to 1, Tor will fetch every consensus flavor, and all server
+    descriptors and authority certificates referenced by those consensuses,
+    except for extra info descriptors. When this option is 1, Tor will also
+    keep fetching descriptors, even when idle.
+    If set to 0, Tor will avoid fetching useless descriptors: flavors that it
+    is not using to build circuits, and authority certificates it does not
+    trust. When Tor hasn't built any application circuits, it will go idle,
+    and stop fetching descriptors. This option is useful if you're using a
+    tor client with an external parser that uses a full consensus.
+    This option fetches all documents except extrainfo descriptors,
+    **DirCache** fetches and serves all documents except extrainfo
+    descriptors, **DownloadExtraInfo*** fetches extrainfo documents, and serves
+    them if **DirCache** is on, and **UseMicrodescriptors** changes the
+    flavour of consensues and descriptors that is fetched and used for
+    building circuits. (Default: 0)
 
 [[HTTPProxy]] **HTTPProxy** __host__[:__port__]::
     Tor will make all its directory requests through this host:port (or host:80
     if port is not specified), rather than connecting directly to any directory
-    servers.
+    servers. (DEPRECATED: As of 0.3.1.0-alpha you should use HTTPSProxy.)
 
 [[HTTPProxyAuthenticator]] **HTTPProxyAuthenticator** __username:password__::
     If defined, Tor will use this username:password for Basic HTTP proxy
     authentication, as in RFC 2617. This is currently the only form of HTTP
     proxy authentication that Tor supports; feel free to submit a patch if you
-    want it to support others.
+    want it to support others. (DEPRECATED: As of 0.3.1.0-alpha you should use
+    HTTPSProxyAuthenticator.)
 
 [[HTTPSProxy]] **HTTPSProxy** __host__[:__port__]::
     Tor will make all its OR (SSL) connections through this host:port (or
@@ -538,7 +589,22 @@ GENERAL OPTIONS
 [[Sandbox]] **Sandbox** **0**|**1**::
     If set to 1, Tor will run securely through the use of a syscall sandbox.
     Otherwise the sandbox will be disabled. The option is currently an
-    experimental feature. (Default: 0)
+    experimental feature. It only works on Linux-based operating systems,
+    and only when Tor has been built with the libseccomp library. This option
+    can not be changed while tor is running.
+ +
+    When the Sandbox is 1, the following options can not be changed when tor
+    is running:
+    Address
+    ConnLimit
+    CookieAuthFile
+    DirPortFrontPage
+    ExtORPortCookieAuthFile
+    Logs
+    ServerDNSResolvConfFile
+    Tor must remain in client or server mode (some changes to ClientOnly and
+    ORPort are not allowed).
+    (Default: 0)
 
 [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]::
     Tor will make all OR connections through the SOCKS 4 proxy at host:port
@@ -555,10 +621,10 @@ GENERAL OPTIONS
     in accordance to RFC 1929. Both username and password must be between 1 and
     255 characters.
 
-[[SocksSocketsGroupWritable]] **SocksSocketsGroupWritable** **0**|**1**::
+[[UnixSocksGroupWritable]] **UnixSocksGroupWritable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read and
-    write unix sockets (e.g. SocksSocket). If the option is set to 1, make
-    the SocksSocket socket readable and writable by the default GID. (Default: 0)
+    write unix sockets (e.g. SocksPort unix:). If the option is set to 1, make
+    the Unix socket readable and writable by the default GID. (Default: 0)
 
 [[KeepalivePeriod]] **KeepalivePeriod** __NUM__::
     To keep firewalls from expiring connections, send a padding keepalive cell
@@ -597,7 +663,8 @@ GENERAL OPTIONS
  +
     The currently recognized domains are: general, crypto, net, config, fs,
     protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge,
-    acct, hist, and handshake.  Domain names are case-insensitive. +
+    acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos.
+    Domain names are case-insensitive. +
  +
     For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends
     to stdout: all handshake messages of any severity, all info-and-higher
@@ -609,7 +676,7 @@ GENERAL OPTIONS
     message currently has at least one domain; most currently have exactly
     one.  This doesn't affect controller log messages. (Default: 0)
 
-[[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**::
+[[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**::
     Unparseable descriptors (e.g. for votes, consensuses, routers) are logged
     in separate files by hash, up to the specified size in total.  Note that
     only files logged during the lifetime of this Tor process count toward the
@@ -621,27 +688,41 @@ GENERAL OPTIONS
     is only useful when you have multiple network interfaces, and you want all
     of Tor's outgoing connections to use a single one. This option may
     be used twice, once with an IPv4 address and once with an IPv6 address.
+    IPv6 addresses should be wrapped in square brackets.
     This setting will be ignored for connections to the loopback addresses
-    (127.0.0.0/8 and ::1).
+    (127.0.0.0/8 and ::1), and is not used for DNS requests as well.
+
+[[OutboundBindAddressOR]] **OutboundBindAddressOR** __IP__::
+    Make all outbound non-exit (relay and other) connections
+    originate from the IP address specified. This option overrides
+    **OutboundBindAddress** for the same IP version. This option may
+    be used twice, once with an IPv4 address and once with an IPv6
+    address. IPv6 addresses should be wrapped in square brackets.
+    This setting will be ignored for connections to the loopback
+    addresses (127.0.0.0/8 and ::1).
+
+[[OutboundBindAddressExit]] **OutboundBindAddressExit** __IP__::
+    Make all outbound exit connections originate from the IP address
+    specified. This option overrides **OutboundBindAddress** for the
+    same IP version. This option may be used twice, once with an IPv4
+    address and once with an IPv6 address.
+    IPv6 addresses should be wrapped in square brackets.
+    This setting will be ignored
+    for connections to the loopback addresses (127.0.0.0/8 and ::1).
 
 [[PidFile]] **PidFile** __FILE__::
     On startup, write our PID to FILE. On clean shutdown, remove
-    FILE.
+    FILE. Can not be changed while tor is running.
 
 [[ProtocolWarnings]] **ProtocolWarnings** **0**|**1**::
     If 1, Tor will log with severity \'warn' various cases of other parties not
     following the Tor specification. Otherwise, they are logged with severity
     \'info'. (Default: 0)
 
-[[PredictedPortsRelevanceTime]] **PredictedPortsRelevanceTime** __NUM__::
-    Set how long, after the client has made an anonymized connection to a
-    given port, we will try to make sure that we build circuits to
-    exits that support that port. The maximum value for this option is 1
-    hour. (Default: 1 hour)
-
 [[RunAsDaemon]] **RunAsDaemon** **0**|**1**::
     If 1, Tor forks and daemonizes to the background. This option has no effect
     on Windows; instead you should use the --service command-line option.
+    Can not be changed while tor is running.
     (Default: 0)
 
 [[LogTimeGranularity]] **LogTimeGranularity** __NUM__::
@@ -658,7 +739,13 @@ GENERAL OPTIONS
 
 [[SyslogIdentityTag]] **SyslogIdentityTag** __tag__::
     When logging to syslog, adds a tag to the syslog identity such that
-    log entries are marked with "Tor-__tag__".  (Default: none)
+    log entries are marked with "Tor-__tag__". Can not be changed while tor is
+    running. (Default: none)
+
+[[AndroidIdentityTag]] **AndroidIdentityTag** __tag__::
+    When logging to Android's logging subsystem, adds a tag to the log identity
+    such that log entries are marked with "Tor-__tag__". Can not be changed while
+    tor is running. (Default: none)
 
 [[SafeLogging]] **SafeLogging** **0**|**1**|**relay**::
     Tor can scrub potentially sensitive strings from log messages (e.g.
@@ -673,6 +760,7 @@ GENERAL OPTIONS
 
 [[User]] **User** __Username__::
     On startup, setuid to this user and setgid to their primary group.
+    Can not be changed while tor is running.
 
 [[KeepBindCapabilities]] **KeepBindCapabilities** **0**|**1**|**auto**::
     On Linux, when we are started as root and we switch our identity using
@@ -680,37 +768,38 @@ GENERAL OPTIONS
     try to retain our ability to bind to low ports.  If this value is 1, we
     try to keep the capability; if it is 0 we do not; and if it is **auto**,
     we keep the capability only if we are configured to listen on a low port.
+    Can not be changed while tor is running.
     (Default: auto.)
 
 [[HardwareAccel]] **HardwareAccel** **0**|**1**::
     If non-zero, try to use built-in (static) crypto hardware acceleration when
-    available. (Default: 0)
+    available. Can not be changed while tor is running. (Default: 0)
 
 [[AccelName]] **AccelName** __NAME__::
     When using OpenSSL hardware crypto acceleration attempt to load the dynamic
     engine of this name. This must be used for any dynamic hardware engine.
-    Names can be verified with the openssl engine command.
+    Names can be verified with the openssl engine command. Can not be changed
+    while tor is running.
 
 [[AccelDir]] **AccelDir** __DIR__::
     Specify this option if using dynamic hardware acceleration and the engine
     implementation library resides somewhere other than the OpenSSL default.
+    Can not be changed while tor is running.
 
 [[AvoidDiskWrites]] **AvoidDiskWrites** **0**|**1**::
     If non-zero, try to write to disk less frequently than we would otherwise.
     This is useful when running on flash memory or other media that support
     only a limited number of writes. (Default: 0)
 
-[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM1__::
+[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM__::
     If this value is set, we override the default algorithm for choosing which
-    circuit's cell to deliver or relay next. When the value is 0, we
-    round-robin between the active circuits on a connection, delivering one
-    cell from each in turn. When the value is positive, we prefer delivering
-    cells from whichever connection has the lowest weighted cell count, where
-    cells are weighted exponentially according to the supplied
-    CircuitPriorityHalflife value (in seconds). If this option is not set at
-    all, we use the behavior recommended in the current consensus
-    networkstatus. This is an advanced option; you generally shouldn't have
-    to mess with it. (Default: not set)
+    circuit's cell to deliver or relay next. It is delivered first to the
+    circuit that has the lowest weighted cell count, where cells are weighted
+    exponentially according to this value (in seconds). If the value is -1, it
+    is taken from the consensus if possible else it will fallback to the
+    default value of 30. Minimum: 1, Maximum: 2147483647. This can be defined
+    as a float value. This is an advanced option; you generally shouldn't have
+    to mess with it. (Default: -1)
 
 [[CountPrivateBandwidth]] **CountPrivateBandwidth** **0**|**1**::
     If this option is set, then Tor's rate-limiting applies not only to
@@ -718,27 +807,63 @@ GENERAL OPTIONS
     127.0.0.1 or 10.0.0.1.  This is mostly useful for debugging
     rate-limiting.  (Default: 0)
 
+[[ExtendByEd25519ID]] **ExtendByEd25519ID** **0**|**1**|**auto**::
+    If this option is set to 1, we always try to include a relay's Ed25519 ID
+    when telling the proceeding relay in a circuit to extend to it.
+    If this option is set to 0, we never include Ed25519 IDs when extending
+    circuits.  If the option is set to "default", we obey a
+    parameter in the consensus document. (Default: auto)
+
+[[NoExec]] **NoExec** **0**|**1**::
+    If this option is set to 1, then Tor will never launch another
+    executable, regardless of the settings of ClientTransportPlugin
+    or ServerTransportPlugin.  Once this option has been set to 1,
+    it cannot be set back to 0 without restarting Tor. (Default: 0)
+
+[[Schedulers]] **Schedulers** **KIST**|**KISTLite**|**Vanilla**::
+    Specify the scheduler type that tor should use. The scheduler is
+    responsible for moving data around within a Tor process. This is an ordered
+    list by priority which means that the first value will be tried first and if
+    unavailable, the second one is tried and so on. It is possible to change
+    these values at runtime. This option mostly effects relays, and most
+    operators should leave it set to its default value.
+    (Default: KIST,KISTLite,Vanilla)
+ +
+    The possible scheduler types are:
+ +
+    **KIST**: Kernel-Informed Socket Transport. Tor will use TCP information
+    from the kernel to make informed decisions regarding how much data to send
+    and when to send it. KIST also handles traffic in batches (see
+    KISTSchedRunInterval) in order to improve traffic prioritization decisions.
+    As implemented, KIST will only work on Linux kernel version 2.6.39 or
+    higher.
+ +
+    **KISTLite**: Same as KIST but without kernel support. Tor will use all
+    the same mechanics as with KIST, including the batching, but its decisions
+    regarding how much data to send will not be as good. KISTLite will work on
+    all kernels and operating systems, and the majority of the benefits of KIST
+    are still realized with KISTLite.
+ +
+    **Vanilla**: The scheduler that Tor used before KIST was implemented. It
+    sends as much data as possible, as soon as possible. Vanilla will work on
+    all kernels and operating systems.
+
+[[KISTSchedRunInterval]] **KISTSchedRunInterval** __NUM__ **msec**::
+    If KIST or KISTLite is used in the Schedulers option, this controls at which
+    interval the scheduler tick is. If the value is 0 msec, the value is taken
+    from the consensus if possible else it will fallback to the default 10
+    msec. Maximum possible value is 100 msec. (Default: 0 msec)
+
+[[KISTSockBufSizeFactor]] **KISTSockBufSizeFactor** __NUM__::
+    If KIST is used in Schedulers, this is a multiplier of the per-socket
+    limit calculation of the KIST algorithm. (Default: 1.0)
+
 CLIENT OPTIONS
 --------------
 
 The following options are useful only for clients (that is, if
-**SocksPort**, **TransPort**, **DNSPort**, or **NATDPort** is non-zero):
-
-[[AllowInvalidNodes]] **AllowInvalidNodes** **entry**|**exit**|**middle**|**introduction**|**rendezvous**|**...**::
-    If some Tor servers are obviously not working right, the directory
-    authorities can manually mark them as invalid, meaning that it's not
-    recommended you use them for entry or exit positions in your circuits. You
-    can opt to use them in some circuit positions, though. The default is
-    "middle,rendezvous", and other choices are not advised.
-
-[[ExcludeSingleHopRelays]] **ExcludeSingleHopRelays** **0**|**1**::
-    This option controls whether circuits built by Tor will include relays with
-    the AllowSingleHopExits flag set to true. If ExcludeSingleHopRelays is set
-    to 0, these relays will be included. Note that these relays might be at
-    higher risk of being seized or observed, so they are not normally
-    included.  Also note that relatively few clients turn off this option,
-    so using these relays might make your client stand out.
-    (Default: 1)
+**SocksPort**, **HTTPTunnelPort**, **TransPort**, **DNSPort**, or
+**NATDPort** is non-zero):
 
 [[Bridge]] **Bridge** [__transport__] __IP__:__ORPort__ [__fingerprint__]::
     When set along with UseBridges, instructs Tor to use the relay at
@@ -753,7 +878,12 @@ The following options are useful only for clients (that is, if
     rather than connecting to the bridge directly. Some transports use a
     transport-specific method to work out the remote address to connect to.
     These transports typically ignore the "IP:ORPort" specified in the bridge
-    line.
+    line.  +
+ +
+    Tor passes any "key=val" settings to the pluggable transport proxy as
+    per-connection arguments when connecting to the bridge.  Consult
+    the documentation of the pluggable transport for details of what
+    arguments it supports.
 
 [[LearnCircuitBuildTimeout]] **LearnCircuitBuildTimeout** **0**|**1**::
     If 0, CircuitBuildTimeout adaptive learning is disabled. (Default: 1)
@@ -766,13 +896,15 @@ The following options are useful only for clients (that is, if
     LearnCircuitBuildTimeout is 0, this value is the only value used.
     (Default: 60 seconds)
 
-[[CircuitIdleTimeout]] **CircuitIdleTimeout** __NUM__::
-    If we have kept a clean (never used) circuit around for NUM seconds, then
-    close it. This way when the Tor client is entirely idle, it can expire all
-    of its circuits, and then expire its TLS connections. Also, if we end up
-    making a circuit that is not useful for exiting any of the requests we're
-    receiving, it won't forever take up a slot in the circuit list. (Default: 1
-    hour)
+[[CircuitsAvailableTimeout]] **CircuitsAvailableTimeout** __NUM__::
+    Tor will attempt to keep at least one open, unused circuit available for
+    this amount of time. This option governs how long idle circuits are kept
+    open, as well as the amount of time Tor will keep a circuit open to each
+    of the recently used ports. This way when the Tor client is entirely
+    idle, it can expire all of its circuits, and then expire its TLS
+    connections. Note that the actual timeout value is uniformly randomized
+    from the specified value to twice that amount. (Default: 30 minutes;
+    Max: 24 hours)
 
 [[CircuitStreamTimeout]] **CircuitStreamTimeout** __NUM__::
     If non-zero, this option overrides our internal timeout schedule for how
@@ -789,6 +921,22 @@ The following options are useful only for clients (that is, if
     and fast enough. The current behavior is simply that Tor is a client
     unless ORPort, ExtORPort, or DirPort are configured.) (Default: 0)
 
+[[ConnectionPadding]] **ConnectionPadding** **0**|**1**|**auto**::
+    This option governs Tor's use of padding to defend against some forms of
+    traffic analysis. If it is set to 'auto', Tor will send padding only
+    if both the client and the relay support it. If it is set to 0, Tor will
+    not send any padding cells. If it is set to 1, Tor will still send padding
+    for client connections regardless of relay support. Only clients may set
+    this option. This option should be offered via the UI to mobile users
+    for use where bandwidth may be expensive.
+    (Default: auto)
+
+[[ReducedConnectionPadding]] **ReducedConnectionPadding** **0**|**1**::
+    If set to 1, Tor will not not hold OR connections open for very long,
+    and will send less padding on these connections. Only clients may set
+    this option. This option should be offered via the UI to mobile users
+    for use where bandwidth may be expensive. (Default: 0)
+
 [[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to avoid when building a circuit. Country codes are
@@ -817,7 +965,7 @@ The following options are useful only for clients (that is, if
 [[ExcludeExitNodes]] **ExcludeExitNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to never use when picking an exit node---that is, a
-    node that delivers traffic for you outside the Tor network.   Note that any
+    node that delivers traffic for you *outside* the Tor network.   Note that any
     node listed in ExcludeNodes is automatically considered to be part of this
     list too.  See
     the **ExcludeNodes** option for more information on how to specify
@@ -834,7 +982,7 @@ The following options are useful only for clients (that is, if
 [[ExitNodes]] **ExitNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to use as exit node---that is, a
-    node that delivers traffic for you outside the Tor network.  See
+    node that delivers traffic for you *outside* the Tor network.  See
     the **ExcludeNodes** option for more information on how to specify nodes. +
  +
     Note that if you list too few nodes here, or if you exclude too many exit
@@ -842,7 +990,7 @@ The following options are useful only for clients (that is, if
     if none of the exits you list allows traffic on port 80 or 443, you won't
     be able to browse the web. +
  +
-    Note also that not every circuit is used to deliver traffic outside of
+    Note also that not every circuit is used to deliver traffic *outside* of
     the Tor network.  It is normal to see non-exit circuits (such as those
     used to connect to hidden services, those that do directory fetches,
     those used for relay reachability self-tests, and so on) that end
@@ -852,7 +1000,7 @@ The following options are useful only for clients (that is, if
     The ExcludeNodes option overrides this option: any node listed in both
     ExitNodes and ExcludeNodes is treated as excluded. +
  +
-    The .exit address notation, if enabled via AllowDotExit, overrides
+    The .exit address notation, if enabled via MapAddress, overrides
     this option.
 
 [[EntryNodes]] **EntryNodes** __node__,__node__,__...__::
@@ -868,16 +1016,16 @@ The following options are useful only for clients (that is, if
     the **ExcludeNodes** option for more information on how to specify nodes.
 
 [[StrictNodes]] **StrictNodes** **0**|**1**::
-    If StrictNodes is set to 1, Tor will treat the ExcludeNodes option as a
-    requirement to follow for all the circuits you generate, even if doing so
-    will break functionality for you.  If StrictNodes is set to 0, Tor will
+    If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
+    as a requirement to follow for all the circuits you generate, even if
+    doing so will break functionality for you (StrictNodes applies to neither
+    ExcludeExitNodes nor to ExitNodes).  If StrictNodes is set to 0, Tor will
     still try to avoid nodes in the ExcludeNodes list, but it will err on the
-    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells
-    Tor that it is okay to use an excluded node when it is *necessary* to
-    perform relay reachability self-tests, connect to
-    a hidden service, provide a hidden service to a client, fulfill a .exit
-    request, upload directory information, or download directory information.
-    (Default: 0)
+    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells Tor
+    that it is okay to use an excluded node when it is *necessary* to perform
+    relay reachability self-tests, connect to a hidden service, provide a
+    hidden service to a client, fulfill a .exit request, upload directory
+    information, or download directory information.  (Default: 0)
 
 [[FascistFirewall]] **FascistFirewall** **0**|**1**::
     If 1, Tor will only create outgoing connections to ORs running on ports
@@ -892,7 +1040,7 @@ The following options are useful only for clients (that is, if
     **FascistFirewall** is set. This option is deprecated; use ReachableAddresses
     instead. (Default: 80, 443)
 
-[[ReachableAddresses]] **ReachableAddresses** __ADDR__[/__MASK__][:__PORT__]...::
+[[ReachableAddresses]] **ReachableAddresses** __IP__[/__MASK__][:__PORT__]...::
     A comma-separated list of IP addresses and ports that your firewall allows
     you to connect to. The format is as for the addresses in ExitPolicy, except
     that "accept" is understood unless "reject" is explicitly provided. For
@@ -901,14 +1049,15 @@ The following options are useful only for clients (that is, if
     99, rejects port 80 connections to net 18, and accepts connections to port
     80 otherwise. (Default: \'accept \*:*'.)
 
-[[ReachableDirAddresses]] **ReachableDirAddresses** __ADDR__[/__MASK__][:__PORT__]...::
+[[ReachableDirAddresses]] **ReachableDirAddresses** __IP__[/__MASK__][:__PORT__]...::
     Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
     these restrictions when fetching directory information, using standard HTTP
     GET requests. If not set explicitly then the value of
     **ReachableAddresses** is used. If **HTTPProxy** is set then these
-    connections will go through that proxy.
+    connections will go through that proxy.   (DEPRECATED: This option has
+    had no effect for some time.)
 
-[[ReachableORAddresses]] **ReachableORAddresses** __ADDR__[/__MASK__][:__PORT__]...::
+[[ReachableORAddresses]] **ReachableORAddresses** __IP__[/__MASK__][:__PORT__]...::
     Like **ReachableAddresses**, a list of addresses and ports. Tor will obey
     these restrictions when connecting to Onion Routers, using TLS/SSL. If not
     set explicitly then the value of **ReachableAddresses** is used. If
@@ -931,24 +1080,6 @@ The following options are useful only for clients (that is, if
     services can be configured to require authorization using the
     **HiddenServiceAuthorizeClient** option.
 
-[[CloseHSClientCircuitsImmediatelyOnTimeout]] **CloseHSClientCircuitsImmediatelyOnTimeout** **0**|**1**::
-    If 1, Tor will close unfinished hidden service client circuits
-    which have not moved closer to connecting to their destination
-    hidden service when their internal state has not changed for the
-    duration of the current circuit-build timeout.  Otherwise, such
-    circuits will be left open, in the hope that they will finish
-    connecting to their destination hidden services.  In either case,
-    another set of introduction and rendezvous circuits for the same
-    destination hidden service will be launched. (Default: 0)
-
-[[CloseHSServiceRendCircuitsImmediatelyOnTimeout]] **CloseHSServiceRendCircuitsImmediatelyOnTimeout** **0**|**1**::
-    If 1, Tor will close unfinished hidden-service-side rendezvous
-    circuits after the current circuit-build timeout.  Otherwise, such
-    circuits will be left open, in the hope that they will finish
-    connecting to their destinations.  In either case, another
-    rendezvous circuit for the same destination client will be
-    launched. (Default: 0)
-
 [[LongLivedPorts]] **LongLivedPorts** __PORTS__::
     A list of ports for services that tend to have long-running connections
     (e.g. chat and interactive shells). Circuits for streams that use these
@@ -1007,7 +1138,8 @@ The following options are useful only for clients (that is, if
     but never attach a new stream to a circuit that is too old.  For hidden
     services, this applies to the __last__ time a circuit was used, not the
     first. Circuits with streams constructed with SOCKS authentication via
-    SocksPorts that have **KeepAliveIsolateSOCKSAuth** ignore this value.
+    SocksPorts that have **KeepAliveIsolateSOCKSAuth** also remain alive
+    for MaxCircuitDirtiness seconds after carrying the last such stream.
     (Default: 10 minutes)
 
 [[MaxClientCircuitsPending]] **MaxClientCircuitsPending** __NUM__::
@@ -1056,7 +1188,9 @@ The following options are useful only for clients (that is, if
         Unsupported and force-disabled when using Unix domain sockets.)
     **IsolateSOCKSAuth**;;
         Don't share circuits with streams for which different
-        SOCKS authentication was provided. (On by default;
+        SOCKS authentication was provided. (For HTTPTunnelPort
+        connections, this option looks at the Proxy-Authorization and
+        X-Tor-Stream-Isolation headers. On by default;
         you can disable it with **NoIsolateSOCKSAuth**.)
     **IsolateClientProtocol**;;
         Don't share circuits with streams using a different protocol.
@@ -1069,8 +1203,9 @@ The following options are useful only for clients (that is, if
         Don't share circuits with streams targeting a different
         destination address.
     **KeepAliveIsolateSOCKSAuth**;;
-        If **IsolateSOCKSAuth** is enabled, keep alive circuits that have
-        streams with SOCKS authentication set indefinitely.
+        If **IsolateSOCKSAuth** is enabled, keep alive circuits while they have
+        at least one stream with SOCKS authentication active. After such a circuit
+        is idle for more than MaxCircuitDirtiness seconds, it can be closed.
     **SessionGroup=**__INT__;;
         If no other isolation rules would prevent it, allow streams
         on this port to share circuits with streams from every other
@@ -1078,6 +1213,7 @@ The following options are useful only for clients (that is, if
         on different SocksPorts, TransPorts, etc are always isolated from one
         another. This option overrides that behavior.)
 
+// Anchor only for formatting, not visible in the man page.
 [[OtherSocksPortFlags]]::
     Other recognized __flags__ for a SocksPort are:
     **NoIPv4Traffic**;;
@@ -1103,7 +1239,7 @@ The following options are useful only for clients (that is, if
         flag is not supported.
     **CacheIPv4DNS**;;
         Tells the client to remember IPv4 DNS answers we receive from exit
-        nodes via this connection. (On by default.)
+        nodes via this connection.
     **CacheIPv6DNS**;;
         Tells the client to remember IPv6 DNS answers we receive from exit
         nodes via this connection.
@@ -1118,8 +1254,8 @@ The following options are useful only for clients (that is, if
         nodes via this connection.
     **UseIPv4Cache**;;
         Tells the client to use any cached IPv4 DNS answers we have when making
-        requests via this connection. (NOTE: This option, along UseIPv6Cache
-        and UseDNSCache, can harm your anonymity, and probably
+        requests via this connection. (NOTE: This option, or UseIPv6Cache
+        or UseDNSCache, can harm your anonymity, and probably
         won't help performance as much as you might expect. Use with care!)
     **UseIPv6Cache**;;
         Tells the client to use any cached IPv6 DNS answers we have when making
@@ -1142,20 +1278,12 @@ The following options are useful only for clients (that is, if
         authentication" when IsolateSOCKSAuth is disabled, or when this
         option is set.
 
+// Anchor only for formatting, not visible in the man page.
+[[SocksPortFlagsMisc]]::
     Flags are processed left to right. If flags conflict, the last flag on the
     line is used, and all earlier flags are ignored. No error is issued for
     conflicting flags.
 
-[[SocksListenAddress]] **SocksListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for connections from Socks-speaking
-    applications. (Default: 127.0.0.1) You can also specify a port (e.g.
-    192.168.0.1:9100). This directive can be specified multiple times to bind
-    to multiple addresses/ports.  (DEPRECATED: As of 0.2.3.x-alpha, you can
-    now use multiple SocksPort entries, and provide addresses for SocksPort
-    entries, so SocksListenAddress no longer has a purpose.  For backward
-    compatibility, SocksListenAddress is only allowed when SocksPort is just
-    a port number.)
-
 [[SocksPolicy]] **SocksPolicy** __policy__,__policy__,__...__::
     Set an entrance policy for this server, to limit who can connect to the
     SocksPort and DNSPort ports. The policies have the same form as exit
@@ -1168,11 +1296,14 @@ The following options are useful only for clients (that is, if
     2 minutes)
 
 [[TokenBucketRefillInterval]] **TokenBucketRefillInterval** __NUM__ [**msec**|**second**]::
-    Set the refill interval of Tor's token bucket to NUM milliseconds.
-    NUM must be between 1 and 1000, inclusive.  Note that the configured
-    bandwidth limits are still expressed in bytes per second: this
+    Set the refill delay interval of Tor's token bucket to NUM milliseconds.
+    NUM must be between 1 and 1000, inclusive.  When Tor is out of bandwidth,
+    on a connection or globally, it will wait up to this long before it tries
+    to use that connection again.
+    Note that bandwidth limits are still expressed in bytes per second: this
     option only affects the frequency with which Tor checks to see whether
-    previously exhausted connections may read again. (Default: 100 msec)
+    previously exhausted connections may read again.
+    Can not be changed while tor is running. (Default: 100 msec)
 
 [[TrackHostExits]] **TrackHostExits** __host__,__.domain__,__...__::
     For each value in the comma separated list, Tor will track recent
@@ -1209,15 +1340,6 @@ The following options are useful only for clients (that is, if
     Authorities, Single Onion Services, and Tor2web clients. In these cases,
     the this option is ignored. (Default: 1)
 
-[[UseEntryGuardsAsDirGuards]] **UseEntryGuardsAsDirGuards** **0**|**1**::
-    If this option is set to 1, and UseEntryGuards is also set to 1,
-    we try to use our entry guards as directory
-    guards, and failing that, pick more nodes to act as our directory guards.
-    This helps prevent an adversary from enumerating clients. It's only
-    available for clients (non-relay, non-bridge) that aren't configured to
-    download any non-default directory material.  It doesn't currently
-    do anything when we lack a live consensus. (Default: 1)
-
 [[GuardfractionFile]] **GuardfractionFile** __FILENAME__::
     V3 authoritative directories only. Configures the location of the
     guardfraction file which contains information about how long relays
@@ -1231,16 +1353,22 @@ The following options are useful only for clients (that is, if
 
 [[NumEntryGuards]] **NumEntryGuards** __NUM__::
     If UseEntryGuards is set to 1, we will try to pick a total of NUM routers
-    as long-term entries for our circuits. If NUM is 0, we try to learn
-    the number from the NumEntryGuards consensus parameter, and default
-    to 3 if the consensus parameter isn't set. (Default: 0)
+    as long-term entries for our circuits. If NUM is 0, we try to learn the
+    number from the guard-n-primary-guards-to-use consensus parameter, and
+    default to 1 if the consensus parameter isn't set. (Default: 0)
+
+[[NumPrimaryGuards]] **NumPrimaryGuards** __NUM__::
+    If UseEntryGuards is set to 1, we will try to pick NUM routers for our
+    primary guard list, which is the set of routers we strongly prefer when
+    connecting to the Tor network. If NUM is 0, we try to learn the number from
+    the guard-n-primary-guards consensus parameter, and default to 3 if the
+    consensus parameter isn't set. (Default: 0)
 
 [[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__::
-    If UseEntryGuardsAsDirectoryGuards is enabled, we try to make sure we
-    have at least NUM routers to use as directory guards. If this option
-    is set to 0, use the value from the NumDirectoryGuards consensus
-    parameter, falling back to the value from NumEntryGuards if the
-    consensus parameter is 0 or isn't set. (Default: 0)
+    If UseEntryGuards is set to 1, we try to make sure we have at least NUM
+    routers to use as directory guards. If this option is set to 0, use the
+    value from the guard-n-primary-dir-guards-to-use consensus parameter, and
+    default to 3 if the consensus parameter isn't set.  (Default: 0)
 
 [[GuardLifetime]] **GuardLifetime**  __N__ **days**|**weeks**|**months**::
     If nonzero, and UseEntryGuards is set, minimum time to keep a guard before
@@ -1262,15 +1390,9 @@ The following options are useful only for clients (that is, if
     helps to determine whether an application using Tor is possibly leaking
     DNS requests. (Default: 0)
 
-[[WarnUnsafeSocks]] **WarnUnsafeSocks** **0**|**1**::
-    When this option is enabled, Tor will warn whenever a request is
-    received that only contains an IP address instead of a hostname. Allowing
-    applications to do DNS resolves themselves is usually a bad idea and
-    can leak your location to attackers. (Default: 1)
+[[VirtualAddrNetworkIPv4]] **VirtualAddrNetworkIPv4** __IPv4Address__/__bits__ +
 
-[[VirtualAddrNetworkIPv4]] **VirtualAddrNetworkIPv4** __Address__/__bits__ +
-
-[[VirtualAddrNetworkIPv6]] **VirtualAddrNetworkIPv6** [__Address__]/__bits__::
+[[VirtualAddrNetworkIPv6]] **VirtualAddrNetworkIPv6** [__IPv6Address__]/__bits__::
     When Tor needs to assign a virtual (unused) address because of a MAPADDRESS
     command from the controller or the AutomapHostsOnResolve feature, Tor
     picks an unassigned address from this range. (Defaults:
@@ -1293,23 +1415,13 @@ The following options are useful only for clients (that is, if
     resolved. This helps trap accidental attempts to resolve URLs and so on.
     (Default: 0)
 
-[[AllowDotExit]] **AllowDotExit** **0**|**1**::
-    If enabled, we convert "www.google.com.foo.exit" addresses on the
-    SocksPort/TransPort/NATDPort into "www.google.com" addresses that exit from
-    the node "foo". Disabled by default since attacking websites and exit
-    relays can use it to manipulate your path selection. (Default: 0)
-
-[[FastFirstHopPK]] **FastFirstHopPK** **0**|**1**|**auto**::
-    When this option is disabled, Tor uses the public key step for the first
-    hop of creating circuits. Skipping it is generally safe since we have
-    already used TLS to authenticate the relay and to establish forward-secure
-    keys. Turning this option off makes circuit building a little
-    slower. Setting this option to "auto" takes advice from the authorities
-    in the latest consensus about whether to use this feature.  +
- +
-    Note that Tor will always use the public key step for the first hop if it's
-    operating as a relay, and it will never use the public key step if it
-    doesn't yet know the onion key of the first hop. (Default: auto)
+[[HTTPTunnelPort]] **HTTPTunnelPort**  \['address':]__port__|**auto** [_isolation flags_]::
+    Open this port to listen for proxy connections using the "HTTP CONNECT"
+    protocol instead of SOCKS. Set this to
+    0 if you don't want to allow "HTTP CONNECT" connections. Set the port
+    to "auto" to have Tor pick a port for you. This directive can be
+    specified multiple times to bind to multiple addresses/ports.  See
+    SOCKSPort for an explanation of isolation flags. (Default: 0)
 
 [[TransPort]] **TransPort**  \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for transparent proxy connections.  Set this to
@@ -1321,43 +1433,31 @@ The following options are useful only for clients (that is, if
     TransPort requires OS support for transparent proxies, such as BSDs' pf or
     Linux's IPTables. If you're planning to use Tor as a transparent proxy for
     a network, you'll want to examine and change VirtualAddrNetwork from the
-    default setting. You'll also want to set the TransListenAddress option for
-    the network you'd like to proxy. (Default: 0)
-
-[[TransListenAddress]] **TransListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for transparent proxy connections. (Default:
-    127.0.0.1). This is useful for exporting a transparent proxy server to an
-    entire network. (DEPRECATED: As of 0.2.3.x-alpha, you can
-    now use multiple TransPort entries, and provide addresses for TransPort
-    entries, so TransListenAddress no longer has a purpose.  For backward
-    compatibility, TransListenAddress is only allowed when TransPort is just
-    a port number.)
+    default setting. (Default: 0)
 
 [[TransProxyType]] **TransProxyType** **default**|**TPROXY**|**ipfw**|**pf-divert**::
     TransProxyType may only be enabled when there is transparent proxy listener
-    enabled.
+    enabled. +
  +
     Set this to "TPROXY" if you wish to be able to use the TPROXY Linux module
     to transparently proxy connections that are configured using the TransPort
-    option. This setting lets the listener on the TransPort accept connections
-    for all addresses, even when the TransListenAddress is configured for an
-    internal address. Detailed information on how to configure the TPROXY
+    option. Detailed information on how to configure the TPROXY
     feature can be found in the Linux kernel source tree in the file
-    Documentation/networking/tproxy.txt.
+    Documentation/networking/tproxy.txt. +
  +
-    Set this option to "ipfw" to use the FreeBSD ipfw interface.
+    Set this option to "ipfw" to use the FreeBSD ipfw interface. +
  +
     On *BSD operating systems when using pf, set this to "pf-divert" to take
     advantage of +divert-to+ rules, which do not modify the packets like
     +rdr-to+ rules do. Detailed information on how to configure pf to use
     +divert-to+ rules can be found in the pf.conf(5) manual page. On OpenBSD,
     +divert-to+ is available to use on versions greater than or equal to
-    OpenBSD 4.4.
+    OpenBSD 4.4. +
  +
     Set this to "default", or leave it unconfigured, to use regular IPTables
-    on Linux, or to use pf +rdr-to+ rules on *BSD systems.
+    on Linux, or to use pf +rdr-to+ rules on *BSD systems. +
  +
-    (Default: "default".)
+    (Default: "default")
 
 [[NATDPort]] **NATDPort** \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for connections from old versions of ipfw (as
@@ -1369,13 +1469,6 @@ The following options are useful only for clients (that is, if
  +
     This option is only for people who cannot use TransPort. (Default: 0)
 
-[[NATDListenAddress]] **NATDListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for NATD connections. (DEPRECATED: As of
-    0.2.3.x-alpha, you can now use multiple NATDPort entries, and provide
-    addresses for NATDPort entries, so NATDListenAddress no longer has a
-    purpose.  For backward compatibility, NATDListenAddress is only allowed
-    when NATDPort is just a port number.)
-
 [[AutomapHostsOnResolve]] **AutomapHostsOnResolve** **0**|**1**::
     When this option is enabled, and we get a request to resolve an address
     that ends with one of the suffixes in **AutomapHostsSuffixes**, we map an
@@ -1396,24 +1489,18 @@ The following options are useful only for clients (that is, if
     addresses/ports. See SocksPort for an explanation of isolation
     flags. (Default: 0)
 
-[[DNSListenAddress]] **DNSListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for DNS connections. (DEPRECATED: As of
-    0.2.3.x-alpha, you can now use multiple DNSPort entries, and provide
-    addresses for DNSPort entries, so DNSListenAddress no longer has a
-    purpose.  For backward compatibility, DNSListenAddress is only allowed
-    when DNSPort is just a port number.)
-
 [[ClientDNSRejectInternalAddresses]] **ClientDNSRejectInternalAddresses** **0**|**1**::
     If true, Tor does not believe any anonymously retrieved DNS answer that
     tells it that an address resolves to an internal address (like 127.0.0.1 or
-    192.168.0.1). This option prevents certain browser-based attacks; don't
-    turn it off unless you know what you're doing. (Default: 1)
+    192.168.0.1). This option prevents certain browser-based attacks; it
+    is not allowed to be set on the default network. (Default: 1)
 
 [[ClientRejectInternalAddresses]] **ClientRejectInternalAddresses** **0**|**1**::
     If true, Tor does not try to fulfill requests to connect to an internal
-    address (like 127.0.0.1 or 192.168.0.1) __unless a exit node is
+    address (like 127.0.0.1 or 192.168.0.1) __unless an exit node is
     specifically requested__ (for example, via a .exit hostname, or a
-    controller request).  (Default: 1)
+    controller request).  If true, multicast DNS hostnames for machines on the
+    local network (of the form *.local) are also rejected.  (Default: 1)
 
 [[DownloadExtraInfo]] **DownloadExtraInfo** **0**|**1**::
     If true, Tor downloads and caches "extra-info" documents. These documents
@@ -1431,11 +1518,6 @@ The following options are useful only for clients (that is, if
     Like WarnPlaintextPorts, but instead of warning about risky port uses, Tor
     will instead refuse to make the connection. (Default: None)
 
-[[AllowSingleHopCircuits]] **AllowSingleHopCircuits** **0**|**1**::
-    When this option is set, the attached Tor controller can use relays
-    that have the **AllowSingleHopExits** option turned on to build
-    one-hop Tor connections.  (Default: 0)
-
 [[OptimisticData]] **OptimisticData** **0**|**1**|**auto**::
     When this option is set, and Tor is using an exit node that supports
     the feature, it will try optimistically to send data to the exit node
@@ -1462,23 +1544,132 @@ The following options are useful only for clients (that is, if
     (Example:
     Tor2webRendezvousPoints Fastyfasty, ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
  +
-    This feature can only be used if Tor2webMode is also enabled.
+    This feature can only be used if Tor2webMode is also enabled. +
  +
     ExcludeNodes have higher priority than Tor2webRendezvousPoints,
     which means that nodes specified in ExcludeNodes will not be
-    picked as RPs.
+    picked as RPs. +
  +
     If no nodes in Tor2webRendezvousPoints are currently available for
     use, Tor will choose a random node when building HS circuits.
 
+[[HSLayer2Nodes]] **HSLayer2Nodes** __node__,__node__,__...__::
+    A list of identity fingerprints, nicknames, country codes, and
+    address patterns of nodes that are allowed to be used as the
+    second hop in all client or service-side Onion Service circuits.
+    This option mitigates attacks where the adversary runs middle nodes
+    and induces your client or service to create many circuits, in order
+    to discover your primary guard node.
+    (Default: Any node in the network may be used in the second hop.)
+ +
+    (Example:
+    HSLayer2Nodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
+ +
+    When this is set, the resulting hidden service paths will
+    look like:
+ +
+        C - G - L2 - M - Rend +
+        C - G - L2 - M - HSDir +
+        C - G - L2 - M - Intro +
+        S - G - L2 - M - Rend +
+        S - G - L2 - M - HSDir +
+        S - G - L2 - M - Intro +
+ +
+    where C is this client, S is the service, G is the Guard node,
+    L2 is a node from this option, and M is a random middle node.
+    Rend, HSDir, and Intro point selection is not affected by this
+    option.
+ +
+    This option may be combined with HSLayer3Nodes to create
+    paths of the form:
+ +
+        C - G - L2 - L3 - Rend +
+        C - G - L2 - L3 - M - HSDir +
+        C - G - L2 - L3 - M - Intro +
+        S - G - L2 - L3 - M - Rend +
+        S - G - L2 - L3 - HSDir +
+        S - G - L2 - L3 - Intro +
+ +
+    ExcludeNodes have higher priority than HSLayer2Nodes,
+    which means that nodes specified in ExcludeNodes will not be
+    picked.
+ +
+    When either this option or HSLayer3Nodes are set, the /16 subnet
+    and node family restrictions are removed for hidden service
+    circuits. Additionally, we allow the guard node to be present
+    as the Rend, HSDir, and IP node, and as the hop before it. This
+    is done to prevent the adversary from inferring information
+    about our guard, layer2, and layer3 node choices at later points
+    in the path.
+ +
+    This option is meant to be managed by a Tor controller such as
+    https://github.com/mikeperry-tor/vanguards that selects and
+    updates this set of nodes for you. Hence it does not do load
+    balancing if fewer than 20 nodes are selected, and if no nodes in
+    HSLayer2Nodes are currently available for use, Tor will not work.
+    Please use extreme care if you are setting this option manually.
+
+[[HSLayer3Nodes]] **HSLayer3Nodes** __node__,__node__,__...__::
+    A list of identity fingerprints, nicknames, country codes, and
+    address patterns of nodes that are allowed to be used as the
+    third hop in all client and service-side Onion Service circuits.
+    This option mitigates attacks where the adversary runs middle nodes
+    and induces your client or service to create many circuits, in order
+    to discover your primary or Layer2 guard nodes.
+    (Default: Any node in the network may be used in the third hop.)
+ +
+    (Example:
+    HSLayer3Nodes ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
+ +
+    When this is set by itself, the resulting hidden service paths
+    will look like: +
+        C - G - M - L3 - Rend +
+        C - G - M - L3 - M - HSDir +
+        C - G - M - L3 - M - Intro +
+        S - G - M - L3 - M - Rend +
+        S - G - M - L3 - HSDir +
+        S - G - M - L3 - Intro +
+    where C is this client, S is the service, G is the Guard node,
+    L2 is a node from this option, and M is a random middle node.
+    Rend, HSDir, and Intro point selection is not affected by this
+    option.
+ +
+    While it is possible to use this option by itself, it should be
+    combined with HSLayer2Nodes to create paths of the form:
+ +
+        C - G - L2 - L3 - Rend +
+        C - G - L2 - L3 - M - HSDir +
+        C - G - L2 - L3 - M - Intro +
+        S - G - L2 - L3 - M - Rend +
+        S - G - L2 - L3 - HSDir +
+        S - G - L2 - L3 - Intro +
+ +
+    ExcludeNodes have higher priority than HSLayer3Nodes,
+    which means that nodes specified in ExcludeNodes will not be
+    picked.
+ +
+    When either this option or HSLayer2Nodes are set, the /16 subnet
+    and node family restrictions are removed for hidden service
+    circuits. Additionally, we allow the guard node to be present
+    as the Rend, HSDir, and IP node, and as the hop before it. This
+    is done to prevent the adversary from inferring information
+    about our guard, layer2, and layer3 node choices at later points
+    in the path.
+  +
+    This option is meant to be managed by a Tor controller such as
+    https://github.com/mikeperry-tor/vanguards that selects and
+    updates this set of nodes for you. Hence it does not do load
+    balancing if fewer than 20 nodes are selected, and if no nodes in
+    HSLayer3Nodes are currently available for use, Tor will not work.
+    Please use extreme care if you are setting this option manually.
+
 [[UseMicrodescriptors]] **UseMicrodescriptors** **0**|**1**|**auto**::
     Microdescriptors are a smaller version of the information that Tor needs
     in order to build its circuits.  Using microdescriptors makes Tor clients
     download less directory information, thus saving bandwidth.  Directory
     caches need to fetch regular descriptors and microdescriptors, so this
-    option doesn't save any bandwidth for them.  If this option is set to
-    "auto" (recommended) then it is on for all clients that do not set
-    FetchUselessDescriptors. (Default: auto)
+    option doesn't save any bandwidth for them.  For legacy reasons, auto is
+    accepted, but it has the same effect as 1. (Default: auto)
 
 [[PathBiasCircThreshold]] **PathBiasCircThreshold** __NUM__ +
 
@@ -1494,7 +1685,7 @@ The following options are useful only for clients (that is, if
     These options override the default behavior of Tor's (**currently
     experimental**) path bias detection algorithm. To try to find broken or
     misbehaving guard nodes, Tor looks for nodes where more than a certain
-    fraction of circuits through that guard fail to get built.
+    fraction of circuits through that guard fail to get built. +
  +
     The PathBiasCircThreshold option controls how many circuits we need to build
     through a guard before we make these checks.  The PathBiasNoticeRate,
@@ -1520,14 +1711,14 @@ The following options are useful only for clients (that is, if
 
 [[PathBiasScaleUseThreshold]] **PathBiasScaleUseThreshold** __NUM__::
     Similar to the above options, these options override the default behavior
-    of Tor's (**currently experimental**) path use bias detection algorithm.
+    of Tor's (**currently experimental**) path use bias detection algorithm. +
  +
     Where as the path bias parameters govern thresholds for successfully
     building circuits, these four path use bias parameters govern thresholds
     only for circuit usage. Circuits which receive no stream usage
     are not counted by this detection algorithm. A used circuit is considered
     successful if it is capable of carrying streams or otherwise receiving
-    well-formed responses to RELAY cells.
+    well-formed responses to RELAY cells. +
  +
     By default, or if a negative value is provided for one of these options,
     Tor uses reasonable defaults from the networkstatus consensus document.
@@ -1552,7 +1743,8 @@ The following options are useful only for clients (that is, if
     server has both. (Tor also prefers an IPv6 DirPort if IPv4Client is set to
     0.) If this option is set to auto, clients prefer IPv4. Other things may
     influence the choice. This option breaks a tie to the favor of IPv6.
-    (Default: auto)
+    (Default: auto)  (DEPRECATED: This option has had no effect for some
+    time.)
 
 [[ClientPreferIPv6ORPort]] **ClientPreferIPv6ORPort** **0**|**1**|**auto**::
     If this option is set to 1, Tor prefers an OR port with an IPv6
@@ -1573,46 +1765,35 @@ The following options are useful only for clients (that is, if
     prevent your Tor client from bootstrapping.  If this option is negative,
     Tor will use a default value chosen by the directory authorities. If the
     directory authorities do not choose a value, Tor will default to 0.6.
-    (Default: -1.)
+    (Default: -1)
 
-[[ClientBootstrapConsensusAuthorityDownloadSchedule]] **ClientBootstrapConsensusAuthorityDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses from authorities
+[[ClientBootstrapConsensusAuthorityDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from authorities
     if they are bootstrapping (that is, they don't have a usable, reasonably
     live consensus). Only used by clients fetching from a list of fallback
     directory mirrors. This schedule is advanced by (potentially concurrent)
     connection attempts, unlike other schedules, which are advanced by
-    connection failures. (Default: 10, 11, 3600, 10800, 25200, 54000,
-    111600, 262800)
+    connection failures. (Default: 6)
 
-[[ClientBootstrapConsensusFallbackDownloadSchedule]] **ClientBootstrapConsensusFallbackDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses from fallback
+[[ClientBootstrapConsensusFallbackDownloadInitialDelay]] **ClientBootstrapConsensusFallbackDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from fallback
     directory mirrors if they are bootstrapping (that is, they don't have a
     usable, reasonably live consensus). Only used by clients fetching from a
     list of fallback directory mirrors. This schedule is advanced by
     (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0, 1, 4, 11, 3600,
-    10800, 25200, 54000, 111600, 262800)
+    which are advanced by connection failures. (Default: 0)
 
-[[ClientBootstrapConsensusAuthorityOnlyDownloadSchedule]] **ClientBootstrapConsensusAuthorityOnlyDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses from authorities
+[[ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses from authorities
     if they are bootstrapping (that is, they don't have a usable, reasonably
     live consensus). Only used by clients which don't have or won't fetch
     from a list of fallback directory mirrors. This schedule is advanced by
     (potentially concurrent) connection attempts, unlike other schedules,
-    which are advanced by connection failures. (Default: 0, 3, 7, 3600,
-    10800, 25200, 54000, 111600, 262800)
-
-[[ClientBootstrapConsensusMaxDownloadTries]] **ClientBootstrapConsensusMaxDownloadTries** __NUM__::
-    Try this many times to download a consensus while bootstrapping using
-    fallback directory mirrors before giving up. (Default: 7)
-
-[[ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries]] **ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries** __NUM__::
-    Try this many times to download a consensus while bootstrapping using
-    authorities before giving up. (Default: 4)
+    which are advanced by connection failures. (Default: 0)
 
 [[ClientBootstrapConsensusMaxInProgressTries]] **ClientBootstrapConsensusMaxInProgressTries** __NUM__::
     Try this many simultaneous connections to download a consensus before
-    waiting for one to complete, timeout, or error out. (Default: 4)
+    waiting for one to complete, timeout, or error out. (Default: 3)
 
 SERVER OPTIONS
 --------------
@@ -1621,19 +1802,13 @@ The following options are useful only for servers (that is, if ORPort
 is non-zero):
 
 [[Address]] **Address** __address__::
-    The IP address or fully qualified domain name of this server (e.g.
-    moria.mit.edu). You can leave this unset, and Tor will guess your IP
-    address.  This IP address is the one used to tell clients and other
-    servers where to find your Tor server; it doesn't affect the IP that your
-    Tor client binds to.  To bind to a different address, use the
-    *ListenAddress and OutboundBindAddress options.
-
-[[AllowSingleHopExits]] **AllowSingleHopExits** **0**|**1**::
-    This option controls whether clients can use this server as a single hop
-    proxy. If set to 1, clients can use this server as an exit even if it is
-    the only hop in the circuit.  Note that most clients will refuse to use
-    servers that set this option, since most clients have
-    ExcludeSingleHopRelays set.  (Default: 0)
+    The IPv4 address of this server, or a fully qualified domain name of
+    this server that resolves to an IPv4 address.  You can leave this
+    unset, and Tor will try to guess your IPv4 address.  This IPv4
+    address is the one used to tell clients and other servers where to
+    find your Tor server; it doesn't affect the address that your server
+    binds to.  To bind to a different address, use the ORPort and
+    OutboundBindAddress options.
 
 [[AssumeReachable]] **AssumeReachable** **0**|**1**::
     This option is used when bootstrapping a new Tor network. If set to 1,
@@ -1648,6 +1823,17 @@ is non-zero):
     server descriptor to the bridge database, rather than
     to the public directory authorities.
 
+[[BridgeDistribution]] **BridgeDistribution** __string__::
+    If set along with BridgeRelay, Tor will include a new line in its
+    bridge descriptor which indicates to the BridgeDB service how it
+    would like its bridge address to be given out. Set it to "none" if
+    you want BridgeDB to avoid distributing your bridge address, or "any" to
+    let BridgeDB decide. (Default: any)
+ +
+    Note: as of Oct 2017, the BridgeDB part of this option is not yet
+    implemented.  Until BridgeDB is updated to obey this option, your
+    bridge will make this request, but it will not (yet) be obeyed.
+
 [[ContactInfo]] **ContactInfo** __email_address__::
     Administrative contact information for this relay or bridge. This line
     can be used to contact you if your relay or bridge is misconfigured or
@@ -1655,13 +1841,18 @@ is non-zero):
     descriptors containing these lines and that Google indexes them, so
     spammers might also collect them. You may want to obscure the fact
     that it's an email address and/or generate a new address for this
-    purpose.
+    purpose. +
+ +
+    ContactInfo **must** be set to a working address if you run more than one
+    relay or bridge.  (Really, everybody running a relay or bridge should set
+    it.)
+
 
 [[ExitRelay]] **ExitRelay** **0**|**1**|**auto**::
     Tells Tor whether to run as an exit relay.  If Tor is running as a
     non-bridge server, and ExitRelay is set to 1, then Tor allows traffic to
     exit according to the ExitPolicy option (or the default ExitPolicy if
-    none is specified).
+    none is specified). +
  +
     If ExitRelay is set to 0, no traffic is allowed to
     exit, and the ExitPolicy option is ignored. +
@@ -1675,8 +1866,8 @@ is non-zero):
     "**accept[6]**|**reject[6]**  __ADDR__[/__MASK__][:__PORT__]". If /__MASK__ is
     omitted then this policy just applies to the host given. Instead of giving
     a host or network you can also use "\*" to denote the universe (0.0.0.0/0
-    and ::/128), or \*4 to denote all IPv4 addresses, and \*6 to denote all
-    IPv6 addresses.
+    and ::/0), or \*4 to denote all IPv4 addresses, and \*6 to denote all IPv6
+    addresses.
     __PORT__ can be a single port number, an interval of ports
     "__FROM_PORT__-__TO_PORT__", or "\*". If __PORT__ is omitted, that means
     "\*". +
@@ -1724,8 +1915,15 @@ is non-zero):
     write your IPv6 rules using accept6/reject6 \*6, and your IPv4 rules using
     accept/reject \*4. If you want to \_replace_ the default exit policy, end
     your exit policy with either a reject \*:* or an accept \*:*. Otherwise,
-    you're \_augmenting_ (prepending to) the default exit policy. The default
-    exit policy is: +
+    you're \_augmenting_ (prepending to) the default exit policy. +
+ +
+    If you want to use a reduced exit policy rather than the default exit
+    policy, set "ReducedExitPolicy 1". If you want to _replace_ the default
+    exit policy with your custom exit policy, end your exit policy with either
+    a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending
+    to) the default or reduced exit policy. +
+ +
+    The default exit policy is:
 
        reject *:25
        reject *:119
@@ -1739,6 +1937,8 @@ is non-zero):
        reject *:6881-6999
        accept *:*
 
+// Anchor only for formatting, not visible in the man page.
+[[ExitPolicyDefault]]::
     Since the default exit policy uses accept/reject *, it applies to both
     IPv4 and IPv6 addresses.
 
@@ -1760,6 +1960,99 @@ is non-zero):
     to disclose.
     (Default: 0)
 
+[[ReducedExitPolicy]] **ReducedExitPolicy** **0**|**1**::
+    If set, use a reduced exit policy rather than the default one. +
+ +
+    The reduced exit policy is an alternative to the default exit policy. It
+    allows as many Internet services as possible while still blocking the
+    majority of TCP ports. Currently, the policy allows approximately 65 ports.
+    This reduces the odds that your node will be used for peer-to-peer
+    applications. +
+ +
+    The reduced exit policy is:
+
+        accept *:20-21
+        accept *:22
+        accept *:23
+        accept *:43
+        accept *:53
+        accept *:79
+        accept *:80-81
+        accept *:88
+        accept *:110
+        accept *:143
+        accept *:194
+        accept *:220
+        accept *:389
+        accept *:443
+        accept *:464
+        accept *:465
+        accept *:531
+        accept *:543-544
+        accept *:554
+        accept *:563
+        accept *:587
+        accept *:636
+        accept *:706
+        accept *:749
+        accept *:873
+        accept *:902-904
+        accept *:981
+        accept *:989-990
+        accept *:991
+        accept *:992
+        accept *:993
+        accept *:994
+        accept *:995
+        accept *:1194
+        accept *:1220
+        accept *:1293
+        accept *:1500
+        accept *:1533
+        accept *:1677
+        accept *:1723
+        accept *:1755
+        accept *:1863
+        accept *:2082
+        accept *:2083
+        accept *:2086-2087
+        accept *:2095-2096
+        accept *:2102-2104
+        accept *:3128
+        accept *:3389
+        accept *:3690
+        accept *:4321
+        accept *:4643
+        accept *:5050
+        accept *:5190
+        accept *:5222-5223
+        accept *:5228
+        accept *:5900
+        accept *:6660-6669
+        accept *:6679
+        accept *:6697
+        accept *:8000
+        accept *:8008
+        accept *:8074
+        accept *:8080
+        accept *:8082
+        accept *:8087-8088
+        accept *:8232-8233
+        accept *:8332-8333
+        accept *:8443
+        accept *:8888
+        accept *:9418
+        accept *:9999
+        accept *:10000
+        accept *:11371
+        accept *:19294
+        accept *:19638
+        accept *:50002
+        accept *:64738
+        reject *:*
+
+    (Default: 0)
+
 [[IPv6Exit]] **IPv6Exit** **0**|**1**::
     If set, and we are an exit node, allow clients to use us for IPv6
     traffic. (Default: 0)
@@ -1768,21 +2061,30 @@ is non-zero):
     If we have more onionskins queued for processing than we can process in
     this amount of time, reject new ones. (Default: 1750 msec)
 
-[[MyFamily]] **MyFamily** __node__,__node__,__...__::
-    Declare that this Tor server is controlled or administered by a group or
-    organization identical or similar to that of the other servers, defined by
-    their identity fingerprints. When two servers both declare
-    that they are in the same \'family', Tor clients will not use them in the
-    same circuit. (Each server only needs to list the other servers in its
-    family; it doesn't need to list itself, but it won't hurt.) Do not list
-    any bridge relay as it would compromise its concealment.
+[[MyFamily]] **MyFamily** __fingerprint__,__fingerprint__,...::
+    Declare that this Tor relay is controlled or administered by a group or
+    organization identical or similar to that of the other relays, defined by
+    their (possibly $-prefixed) identity fingerprints.
+    This option can be repeated many times, for
+    convenience in defining large families: all fingerprints in all MyFamily
+    lines are merged into one list.
+    When two relays both declare that they are in the
+    same \'family', Tor clients will not use them in the same circuit. (Each
+    relay only needs to list the other servers in its family; it doesn't need to
+    list itself, but it won't hurt if it does.) Do not list any bridge relay as it would
+    compromise its concealment. +
  +
     When listing a node, it's better to list it by fingerprint than by
-    nickname: fingerprints are more reliable.
+    nickname: fingerprints are more reliable. +
+ +
+    If you run more than one relay, the MyFamily option on each relay
+    **must** list all other relays, as described above.
 
 [[Nickname]] **Nickname** __name__::
     Set the server's nickname to \'name'. Nicknames must be between 1 and 19
     characters inclusive, and must contain only the characters [a-zA-Z0-9].
+    If not set, **Unnamed** will be used. Relays can always be uniquely identified
+    by their identity fingerprints.
 
 [[NumCPUs]] **NumCPUs** __num__::
     How many processes to use at once for decrypting onionskins and other
@@ -1793,62 +2095,45 @@ is non-zero):
     Advertise this port to listen for connections from Tor clients and
     servers.  This option is required to be a Tor server.
     Set it to "auto" to have Tor pick a port for you. Set it to 0 to not
-    run an ORPort at all. This option can occur more than once. (Default: 0)
-+
+    run an ORPort at all. This option can occur more than once. (Default: 0) +
+ +
     Tor recognizes these flags on each ORPort:
-    **NoAdvertise**::
+    **NoAdvertise**;;
         By default, we bind to a port and tell our users about it. If
         NoAdvertise is specified, we don't advertise, but listen anyway.  This
         can be useful if the port everybody will be connecting to (for
         example, one that's opened on our firewall) is somewhere else.
-    **NoListen**::
+    **NoListen**;;
         By default, we bind to a port and tell our users about it. If
         NoListen is specified, we don't bind, but advertise anyway.  This
         can be useful if something else  (for example, a firewall's port
         forwarding configuration) is causing connections to reach us.
-    **IPv4Only**::
+    **IPv4Only**;;
         If the address is absent, or resolves to both an IPv4 and an IPv6
         address, only listen to the IPv4 address.
-    **IPv6Only**::
+    **IPv6Only**;;
         If the address is absent, or resolves to both an IPv4 and an IPv6
         address, only listen to the IPv6 address.
-+
+
+// Anchor only for formatting, not visible in the man page.
+[[ORPortFlagsExclusive]]::
     For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and
     IPv4Only and IPv6Only are mutually exclusive.
 
-[[ORListenAddress]] **ORListenAddress** __IP__[:__PORT__]::
-    Bind to this IP address to listen for connections from Tor clients and
-    servers. If you specify a port, bind to this port rather than the one
-    specified in ORPort. (Default: 0.0.0.0) This directive can be specified
-    multiple times to bind to multiple addresses/ports.
-+
-    This option is deprecated; you can get the same behavior with ORPort now
-    that it supports NoAdvertise and explicit addresses.
-
-[[PortForwarding]] **PortForwarding** **0**|**1**::
-    Attempt to automatically forward the DirPort and ORPort on a NAT router
-    connecting this Tor server to the Internet. If set, Tor will try both
-    NAT-PMP (common on Apple routers) and UPnP (common on routers from other
-    manufacturers). (Default: 0)
-
-[[PortForwardingHelper]] **PortForwardingHelper** __filename__|__pathname__::
-    If PortForwarding is set, use this executable to configure the forwarding.
-    If set to a filename, the system path will be searched for the executable.
-    If set to a path, only the specified path will be executed.
-    (Default: tor-fw-helper)
-
 [[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**::
     This option specifies which descriptors Tor will publish when acting as
     a relay. You can
-    choose multiple arguments, separated by commas.
+    choose multiple arguments, separated by commas. +
  +
     If this option is set to 0, Tor will not publish its
     descriptors to any directories. (This is useful if you're testing
-    out your server, or if you're using a Tor controller that handles directory
-    publishing for you.) Otherwise, Tor will publish its descriptors of all
-    type(s) specified. The default is "1",
-    which means "if running as a server, publish the
-    appropriate descriptors to the authorities".
+    out your server, or if you're using a Tor controller that handles
+    directory publishing for you.) Otherwise, Tor will publish its
+    descriptors of all type(s) specified. The default is "1", which
+    means "if running as a relay or bridge, publish descriptors to the
+    appropriate authorities". Other possibilities are "v3", meaning
+    "publish as if you're a relay", and "bridge", meaning "publish as
+    if you're a bridge".
 
 [[ShutdownWaitLength]] **ShutdownWaitLength** __NUM__::
     When we get a SIGINT and we're a server, we begin shutting down:
@@ -1868,7 +2153,12 @@ is non-zero):
     to 0 will disable the heartbeat. Otherwise, it must be at least 30
     minutes. (Default: 6 hours)
 
-[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**|**TBytes**::
+[[MainloopStats]] **MainloopStats** **0**|**1**::
+    Log main loop statistics every **HeartbeatPeriod** seconds. This is a log
+    level __notice__ message designed to help developers instrumenting Tor's
+    main event loop. (Default: 0)
+
+[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Limits the max number of bytes sent and received within a set time period
     using a given calculation rule (see: AccountingStart, AccountingRule).
     Useful if you need to stay under a specific bandwidth. By default, the
@@ -1898,15 +2188,16 @@ is non-zero):
     (Default: max)
 
 [[AccountingStart]] **AccountingStart** **day**|**week**|**month** [__day__] __HH:MM__::
-    Specify how long accounting periods last. If **month** is given, each
-    accounting period runs from the time __HH:MM__ on the __dayth__ day of one
-    month to the same day and time of the next. (The day must be between 1 and
-    28.) If **week** is given, each accounting period runs from the time __HH:MM__
-    of the __dayth__ day of one week to the same day and time of the next week,
-    with Monday as day 1 and Sunday as day 7. If **day** is given, each
-    accounting period runs from the time __HH:MM__ each day to the same time on
-    the next day. All times are local, and given in 24-hour time. (Default:
-    "month 1 0:00")
+    Specify how long accounting periods last. If **month** is given,
+    each accounting period runs from the time __HH:MM__ on the __dayth__ day of one
+    month to the same day and time of the next. The relay will go at full speed,
+    use all the quota you specify, then hibernate for the rest of the period. (The
+    day must be between 1 and 28.) If **week** is given, each accounting period
+    runs from the time __HH:MM__ of the __dayth__ day of one week to the same day
+    and time of the next week, with Monday as day 1 and Sunday as day 7. If **day**
+    is given, each accounting period runs from the time __HH:MM__ each day to the
+    same time on the next day. All times are local, and given in 24-hour time.
+    (Default: "month 1 0:00")
 
 [[RefuseUnknownExits]] **RefuseUnknownExits** **0**|**1**|**auto**::
     Prevent nodes that don't appear in the consensus from exiting using this
@@ -1942,7 +2233,7 @@ is non-zero):
     correct this. This option only affects name lookups that your server does
     on behalf of clients. (Default: 1)
 
-[[ServerDNSTestAddresses]] **ServerDNSTestAddresses** __address__,__address__,__...__::
+[[ServerDNSTestAddresses]] **ServerDNSTestAddresses** __hostname__,__hostname__,__...__::
     When we're detecting DNS hijacking, make sure that these __valid__ addresses
     aren't getting redirected. If they are, then our DNS is completely useless,
     and we'll reset our exit policy to "reject \*:*". This option only affects
@@ -1976,12 +2267,6 @@ is non-zero):
 [[GeoIPv6File]] **GeoIPv6File** __filename__::
     A filename containing IPv6 GeoIP data, for use with by-country statistics.
 
-[[TLSECGroup]] **TLSECGroup** **P224**|**P256**::
-    What EC group should we try to use for incoming TLS connections?
-    P224 is faster, but makes us stand out more. Has no effect if
-    we're a client, or if our OpenSSL version lacks support for ECDHE.
-    (Default: P256)
-
 [[CellStatistics]] **CellStatistics** **0**|**1**::
     Relays only.
     When this option is enabled, Tor collects statistics about cell
@@ -1992,6 +2277,15 @@ is non-zero):
     If ExtraInfoStatistics is enabled, it will published as part of
     extra-info document. (Default: 0)
 
+[[PaddingStatistics]] **PaddingStatistics** **0**|**1**::
+    Relays and bridges only.
+    When this option is enabled, Tor collects statistics for padding cells
+    sent and received by this relay, in addition to total cell counts.
+    These statistics are rounded, and omitted if traffic is low. This
+    information is important for load balancing decisions related to padding.
+    If ExtraInfoStatistics is enabled, it will be published
+    as a part of extra-info document. (Default: 1)
+
 [[DirReqStatistics]] **DirReqStatistics** **0**|**1**::
     Relays and bridges only.
     When this option is enabled, a Tor directory writes statistics on the
@@ -2079,11 +2373,28 @@ is non-zero):
     ed25519 master identity key, as well as the corresponding temporary
     signing keys and certificates. (Default: 0)
 
+[[KeyDirectory]] **KeyDirectory** __DIR__::
+    Store secret keys in DIR. Can not be changed while tor is
+    running.
+    (Default: the "keys" subdirectory of DataDirectory.)
+
+[[KeyDirectoryGroupReadable]] **KeyDirectoryGroupReadable** **0**|**1**::
+    If this option is set to 0, don't allow the filesystem group to read the
+    KeywDirectory. If the option is set to 1, make the KeyDirectory readable
+    by the default GID. (Default: 0)
+
+[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
+    Tells an authority, or other node tracking node reliability and history,
+    that fine-grained information about nodes can be discarded when it hasn't
+    changed for a given amount of time.  (Default: 24 hours)
+
+
 DIRECTORY SERVER OPTIONS
 ------------------------
 
-The following options are useful only for directory servers (that is,
-if DirPort is non-zero):
+The following options are useful only for directory servers. (Relays with
+enough bandwidth automatically become directory servers; see DirCache for
+details.)
 
 [[DirPortFrontPage]] **DirPortFrontPage** __FILENAME__::
     When this option is set, it takes an HTML file and publishes it as "/" on
@@ -2095,19 +2406,10 @@ if DirPort is non-zero):
     If this option is nonzero, advertise the directory service on this port.
     Set it to "auto" to have Tor pick a port for you.  This option can occur
     more than once, but only one advertised DirPort is supported: all
-    but one DirPort must have the **NoAdvertise** flag set. (Default: 0)
-+
+    but one DirPort must have the **NoAdvertise** flag set. (Default: 0) +
+ +
     The same flags are supported here as are supported by ORPort.
 
-[[DirListenAddress]] **DirListenAddress** __IP__[:__PORT__]::
-    Bind the directory service to this address. If you specify a port, bind to
-    this port rather than the one specified in DirPort.  (Default: 0.0.0.0)
-    This directive can be specified multiple times  to bind to multiple
-    addresses/ports.
-+
-    This option is deprecated; you can get the same behavior with DirPort now
-    that it supports NoAdvertise and explicit addresses.
-
 [[DirPolicy]] **DirPolicy** __policy__,__policy__,__...__::
     Set an entrance policy for this server, to limit who can connect to the
     directory ports. The policies have the same form as exit policies above,
@@ -2115,10 +2417,152 @@ if DirPort is non-zero):
     some entry in the policy is accepted.
 
 [[DirCache]] **DirCache** **0**|**1**::
-    When this option is set, Tor caches all current directory documents and
-    accepts client requests for them. Setting DirPort is not required for this,
-    because clients connect via the ORPort by default. Setting either DirPort
-    or BridgeRelay and setting DirCache to 0 is not supported.  (Default: 1)
+    When this option is set, Tor caches all current directory documents except
+    extra info documents, and accepts client requests for them. If
+    **DownloadExtraInfo** is set, cached extra info documents are also cached.
+    Setting **DirPort** is not required for **DirCache**, because clients
+    connect via the ORPort by default. Setting either DirPort or BridgeRelay
+    and setting DirCache to 0 is not supported.  (Default: 1)
+
+[[MaxConsensusAgeForDiffs]] **MaxConsensusAgeForDiffs**  __N__ **minutes**|**hours**|**days**|**weeks**::
+    When this option is nonzero, Tor caches will not try to generate
+    consensus diffs for any consensus older than this amount of time.
+    If this option is set to zero, Tor will pick a reasonable default from
+    the current networkstatus document.  You should not set this
+    option unless your cache is severely low on disk space or CPU.
+    If you need to set it, keeping it above 3 or 4 hours will help clients
+    much more than setting it to zero.
+    (Default: 0)
+
+
+DENIAL OF SERVICE MITIGATION OPTIONS
+------------------------------------
+
+Tor has three built-in mitigation options that can be individually
+enabled/disabled and fine-tuned, but by default Tor directory authorities will
+define reasonable values for relays and no explicit configuration is required
+to make use of these protections. The mitigations take place at relays,
+and are as follows:
+
+  1. If a single client address makes too many concurrent connections (this is
+     configurable via DoSConnectionMaxConcurrentCount), hang up on further
+     connections.
+ +
+  2. If a single client IP address (v4 or v6) makes circuits too quickly
+     (default values are more than 3 per second, with an allowed burst of 90,
+     see DoSCircuitCreationRate and DoSCircuitCreationBurst) while also having
+     too many connections open (default is 3, see
+     DoSCircuitCreationMinConnections), tor will refuse any new circuit (CREATE
+     cells) for the next while (random value between 1 and 2 hours).
+ +
+  3. If a client asks to establish a rendezvous point to you directly (ex:
+     Tor2Web client), ignore the request.
+
+These defenses can be manually controlled by torrc options, but relays will
+also take guidance from consensus parameters using these same names, so there's
+no need to configure anything manually. In doubt, do not change those values.
+
+The values set by the consensus, if any, can be found here:
+https://consensus-health.torproject.org/#consensusparams
+
+If any of the DoS mitigations are enabled, a heartbeat message will appear in
+your log at NOTICE level which looks like:
+
+    DoS mitigation since startup: 429042 circuits rejected, 17 marked addresses.
+    2238 connections closed. 8052 single hop clients refused.
+
+The following options are useful only for a public relay. They control the
+Denial of Service mitigation subsystem described above.
+
+[[DoSCircuitCreationEnabled]] **DoSCircuitCreationEnabled** **0**|**1**|**auto**::
+
+    Enable circuit creation DoS mitigation. If set to 1 (enabled), tor will
+    cache client IPs along with statistics in order to detect circuit DoS
+    attacks. If an address is positively identified, tor will activate
+    defenses against the address. See the DoSCircuitCreationDefenseType option
+    for more details.  This is a client to relay detection only. "auto" means
+    use the consensus parameter. If not defined in the consensus, the value is 0.
+    (Default: auto)
+
+[[DoSCircuitCreationMinConnections]] **DoSCircuitCreationMinConnections** __NUM__::
+
+    Minimum threshold of concurrent connections before a client address can be
+    flagged as executing a circuit creation DoS. In other words, once a client
+    address reaches the circuit rate and has a minimum of NUM concurrent
+    connections, a detection is positive. "0" means use the consensus
+    parameter. If not defined in the consensus, the value is 3.
+    (Default: 0)
+
+[[DoSCircuitCreationRate]] **DoSCircuitCreationRate** __NUM__::
+
+    The allowed circuit creation rate per second applied per client IP
+    address. If this option is 0, it obeys a consensus parameter. If not
+    defined in the consensus, the value is 3.
+    (Default: 0)
+
+[[DoSCircuitCreationBurst]] **DoSCircuitCreationBurst** __NUM__::
+
+    The allowed circuit creation burst per client IP address. If the circuit
+    rate and the burst are reached, a client is marked as executing a circuit
+    creation DoS. "0" means use the consensus parameter. If not defined in the
+    consensus, the value is 90.
+    (Default: 0)
+
+[[DoSCircuitCreationDefenseType]] **DoSCircuitCreationDefenseType** __NUM__::
+
+    This is the type of defense applied to a detected client address. The
+    possible values are:
+ +
+      1: No defense.
+ +
+      2: Refuse circuit creation for the DoSCircuitCreationDefenseTimePeriod period of time.
+ +
+    "0" means use the consensus parameter. If not defined in the consensus, the value is 2.
+    (Default: 0)
+
+[[DoSCircuitCreationDefenseTimePeriod]] **DoSCircuitCreationDefenseTimePeriod** __N__ **seconds**|**minutes**|**hours**::
+
+    The base time period in seconds that the DoS defense is activated for. The
+    actual value is selected randomly for each activation from N+1 to 3/2 * N.
+    "0" means use the consensus parameter. If not defined in the consensus,
+    the value is 3600 seconds (1 hour).
+    (Default: 0)
+
+[[DoSConnectionEnabled]] **DoSConnectionEnabled** **0**|**1**|**auto**::
+
+    Enable the connection DoS mitigation. If set to 1 (enabled), for client
+    address only, this allows tor to mitigate against large number of
+    concurrent connections made by a single IP address. "auto" means use the
+    consensus parameter. If not defined in the consensus, the value is 0.
+    (Default: auto)
+
+[[DoSConnectionMaxConcurrentCount]] **DoSConnectionMaxConcurrentCount** __NUM__::
+
+    The maximum threshold of concurrent connection from a client IP address.
+    Above this limit, a defense selected by DoSConnectionDefenseType is
+    applied. "0" means use the consensus parameter. If not defined in the
+    consensus, the value is 100.
+    (Default: 0)
+
+[[DoSConnectionDefenseType]] **DoSConnectionDefenseType** __NUM__::
+
+    This is the type of defense applied to a detected client address for the
+    connection mitigation. The possible values are:
+ +
+      1: No defense.
+ +
+      2: Immediately close new connections.
+ +
+    "0" means use the consensus parameter. If not defined in the consensus, the value is 2.
+    (Default: 0)
+
+[[DoSRefuseSingleHopClientRendezvous]] **DoSRefuseSingleHopClientRendezvous** **0**|**1**|**auto**::
+
+    Refuse establishment of rendezvous points for single hop clients. In other
+    words, if a client directly connects to the relay and sends an
+    ESTABLISH_RENDEZVOUS cell, it is silently dropped. "auto" means use the
+    consensus parameter. If not defined in the consensus, the value is 0.
+    (Default: auto)
 
 
 DIRECTORY AUTHORITY SERVER OPTIONS
@@ -2199,7 +2643,7 @@ on the public Tor network.
 [[AuthDirBadExit]] **AuthDirBadExit** __AddressPattern...__::
     Authoritative directories only. A set of address patterns for servers that
     will be listed as bad exits in any network status document this authority
-    publishes, if **AuthDirListBadExits** is set.
+    publishes, if **AuthDirListBadExits** is set. +
  +
     (The address pattern syntax here and in the options below
     is the same as for exit policies, except that you don't need to say
@@ -2237,26 +2681,22 @@ on the public Tor network.
     list as acceptable on a single IP address. Set this to "0" for "no limit".
     (Default: 2)
 
-[[AuthDirMaxServersPerAuthAddr]] **AuthDirMaxServersPerAuthAddr** __NUM__::
-    Authoritative directories only. Like AuthDirMaxServersPerAddr, but applies
-    to addresses shared with directory authorities. (Default: 5)
-
-[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Authoritative directories only. If non-zero, always vote the
     Fast flag for any relay advertising this amount of capacity or
     more. (Default: 100 KBytes)
 
-[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Authoritative directories only. If non-zero, this advertised capacity
     or more is always sufficient to satisfy the bandwidth requirement
-    for the Guard flag. (Default: 250 KBytes)
+    for the Guard flag. (Default: 2 MBytes)
 
 [[AuthDirPinKeys]] **AuthDirPinKeys** **0**|**1**::
     Authoritative directories only. If non-zero, do not allow any relay to
     publish a descriptor if any other relay has reserved its <Ed25519,RSA>
     identity keypair. In all cases, Tor records every keypair it accepts
     in a journal if it is new, or if it differs from the most recently
-    accepted pinning for one of the keys it contains. (Default: 0)
+    accepted pinning for one of the keys it contains. (Default: 1)
 
 [[AuthDirSharedRandomness]] **AuthDirSharedRandomness** **0**|**1**::
     Authoritative directories only. Switch for the shared random protocol.
@@ -2264,6 +2704,13 @@ on the public Tor network.
     (default), the flag "shared-rand-participate" is added to the authority
     vote indicating participation in the protocol. (Default: 1)
 
+[[AuthDirTestEd25519LinkKeys]] **AuthDirTestEd25519LinkKeys**  **0**|**1**::
+    Authoritative directories only. If this option is set to 0, then we treat
+    relays as "Running" if their RSA key is correct when we probe them,
+    regardless of their Ed25519 key. We should only ever set this option to 0
+    if there is some major bug in Ed25519 link authentication that causes us
+    to label all the relays as not Running.  (Default: 1)
+
 [[BridgePassword]] **BridgePassword** __Password__::
     If set, contains an HTTP authenticator that tells a bridge authority to
     serve all requested bridge information. Used by the (only partially
@@ -2310,16 +2757,31 @@ on the public Tor network.
     different identity.  This feature is used to migrate directory authority
     keys in the event of a compromise.  (Default: 0)
 
-[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
-    Tells an authority, or other node tracking node reliability and history,
-    that fine-grained information about nodes can be discarded when it hasn't
-    changed for a given amount of time.  (Default: 24 hours)
-
 [[AuthDirHasIPv6Connectivity]] **AuthDirHasIPv6Connectivity** **0**|**1**::
     Authoritative directories only. When set to 0, OR ports with an
-    IPv6 address are being accepted without reachability testing.
-    When set to 1, IPv6 OR ports are being tested just like IPv4 OR
-    ports. (Default: 0)
+    IPv6 address are not included in the authority's votes. When set to 1,
+    IPv6 OR ports are tested for reachability like IPv4 OR ports. If the
+    reachability test succeeds, the authority votes for the IPv6 ORPort, and
+    votes Running for the relay. If the reachability test fails, the authority
+    does not vote for the IPv6 ORPort, and does not vote Running (Default: 0)  +
++
+    The content of the consensus depends on the number of voting authorities
+    that set AuthDirHasIPv6Connectivity:
+
+      If no authorities set AuthDirHasIPv6Connectivity 1, there will be no
+      IPv6 ORPorts in the consensus.
+
+      If a minority of authorities set AuthDirHasIPv6Connectivity 1,
+      unreachable IPv6 ORPorts will be removed from the consensus. But the
+      majority of IPv4-only authorities will still vote the relay as Running.
+      Reachable IPv6 ORPort lines will be included in the consensus
+
+      If a majority of voting authorities set AuthDirHasIPv6Connectivity 1,
+      relays with unreachable IPv6 ORPorts will not be listed as Running.
+      Reachable IPv6 ORPort lines will be included in the consensus
+      (To ensure that any valid majority will vote relays with unreachable
+      IPv6 ORPorts not Running, 75% of authorities must set
+      AuthDirHasIPv6Connectivity 1.)
 
 [[MinMeasuredBWsForAuthToIgnoreAdvertised]] **MinMeasuredBWsForAuthToIgnoreAdvertised** __N__::
     A total value, in abstract bandwidth units, describing how much
@@ -2335,9 +2797,9 @@ The following options are used to configure a hidden service.
 [[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__::
     Store data files for a hidden service in DIRECTORY. Every hidden service
     must have a separate directory. You may use this option  multiple times to
-    specify multiple services. DIRECTORY must be an existing directory.
+    specify multiple services. If DIRECTORY does not exist, Tor will create it.
     (Note: in current versions of Tor, if DIRECTORY is a relative path,
-    it will be relative to current
+    it will be relative to the current
     working directory of Tor instance, not to its DataDirectory.  Do not
     rely on this behavior; it is not guaranteed to remain the same in future
     versions.)
@@ -2360,9 +2822,9 @@ The following options are used to configure a hidden service.
     you're using a Tor controller that handles hidserv publishing for you.
     (Default: 1)
 
-[[HiddenServiceVersion]] **HiddenServiceVersion** __version__,__version__,__...__::
+[[HiddenServiceVersion]] **HiddenServiceVersion** **2**|**3**::
     A list of rendezvous service descriptor versions to publish for the hidden
-    service. Currently, only version 2 is supported. (Default: 2)
+    service. Currently, versions 2 and 3 are supported. (Default: 2)
 
 [[HiddenServiceAuthorizeClient]] **HiddenServiceAuthorizeClient** __auth-type__ __client-name__,__client-name__,__...__::
     If configured, the hidden service is accessible for authorized clients
@@ -2374,7 +2836,8 @@ The following options are used to configure a hidden service.
     spaces). If this option is set, the hidden service is not accessible for
     clients without authorization any more. Generated authorization data can be
     found in the hostname file. Clients need to put this authorization data in
-    their configuration file using **HidServAuth**.
+    their configuration file using **HidServAuth**. This option is only for v2
+    services.
 
 [[HiddenServiceAllowUnknownPorts]] **HiddenServiceAllowUnknownPorts** **0**|**1**::
    If set to 1, then connections to unrecognized ports do not cause the
@@ -2384,8 +2847,8 @@ The following options are used to configure a hidden service.
 
 [[HiddenServiceMaxStreams]] **HiddenServiceMaxStreams** __N__::
    The maximum number of simultaneous streams (connections) per rendezvous
-   circuit. (Setting this to 0 will allow an unlimited number of simultanous
-   streams.) (Default: 0)
+   circuit. The maximum value allowed is 65535. (Setting this to 0 will allow
+   an unlimited number of simultaneous streams.) (Default: 0)
 
 [[HiddenServiceMaxStreamsCloseCircuit]] **HiddenServiceMaxStreamsCloseCircuit** **0**|**1**::
    If set to 1, then exceeding **HiddenServiceMaxStreams** will cause the
@@ -2394,8 +2857,10 @@ The following options are used to configure a hidden service.
 
 [[RendPostPeriod]] **RendPostPeriod** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
     Every time the specified period elapses, Tor uploads any rendezvous
-    service descriptors to the directory servers. This information  is also
-    uploaded whenever it changes. (Default: 1 hour)
+    service descriptors to the directory servers. This information is also
+    uploaded whenever it changes. Minimum value allowed is 10 minutes and
+    maximum is 3.5 days. This option is only for v2 services.
+    (Default: 1 hour)
 
 [[HiddenServiceDirGroupReadable]] **HiddenServiceDirGroupReadable** **0**|**1**::
     If this option is set to 1, allow the filesystem group to read the
@@ -2405,7 +2870,7 @@ The following options are used to configure a hidden service.
 
 [[HiddenServiceNumIntroductionPoints]] **HiddenServiceNumIntroductionPoints** __NUM__::
     Number of introduction points the hidden service will have. You can't
-    have more than 10. (Default: 3)
+    have more than 10 for v2 service and 20 for v3. (Default: 3)
 
 [[HiddenServiceSingleHopMode]] **HiddenServiceSingleHopMode** **0**|**1**::
     **Experimental - Non Anonymous** Hidden Services on a tor instance in
@@ -2417,20 +2882,20 @@ The following options are used to configure a hidden service.
     Single Onion Service. One-hop circuits make Single Onion servers easily
     locatable, but clients remain location-anonymous. However, the fact that a
     client is accessing a Single Onion rather than a Hidden Service may be
-    statistically distinguishable.
-
+    statistically distinguishable. +
+ +
     **WARNING:** Once a hidden service directory has been used by a tor
     instance in HiddenServiceSingleHopMode, it can **NEVER** be used again for
     a hidden service. It is best practice to create a new hidden service
     directory, key, and address for each new Single Onion Service and Hidden
     Service. It is not possible to run Single Onion Services and Hidden
     Services from the same tor instance: they should be run on different
-    servers with different IP addresses.
-
+    servers with different IP addresses. +
+ +
     HiddenServiceSingleHopMode requires HiddenServiceNonAnonymousMode to be set
     to 1. Since a Single Onion service is non-anonymous, you can not configure
     a SOCKSPort on a tor instance that is running in
-    **HiddenServiceSingleHopMode**.
+    **HiddenServiceSingleHopMode**. Can not be changed while tor is running.
     (Default: 0)
 
 [[HiddenServiceNonAnonymousMode]] **HiddenServiceNonAnonymousMode** **0**|**1**::
@@ -2438,103 +2903,8 @@ The following options are used to configure a hidden service.
     non-anonymous HiddenServiceSingleHopMode. Enables direct connections in the
     server-side hidden service protocol.  If you are using this option,
     you need to disable all client-side services on your Tor instance,
-    including setting SOCKSPort to "0".
-    (Default: 0)
-
-DENIAL OF SERVICE MITIGATION OPTIONS
-------------------------------------
-
-The following options are useful only for a public relay. They control the
-Denial of Service mitigation subsystem.
-
-[[DoSCircuitCreationEnabled]] **DoSCircuitCreationEnabled** **0**|**1**|**auto**::
-
-    Enable circuit creation DoS mitigation. If enabled, tor will cache client
-    IPs along with statistics in order to detect circuit DoS attacks. If an
-    address is positively identified, tor will activate defenses against the
-    address. See the DoSCircuitCreationDefenseType option for more details.
-    This is a client to relay detection only. "auto" means use the consensus
-    parameter. If not defined in the consensus, the value is 0.
-    (Default: auto)
-
-[[DoSCircuitCreationMinConnections]] **DoSCircuitCreationMinConnections** __NUM__::
-
-    Minimum threshold of concurrent connections before a client address can be
-    flagged as executing a circuit creation DoS. In other words, once a client
-    address reaches the circuit rate and has a minimum of NUM concurrent
-    connections, a detection is positive. "0" means use the consensus
-    parameter. If not defined in the consensus, the value is 3.
-    (Default: 0)
-
-[[DoSCircuitCreationRate]] **DoSCircuitCreationRate** __NUM__::
-
-    The allowed circuit creation rate per second applied per client IP
-    address. If this option is 0, it obeys a consensus parameter. If not
-    defined in the consensus, the value is 3.
-    (Default: 0)
-
-[[DoSCircuitCreationBurst]] **DoSCircuitCreationBurst** __NUM__::
-
-    The allowed circuit creation burst per client IP address. If the circuit
-    rate and the burst are reached, a client is marked as executing a circuit
-    creation DoS. "0" means use the consensus parameter. If not defined in the
-    consensus, the value is 90.
-    (Default: 0)
-
-[[DoSCircuitCreationDefenseType]] **DoSCircuitCreationDefenseType** __NUM__::
-
-    This is the type of defense applied to a detected client address. The
-    possible values are:
-
-      1: No defense.
-      2: Refuse circuit creation for the DoSCircuitCreationDefenseTimePeriod period of time.
-+
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 2.
-    (Default: 0)
-
-[[DoSCircuitCreationDefenseTimePeriod]] **DoSCircuitCreationDefenseTimePeriod** __N__ **seconds**|**minutes**|**hours**::
-
-    The base time period in seconds that the DoS defense is activated for. The
-    actual value is selected randomly for each activation from N+1 to 3/2 * N.
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 3600 seconds (1 hour).  (Default: 0)
-
-[[DoSConnectionEnabled]] **DoSConnectionEnabled** **0**|**1**|**auto**::
-
-    Enable the connection DoS mitigation. For client address only, this allows
-    tor to mitigate against large number of concurrent connections made by a
-    single IP address. "auto" means use the consensus parameter. If not
-    defined in the consensus, the value is 0.
-    (Default: auto)
-
-[[DoSConnectionMaxConcurrentCount]] **DoSConnectionMaxConcurrentCount** __NUM__::
-
-    The maximum threshold of concurrent connection from a client IP address.
-    Above this limit, a defense selected by DoSConnectionDefenseType is
-    applied. "0" means use the consensus parameter. If not defined in the
-    consensus, the value is 100.
-    (Default: 0)
-
-[[DoSConnectionDefenseType]] **DoSConnectionDefenseType** __NUM__::
-
-    This is the type of defense applied to a detected client address for the
-    connection mitigation. The possible values are:
-
-      1: No defense.
-      2: Immediately close new connections.
-+
-    "0" means use the consensus parameter. If not defined in the consensus,
-    the value is 2.
-    (Default: 0)
-
-[[DoSRefuseSingleHopClientRendezvous]] **DoSRefuseSingleHopClientRendezvous** **0**|**1**|**auto**::
-
-    Refuse establishment of rendezvous points for single hop clients. In other
-    words, if a client directly connects to the relay and sends an
-    ESTABLISH_RENDEZVOUS cell, it is silently dropped. "auto" means use the
-    consensus parameter. If not defined in the consensus, the value is 0.
-    (Default: auto)
+    including setting SOCKSPort to "0". Can not be changed while tor is
+    running. (Default: 0)
 
 TESTING NETWORK OPTIONS
 -----------------------
@@ -2554,14 +2924,9 @@ The following options are used for running a testing Tor network.
        AssumeReachable 1
        AuthDirMaxServersPerAddr 0
        AuthDirMaxServersPerAuthAddr 0
-       ClientBootstrapConsensusAuthorityDownloadSchedule 0, 2,
-          4 (for 40 seconds), 8, 16, 32, 60
-       ClientBootstrapConsensusFallbackDownloadSchedule 0, 1,
-          4 (for 40 seconds), 8, 16, 32, 60
-       ClientBootstrapConsensusAuthorityOnlyDownloadSchedule 0, 1,
-          4 (for 40 seconds), 8, 16, 32, 60
-       ClientBootstrapConsensusMaxDownloadTries 80
-       ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries 80
+       ClientBootstrapConsensusAuthorityDownloadInitialDelay 0
+       ClientBootstrapConsensusFallbackDownloadInitialDelay 0
+       ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay 0
        ClientDNSRejectInternalAddresses 0
        ClientRejectInternalAddresses 0
        CountPrivateBandwidth 1
@@ -2576,20 +2941,16 @@ The following options are used for running a testing Tor network.
        TestingV3AuthInitialDistDelay 20 seconds
        TestingAuthDirTimeToLearnReachability 0 minutes
        TestingEstimatedDescriptorPropagationTime 0 minutes
-       TestingServerDownloadSchedule 0, 0, 0, 5, 10, 15, 20, 30, 60
-       TestingClientDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
-       TestingServerConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
-       TestingClientConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60
-       TestingBridgeDownloadSchedule 60, 30, 30, 60
+       TestingServerDownloadInitialDelay 0
+       TestingClientDownloadInitialDelay 0
+       TestingServerConsensusDownloadInitialDelay 0
+       TestingClientConsensusDownloadInitialDelay 0
+       TestingBridgeDownloadInitialDelay 10
+       TestingBridgeBootstrapDownloadInitialDelay 0
        TestingClientMaxIntervalWithoutRequest 5 seconds
        TestingDirConnectionMaxStall 30 seconds
-       TestingConsensusMaxDownloadTries 80
-       TestingDescriptorMaxDownloadTries 80
-       TestingMicrodescMaxDownloadTries 80
-       TestingCertMaxDownloadTries 80
        TestingEnableConnBwEvent 1
        TestingEnableCellStatsEvent 1
-       TestingEnableTbEmptyEvent 1
 
 [[TestingV3AuthInitialVotingInterval]] **TestingV3AuthInitialVotingInterval** __N__ **minutes**|**hours**::
     Like V3AuthVotingInterval, but for initial voting interval before the first
@@ -2620,33 +2981,35 @@ The following options are used for running a testing Tor network.
     time. Changing this requires that **TestingTorNetwork** is set. (Default:
     10 minutes)
 
-[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Minimum value for the Fast flag.  Overrides the ordinary minimum taken
     from the consensus when TestingTorNetwork is set. (Default: 0.)
 
-[[TestingServerDownloadSchedule]] **TestingServerDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when servers should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 0, 60, 60, 120,
-    300, 900, 2147483647)
+[[TestingServerDownloadInitialDelay]] **TestingServerDownloadInitialDelay** __N__::
+    Initial delay in seconds for when servers should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingClientDownloadSchedule]] **TestingClientDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download things in general. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
-    2147483647)
+[[TestingClientDownloadInitialDelay]] **TestingClientDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download things in general. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingServerConsensusDownloadSchedule]] **TestingServerConsensusDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when servers should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
-    1800, 1800, 1800, 1800, 1800, 3600, 7200)
+[[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__::
+    Initial delay in seconds for when servers should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingClientConsensusDownloadSchedule]] **TestingClientConsensusDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download consensuses. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600,
-    1800, 3600, 3600, 3600, 10800, 21600, 43200)
+[[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download consensuses. Changing this
+    requires that **TestingTorNetwork** is set. (Default: 0)
 
-[[TestingBridgeDownloadSchedule]] **TestingBridgeDownloadSchedule** __N__,__N__,__...__::
-    Schedule for when clients should download bridge descriptors. Changing this
-    requires that **TestingTorNetwork** is set. (Default: 3600, 900, 900, 3600)
+[[TestingBridgeDownloadInitialDelay]] **TestingBridgeDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download each bridge descriptor when they
+    know that one or more of their configured bridges are running. Changing
+    this requires that **TestingTorNetwork** is set. (Default: 10800)
+
+[[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__::
+    Initial delay in seconds for when clients should download each bridge descriptor when they
+    have just started, or when they can not contact any of their bridges.
+    Changing this requires that **TestingTorNetwork** is set. (Default: 0)
 
 [[TestingClientMaxIntervalWithoutRequest]] **TestingClientMaxIntervalWithoutRequest** __N__ **seconds**|**minutes**::
     When directory clients have only a few descriptors to request, they batch
@@ -2659,27 +3022,11 @@ The following options are used for running a testing Tor network.
     Changing this requires that **TestingTorNetwork** is set. (Default:
     5 minutes)
 
-[[TestingConsensusMaxDownloadTries]] **TestingConsensusMaxDownloadTries** __NUM__::
-    Try this many times to download a consensus before giving up. Changing
-    this requires that **TestingTorNetwork** is set. (Default: 8)
-
-[[TestingDescriptorMaxDownloadTries]] **TestingDescriptorMaxDownloadTries** __NUM__::
-    Try this often to download a server descriptor before giving up.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 8)
-
-[[TestingMicrodescMaxDownloadTries]] **TestingMicrodescMaxDownloadTries** __NUM__::
-    Try this often to download a microdesc descriptor before giving up.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 8)
-
-[[TestingCertMaxDownloadTries]] **TestingCertMaxDownloadTries** __NUM__::
-    Try this often to download a v3 authority certificate before giving up.
-    Changing this requires that **TestingTorNetwork** is set. (Default: 8)
-
 [[TestingDirAuthVoteExit]] **TestingDirAuthVoteExit** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and
     address patterns of nodes to vote Exit for regardless of their
     uptime, bandwidth, or exit policy. See the **ExcludeNodes**
-    option for more information on how to specify nodes.
+    option for more information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set. See the **ExcludeNodes** option for more
@@ -2688,7 +3035,7 @@ The following options are used for running a testing Tor network.
 [[TestingDirAuthVoteExitIsStrict]] **TestingDirAuthVoteExitIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Exit flag unless it is specified
     in the **TestingDirAuthVoteExit** list, regardless of its uptime, bandwidth,
-    or exit policy.
+    or exit policy. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2697,14 +3044,14 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote Guard for regardless of their
     uptime and bandwidth. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+    information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
 [[TestingDirAuthVoteGuardIsStrict]] **TestingDirAuthVoteGuardIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Guard flag unless it is specified
-    in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth.
+    in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2713,14 +3060,14 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote HSDir for regardless of their
     uptime and DirPort. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+    information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     must be set.
 
 [[TestingDirAuthVoteHSDirIsStrict]] **TestingDirAuthVoteHSDirIsStrict** **0**|**1** ::
     If True (1), a node will never receive the HSDir flag unless it is specified
-    in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort.
+    in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2735,12 +3082,7 @@ The following options are used for running a testing Tor network.
     events.  Changing this requires that **TestingTorNetwork** is set.
     (Default: 0)
 
-[[TestingEnableTbEmptyEvent]] **TestingEnableTbEmptyEvent** **0**|**1**::
-    If this option is set, then Tor controllers may register for TB_EMPTY
-    events.  Changing this requires that **TestingTorNetwork** is set.
-    (Default: 0)
-
-[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Sets a lower-bound for assigning an exit flag when running as an
     authority on a testing network. Overrides the usual default lower bound
     of 4 KB. (Default: 0)
@@ -2764,6 +3106,19 @@ The following options are used for running a testing Tor network.
     we replace it and issue a new key?
     (Default: 3 hours for link and auth; 1 day for signing.)
 
+NON-PERSISTENT OPTIONS
+----------------------
+
+These options are not saved to the torrc file by the "SAVECONF" controller
+command.  Other options of this type are documented in control-spec.txt,
+section 5.4. End-users should mostly ignore them.
+
+[[UnderscorePorts]] **\_\_ControlPort**, **\_\_DirPort**, **\_\_DNSPort**, **\_\_ExtORPort**, **\_\_NATDPort**, **\_\_ORPort**, **\_\_SocksPort**, **\_\_TransPort**::
+    These underscore-prefixed options are variants of the regular Port
+    options.  They behave the same, except they are not saved to the
+    torrc file by the controller's SAVECONF command.
+
+
 SIGNALS
 -------
 
@@ -2810,32 +3165,35 @@ FILES
 **@LOCALSTATEDIR@/lib/tor/**::
     The tor process stores keys and other data here.
 
-__DataDirectory__**/cached-status/**::
-    The most recently downloaded network status document for each authority.
-    Each file holds one such document; the filenames are the hexadecimal
-    identity key fingerprints of the directory authorities.  Mostly obsolete.
 
-__DataDirectory__**/cached-certs**::
+__CacheDirectory__**/cached-certs**::
     This file holds downloaded directory key certificates that are used to
     verify authenticity of documents generated by Tor directory authorities.
 
-__DataDirectory__**/cached-consensus** and/or **cached-microdesc-consensus**::
+__CacheDirectory__**/cached-consensus** and/or **cached-microdesc-consensus**::
     The most recent consensus network status document we've downloaded.
 
-__DataDirectory__**/cached-descriptors** and **cached-descriptors.new**::
+__CacheDirectory__**/cached-descriptors** and **cached-descriptors.new**::
     These files hold downloaded router statuses. Some routers may appear more
     than once; if so, the most recently published descriptor is used. Lines
     beginning with @-signs are annotations that contain more information about
     a given router. The ".new" file is an append-only journal; when it gets
     too large, all entries are merged into a new cached-descriptors file.
 
-__DataDirectory__**/cached-microdescs** and **cached-microdescs.new**::
+__CacheDirectory__**/cached-extrainfo** and **cached-extrainfo.new**::
+   As "cached-descriptors", but holds optionally-downloaded "extra-info"
+   documents. Relays use these documents to send inessential information
+   about statistics, bandwidth history, and network health to the
+   authorities. They aren't fetched by default; see the DownloadExtraInfo
+   option for more info.
+
+__CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**::
     These files hold downloaded microdescriptors.  Lines beginning with
     @-signs are annotations that contain more information about a given
     router. The ".new" file is an append-only journal; when it gets too
     large, all entries are merged into a new cached-microdescs file.
 
-__DataDirectory__**/cached-routers** and **cached-routers.new**::
+__CacheDirectory__**/cached-routers** and **cached-routers.new**::
     Obsolete versions of cached-descriptors and cached-descriptors.new. When
     Tor can't find the newer files, it looks here instead.
 
@@ -2843,18 +3201,27 @@ __DataDirectory__**/state**::
     A set of persistent key-value mappings. These are documented in
     the file. These include:
             - The current entry guards and their status.
-            - The current bandwidth accounting values (unused so far; see
-            below).
+            - The current bandwidth accounting values.
             - When the file was last written
             - What version of Tor generated the state file
             - A short history of bandwidth usage, as produced in the server
             descriptors.
 
+__DataDirectory__**/sr-state**::
+    Authority only. State file used to record information about the current
+    status of the shared-random-value voting state.
+
+__CacheDirectory__**/diff-cache**::
+    Directory cache only. Holds older consensuses, and diffs from older
+    consensuses to the most recent consensus of each type, compressed
+    in various ways. Each file contains a set of key-value arguments
+    describing its contents, followed by a single NUL byte, followed by the
+    main file contents.
+
 __DataDirectory__**/bw_accounting**::
     Used to track bandwidth accounting values (when the current period starts
     and ends; how much has been read and written so far this period). This file
-    is obsolete, and the data is now stored in the \'state' file as well. Only
-    used when bandwidth accounting is enabled.
+    is obsolete, and the data is now stored in the \'state' file instead.
 
 __DataDirectory__**/control_auth_cookie**::
     Used for cookie authentication with the controller. Location can be
@@ -2867,63 +3234,71 @@ __DataDirectory__**/lock**::
     directory. If access to this file is locked, data directory is already
     in use by Tor.
 
-__DataDirectory__**/keys/***::
-    Only used by servers. Holds identity keys and onion keys.
+__DataDirectory__**/key-pinning-journal**::
+    Used by authorities. A line-based file that records mappings between
+    RSA1024 identity keys and Ed25519 identity keys. Authorities enforce
+    these mappings, so that once a relay has picked an Ed25519 key, stealing
+    or factoring the RSA1024 key will no longer let an attacker impersonate
+    the relay.
 
-__DataDirectory__**/keys/authority_identity_key**::
+__KeyDirectory__**/authority_identity_key**::
     A v3 directory authority's master identity key, used to authenticate its
     signing key. Tor doesn't use this while it's running. The tor-gencert
     program uses this. If you're running an authority, you should keep this
     key offline, and not actually put it here.
 
-__DataDirectory__**/keys/authority_certificate**::
+__KeyDirectory__**/authority_certificate**::
     A v3 directory authority's certificate, which authenticates the authority's
     current vote- and consensus-signing key using its master identity key.
     Only directory authorities use this file.
 
-__DataDirectory__**/keys/authority_signing_key**::
+__KeyDirectory__**/authority_signing_key**::
     A v3 directory authority's signing key, used to sign votes and consensuses.
     Only directory authorities use this file.  Corresponds to the
     **authority_certificate** cert.
 
-__DataDirectory__**/keys/legacy_certificate**::
+__KeyDirectory__**/legacy_certificate**::
     As authority_certificate: used only when V3AuthUseLegacyKey is set.
     See documentation for V3AuthUseLegacyKey.
 
-__DataDirectory__**/keys/legacy_signing_key**::
+__KeyDirectory__**/legacy_signing_key**::
     As authority_signing_key: used only when V3AuthUseLegacyKey is set.
     See documentation for V3AuthUseLegacyKey.
 
-__DataDirectory__**/keys/secret_id_key**::
+__KeyDirectory__**/secret_id_key**::
     A relay's RSA1024 permanent identity key, including private and public
     components.  Used to sign router descriptors, and to sign other keys.
 
-__DataDirectory__**/keys/ed25519_master_id_public_key**::
+__KeyDirectory__**/ed25519_master_id_public_key**::
     The public part of a relay's Ed25519 permanent identity key.
 
-__DataDirectory__**/keys/ed25519_master_id_secret_key**::
+__KeyDirectory__**/ed25519_master_id_secret_key**::
     The private part of a relay's Ed25519 permanent identity key.  This key
     is used to sign the medium-term ed25519 signing key.  This file can be
     kept offline, or kept encrypted. If so, Tor will not be able to generate
     new signing keys itself; you'll need to use tor --keygen yourself to do
     so.
 
-__DataDirectory__**/keys/ed25519_signing_secret_key**::
+__KeyDirectory__**/ed25519_signing_secret_key**::
     The private and public components of a relay's medium-term Ed25519 signing
     key. This key is authenticated by the Ed25519 master key, in turn
     authenticates other keys (and router descriptors).
 
-__DataDirectory__**/keys/ed25519_signing_cert**::
+__KeyDirectory__**/ed25519_signing_cert**::
     The certificate which authenticates "ed25519_signing_secret_key" as
     having been signed by the Ed25519 master key.
 
-__DataDirectory__**/keys/secret_onion_key**::
+__KeyDirectory__**/secret_onion_key** and **secret_onion_key.old**::
     A relay's RSA1024 short-term onion key. Used to decrypt old-style ("TAP")
-    circuit extension requests.
+    circuit extension requests. The ".old" file holds the previously
+    generated key, which the relay uses to handle any requests that were
+    made by clients that didn't have the new one.
 
-__DataDirectory__**/keys/secret_onion_key_ntor**::
+__KeyDirectory__**/secret_onion_key_ntor** and **secret_onion_key_ntor.old**::
     A relay's Curve25519 short-term onion key. Used to handle modern ("ntor")
-    circuit extension requests.
+    circuit extension requests. The ".old" file holds the previously
+    generated key, which the relay uses to handle any requests that were
+    made by clients that didn't have the new one.
 
 __DataDirectory__**/fingerprint**::
     Only used by servers. Holds the fingerprint of the server's identity key.
@@ -2932,15 +3307,25 @@ __DataDirectory__**/hashed-fingerprint**::
     Only used by bridges. Holds the hashed fingerprint of the bridge's
     identity key. (That is, the hash of the hash of the identity key.)
 
+__DataDirectory__**/approved-routers**::
+    Only used by authoritative directory servers. This file lists
+    the status of routers by their identity fingerprint.
+    Each line lists a status and a fingerprint separated by
+    whitespace. See your **fingerprint** file in the __DataDirectory__ for an
+    example line. If the status is **!reject** then descriptors from the
+    given identity (fingerprint) are rejected by this server. If it is
+    **!invalid** then descriptors are accepted but marked in the directory as
+    not valid, that is, not recommended.
+
 __DataDirectory__**/v3-status-votes**::
     Only for v3 authoritative directory servers. This file contains
     status votes from all the authoritative directory servers.
 
-__DataDirectory__**/unverified-consensus**::
+__CacheDirectory__**/unverified-consensus**::
     This file contains a network consensus document that has been downloaded,
     but which we didn't have the right certificates to check yet.
 
-__DataDirectory__**/unverified-microdesc-consensus**::
+__CacheDirectory__**/unverified-microdesc-consensus**::
     This file contains a microdescriptor-flavored network consensus document
     that has been downloaded, but which we didn't have the right certificates
     to check yet.
@@ -2978,15 +3363,29 @@ __DataDirectory__**/stats/conn-stats**::
     Only used by servers. This file is used to collect approximate connection
     history (number of active connections over time).
 
+__DataDirectory__**/stats/hidserv-stats**::
+    Only used by servers. This file is used to collect approximate counts
+    of what fraction of the traffic is hidden service rendezvous traffic, and
+    approximately how many hidden services the relay has seen.
+
 __DataDirectory__**/networkstatus-bridges**::
     Only used by authoritative bridge directories. Contains information
     about bridges that have self-reported themselves to the bridge
     authority.
 
+__DataDirectory__**/approved-routers**::
+    Authorities only. This file is used to configure which relays are
+    known to be valid, invalid, and so forth.
+
 __HiddenServiceDirectory__**/hostname**::
     The <base32-encoded-fingerprint>.onion domain name for this hidden service.
     If the hidden service is restricted to authorized clients only, this file
     also contains authorization data for all clients.
+ +
+    Note that clients will ignore any extra subdomains prepended to a hidden
+    service hostname. So if you have "xyz.onion" as your hostname, you
+    can tell clients to connect to "www.xyz.onion" or "irc.xyz.onion"
+    for virtual-hosting purposes.
 
 __HiddenServiceDirectory__**/private_key**::
     The private key for this hidden service.
diff --git a/doc/torify.1.txt b/doc/torify.1.txt
index 8dca901317..7e49081cfc 100644
--- a/doc/torify.1.txt
+++ b/doc/torify.1.txt
@@ -17,25 +17,23 @@ SYNOPSIS
 
 DESCRIPTION
 -----------
-**torify** is a simple wrapper that attempts to find the best underlying Tor
-wrapper available on a system. It calls torsocks with a tor specific
-configuration file. +
+**torify** is a simple wrapper that calls torsocks with a tor-specific
+configuration file.
 
-torsocks is an improved wrapper that explicitly rejects UDP, safely resolves DNS
-lookups and properly socksifies your TCP connections. +
-
-Please note that since both method use LD_PRELOAD, torify cannot be applied to
-suid binaries.
+It is provided for backward compatibility; instead you should use torsocks.
 
 WARNING
 -------
-When used with torsocks, torify should not leak DNS requests or UDP data. +
+When used with torsocks, torify should not leak DNS requests or UDP data.
+
+torify can leak ICMP data.
 
-Both will leak ICMP data.
+torify will not ensure that different requests are processed on
+different circuits.
 
 SEE ALSO
 --------
-**tor**(1), **tor-resolve**(1), **torsocks**(1)
+**tor**(1), **torsocks**(1)
 
 AUTHORS
 -------
diff --git a/doc/torrc_format.txt b/doc/torrc_format.txt
index 7a809901d9..7a4a92a663 100644
--- a/doc/torrc_format.txt
+++ b/doc/torrc_format.txt
@@ -18,9 +18,10 @@ does, not what it should do.
    ; specified in RFC5234.
 
    ; A file is interpreted as every Entry in the file, in order.
-   TorrcFile = *Line
+   TorrcFile = *Line [ UnterminatedLine ]
 
-   Line = BlankLine / Entry
+   Line = BlankLine LF / Entry LF
+   UnterminatedLine = BlankLine / Entry
 
    BlankLine =  *WSP OptComment LF
    BlankLine =/ *WSP LF
@@ -69,6 +70,12 @@ does, not what it should do.
    ; Anything besides NUL and LF
    NonLF = %x01-%x09 / %x0b - %xff
 
+   ; Note that on windows, we open our configuration files in "text" mode,
+   ; which causes CRLF pairs to be interpreted as LF.  So, on windows:
+   ;         LF = [ %x0d ] %x0a
+   ; but everywhere else,
+   LF = %0x0a
+
    OCTDIG = '0' - '7'
 
    KC = Any character except an isspace() character or '#' or NUL
@@ -175,7 +182,7 @@ and\
 friends
 
 # Backslashes in the middle of a line are included as-is.  The key of
-# this one is "Too" and the value is "Many\\Backsl\ashes here" (with
+# this one is "Too" and the value is "Many\\Backsl\ashes \here" (with
 # backslashes in that last string as-is)
 Too \
 Many\\\
@@ -185,7 +192,7 @@ here
 # And here's the really yucky part. If a comment appears in a multi-line
 # entry, the entry is still able to continue on the next line, as in the
 # following, where the key is "This" and the value is
-# "entry   and some        are  silly"
+# "entry        and some        are  silly"
 This entry      \
  # has comments \
  and some       \
diff --git a/link_rust.sh.in b/link_rust.sh.in
new file mode 100644
index 0000000000..59f4142baa
--- /dev/null
+++ b/link_rust.sh.in
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# A linker script used when building Rust tests.  Autoconf makes link_rust.sh
+# from link_rust_sh.in, and uses it to pass extra options to the linker
+# when linking Rust stuff.
+#
+# We'd like to remove the need for this, but build.rs doesn't let us pass
+# -static-libasan and -static-libubsan to the linker.
+
+$CCLD @RUST_LINKER_OPTIONS@ "$@"
diff --git a/m4/ax_check_sign.m4 b/m4/ax_check_sign.m4
index 104b17014c..d67e114dba 100644
--- a/m4/ax_check_sign.m4
+++ b/m4/ax_check_sign.m4
@@ -41,8 +41,8 @@ AU_ALIAS([VL_CHECK_SIGN], [AX_CHECK_SIGN])
 AC_DEFUN([AX_CHECK_SIGN], [
  typename=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g"`
  AC_CACHE_CHECK([whether $1 is signed], ax_cv_decl_${typename}_signed, [
-   AC_TRY_COMPILE([$4],
-     [ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ],
+   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]],
+     [[ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ]])],
      [ eval "ax_cv_decl_${typename}_signed=\"yes\"" ],
      [ eval "ax_cv_decl_${typename}_signed=\"no\"" ])])
  symbolname=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g" | tr "a-z" "A-Z"`
diff --git a/m4/pc_from_ucontext.m4 b/m4/pc_from_ucontext.m4
index 8a9dc459e6..9b66bf752c 100644
--- a/m4/pc_from_ucontext.m4
+++ b/m4/pc_from_ucontext.m4
@@ -79,29 +79,29 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      if ! $pc_field_found; then
        # Prefer sys/ucontext.h to ucontext.h, for OS X's sake.
        if test "x$ac_cv_header_cygwin_signal_h" = xyes; then
-         AC_TRY_COMPILE([#include <cygwin/signal.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <cygwin/signal.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_sys_ucontext_h" = xyes; then
-         AC_TRY_COMPILE([#include <sys/ucontext.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/ucontext.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_ucontext_h" = xyes; then
-         AC_TRY_COMPILE([#include <ucontext.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ucontext.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        else     # hope some standard header gives it to us
-         AC_TRY_COMPILE([],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
@@ -114,8 +114,8 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      pc_fields="$pc_fields sc_rip"  # OpenBSD (x86_64)
      for pc_field in $pc_fields; do
        if ! $pc_field_found; then
-         AC_TRY_COMPILE([#include <signal.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
diff --git a/scripts/coccinelle/ceil_div.cocci b/scripts/coccinelle/ceil_div.cocci
new file mode 100644
index 0000000000..00843e82c3
--- /dev/null
+++ b/scripts/coccinelle/ceil_div.cocci
@@ -0,0 +1,6 @@
+@@
+expression n, d;
+@@
+
+- (((n) + (d) - 1) / (d))
++ CEIL_DIV(n, d)
diff --git a/scripts/coccinelle/test-operator-cleanup b/scripts/coccinelle/test-operator-cleanup
new file mode 100755
index 0000000000..e7822542a4
--- /dev/null
+++ b/scripts/coccinelle/test-operator-cleanup
@@ -0,0 +1,11 @@
+#!/usr/bin/perl -w -p -i
+
+next if m#^ */\*# or m#^ *\* #;
+
+s/<([,)])/OP_LT$1/;
+s/(?<=[\s,])>([,)])/OP_GT$1/;
+#s/>([,)])/OP_GT$1/;
+s/==([,)])/OP_EQ$1/;
+s/>=([,)])/OP_GE$1/;
+s/<=([,)])/OP_LE$1/;
+s/!=([,)])/OP_NE$1/;
diff --git a/scripts/coccinelle/test_assert_int.cocci b/scripts/coccinelle/test_assert_int.cocci
new file mode 100644
index 0000000000..80e86b4f3c
--- /dev/null
+++ b/scripts/coccinelle/test_assert_int.cocci
@@ -0,0 +1,49 @@
+@@
+int e;
+constant c;
+@@
+
+(
+- tt_assert(e == c)
++ tt_int_op(e, OP_EQ, c)
+|
+- tt_assert(e != c)
++ tt_int_op(e, OP_NE, c)
+|
+- tt_assert(e < c)
++ tt_int_op(e, OP_LT, c)
+|
+- tt_assert(e <= c)
++ tt_int_op(e, OP_LE, c)
+|
+- tt_assert(e > c)
++ tt_int_op(e, OP_GT, c)
+|
+- tt_assert(e >= c)
++ tt_int_op(e, OP_GE, c)
+)
+
+@@
+unsigned int e;
+constant c;
+@@
+
+(
+- tt_assert(e == c)
++ tt_uint_op(e, OP_EQ, c)
+|
+- tt_assert(e != c)
++ tt_uint_op(e, OP_NE, c)
+|
+- tt_assert(e < c)
++ tt_uint_op(e, OP_LT, c)
+|
+- tt_assert(e <= c)
++ tt_uint_op(e, OP_LE, c)
+|
+- tt_assert(e > c)
++ tt_uint_op(e, OP_GT, c)
+|
+- tt_assert(e >= c)
++ tt_uint_op(e, OP_GE, c)
+)
diff --git a/scripts/coccinelle/test_assert_null.cocci b/scripts/coccinelle/test_assert_null.cocci
new file mode 100644
index 0000000000..3d66e1ee0b
--- /dev/null
+++ b/scripts/coccinelle/test_assert_null.cocci
@@ -0,0 +1,11 @@
+@@
+expression * e;
+@@
+
+(
+- tt_assert(e != NULL)
++ tt_ptr_op(e, OP_NE, NULL)
+|
+- tt_assert(e == NULL)
++ tt_ptr_op(e, OP_EQ, NULL)
+)
diff --git a/scripts/coccinelle/test_assert_zero.cocci b/scripts/coccinelle/test_assert_zero.cocci
new file mode 100644
index 0000000000..09feaa5fb4
--- /dev/null
+++ b/scripts/coccinelle/test_assert_zero.cocci
@@ -0,0 +1,5 @@
+@@
+@@
+
+- tt_assert(0)
++ tt_abort()
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
new file mode 100755
index 0000000000..5bef22d900
--- /dev/null
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -0,0 +1,168 @@
+#!/usr/bin/python
+
+FUZZERS = """
+	consensus
+	descriptor
+	diff
+	diff-apply
+	extrainfo
+	hsdescv2
+	hsdescv3
+	http
+	http-connect
+	iptsv2
+	microdesc
+	vrs
+"""
+
+
+PREAMBLE = r"""
+FUZZING_CPPFLAGS = \
+	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+FUZZING_CFLAGS = \
+	$(AM_CFLAGS) $(TEST_CFLAGS)
+FUZZING_LDFLAG = \
+	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+FUZZING_LIBS = \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a \
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIBEVENT_LIBS@ \
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	@TOR_ZSTD_LIBS@
+
+oss-fuzz-prereqs: \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a
+
+noinst_HEADERS += \
+	src/test/fuzz/fuzzing.h
+
+LIBFUZZER = -lFuzzer
+LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
+LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
+LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
+
+LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
+"""
+
+POSTAMBLE = r"""
+noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
+noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
+oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
+fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
+
+test-fuzz-corpora: $(FUZZERS)
+	$(top_srcdir)/src/test/fuzz_static_testcases.sh
+"""
+
+########### No user serviceable parts will follow.
+
+PREAMBLE = PREAMBLE.strip()
+POSTAMBLE = POSTAMBLE.strip()  # If I use it, it's a word!
+FUZZERS = FUZZERS.split()
+FUZZERS.sort()
+
+WARNING = """
+# This file was generated by fuzzing_include_am.py; do not hand-edit unless
+# you enjoy having your changes erased.
+""".strip()
+
+print(WARNING)
+
+print(PREAMBLE)
+
+print("\n# ===== AFL fuzzers")
+
+def get_id_name(s):
+    return s.replace("-", "_")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_{name}_SOURCES = \\
+	src/test/fuzz/fuzzing_common.c \\
+	src/test/fuzz/fuzz_{name}.c
+src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS)
+endif
+""".format(name=idname))
+
+print("if UNITTESTS_ENABLED")
+print("FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+print("endif")
+
+print("\n# ===== libfuzzer")
+print("\nif LIBFUZZER_ENABLED")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_{name}_SOURCES = \\
+	$(src_test_fuzz_fuzz_{name}_SOURCES)
+src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS)
+endif
+""".format(name=idname))
+
+print("LIBFUZZER_FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/lf-fuzz-{name}".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("""
+else
+LIBFUZZER_FUZZERS =
+endif""")
+
+print("\n# ===== oss-fuzz\n")
+print("if OSS_FUZZ_ENABLED")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\
+	$(src_test_fuzz_fuzz_{name}_SOURCES)
+src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+""".format(name=idname))
+
+print("OSS_FUZZ_FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/liboss-fuzz-{name}.a".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("""
+else
+OSS_FUZZ_FUZZERS =
+endif""")
+
+print("")
+
+print(POSTAMBLE)
diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py
index 0dca8a6734..7ea39c540d 100755
--- a/scripts/codegen/gen_server_ciphers.py
+++ b/scripts/codegen/gen_server_ciphers.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc
+# Copyright 2014-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 # This script parses openssl headers to find ciphersuite names, determines
@@ -13,13 +13,13 @@ import sys
 
 EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ]
 BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_",
-              "_SEED_", "_CAMELLIA_", "_NULL" ]
+              "_SEED_", "_CAMELLIA_", "_NULL",
+              "_CCM_8", "_DES_", ]
 
 # these never get #ifdeffed.
 MANDATORY = [
     "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA",
     "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA",
-    "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA",
 ]
 
 def find_ciphers(filename):
@@ -48,15 +48,23 @@ def usable_cipher(ciph):
 # All fields we sort on, in order of priority.
 FIELDS = [ 'cipher', 'fwsec', 'mode',  'digest', 'bitlength' ]
 # Map from sorted fields to recognized value in descending order of goodness
-FIELD_VALS = { 'cipher' : [ 'AES', 'DES'],
+FIELD_VALS = { 'cipher' : [ 'AES', 'CHACHA20' ],
                'fwsec' : [ 'ECDHE', 'DHE' ],
-               'mode' : [ 'GCM', 'CBC' ],
-               'digest' : [ 'SHA384', 'SHA256', 'SHA' ],
+               'mode' : [ 'POLY1305', 'GCM', 'CCM', 'CBC', ],
+               'digest' : [ 'n/a', 'SHA384', 'SHA256', 'SHA', ],
                'bitlength' : [ '256', '128', '192' ],
 }
 
 class Ciphersuite(object):
     def __init__(self, name, fwsec, cipher, bitlength, mode, digest):
+        if fwsec == 'EDH':
+            fwsec = 'DHE'
+
+        if mode in [ '_CBC3', '_CBC', '' ]:
+            mode = 'CBC'
+        elif mode == '_GCM':
+            mode = 'GCM'
+
         self.name = name
         self.fwsec = fwsec
         self.cipher = cipher
@@ -74,42 +82,50 @@ class Ciphersuite(object):
 def parse_cipher(ciph):
     m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph)
 
-    if not m:
-        print "/* Couldn't parse %s ! */"%ciph
-        return None
+    if m:
+        fwsec, cipher, bits, mode, digest = m.groups()
+        return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
-    fwsec, cipher, bits, mode, digest = m.groups()
-    if fwsec == 'EDH':
-        fwsec = 'DHE'
+    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)_CCM', ciph)
+    if m:
+        fwsec, cipher, bits = m.groups()
+        return Ciphersuite(ciph, fwsec, cipher, bits, "CCM", "n/a")
 
-    if mode in [ '_CBC3', '_CBC', '' ]:
-        mode = 'CBC'
-    elif mode == '_GCM':
-        mode = 'GCM'
+    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_CHACHA20_POLY1305', ciph)
+    if m:
+        fwsec, = m.groups()
+        return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a")
+
+    print "/* Couldn't parse %s ! */"%ciph
+    return None
 
-    return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
 ALL_CIPHERS = []
 
 for fname in sys.argv[1:]:
-    ALL_CIPHERS += (parse_cipher(c)
-                           for c in find_ciphers(fname)
-                           if usable_cipher(c) )
+    for c in find_ciphers(fname):
+        if usable_cipher(c):
+            parsed = parse_cipher(c)
+            if parsed != None:
+                ALL_CIPHERS.append(parsed)
 
 ALL_CIPHERS.sort(key=Ciphersuite.sort_key)
 
+indent = " "*7
+
 for c in ALL_CIPHERS:
     if c is ALL_CIPHERS[-1]:
-        colon = ';'
+        colon = ''
     else:
         colon = ' ":"'
 
     if c.name in MANDATORY:
-        print "       /* Required */"
-        print '       %s%s'%(c.name,colon)
+        print "%s/* Required */"%indent
+        print '%s%s%s'%(indent,c.name,colon)
     else:
         print "#ifdef %s"%c.name
-        print '       %s%s'%(c.name,colon)
+        print '%s%s%s'%(indent,c.name,colon)
         print "#endif"
 
+print '%s;'%indent
 
diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py
index e673ec7dc6..946957ac77 100644..100755
--- a/scripts/codegen/get_mozilla_ciphers.py
+++ b/scripts/codegen/get_mozilla_ciphers.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # coding=utf-8
-# Copyright 2011-2015, The Tor Project, Inc
+# Copyright 2011-2017, The Tor Project, Inc
 # original version by Arturo Filastò
 # See LICENSE for licensing information
 
@@ -127,9 +127,9 @@ for k, v in enabled_ciphers.items():
 #oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h',
 #               '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h',
 #               '/usr/include/openssl/tls1.h')
-oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h',
-               'ssl/ssl2.h', 'ssl/ssl23.h',
-               'ssl/tls1.h')
+oSSLinclude = ['ssl3.h', 'ssl.h'
+               'ssl2.h', 'ssl23.h',
+               'tls1.h']
 
 #####
 # This reads the hex code for the ciphers that are used by firefox.
@@ -155,9 +155,12 @@ for x in used_ciphers:
 openssl_macro_by_hex = {}
 all_openssl_macros = {}
 for fl in oSSLinclude:
-    fp = open(ossl(fl), 'r')
+    fname = ossl("include/openssl/"+fl)
+    if not os.path.exists(fname):
+        continue
+    fp = open(fname, 'r')
     for line in fp.readlines():
-        m = re.match('#define\s+(\S+)\s+(\S+)', line)
+        m = re.match('# *define\s+(\S+)\s+(\S+)', line)
         if m:
             value,key = m.groups()
             if key.startswith('0x') and "_CK_" in value:
diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py
index d4ba21efae..8d9d4edaaf 100644
--- a/scripts/codegen/makedesc.py
+++ b/scripts/codegen/makedesc.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc.
+# Copyright 2014-2017, The Tor Project, Inc.
 # See LICENSE for license information
 
 # This is a kludgey python script that uses ctypes and openssl to sign
diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh
index d2669931e9..428804342a 100755
--- a/scripts/codegen/run_trunnel.sh
+++ b/scripts/codegen/run_trunnel.sh
@@ -5,7 +5,13 @@ if test "x$TRUNNEL_PATH" != "x"; then
   export PYTHONPATH
 fi
 
-python -m trunnel --require-version=1.4 ./src/trunnel/*.trunnel
+OPTIONS="--require-version=1.5.1"
 
-python -m trunnel --require-version=1.4 --write-c-files --target-dir=./src/ext/trunnel/
+# Get all .trunnel files recursively from that directory so we can support
+# multiple sub-directories.
+for file in `find ./src/trunnel/ -name '*.trunnel'`; do
+  python -m trunnel ${OPTIONS} $file
+done
+
+python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/
 
diff --git a/scripts/maint/analyze_callgraph.py b/scripts/maint/analyze_callgraph.py
deleted file mode 100755
index 8ce5827f07..0000000000
--- a/scripts/maint/analyze_callgraph.py
+++ /dev/null
@@ -1,259 +0,0 @@
-#!/usr/bin/python
-
-import re
-import sys
-import copy
-import cPickle
-import os
-
-class Parser:
-  def __init__(self):
-    self.calls = {}
-    self.definedIn = {}
-
-  def enter_func(self, name):
-    if self.infunc and not self.extern and self.calledfns:
-      if self.infunc in self.definedIn:
-        #print "{}: {} or {}?".format(
-        #  self.infunc, self.definedIn[self.infunc], self.module)
-        self.definedIn[self.infunc] = 'nil'
-      else:
-        self.definedIn[self.infunc] = self.module
-      self.calls.setdefault(self.infunc, set()).update( self.calledfns )
-
-    self.calledfns = set()
-    self.infunc = name
-    self.extern = False
-
-  def parse_callgraph_file(self, inp, module):
-    self.infunc = None
-    self.extern = False
-    self.calledfns = set()
-    self.module = module
-
-    for line in inp:
-       m = re.match(r"Call graph node for function: '([^']+)'", line)
-       if m:
-           self.enter_func(m.group(1))
-           continue
-       m = re.match(r"  CS<[^>]+> calls external node", line)
-       if m:
-           self.extern = True
-       m = re.match(r"  CS<[^>]+> calls function '([^']+)'", line)
-       if m:
-           self.calledfns.add(m.group(1))
-    self.enter_func(None)
-
-  def extract_callgraph(self):
-    c = self.calls
-    self.calls = {}
-    return c
-
-
-def transitive_closure(g):
-    passno = 0
-    changed = True
-    g = copy.deepcopy(g)
-    import random
-    while changed:
-      passno += 1
-      changed = False
-      keys = g.keys()
-      idx = 0
-      for k in keys:
-         idx += 1
-         print "Pass %d/?: %d/%d\r" %(passno, idx, len(keys)),
-         sys.stdout.flush()
-         newset = g[k].copy()
-         for fn in g[k]:
-            newset.update(g.get(fn, set()))
-         if len(newset) != len(g[k]):
-            g[k].update( newset )
-            changed = True
-
-      print
-
-    return g
-
-def strongly_connected_components(g):
-  # From https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm, done stupidly.
-  index_of = {}
-  index = [ 0 ]
-  lowlink = {}
-  S = []
-  onStack = set()
-
-  all_sccs = []
-
-  def strongconnect(fn):
-    index_of[fn] = index[0]
-    lowlink[fn] = index[0]
-    index[0] += 1
-    S.append(fn)
-    onStack.add(fn)
-
-    for w in g.get(fn, []):
-      if w not in index_of:
-        strongconnect(w)
-        lowlink[fn] = min(lowlink[fn], lowlink[w])
-      elif w in onStack:
-        lowlink[fn] = min(lowlink[fn], index_of[w])
-
-    if lowlink[fn] == index_of[fn]:
-      this_scc = []
-      all_sccs.append(this_scc)
-      while True:
-        w = S.pop()
-        onStack.remove(w)
-        this_scc.append(w)
-        if w == fn:
-          break
-
-  for v in g.keys():
-    if v not in index_of:
-      strongconnect(v)
-
-  return all_sccs
-
-def biggest_component(sccs):
-  return max(len(c) for c in sccs)
-
-def connection_bottlenecks(callgraph):
-
-  callers = {}
-  for fn in callgraph:
-    for fn2 in callgraph[fn]:
-      callers.setdefault(fn2, set()).add(fn)
-
-  components = strongly_connected_components(callgraph)
-  components.sort(key=len)
-  big_component_fns = components[-1]
-  size = len(big_component_fns)
-
-  function_bottlenecks = fn_results = []
-
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    idx += 1
-    print "Pass 1/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-    cg2 = copy.deepcopy(callgraph)
-    del cg2[fn]
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn) )
-
-  print
-  bcf_set = set(big_component_fns)
-
-  call_bottlenecks = fn_results = []
-  result_set = set()
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    fn_callers = callers[fn].intersection(bcf_set)
-    idx += 1
-    if len(fn_callers) != 1:
-      continue
-
-    print "Pass 2/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-
-    caller = fn_callers.pop()
-    assert len(fn_callers) == 0
-    cg2 = copy.deepcopy(callgraph)
-    cg2[caller].remove(fn)
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn, "called by", caller) )
-    result_set.add( (caller, fn) )
-
-  print
-
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    fn_calls = callgraph[fn].intersection(bcf_set)
-    idx += 1
-    if len(fn_calls) != 1:
-      continue
-
-    print "Pass 3/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-
-    callee = fn_calls.pop()
-    if (fn, callee) in result_set:
-      continue
-
-    assert len(fn_calls) == 0
-    cg2 = copy.deepcopy(callgraph)
-    cg2[fn].remove(callee)
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), callee, "called by", fn) )
-
-  print
-
-
-  return (function_bottlenecks, call_bottlenecks)
-
-if __name__ == '__main__':
-    p = Parser()
-    for fname in sys.argv[1:]:
-      modname = re.sub(r'.*/', '', fname).replace('.callgraph', '.c')
-      with open(fname, 'r') as f:
-        p.parse_callgraph_file(f, modname)
-
-    sys.stdout.flush()
-
-    print "Building callgraph"
-    callgraph = p.extract_callgraph()
-    inModule = p.definedIn
-
-    print "Deriving module callgraph"
-    modCallgraph = {}
-    for fn in callgraph:
-      fnMod = inModule[fn]
-      for called in callgraph[fn]:
-        try:
-          calledMod = inModule[called]
-        except KeyError:
-            continue
-        modCallgraph.setdefault(fnMod, set()).add(calledMod)
-    del modCallgraph['nil']
-
-    print "Finding strongly connected components"
-    sccs = strongly_connected_components(callgraph)
-
-    print "Finding the transitive closure of the callgraph.."
-    closure = transitive_closure(callgraph)
-
-    print "Finding bottlenecks..."
-    bottlenecks = connection_bottlenecks(callgraph)
-
-    print "Finding module SCCs"
-    modSCCS = strongly_connected_components(modCallgraph)
-
-    print "Finding module TC"
-    modTC = transitive_closure(modCallgraph)
-
-    print "Finding module bottlenecks"
-    modB = connection_bottlenecks(modCallgraph)
-
-    data = {
-      'callgraph' : callgraph,
-      'sccs' : sccs,
-      'closure' : closure,
-      'bottlenecks' : bottlenecks,
-      'modules' : p.definedIn,
-      'modItems' : {
-        'callgraph' : modCallgraph,
-        'sccs' : modSCCS,
-        'closure' : modTC,
-        'bottlenecks' : modB,
-      }
-    }
-
-    with open('callgraph.pkl', 'w') as f:
-      cPickle.dump(data, f)
-
-
-
diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives
new file mode 100755
index 0000000000..368d842e2e
--- /dev/null
+++ b/scripts/maint/annotate_ifdef_directives
@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# Copyright (c) 2017, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+import re
+
+LINE_OBVIOUSNESS_LIMIT = 4
+
+class Problem(Exception):
+    pass
+
+def uncomment(s):
+    s = re.sub(r'//.*','',s)
+    s = re.sub(r'/\*.*','',s)
+    return s.strip()
+
+def translate(f_in, f_out):
+    whole_file = []
+    stack = []
+    cur_level = whole_file
+    lineno = 0
+    for line in f_in:
+        lineno += 1
+        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
+                     line)
+        if not m:
+            f_out.write(line)
+            continue
+        command,rest = m.groups()
+        if command in ("if", "ifdef", "ifndef"):
+            # The #if directive pushes us one level lower on the stack.
+            if command == 'ifdef':
+                rest = "defined(%s)"%uncomment(rest)
+            elif command == 'ifndef':
+                rest = "!defined(%s)"%uncomment(rest)
+            elif rest.endswith("\\"):
+                rest = rest[:-1]+"..."
+
+            rest = uncomment(rest)
+
+            new_level = [ (command, rest, lineno) ]
+            stack.append(cur_level)
+            cur_level = new_level
+            f_out.write(line)
+        elif command in ("else", "elif"):
+            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
+                raise Problem("Unexpected #%s on %d"% (command,lineno))
+            if (len(cur_level) == 1 and command == 'else' and
+                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
+                f_out.write("#else /* !(%s) */\n"%cur_level[0][1])
+            else:
+                f_out.write(line)
+            cur_level.append((command, rest, lineno))
+        else:
+            assert command == 'endif'
+            if len(stack) == 0:
+                raise Problem("Unmatched #%s on %s"% (command,lineno))
+            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
+                f_out.write(line)
+            elif len(cur_level) == 1 or (
+                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
+                f_out.write("#endif /* %s */\n"%cur_level[0][1])
+            else:
+                f_out.write("#endif /* %s || ... */\n"%cur_level[0][1])
+            cur_level = stack.pop()
+    if len(stack) or cur_level != whole_file:
+        raise Problem("Missing #endif")
+
+import sys,os
+for fn in sys.argv[1:]:
+    with open(fn+"_OUT", 'w') as output_file:
+        translate(open(fn, 'r'), output_file)
+    os.rename(fn+"_OUT", fn)
+
diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl
index 5af95a27e5..a12370f21f 100755
--- a/scripts/maint/checkSpace.pl
+++ b/scripts/maint/checkSpace.pl
@@ -1,48 +1,59 @@
-#!/usr/bin/perl -w
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+my $found = 0;
+sub msg {
+  $found = 1;
+  print "$_[0]";
+}
+
+my $C = 0;
 
 if ($ARGV[0] =~ /^-/) {
-    $lang = shift @ARGV;
+    my $lang = shift @ARGV;
     $C = ($lang eq '-C');
-#    $TXT = ($lang eq '-txt');
 }
 
-for $fn (@ARGV) {
+for my $fn (@ARGV) {
     open(F, "$fn");
-    $lastnil = 0;
-    $lastline = "";
-    $incomment = 0;
+    my $lastnil = 0;
+    my $lastline = "";
+    my $incomment = 0;
+    my $in_func_head = 0;
     while (<F>) {
         ## Warn about windows-style newlines.
-	#    (We insist on lines that end with a single LF character, not
-	#    CR LF.)
+        #    (We insist on lines that end with a single LF character, not
+        #    CR LF.)
         if (/\r/) {
-            print "       CR:$fn:$.\n";
+            msg "       CR:$fn:$.\n";
         }
         ## Warn about tabs.
-	#    (We only use spaces)
+        #    (We only use spaces)
         if (/\t/) {
-            print "      TAB:$fn:$.\n";
+            msg "      TAB:$fn:$.\n";
         }
         ## Warn about labels that don't have a space in front of them
-	#    (We indent every label at least one space)
+        #    (We indent every label at least one space)
         if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
-            print "nosplabel:$fn:$.\n";
+            msg "nosplabel:$fn:$.\n";
         }
         ## Warn about trailing whitespace.
-	#    (We don't allow whitespace at the end of the line; make your
-	#    editor highlight it for you so you can stop adding it in.)
+        #    (We don't allow whitespace at the end of the line; make your
+        #    editor highlight it for you so you can stop adding it in.)
         if (/ +$/) {
-            print "Space\@EOL:$fn:$.\n";
+            msg "Space\@EOL:$fn:$.\n";
         }
         ## Warn about control keywords without following space.
-	#    (We put a space after every 'if', 'while', 'for', 'switch', etc)
+        #    (We put a space after every 'if', 'while', 'for', 'switch', etc)
         if ($C && /\s(?:if|while|for|switch)\(/) {
-            print "      KW(:$fn:$.\n";
+            msg "      KW(:$fn:$.\n";
         }
         ## Warn about #else #if instead of #elif.
         #    (We only allow #elif)
         if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) {
-            print " #else#if:$fn:$.\n";
+            msg " #else#if:$fn:$.\n";
         }
         ## Warn about some K&R violations
         #    (We use K&R-style C, where open braces go on the same line as
@@ -53,23 +64,23 @@ for $fn (@ARGV) {
         #            other stuff;
         #          }
         if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and
-	    $lastline !~ /\{$/) {
-            print "non-K&R {:$fn:$.\n";
-	}
+            $lastline !~ /\{$/) {
+            msg "non-K&R {:$fn:$.\n";
+        }
         if (/^\s*else/ and $lastline =~ /\}$/) {
-	    print "  }\\nelse:$fn:$.\n";
-	}
+            msg "  }\\nelse:$fn:$.\n";
+        }
         $lastline = $_;
         ## Warn about unnecessary empty lines.
         #   (Don't put an empty line before a line that contains nothing
         #   but a closing brace.)
         if ($lastnil && /^\s*}\n/) {
-            print "  UnnecNL:$fn:$.\n";
+            msg "  UnnecNL:$fn:$.\n";
         }
         ## Warn about multiple empty lines.
         #   (At most one blank line in a row.)
         if ($lastnil && /^$/) {
-            print " DoubleNL:$fn:$.\n";
+            msg " DoubleNL:$fn:$.\n";
         } elsif (/^$/) {
             $lastnil = 1;
         } else {
@@ -79,7 +90,7 @@ for $fn (@ARGV) {
         ## accept double-line lines.
         #   (Don't make lines wider than 80 characters, including newline.)
         if (/^.{80}/) {
-            print "     Wide:$fn:$.\n";
+            msg "     Wide:$fn:$.\n";
         }
         ### Juju to skip over comments and strings, since the tests
         ### we're about to do are okay there.
@@ -102,48 +113,52 @@ for $fn (@ARGV) {
             s!"(?:[^\"]+|\\.)*"!"X"!g;
             next if /^\#/;
             ## Warn about C++-style comments.
-	    #   (Use C style comments only.)
+            #   (Use C style comments only.)
             if (m!//!) {
-                #    print "       //:$fn:$.\n";
+                #    msg "       //:$fn:$.\n";
                 s!//.*!!;
             }
             ## Warn about unquoted braces preceded by non-space.
-	    #   (No character except a space should come before a {)
+            #   (No character except a space should come before a {)
             if (/([^\s'])\{/) {
-                print "       $1\{:$fn:$.\n";
+                msg "       $1\{:$fn:$.\n";
             }
+            ## Warn about double semi-colons at the end of a line.
+            if (/;;$/) {
+                msg "       double semi-colons at the end of $. in $fn\n"
+            }            
             ## Warn about multiple internal spaces.
             #if (/[^\s,:]\s{2,}[^\s\\=]/) {
-            #    print "     X  X:$fn:$.\n";
+            #    msg "     X  X:$fn:$.\n";
             #}
             ## Warn about { with stuff after.
             #s/\s+$//;
             #if (/\{[^\}\\]+$/) {
-            #    print "     {X:$fn:$.\n";
+            #    msg "     {X:$fn:$.\n";
             #}
             ## Warn about function calls with space before parens.
-	    #   (Don't put a space between the name of a function and its
-	    #   arguments.)
+            #   (Don't put a space between the name of a function and its
+            #   arguments.)
             if (/(\w+)\s\(([A-Z]*)/) {
                 if ($1 ne "if" and $1 ne "while" and $1 ne "for" and
                     $1 ne "switch" and $1 ne "return" and $1 ne "int" and
                     $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and
                     $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and
-                    $1 ne "size_t" and $1 ne "double" and
+                    $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and
                     $1 ne "workqueue_reply_t") {
-                    print "     fn ():$fn:$.\n";
+                    msg "     fn ():$fn:$.\n";
                 }
             }
             ## Warn about functions not declared at start of line.
-	    #    (When you're declaring functions, put "static" and "const"
-	    #    and the return type on one line, and the function name at
-	    #    the start of a new line.)
+            #    (When you're declaring functions, put "static" and "const"
+            #    and the return type on one line, and the function name at
+            #    the start of a new line.)
             if ($in_func_head ||
                 ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ &&
                  ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ &&
                  ! /= *\{$/ && ! /;$/)) {
                 if (/.\{$/){
-                    print "fn() {:$fn:$.\n";
+                    msg "fn() {:$fn:$.\n";
                     $in_func_head = 0;
                 } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) {
                     $in_func_head = -1; # started with tp fn
@@ -151,31 +166,33 @@ for $fn (@ARGV) {
                     $in_func_head = 0;
                 } elsif (/\{/) {
                     if ($in_func_head == -1) {
-                        print "tp fn():$fn:$.\n";
+                        msg "tp fn():$fn:$.\n";
                     }
                     $in_func_head = 0;
                 }
             }
 
-	    ## Check for forbidden functions except when they are
-	    # explicitly permitted
-	    if (/\bassert\(/ && not /assert OK/) {
-		print "assert :$fn:$.   (use tor_assert)\n";
-	    }
-	    if (/\bmemcmp\(/ && not /memcmp OK/) {
-		print "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
-	    }
-	    # always forbidden.
-	    if (not / OVERRIDE /) {
-		if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
-		    print "$& :$fn:$.\n";
-		}
-		if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
-		    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
-		    print "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
-		}
-	    }
+            ## Check for forbidden functions except when they are
+            # explicitly permitted
+            if (/\bassert\(/ && not /assert OK/) {
+                msg "assert :$fn:$.   (use tor_assert)\n";
+            }
+            if (/\bmemcmp\(/ && not /memcmp OK/) {
+                msg "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
+            }
+            # always forbidden.
+            if (not /\ OVERRIDE\ /) {
+                if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
+                    msg "$& :$fn:$.\n";
+                }
+                if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
+                    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
+                    msg "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
+                }
+            }
         }
     }
     close(F);
 }
+
+exit $found;
diff --git a/scripts/maint/display_callgraph.py b/scripts/maint/display_callgraph.py
deleted file mode 100755
index c9001c6d96..0000000000
--- a/scripts/maint/display_callgraph.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/python
-
-import cPickle
-
-data = cPickle.load(open("callgraph.pkl"))
-
-# data = data['modItems']
-
-callgraph = data['callgraph']
-closure = data['closure']
-sccs = data['sccs']
-fn_bottle, call_bottle = data['bottlenecks']
-
-for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems())):
-   print "%s can reach %s other functions." %(fn, n_reachable)
-
-
-c = [ (len(component), component) for component in sccs ]
-c.sort()
-
-print "\n================================"
-
-for n, component in c:
-   if n < 2:
-      continue
-   print "Strongly connected component of size %d:"%n
-   print component
-
-
-print "\n================================"
-
-print "====== Number of functions pulled into blob, by function in blob."
-fn_bottle.sort()
-for n, fn in fn_bottle[-30:]:
-   print "%3d: %s"%(n, fn)
-
-print "====== Number of functions pulled into blob, by call in blob."
-call_bottle.sort()
-for n, fn1, _, fn2 in call_bottle[-30:]:
-   print "%3d: %s -> %s "%(n, fn2, fn1)
-
diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist
index c9fd8a9236..a118cb5919 100644
--- a/scripts/maint/fallback.blacklist
+++ b/scripts/maint/fallback.blacklist
@@ -3,6 +3,8 @@
 # Format:
 # [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ...
 #   [ ipv6=<IPv6>[:<IPv6 ORPort>] ]
+# or use:
+# scripts/maint/generateFallbackDirLine.py fingerprint ...
 #
 # If a sufficiently specific group of attributes matches, the directory mirror
 # will be excluded: (each group is listed on its own line)
@@ -27,11 +29,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html
 80.82.215.199:80 orport=443 id=3BEFAB76461B6B99DCF34C285E933562F5712AE4 ipv6=[2001:4ba0:cafe:a18::1]:443
 
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008515.html
-# later opt-out in
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008521.html
-5.9.158.75:80 orport=443 id=F1BE15429B3CE696D6807F4D4A58B1BFEC45C822 ipv6=[2a01:4f8:190:514a::2]:443
-
 # Email sent directly to teor, verified using relay contact info
 5.34.183.168:80 orport=443 id=601C92108A568742A7A6D9473FE3A414F7149070
 217.12.199.208:8080 orport=22 id=BCFB0933367D626715DA32A147F417194A5D48D6
@@ -40,14 +37,6 @@
 62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001
 62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101
 
-# these fallback candidates fail the consensus download test in a way that
-# causes stem to hang (and not respond to ^C, at least on OS X)
-# (Is something sending weird responses to DirPort traffic?)
-#217.23.14.190:1194
-#151.80.164.147:80
-#148.251.255.92:80
-#78.142.19.59:80
-
 # Email sent directly to teor, verified using relay contact info
 216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7
 216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6
@@ -80,9 +69,6 @@
 185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869
 
 # Email sent directly to teor, verified using relay contact info
-62.210.82.44:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:3d7c::]:21
-
-# Email sent directly to teor, verified using relay contact info
 46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443
 
 # Email sent directly to teor, verified using relay contact info
@@ -132,7 +118,6 @@
 85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001
 148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001
 62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001
-212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB
 
 # Email sent directly to teor, verified using relay contact info
 74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0
@@ -206,24 +191,42 @@
 # Email sent directly to teor, verified using relay contact info
 88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E
 
-# Fallback was on 0.2.8.2-alpha list, but changed fingerprint before 0.2.8.5
-46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686 ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050
-# Also blacklist anything with the new fingerprint
-id=9C8A123081EFBE022EF795630F447839DDFDDDEC
+# Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6
+37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001
 
-# Fallbacks were on 0.2.8.2-alpha list, but downloads were slow before 0.2.8.5
-185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE
-178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C
+# Email sent directly to teor, verified using relay contact info
+195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
 
-# Fallback was on 0.2.8.2-alpha list, but changed address before 0.2.8.5
-84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34
-# Also blacklist anything with the new address
-84.216.235.55:9030 orport=443
+# Fallback was on 0.2.8.6 list, but opted-out before 0.2.9
+144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
 
-# Fallbacks were on 0.2.8.2-alpha list, but disappeared before 0.2.8.5
-81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820 ipv6=[2a02:180:1:1::517:11ab]:443
-51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D
-185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
+# no dirport
+86.107.110.34:0 orport=9001 id=A0E3D30A660DB70CA0B6D081BA54D094DED6F28D
+94.242.59.147:80 orport=9001 id=674DCBB0D9C1C4C4DBFB4A9AE024AF59FE4E7F46 ipv6=[2a00:1838:35:42::b648]:9001
 
-# Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6
-37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001
+# Email sent directly to teor, verified using relay contact info
+167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
+
+# Email sent directly to teor, verified using relay contact info
+163.172.35.245:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
+
+# Email sent directly to teor, verified using relay contact info
+104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
+
+# Emails sent directly to teor, verified using relay contact info
+217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443
+
+# Emails sent directly to teor, verified using relay contact info
+195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
+195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
+
+# Email sent directly to teor, verified using relay contact info
+5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
+
+#https://lists.torproject.org/pipermail/tor-relays/2017-May/012281.html
+62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
+62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
+
+# Email sent directly to teor
+212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
index c801e46b15..e9158e1280 100644
--- a/scripts/maint/fallback.whitelist
+++ b/scripts/maint/fallback.whitelist
@@ -2,6 +2,8 @@
 #
 # Format:
 # IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
+# or use:
+# scripts/maint/generateFallbackDirLine.py fingerprint ...
 #
 # All attributes must match for the directory mirror to be included.
 # If the fallback has an ipv6 key, the whitelist line must also have
@@ -28,8 +30,9 @@
 # <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html
-78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE
-131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8
+# https://trac.torproject.org/projects/tor/ticket/22321#comment:22
+78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3
+131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html
 5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050
@@ -37,8 +40,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
-62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
 212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
 212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
 
@@ -49,27 +50,26 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
 167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
 
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html
-170.130.1.7:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
-
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
 # Email sent directly to teor, verified using relay contact info
 91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html
-5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33
+91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
-# Sent additional email to teor with more relays
-178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422
-178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377
-178.254.20.134:9030 orport=9001 id=2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC
-178.254.44.135:80 orport=443 id=AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2
-178.254.13.126:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE
-178.254.13.126:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5
+# Sent additional emails to teor with updated relays
+81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22
+# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus
+178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31
+178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32
+81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60
+81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60
+85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61
+85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63
+81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62
+81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html
 51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109
@@ -98,33 +98,28 @@
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
 
-# Emails sent directly to teor, verified using relay contact info
-217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05
-
 # Email sent directly to teor, verified using relay contact info
 94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
 
-# https://twitter.com/binarytenshi/status/717952514327453697
-94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073
-
 # Email sent directly to teor, verified using relay contact info
-171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
 171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
-171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576
+171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
 171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
-# OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576
-#171.25.193.132:80 orport=443 id=01C67E0CA8F97111E652C7564CB3204361FFFAB8
-# OK, but same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
-#171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
+# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
+171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
 
 # Email sent directly to teor, verified using relay contact info
-212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68
+212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001
 93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
-46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001
+46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001
 
 # Email sent directly to teor, verified using relay contact info
-85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE
-185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216
+85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01
+185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051
+# This fallback opted-in in previous releases, then changed its details,
+# and so we blacklisted it. Now we want to whitelist changes.
+# Assume details update is permanent
+185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06
 
 # Email sent directly to teor, verified using relay contact info
 185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778
@@ -154,31 +149,20 @@
 178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
 
 # Email sent directly to teor, verified using relay contact info
-195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64
-51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132
 5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
 
 # Email sent directly to teor, verified using relay contact info
-109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
-109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
-109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
-77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
-109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
-77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
-109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
-77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
-109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
-62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
-109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
-109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
-109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
-77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
-109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
-77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
-109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
-77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
-109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
+109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52
+109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C
+109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24
+109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18
+109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281
+109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9
 62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
+# Assume details update is permanent
+77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass
+77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart
+77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia
 
 # https://twitter.com/biotimylated/status/718994247500718080
 212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
@@ -215,9 +199,7 @@
 
 # Email sent directly to teor, verified using relay contact info
 86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
-
-# Email sent directly to teor, verified using relay contact info
-144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
+86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B
 
 # Email sent directly to teor, verified using relay contact info
 193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
@@ -232,20 +214,19 @@
 # Email sent directly to teor, verified using relay contact info
 89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C
 
-# Email sent directly to teor, verified using relay contact info
-212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001
+# Email sent directly to teor
+# Assume details update is permanent
+212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans
 
 # Email sent directly to teor, verified using relay contact info
 46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB
 
 # Email sent directly to teor, verified using relay contact info
-217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B
-
-# Email sent directly to teor, verified using relay contact info
 193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524
 
 # Email sent directly to teor, verified using relay contact info
-197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626
+# Assume details update is permanent
+197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator
 
 # Email sent directly to teor, verified using relay contact info
 185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C
@@ -256,7 +237,7 @@
 193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E
 
 # Email sent directly to teor, verified using relay contact info
-144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
+138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
 
 # Email sent directly to teor, verified using relay contact info
 37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811
@@ -274,12 +255,12 @@
 134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE
 
 # Email sent directly to teor, verified using relay contact info
-81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
+173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
 
 # Email sent directly to teor, verified using relay contact info
 178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
-188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
-198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050
+188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
+198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050
 159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
 
 # Email sent directly to teor, verified using relay contact info
@@ -301,9 +282,6 @@
 212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
 
 # Email sent directly to teor, verified using relay contact info
-176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22
-
-# Email sent directly to teor, verified using relay contact info
 192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
 192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
 51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A
@@ -318,7 +296,7 @@
 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6
+5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443
 
 # Email sent directly to teor, verified using relay contact info
 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
@@ -328,17 +306,15 @@
 46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7
 
 # Email sent directly to teor, verified using relay contact info
-37.187.102.108:9090 orport=5550 id=F4263275CF54A6836EE7BD527B1328836A6F06E1
-212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C
+37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe
+212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe
 
 # Email sent directly to teor, verified using relay contact info
-195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197
+212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
 
-# Email sent directly to teor, verified using relay contact info
-62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
-
-# Email sent directly to teor, verified using relay contact info
-37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2
+# Email sent directly to teor
+37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ
+5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE
 
 # Email sent directly to teor, verified using relay contact info
 37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443
@@ -356,9 +332,6 @@
 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
-
-# Email sent directly to teor, verified using relay contact info
 198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
 
 # Email sent directly to teor, verified using relay contact info
@@ -405,14 +378,6 @@
 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
 
 # Email sent directly to teor, verified using relay contact info
-# Suitable, check with operator before adding
-#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
-212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60
-163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443
-# Suitable, check with operator before adding
-#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942
-
-# Email sent directly to teor, verified using relay contact info
 46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
 178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794
 
@@ -440,11 +405,8 @@
 # Email sent directly to teor, verified using relay contact info
 5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265
 
-# Email sent directly to teor, verified using relay contact info
-188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484
+# Email sent directly to teor
+188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy
 
 # Email sent directly to teor, verified using relay contact info
 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
@@ -456,12 +418,10 @@
 46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
 46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
 46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451
+163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
 
 # Email sent directly to teor, verified using relay contact info
 176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
-195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
-195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
 
 # Email sent directly to teor, verified using relay contact info
 46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
@@ -485,11 +445,10 @@
 5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
 
 # Email sent directly to teor, verified using relay contact info
-167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
-
-# Email sent directly to teor, verified using relay contact info
-192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
-167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+# Updated details from atlas based on ticket #20010
+163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
+163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2
 
 # Email sent directly to teor, verified using relay contact info
 204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
@@ -498,13 +457,10 @@
 151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
 
 # Email sent directly to teor, verified using relay contact info
-194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C
-
-# Email sent directly to teor, verified using relay contact info
 185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A
+5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
 
 # Email sent directly to teor, verified using relay contact info
 89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
@@ -513,7 +469,7 @@
 92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
 
 # Email sent directly to teor, verified using relay contact info
-80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001
+80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001
 
 # Email sent directly to teor, verified using relay contact info
 83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
@@ -522,9 +478,6 @@
 95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
 
 # Email sent directly to teor, verified using relay contact info
-176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C
-
-# Email sent directly to teor, verified using relay contact info
 128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
 
 # Email sent directly to teor, verified using relay contact info
@@ -540,16 +493,17 @@
 80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
 
 # Email sent directly to teor, verified using relay contact info
-185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A
+185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088
 
-# Email sent directly to teor, verified using relay contact info
-149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76
+# Email sent directly to teor
+149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne
 
 # Email sent directly to teor, verified using relay contact info
-81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243
+81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80
 
 # Email sent directly to teor, verified using relay contact info
-195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993
+# IPv6 address unreliable
+195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993
 138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
 81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
 134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
@@ -563,7 +517,7 @@
 217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
 81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
 46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
-91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993
+91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993
 163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
 188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
 81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
@@ -580,18 +534,17 @@
 # Email sent directly to teor, verified using relay contact info
 212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001
 
-# Email sent directly to teor, verified using relay contact info
-176.158.132.12:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686
+# Email sent directly to teor
+176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld
 
 # Email sent directly to teor, verified using relay contact info
 91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
 
 # Email sent directly to teor, verified using relay contact info
-# Awaiting confirmation of new ORPort from relay operator
 80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
 
-# Email sent directly to teor, verified using relay contact info
-163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D
+# Email sent directly to teor
+163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra
 
 # Email sent directly to teor, verified using relay contact info
 97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
@@ -603,10 +556,11 @@
 178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
 
 # Email sent directly to teor, verified using relay contact info
-195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
+163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6
 
 # Email sent directly to teor, verified using relay contact info
 185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
+185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
 
 # Email sent directly to teor, verified using relay contact info
 164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
@@ -617,13 +571,15 @@
 178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
 
 # Email sent directly to teor, verified using relay contact info
-91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
+# Very low bandwidth, stale consensues, excluded to cut down on warnings
+#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
 
 # Email sent directly to teor, verified using relay contact info
-167.114.113.48:9030 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
+167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
 
 # Email sent directly to teor, verified using relay contact info
-79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6
+# Assume details update is permanent
+213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium
 
 # Email sent directly to teor, verified using relay contact info
 95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443
@@ -652,10 +608,13 @@
 # Email sent directly to teor, verified using relay contact info
 31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798
 
-# Email sent directly to teor, verified using relay contact info
-96.47.231.214:9030 orport=8080 id=F843CB5729575D76FF1FFBB2179BDCF52C0C6387
-192.99.246.48:9030 orport=9001 id=CD6B149BED1BB254EF6DFF9D75DDB11E7F8A38A4 ipv6=[2607:5300:100:200::de3]:9002
-192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2605:e200:d00c:c01d::1111]:9002
+# Email sent directly to teor
+192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo
+192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer
+192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo
+192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall
+192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat
+192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi
 
 # Email sent directly to teor, verified using relay contact info
 136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664
@@ -666,8 +625,8 @@
 
 # Email sent directly to teor, verified using relay contact info
 192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-# OK, but same machine as ED2338CAC2711B3E331392E1ED2831219B794024
-#192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
+# same machine as ED2338CAC2711B3E331392E1ED2831219B794024
+192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
 
 # https://twitter.com/kosjoli/status/719507270904758272
 85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A
@@ -675,20 +634,18 @@
 46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
 
 # Email sent directly to teor, verified using relay contact info
-78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
-
-# Email sent directly to teor, verified using relay contact info
 185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
 
 # Email sent directly to teor, verified using relay contact info
 108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
 
 # Email sent directly to teor, verified using relay contact info
-91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99
+91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6
 
 # Email sent directly to teor, verified using relay contact info
 185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
 192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
+128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D
 
 # Email sent directly to teor, verified using relay contact info
 185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
@@ -719,13 +676,14 @@
 166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
 
 # Emails sent directly to teor, verified using relay contact info
-#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
+69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
 
 # Email sent directly to teor, verified using relay contact info
 213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
 
 # Email sent directly to teor, verified using relay contact info
-188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601
+# Assume details update is permanent
+188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik
 
 # Email sent directly to teor, verified using relay contact info
 88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001
@@ -743,20 +701,44 @@
 # Email sent directly to teor, verified using relay contact info
 107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA
 
-# Email sent directly to teor, verified using relay contact info
-192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC
-
-# Email sent directly to teor, verified using relay contact info
-163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A
-163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4
-163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
+# Email sent directly to teor
+193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001
+
+# Email sent directly to teor
+185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit
+185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail
+185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog
+185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse
+185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse
+185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse
+185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel
+185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit
+185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka
+185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia
+185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk
+185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco
+185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia
+185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver
+185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae
+185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika
+185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine
+185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa
+185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka
+185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes
+185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria
+185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa
+185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny
+185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika
+185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara
+185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit
+185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil
+185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare
+185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig
+185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat
 
 # Email sent directly to teor, verified using relay contact info
 64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2
 
-# Email sent directly to teor, verified using relay contact info
-212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110
-
 # Emails sent directly to teor, verified using relay contact info
 51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF
 
@@ -768,3 +750,247 @@
 
 # Email sent directly to teor, verified using relay contact info
 62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE
+
+# Email sent directly to teor, verified using relay contact info
+51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63
+
+# Email sent directly to teor, verified using relay contact info
+163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443
+
+# Email sent directly to teor, verified using relay contact info
+46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001
+
+# Email sent directly to teor, verified using relay contact info
+199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3
+199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6
+199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29
+199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813
+199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338
+199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060
+199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C
+199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616
+199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971
+199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241
+199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0
+199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD
+199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887
+199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B
+199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A
+199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F
+199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2
+199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0
+199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51
+199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
+86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022
+
+# Email sent directly to teor, verified using relay contact info
+178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443
+
+# Email sent directly to teor, verified using relay contact info
+31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
+# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
+31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D
+31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1
+31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148
+
+# Email sent directly to teor, verified using relay contact info
+185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html
+94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger
+
+# Email sent directly to teor, verified using relay contact info
+185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001
+
+# Email sent directly to teor, verified using relay contact info
+# The e84 part of the IPv6 address does not have a leading 0 in the consensus
+81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A
+
+# Email sent directly to teor, verified using relay contact info
+217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001
+
+# Email sent directly to teor, verified using relay contact info
+212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20
+163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24
+163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7
+
+# Email sent directly to teor, verified using relay contact info
+163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438
+195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6
+
+# Email sent directly to teor, verified using relay contact info
+185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
+185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270
+31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F
+
+# Email sent directly to teor, verified using relay contact info
+89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001
+
+# Email sent directly to teor, verified using relay contact info
+95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
+
+# Email sent directly to teor, verified using relay contact info
+85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
+
+# email sent directly to teor
+72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet
+
+# Email sent directly to teor, verified using relay contact info
+5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+# Assume details update is permanent
+5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel
+
+# https://trac.torproject.org/projects/tor/ticket/22527#comment:1
+199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443
+
+# https://trac.torproject.org/projects/tor/ticket/24695
+163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac
+
+# Email sent directly to teor
+54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2
+
+# Email sent directly to teor
+62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004
+62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005
+91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002
+91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001
+51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006
+51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007
+
+# Email sent directly to teor
+104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11
+
+# Email sent directly to teor
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html
+178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova
+178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2
+
+# Email sent directly to teor
+163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego
+
+# Email sent directly to teor
+46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html
+199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13
+199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26
+199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18
+199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36
+199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14
+199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43
+199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27
+199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33
+199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40
+199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30
+199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12
+199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5
+199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22
+199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8
+199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7
+199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6
+199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37
+199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45
+199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39
+199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48
+199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20
+199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35
+199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50
+199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21
+199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16
+199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47
+199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15
+199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29
+199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44
+199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9
+199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31
+199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41
+199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10
+199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11
+199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46
+199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25
+199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38
+199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4
+199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32
+199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2
+199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28
+199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19
+199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34
+199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49
+199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3
+199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23
+199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1
+199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24
+199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42
+199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html
+5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01
+217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02
+193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03
+149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html
+104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html
+139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html
+172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane
+
+# Email sent directly to teor
+62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine
+
+# Email sent directly to teor
+80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html
+51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1
+51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2
+
+# Email sent directly to teor
+185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone
+
+# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html
+24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia
+
+# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html
+128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit
+
+# Email sent directly to teor
+82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01
+109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02
+51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04
+
+# Email sent directly to teor
+80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02
+
+# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html
+37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX
+
+# These fallbacks opted-in in previous releases, then changed their details,
+# and so we blacklisted them. Now we want to whitelist changes.
+# Assume details update is permanent
+85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme
+176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust
+
+# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html
+82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
index e909fc550a..c5a0cfc81b 100755
--- a/scripts/maint/format_changelog.py
+++ b/scripts/maint/format_changelog.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2015, The Tor Project, Inc.
+# Copyright (c) 2014-2017, The Tor Project, Inc.
 # See LICENSE for licensing information
 #
 # This script reformats a section of the changelog to wrap everything to
@@ -205,6 +205,8 @@ def head_score(s):
         score = -300
     elif lw.startswith("deprecated version"):
         score = -200
+    elif lw.startswith("directory auth"):
+        score = -150
     elif (('new' in lw and 'requirement' in lw) or
           ('new' in lw and 'dependenc' in lw) or
           ('build' in lw and 'requirement' in lw) or
diff --git a/scripts/maint/generateFallbackDirLine.py b/scripts/maint/generateFallbackDirLine.py
new file mode 100755
index 0000000000..b856c938bf
--- /dev/null
+++ b/scripts/maint/generateFallbackDirLine.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# Generate a fallback directory whitelist/blacklist line for every fingerprint
+# passed as an argument.
+#
+# Usage:
+# generateFallbackDirLine.py fingerprint ...
+
+import sys
+import urllib2
+
+import stem.descriptor.remote
+import stem.util.tor_tools
+
+if len(sys.argv) <= 1:
+  print('Usage: %s fingerprint ...' % sys.argv[0])
+  sys.exit(1)
+
+for fingerprint in sys.argv[1:]:
+  if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
+    print("'%s' isn't a valid relay fingerprint" % fingerprint)
+    sys.exit(1)
+
+  try:
+    desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0]
+  except urllib2.HTTPError as exc:
+    if exc.code == 404:
+      print('# %s not found in recent descriptors' % fingerprint)
+      continue
+    else:
+      raise
+
+  if not desc.dir_port:
+    print("# %s needs a DirPort" % fingerprint)
+  else:
+    ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6]
+    ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else ''
+    print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname))
diff --git a/scripts/maint/generate_callgraph.sh b/scripts/maint/generate_callgraph.sh
deleted file mode 100755
index c6b33c0aea..0000000000
--- a/scripts/maint/generate_callgraph.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-C_FILES=`echo src/common/*.c src/or/*.c src/tools/*.c`
-CFLAGS="-Isrc/ext/trunnel -Isrc/trunnel -I. -Isrc/ext -Isrc/common -DLOCALSTATEDIR=\"\" -DSHARE_DATADIR=\"\" -Dinline="
-
-mkdir -p callgraph/src/common
-mkdir -p callgraph/src/or
-mkdir -p callgraph/src/tools
-
-for fn in $C_FILES; do
-  echo $fn
-  clang $CFLAGS  -S -emit-llvm -fno-inline -o - $fn  | \
-    opt -analyze -print-callgraph >/dev/null 2> "callgraph/${fn}allgraph"
-done
diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py
index b63a4eb3a1..d5b8fcae5c 100755
--- a/scripts/maint/lintChanges.py
+++ b/scripts/maint/lintChanges.py
@@ -7,7 +7,7 @@ import re
 import os
 
 
-KNOWN_GROUPS=set([
+KNOWN_GROUPS = set([
     "Minor bugfix",
     "Minor bugfixes",
     "Major bugfix",
@@ -20,7 +20,20 @@ KNOWN_GROUPS=set([
     "Testing",
     "Documentation",
     "Code simplification and refactoring",
-    "Removed features"])
+    "Removed features",
+    "Deprecated features",
+    "Directory authority changes"])
+
+NEEDS_SUBCATEGORIES = set([
+    "Minor bugfix",
+    "Minor bugfixes",
+    "Major bugfix",
+    "Major bugfixes",
+    "Minor feature",
+    "Minor features",
+    "Major feature",
+    "Major features",
+    ])
 
 def lintfile(fname):
     have_warned = []
@@ -43,17 +56,13 @@ def lintfile(fname):
     if bugnum and bugnum not in contents:
         warn("bug number {} does not appear".format(bugnum))
 
-    lines = contents.split("\n")
-
     m = re.match(r'^[ ]{2}o ([^\(:]*)([^:]*):', contents)
     if not m:
-        warn("header not in format expected")
+        warn("Header not in format expected. ('  o Foo:' or '  o Foo (Bar):')")
     elif m.group(1).strip() not in KNOWN_GROUPS:
-        warn("Weird header: %r"%m.group(1))
-    elif ( ("bugfix" in m.group(1) or "feature" in m.group(1)) and
-           ("Removed" not in m.group(1)) and
-           '(' not in m.group(2)):
-        warn("Missing subcategory on %s"%m.group(1))
+        warn("Unrecognized header: %r" % m.group(1))
+    elif (m.group(1) in NEEDS_SUBCATEGORIES and '(' not in m.group(2)):
+        warn("Missing subcategory on %r" % m.group(1))
 
     if m:
         isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower())
@@ -63,23 +72,46 @@ def lintfile(fname):
     contents = " ".join(contents.split())
 
     if re.search(r'\#\d{2,}', contents):
-        warn("don't use a # before ticket numbers")
+        warn("Don't use a # before ticket numbers. ('bug 1234' not '#1234')")
 
     if isBug and not re.search(r'(\d+)', contents):
-        warn("bugfix does not mention a number")
-    elif isBug and not re.search(r'Fixes ([a-z ]*)bug (\d+)', contents):
-        warn("bugfix does not say 'Fixes bug XXX'")
+        warn("Ticket marked as bugfix, but does not mention a number.")
+    elif isBug and not re.search(r'Fixes ([a-z ]*)bugs? (\d+)', contents):
+        warn("Ticket marked as bugfix, but does not say 'Fixes bug XXX'")
 
     if re.search(r'[bB]ug (\d+)', contents):
         if not re.search(r'[Bb]ugfix on ', contents):
-            warn("bugfix does not say 'bugfix on X.Y.Z'")
-        elif not re.search('[fF]ixes ([a-z ]*)bug (\d+); bugfix on ',
+            warn("Bugfix does not say 'bugfix on X.Y.Z'")
+        elif not re.search('[fF]ixes ([a-z ]*)bugs? (\d+)((, \d+)* and \d+)?; bugfix on ',
                            contents):
-            warn("bugfix incant is not semicoloned")
-
+            warn("Bugfix does not say 'Fixes bug X; bugfix on Y'")
+        elif re.search('tor-([0-9]+)', contents):
+            warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)")
+
+    return have_warned != []
+
+def files(args):
+    """Walk through the arguments: for directories, yield their contents;
+       for files, just yield the files. Only search one level deep, because
+       that's how the changes directory is laid out."""
+    for f in args:
+        if os.path.isdir(f):
+            for item in os.listdir(f):
+                if item.startswith("."): #ignore dotfiles
+                    continue
+                yield os.path.join(f, item)
+        else:
+            yield f
 
 if __name__ == '__main__':
-    for fname in sys.argv[1:]:
+    problems = 0
+    for fname in files(sys.argv[1:]):
         if fname.endswith("~"):
             continue
-        lintfile(fname)
+        if lintfile(fname):
+            problems += 1
+
+    if problems:
+        sys.exit(1)
+    else:
+        sys.exit(0)
diff --git a/scripts/maint/lookupFallbackDirContact.py b/scripts/maint/lookupFallbackDirContact.py
new file mode 100755
index 0000000000..14c53d1282
--- /dev/null
+++ b/scripts/maint/lookupFallbackDirContact.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+# Lookup fallback directory contact lines for every fingerprint passed as an
+# argument.
+#
+# Usage:
+# lookupFallbackDirContact.py fingerprint ...
+
+import sys
+
+import stem.descriptor.remote as remote
+
+if len(sys.argv) <= 1:
+  print "Usage: {} fingerprint ...".format(sys.argv[0])
+  sys.exit(-1)
+
+# we need descriptors, because the consensus does not have contact infos
+descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run()
+
+descriptor_list_fingerprints = []
+for d in descriptor_list:
+  assert d.fingerprint in sys.argv[1:]
+  descriptor_list_fingerprints.append(d.fingerprint)
+  print "{} {}".format(d.fingerprint, d.contact)
+
+for fingerprint in sys.argv[1:]:
+  if fingerprint not in descriptor_list_fingerprints:
+    print "{} not found in current descriptors".format(fingerprint)
diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py
index 43f5b6eb16..53d3d902eb 100755
--- a/scripts/maint/redox.py
+++ b/scripts/maint/redox.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 #
-#  Copyright (c) 2008-2015, The Tor Project, Inc.
+#  Copyright (c) 2008-2017, The Tor Project, Inc.
 #  See LICENSE for licensing information.
 #
 # Hi!
@@ -101,7 +101,7 @@ def read():
 
 def findline(lines, lineno, ident):
     """Given a list of all the lines in the file (adjusted so 1-indexing works),
-       a line number that ident is alledgedly on, and ident, I figure out
+       a line number that ident is allegedly on, and ident, I figure out
        the line where ident was really declared."""
     lno = lineno
     for lineno in xrange(lineno, 0, -1):
diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh
new file mode 100755
index 0000000000..efb8706fea
--- /dev/null
+++ b/scripts/maint/run_calltool.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# You can find calltool at https://gitweb.torproject.org/user/nickm/calltool.git
+
+set -e
+
+if test "x$CALLTOOL_PATH" != "x"; then
+  PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}"
+  export PYTHONPATH
+fi
+
+mkdir -p callgraph
+
+SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgraph module_scc module_scc_weaklinks"
+
+for calculation in $SUBITEMS; do
+  echo "======== $calculation"
+  python -m calltool $calculation > callgraph/$calculation
+done
+
+echo <<EOF > callgraph/README
+This directory holds output from calltool, as run on Tor.  For more
+information about each of these files, see the NOTES and README files in
+the calltool distribution.
+
+You can find calltool at
+    https://gitweb.torproject.org/user/nickm/calltool.git
+EOF
+
diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py
index d6ec0e269d..22e40fd369 100755
--- a/scripts/maint/sortChanges.py
+++ b/scripts/maint/sortChanges.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2015, The Tor Project, Inc.
+# Copyright (c) 2014-2017, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 """This script sorts a bunch of changes files listed on its command
diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl
index 8bd6a18210..beb0b8f26e 100755
--- a/scripts/maint/updateCopyright.pl
+++ b/scripts/maint/updateCopyright.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -i -w -p
 
-$NEWYEAR=2016;
+$NEWYEAR=2017;
 
-s/Copyright(.*) (201[^6]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
+s/Copyright(.*) (201[^7]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
 
 s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/;
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
index 110ecda64c..b093463e08 100755
--- a/scripts/maint/updateFallbackDirs.py
+++ b/scripts/maint/updateFallbackDirs.py
@@ -1,20 +1,25 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
-# Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc
+# Usage:
+#
+# Regenerate the list:
+# scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc 2> fallback_dirs.log
+#
+# Check the existing list:
+# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log
+# mv fallback_dirs.inc.ok src/or/fallback_dirs.inc
 #
 # This script should be run from a stable, reliable network connection,
 # with no other network activity (and not over tor).
 # If this is not possible, please disable:
 # PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS
 #
-# Needs dateutil (and potentially other python packages)
-# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem .
+# Needs dateutil, stem, and potentially other python packages.
 # Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
-# for netblock analysis, in PYTHONPATH, or just
-# ln -s ../py2-ipaddress-3.4.1/ipaddress.py .
+# for netblock analysis.
 #
 # Then read the logs to make sure the fallbacks aren't dominated by a single
-# netblock or port
+# netblock or port.
 
 # Script by weasel, April 2015
 # Portions by gsathya & karsten, 2013
@@ -37,16 +42,13 @@ import dateutil.parser
 # bson_lazy provides bson
 #from bson import json_util
 import copy
+import re
 
-from stem.descriptor.remote import DescriptorDownloader
+from stem.descriptor import DocumentHandler
+from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS
 
 import logging
-# INFO tells you why each relay was included or excluded
-# WARN tells you about potential misconfigurations and relay detail changes
-logging.basicConfig(level=logging.WARNING)
 logging.root.name = ''
-# INFO tells you about each consensus download attempt
-logging.getLogger('stem').setLevel(logging.WARNING)
 
 HAVE_IPADDRESS = False
 try:
@@ -64,6 +66,17 @@ except ImportError:
 
 ## Top-Level Configuration
 
+# We use semantic versioning: https://semver.org
+# In particular:
+# * major changes include removing a mandatory field, or anything else that
+#   would break an appropriately tolerant parser,
+# * minor changes include adding a field,
+# * patch changes include changing header comments or other unstructured
+#   content
+FALLBACK_FORMAT_VERSION = '2.0.0'
+SECTION_SEPARATOR_BASE = '====='
+SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */'
+
 # Output all candidate fallbacks, or only output selected fallbacks?
 OUTPUT_CANDIDATES = False
 
@@ -80,7 +93,28 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
 # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
 PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
 
-# Output fallback name, flags, and ContactInfo in a C comment?
+# Must relays be running now?
+MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
+                       or PERFORM_IPV6_DIRPORT_CHECKS)
+
+# Clients have been using microdesc consensuses by default for a while now
+DOWNLOAD_MICRODESC_CONSENSUS = True
+
+# If a relay delivers an expired consensus, if it expired less than this many
+# seconds ago, we still allow the relay. This should never be less than -90,
+# as all directory mirrors should have downloaded a consensus 90 minutes
+# before it expires. It should never be more than 24 hours, because clients
+# reject consensuses that are older than REASONABLY_LIVE_TIME.
+# For the consensus expiry check to be accurate, the machine running this
+# script needs an accurate clock.
+#
+# Relays on 0.3.0 and later return a 404 when they are about to serve an
+# expired consensus. This makes them fail the download check.
+# We use a tolerance of 0, so that 0.2.x series relays also fail the download
+# check if they serve an expired consensus.
+CONSENSUS_EXPIRY_TOLERANCE = 0 
+
+# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
 OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
 
 # Output matching ContactInfo in fallbacks list or the blacklist?
@@ -88,6 +122,12 @@ OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
 CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
 CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False
 
+# How the list should be sorted:
+# fingerprint: is useful for stable diffs of fallback lists
+# measured_bandwidth: is useful when pruning the list based on bandwidth
+# contact: is useful for contacting operators once the list has been pruned
+OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
+
 ## OnionOO Settings
 
 ONIONOO = 'https://onionoo.torproject.org/'
@@ -121,24 +161,33 @@ BLACKLIST_EXCLUDES_WHITELIST_ENTRIES = True
 
 WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
 BLACKLIST_FILE_NAME = 'scripts/maint/fallback.blacklist'
+FALLBACK_FILE_NAME  = 'src/or/fallback_dirs.inc'
 
 # The number of bytes we'll read from a filter file before giving up
 MAX_LIST_FILE_SIZE = 1024 * 1024
 
 ## Eligibility Settings
 
-# Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted
-# This causes OnionOO to (correctly) reset its stability timer
-# This issue will be fixed in 0.2.7.7 and 0.2.8.2
-# Until then, the CUTOFFs below ensure a decent level of stability.
-ADDRESS_AND_PORT_STABLE_DAYS = 7
-# What time-weighted-fraction of these flags must FallbackDirs
-# Equal or Exceed?
-CUTOFF_RUNNING = .95
-CUTOFF_V2DIR = .95
-CUTOFF_GUARD = .95
-# What time-weighted-fraction of these flags must FallbackDirs
-# Equal or Fall Under?
+# Require fallbacks to have the same address and port for a set amount of time
+# We used to have this at 1 week, but that caused many fallback failures, which
+# meant that we had to rebuild the list more often. We want fallbacks to be
+# stable for 2 years, so we set it to a few months.
+#
+# If a relay changes address or port, that's it, it's not useful any more,
+# because clients can't find it
+ADDRESS_AND_PORT_STABLE_DAYS = 90
+# We ignore relays that have been down for more than this period
+MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
+# FallbackDirs must have a time-weighted-fraction that is greater than or
+# equal to:
+# Mirrors that are down half the time are still useful half the time
+CUTOFF_RUNNING = .50
+CUTOFF_V2DIR = .50
+# Guard flags are removed for some time after a relay restarts, so we ignore
+# the guard flag.
+CUTOFF_GUARD = .00
+# FallbackDirs must have a time-weighted-fraction that is less than or equal
+# to:
 # .00 means no bad exits
 PERMITTED_BADEXIT = .00
 
@@ -155,28 +204,41 @@ ONIONOO_SCALE_ONE = 999.
 _FB_POG = 0.2
 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
 
-# We want exactly 100 fallbacks for the initial release
-# This gives us scope to add extra fallbacks to the list as needed
 # Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100
-# Emit a C #error if the number of fallbacks is below
-MIN_FALLBACK_COUNT = 100
+MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
+# Emit a C #error if the number of fallbacks is less than expected
+MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
+
+# The maximum number of fallbacks on the same address, contact, or family
+#
+# With 150 fallbacks, this means each operator sees 5% of client bootstraps.
+# For comparison:
+#  - We try to limit guard and exit operators to 5% of the network
+#  - The directory authorities used to see 11% of client bootstraps each
+#
+# We also don't want too much of the list to go down if a single operator
+# has to move all their relays.
+MAX_FALLBACKS_PER_IP = 1
+MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
+MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
+MAX_FALLBACKS_PER_CONTACT = 7
+MAX_FALLBACKS_PER_FAMILY = 7
 
 ## Fallback Bandwidth Requirements
 
-# Any fallback with the Exit flag has its bandwidth multipled by this fraction
+# Any fallback with the Exit flag has its bandwidth multiplied by this fraction
 # to make sure we aren't further overloading exits
 # (Set to 1.0, because we asked that only lightly loaded exits opt-in,
 # and the extra load really isn't that much for large relays.)
 EXIT_BANDWIDTH_FRACTION = 1.0
 
 # If a single fallback's bandwidth is too low, it's pointless adding it
-# We expect fallbacks to handle an extra 30 kilobytes per second of traffic
-# Make sure they can support a hundred times the expected extra load
-# (Use 102.4 to make it come out nicely in MB/s)
+# We expect fallbacks to handle an extra 10 kilobytes per second of traffic
+# Make sure they can support fifty times the expected extra load
+#
 # We convert this to a consensus weight before applying the filter,
 # because all the bandwidth amounts are specified by the relay
-MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0
+MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0
 
 # Clients will time out after 30 seconds trying to download a consensus
 # So allow fallback directories half that to deliver a consensus
@@ -188,21 +250,6 @@ CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
 # This avoids delisting a relay due to transient network conditions
 CONSENSUS_DOWNLOAD_RETRY = True
 
-## Fallback Weights for Client Selection
-
-# All fallback weights are equal, and set to the value below
-# Authorities are weighted 1.0 by default
-# Clients use these weights to select fallbacks and authorities at random
-# If there are 100 fallbacks and 9 authorities:
-#  - each fallback is chosen with probability 10.0/(10.0*100 + 1.0*9) ~= 0.99%
-#  - each authority is chosen with probability 1.0/(10.0*100 + 1.0*9) ~= 0.09%
-# A client choosing a bootstrap directory server will choose a fallback for
-# 10.0/(10.0*100 + 1.0*9) * 100 = 99.1% of attempts, and an authority for
-# 1.0/(10.0*100 + 1.0*9) * 9 = 0.9% of attempts.
-# (This disregards the bootstrap schedules, where clients start by choosing
-# from fallbacks & authoritites, then later choose from only authorities.)
-FALLBACK_OUTPUT_WEIGHT = 10.0
-
 ## Parsing Functions
 
 def parse_ts(t):
@@ -242,6 +289,10 @@ def cleanse_c_multiline_comment(raw_string):
   bad_char_list = '*/'
   # Prevent a malicious string from using C nulls
   bad_char_list += '\0'
+  # Avoid confusing parsers by making sure there is only one comma per fallback
+  bad_char_list += ','
+  # Avoid confusing parsers by making sure there is only one equals per field
+  bad_char_list += '='
   # Be safer by removing bad characters entirely
   cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
   # Some compilers may further process the content of comments
@@ -262,6 +313,10 @@ def cleanse_c_string(raw_string):
   bad_char_list += '\\'
   # Prevent a malicious string from using C nulls
   bad_char_list += '\0'
+  # Avoid confusing parsers by making sure there is only one comma per fallback
+  bad_char_list += ','
+  # Avoid confusing parsers by making sure there is only one equals per field
+  bad_char_list += '='
   # Be safer by removing bad characters entirely
   cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
   # Some compilers may further process the content of strings
@@ -329,6 +384,15 @@ def read_from_file(file_name, max_len):
                  )
   return None
 
+def parse_fallback_file(file_name):
+  file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
+  file_data = cleanse_unprintable(file_data)
+  file_data = remove_bad_chars(file_data, '\n"\0')
+  file_data = re.sub('/\*.*?\*/', '', file_data)
+  file_data = file_data.replace(',', '\n')
+  file_data = file_data.replace(' weight=10', '')
+  return file_data
+
 def load_possibly_compressed_response_json(response):
     if response.info().get('Content-Encoding') == 'gzip':
       buf = StringIO.StringIO( response.read() )
@@ -367,8 +431,8 @@ def onionoo_fetch(what, **kwargs):
   params = kwargs
   params['type'] = 'relay'
   #params['limit'] = 10
-  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS,)
-  params['last_seen_days'] = '-7'
+  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
+  params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
   params['flag'] = 'V2Dir'
   url = ONIONOO + what + '?' + urllib.urlencode(params)
 
@@ -491,12 +555,14 @@ class Candidate(object):
       details['flags'] = []
     if (not 'advertised_bandwidth' in details
         or details['advertised_bandwidth'] is None):
-      # relays without advertised bandwdith have it calculated from their
+      # relays without advertised bandwidth have it calculated from their
       # consensus weight
       details['advertised_bandwidth'] = 0
     if (not 'effective_family' in details
         or details['effective_family'] is None):
       details['effective_family'] = []
+    if not 'platform' in details:
+      details['platform'] = None
     details['last_changed_address_or_port'] = parse_ts(
                                       details['last_changed_address_or_port'])
     self._data = details
@@ -511,6 +577,8 @@ class Candidate(object):
     self._compute_ipv6addr()
     if not self.has_ipv6():
       logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
+    self._compute_version()
+    self._extra_info_cache = None
 
   def _stable_sort_or_addresses(self):
     # replace self._data['or_addresses'] with a stable ordering,
@@ -623,6 +691,59 @@ class Candidate(object):
         self.ipv6orport = int(port)
         return
 
+  def _compute_version(self):
+    # parse the version out of the platform string
+    # The platform looks like: "Tor 0.2.7.6 on Linux"
+    self._data['version'] = None
+    if self._data['platform'] is None:
+      return
+    # be tolerant of weird whitespacing, use a whitespace split
+    tokens = self._data['platform'].split()
+    for token in tokens:
+      vnums = token.split('.')
+      # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
+      if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
+          vnums[2].isdigit()):
+        self._data['version'] = token
+        return
+
+  # From #20509
+  # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
+  # and version 0.3.0.0-alpha-dev
+  # Exhaustive lists are hard to get wrong
+  STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
+                              '0.2.9.2-alpha',
+                              '0.2.9.2-alpha-dev',
+                              '0.2.9.3-alpha',
+                              '0.2.9.3-alpha-dev',
+                              '0.2.9.4-alpha',
+                              '0.2.9.4-alpha-dev',
+                              '0.3.0.0-alpha-dev'
+                              ]
+
+  def is_valid_version(self):
+    # call _compute_version before calling this
+    # is the version of the relay a version we want as a fallback?
+    # checks both recommended versions and bug #20499 / #20509
+    #
+    # if the relay doesn't have a recommended version field, exclude the relay
+    if not self._data.has_key('recommended_version'):
+      log_excluded('%s not a candidate: no recommended_version field',
+                   self._fpr)
+      return False
+    if not self._data['recommended_version']:
+      log_excluded('%s not a candidate: version not recommended', self._fpr)
+      return False
+    # if the relay doesn't have version field, exclude the relay
+    if not self._data.has_key('version'):
+      log_excluded('%s not a candidate: no version field', self._fpr)
+      return False
+    if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
+      logging.warning('%s not a candidate: version delivers stale consensuses',
+                      self._fpr)
+      return False
+    return True
+
   @staticmethod
   def _extract_generic_history(history, which='unknown'):
     # given a tree like this:
@@ -767,41 +888,42 @@ class Candidate(object):
       self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
 
   def is_candidate(self):
-    must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS
-                           or PERFORM_IPV6_DIRPORT_CHECKS)
-    if (must_be_running_now and not self.is_running()):
-      logging.info('%s not a candidate: not running now, unable to check ' +
-                   'DirPort consensus download', self._fpr)
-      return False
-    if (self._data['last_changed_address_or_port'] >
-        self.CUTOFF_ADDRESS_AND_PORT_STABLE):
-      logging.info('%s not a candidate: changed address/port recently (%s)',
-                   self._fpr, self._data['last_changed_address_or_port'])
-      return False
-    if self._running < CUTOFF_RUNNING:
-      logging.info('%s not a candidate: running avg too low (%lf)',
-                   self._fpr, self._running)
-      return False
-    if self._v2dir < CUTOFF_V2DIR:
-      logging.info('%s not a candidate: v2dir avg too low (%lf)',
-                   self._fpr, self._v2dir)
-      return False
-    if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
-      logging.info('%s not a candidate: badexit avg too high (%lf)',
-                   self._fpr, self._badexit)
-      return False
-    # if the relay doesn't report a version, also exclude the relay
-    if (not self._data.has_key('recommended_version')
-        or not self._data['recommended_version']):
-      logging.info('%s not a candidate: version not recommended', self._fpr)
-      return False
-    if self._guard < CUTOFF_GUARD:
-      logging.info('%s not a candidate: guard avg too low (%lf)',
-                   self._fpr, self._guard)
-      return False
-    if (not self._data.has_key('consensus_weight')
-        or self._data['consensus_weight'] < 1):
-      logging.info('%s not a candidate: consensus weight invalid', self._fpr)
+    try:
+      if (MUST_BE_RUNNING_NOW and not self.is_running()):
+        log_excluded('%s not a candidate: not running now, unable to check ' +
+                     'DirPort consensus download', self._fpr)
+        return False
+      if (self._data['last_changed_address_or_port'] >
+          self.CUTOFF_ADDRESS_AND_PORT_STABLE):
+        log_excluded('%s not a candidate: changed address/port recently (%s)',
+                     self._fpr, self._data['last_changed_address_or_port'])
+        return False
+      if self._running < CUTOFF_RUNNING:
+        log_excluded('%s not a candidate: running avg too low (%lf)',
+                     self._fpr, self._running)
+        return False
+      if self._v2dir < CUTOFF_V2DIR:
+        log_excluded('%s not a candidate: v2dir avg too low (%lf)',
+                     self._fpr, self._v2dir)
+        return False
+      if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
+        log_excluded('%s not a candidate: badexit avg too high (%lf)',
+                     self._fpr, self._badexit)
+        return False
+      # this function logs a message depending on which check fails
+      if not self.is_valid_version():
+        return False
+      if self._guard < CUTOFF_GUARD:
+        log_excluded('%s not a candidate: guard avg too low (%lf)',
+                     self._fpr, self._guard)
+        return False
+      if (not self._data.has_key('consensus_weight')
+          or self._data['consensus_weight'] < 1):
+        log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
+        return False
+    except BaseException as e:
+      logging.warning("Exception %s when checking if fallback is a candidate",
+                      str(e))
       return False
     return True
 
@@ -878,26 +1000,26 @@ class Candidate(object):
       for key in entry:
         value = entry[key]
         if key == 'id' and value == self._fpr:
-          logging.info('%s is in the blacklist: fingerprint matches',
+          log_excluded('%s is in the blacklist: fingerprint matches',
                        self._fpr)
           return True
         if key == 'ipv4' and value == self.dirip:
           # if the dirport is present, check it too
           if entry.has_key('dirport'):
             if int(entry['dirport']) == self.dirport:
-              logging.info('%s is in the blacklist: IPv4 (%s) and ' +
+              log_excluded('%s is in the blacklist: IPv4 (%s) and ' +
                            'DirPort (%d) match', self._fpr, self.dirip,
                            self.dirport)
               return True
           # if the orport is present, check it too
           elif entry.has_key('orport'):
             if int(entry['orport']) == self.orport:
-              logging.info('%s is in the blacklist: IPv4 (%s) and ' +
+              log_excluded('%s is in the blacklist: IPv4 (%s) and ' +
                            'ORPort (%d) match', self._fpr, self.dirip,
                            self.orport)
               return True
           else:
-            logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' +
+            log_excluded('%s is in the blacklist: IPv4 (%s) matches, and ' +
                          'entry has no DirPort or ORPort', self._fpr,
                          self.dirip)
             return True
@@ -911,19 +1033,19 @@ class Candidate(object):
             # if the dirport is present, check it too
             if entry.has_key('dirport'):
               if int(entry['dirport']) == self.dirport:
-                logging.info('%s is in the blacklist: IPv6 (%s) and ' +
+                log_excluded('%s is in the blacklist: IPv6 (%s) and ' +
                              'DirPort (%d) match', self._fpr, ipv6,
                              self.dirport)
                 return True
             # we've already checked the ORPort, it's part of entry['ipv6']
             else:
-              logging.info('%s is in the blacklist: IPv6 (%s) matches, and' +
+              log_excluded('%s is in the blacklist: IPv6 (%s) matches, and' +
                            'entry has no DirPort', self._fpr, ipv6)
               return True
         elif (key == 'ipv6' or self.has_ipv6()):
           # only log if the fingerprint matches but the IPv6 doesn't
           if entry.has_key('id') and entry['id'] == self._fpr:
-            logging.info('%s skipping IPv6 blacklist comparison: relay ' +
+            log_excluded('%s skipping IPv6 blacklist comparison: relay ' +
                          'has%s IPv6%s, but entry has%s IPv6%s', self._fpr,
                          '' if self.has_ipv6() else ' no',
                          (' (' + ipv6 + ')') if self.has_ipv6() else  '',
@@ -1062,42 +1184,63 @@ class Candidate(object):
         return True
     return False
 
-  # report how long it takes to download a consensus from dirip:dirport
+  # log how long it takes to download a consensus from dirip:dirport
+  # returns True if the download failed, False if it succeeded within max_time
   @staticmethod
-  def fallback_consensus_download_speed(dirip, dirport, nickname, max_time):
+  def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
+                                        max_time):
     download_failed = False
-    downloader = DescriptorDownloader()
-    start = datetime.datetime.utcnow()
     # some directory mirrors respond to requests in ways that hang python
     # sockets, which is why we log this line here
-    logging.info('Initiating consensus download from %s (%s:%d).', nickname,
-                 dirip, dirport)
+    logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
+                 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
+                 nickname, dirip, dirport, fingerprint)
     # there appears to be about 1 second of overhead when comparing stem's
     # internal trace time and the elapsed time calculated here
     TIMEOUT_SLOP = 1.0
+    start = datetime.datetime.utcnow()
     try:
-      downloader.get_consensus(endpoints = [(dirip, dirport)],
-                               timeout = (max_time + TIMEOUT_SLOP),
-                               validate = True,
-                               retries = 0,
-                               fall_back_to_authority = False).run()
+      consensus = get_consensus(
+                              endpoints = [(dirip, dirport)],
+                              timeout = (max_time + TIMEOUT_SLOP),
+                              validate = True,
+                              retries = 0,
+                              fall_back_to_authority = False,
+                              document_handler = DocumentHandler.BARE_DOCUMENT,
+                              microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
+                                ).run()[0]
+      end = datetime.datetime.utcnow()
+      time_since_expiry = (end - consensus.valid_until).total_seconds()
     except Exception, stem_error:
-      logging.info('Unable to retrieve a consensus from %s: %s', nickname,
+      end = datetime.datetime.utcnow()
+      log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
                     stem_error)
       status = 'error: "%s"' % (stem_error)
       level = logging.WARNING
       download_failed = True
-    elapsed = (datetime.datetime.utcnow() - start).total_seconds()
-    if elapsed > max_time:
+    elapsed = (end - start).total_seconds()
+    if download_failed:
+      # keep the error failure status, and avoid using the variables
+      pass
+    elif elapsed > max_time:
       status = 'too slow'
       level = logging.WARNING
       download_failed = True
+    elif (time_since_expiry > 0):
+      status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
+      if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE:
+        status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE)
+        level = logging.INFO
+      else:
+        status += ', invalid'
+        level = logging.WARNING
+        download_failed = True
     else:
       status = 'ok'
       level = logging.DEBUG
-    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' +
+    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
                  'max download time %0.1fs.', elapsed, status, nickname,
-                 dirip, dirport, max_time)
+                 dirip, dirport, fingerprint, max_time)
     return download_failed
 
   # does this fallback download the consensus fast enough?
@@ -1109,12 +1252,14 @@ class Candidate(object):
       ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
                                                 self.dirport,
                                                 self._data['nickname'],
+                                                self._fpr,
                                                 CONSENSUS_DOWNLOAD_SPEED_MAX)
     if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
       # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
       ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
                                                 self.dirport,
                                                 self._data['nickname'],
+                                                self._fpr,
                                                 CONSENSUS_DOWNLOAD_SPEED_MAX)
     return ((not ipv4_failed) and (not ipv6_failed))
 
@@ -1151,6 +1296,7 @@ class Candidate(object):
     # /*
     # nickname
     # flags
+    # adjusted bandwidth, consensus weight
     # [contact]
     # [identical contact counts]
     # */
@@ -1162,6 +1308,13 @@ class Candidate(object):
     s += 'Flags: '
     s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
     s += '\n'
+    # this is an adjusted bandwidth, see calculate_measured_bandwidth()
+    bandwidth = self._data['measured_bandwidth']
+    weight = self._data['consensus_weight']
+    s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
+        bandwidth/(1024.0*1024.0),
+        weight)
+    s += '\n'
     if self._data['contact'] is not None:
       s += cleanse_c_multiline_comment(self._data['contact'])
       if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT:
@@ -1183,6 +1336,7 @@ class Candidate(object):
       s += '\n'
     s += '*/'
     s += '\n'
+    return s
 
   # output the fallback info C string for this fallback
   # this is the text that would go after FallbackDir in a torrc
@@ -1190,8 +1344,14 @@ class Candidate(object):
   # comment-out the returned string
   def fallbackdir_info(self, dl_speed_ok):
     # "address:dirport orport=port id=fingerprint"
+    # (insert additional madatory fields here)
     # "[ipv6=addr:orport]"
-    # "weight=FALLBACK_OUTPUT_WEIGHT",
+    # (insert additional optional fields here)
+    # /* nickname=name */
+    # /* extrainfo={0,1} */
+    # (insert additional comment fields here)
+    # /* ===== */
+    # ,
     #
     # Do we want a C string, or a commented-out string?
     c_string = dl_speed_ok
@@ -1212,10 +1372,34 @@ class Candidate(object):
             self.orport,
             cleanse_c_string(self._fpr))
     s += '\n'
+    # (insert additional madatory fields here)
     if self.has_ipv6():
       s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
       s += '\n'
-    s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT)
+    # (insert additional optional fields here)
+    if not comment_string:
+      s += '/* '
+    s += 'nickname=%s'%(cleanse_c_string(self._data['nickname']))
+    if not comment_string:
+      s += ' */'
+    s += '\n'
+    # if we know that the fallback is an extrainfo cache, flag it
+    # and if we don't know, assume it is not
+    if not comment_string:
+      s += '/* '
+    s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0)
+    if not comment_string:
+      s += ' */'
+    s += '\n'
+    # (insert additional comment fields here)
+    # The terminator and comma must be the last line in each fallback entry
+    if not comment_string:
+      s += '/* '
+    s += SECTION_SEPARATOR_BASE
+    if not comment_string:
+      s += ' */'
+    s += '\n'
+    s += ','
     if comment_string:
       s += '\n'
       s += '*/'
@@ -1251,7 +1435,8 @@ class CandidateList(dict):
     d = fetch('details',
         fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
                 'consensus_weight,advertised_bandwidth,or_addresses,' +
-                'dir_address,recommended_version,flags,effective_family'))
+                'dir_address,recommended_version,flags,effective_family,' +
+                'platform'))
     logging.debug('Loading details document done.')
 
     if not 'relays' in d: raise Exception("No relays found in document.")
@@ -1297,13 +1482,12 @@ class CandidateList(dict):
     self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
                         reverse=True)
 
-  # sort fallbacks by their fingerprint, lowest to highest
-  # this is useful for stable diffs of fallback lists
-  def sort_fallbacks_by_fingerprint(self):
-    self.fallbacks.sort(key=lambda f: f._fpr)
+  # sort fallbacks by the data field data_field, lowest to highest
+  def sort_fallbacks_by(self, data_field):
+    self.fallbacks.sort(key=lambda f: f._data[data_field])
 
   @staticmethod
-  def load_relaylist(file_name):
+  def load_relaylist(file_obj):
     """ Read each line in the file, and parse it like a FallbackDir line:
         an IPv4 address and optional port:
           <IPv4 address>:<port>
@@ -1318,8 +1502,9 @@ class CandidateList(dict):
         (of string -> string key/value pairs),
         and these dictionaries are placed in an array.
         comments start with # and are ignored """
+    file_data = file_obj['data']
+    file_name = file_obj['name']
     relaylist = []
-    file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
     if file_data is None:
       return relaylist
     for line in file_data.split('\n'):
@@ -1360,12 +1545,12 @@ class CandidateList(dict):
     return relaylist
 
   # apply the fallback whitelist and blacklist
-  def apply_filter_lists(self):
+  def apply_filter_lists(self, whitelist_obj, blacklist_obj):
     excluded_count = 0
     logging.debug('Applying whitelist and blacklist.')
     # parse the whitelist and blacklist
-    whitelist = self.load_relaylist(WHITELIST_FILE_NAME)
-    blacklist = self.load_relaylist(BLACKLIST_FILE_NAME)
+    whitelist = self.load_relaylist(whitelist_obj)
+    blacklist = self.load_relaylist(blacklist_obj)
     filtered_fallbacks = []
     for f in self.fallbacks:
       in_whitelist = f.is_in_whitelist(whitelist)
@@ -1385,7 +1570,7 @@ class CandidateList(dict):
       elif in_blacklist:
         # exclude
         excluded_count += 1
-        logging.info('Excluding %s: in blacklist.', f._fpr)
+        log_excluded('Excluding %s: in blacklist.', f._fpr)
       else:
         if INCLUDE_UNLISTED_ENTRIES:
           # include
@@ -1393,7 +1578,7 @@ class CandidateList(dict):
         else:
           # exclude
           excluded_count += 1
-          logging.info('Excluding %s: in neither blacklist nor whitelist.',
+          log_excluded('Excluding %s: in neither blacklist nor whitelist.',
                        f._fpr)
     self.fallbacks = filtered_fallbacks
     return excluded_count
@@ -1404,7 +1589,7 @@ class CandidateList(dict):
                                                 excluded_count, initial_count)
 
   # calculate each fallback's measured bandwidth based on the median
-  # consensus weight to advertised bandwdith ratio
+  # consensus weight to advertised bandwidth ratio
   def calculate_measured_bandwidth(self):
     self.sort_fallbacks_by_cw_to_bw_factor()
     median_fallback = self.fallback_median(True)
@@ -1429,8 +1614,8 @@ class CandidateList(dict):
         # the bandwidth we log here is limited by the relay's consensus weight
         # as well as its adverttised bandwidth. See set_measured_bandwidth
         # for details
-        logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' +
-                     'be at least %.1fMB/s', f._fpr,
+        log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
+                     'must be at least %.1fMByte/s', f._fpr,
                      f._data['measured_bandwidth']/(1024.0*1024.0),
                      MIN_BANDWIDTH/(1024.0*1024.0))
     self.fallbacks = above_min_bw_fallbacks
@@ -1470,49 +1655,85 @@ class CandidateList(dict):
     else:
       return None
 
-  # does exclusion_list contain attribute?
+  # return a new bag suitable for storing attributes
+  @staticmethod
+  def attribute_new():
+    return dict()
+
+  # get the count of attribute in attribute_bag
+  # if attribute is None or the empty string, return 0
+  @staticmethod
+  def attribute_count(attribute, attribute_bag):
+    if attribute is None or attribute == '':
+      return 0
+    if attribute not in attribute_bag:
+      return 0
+    return attribute_bag[attribute]
+
+  # does attribute_bag contain more than max_count instances of attribute?
   # if so, return False
   # if not, return True
-  # if attribute is None or the empty string, always return True
+  # if attribute is None or the empty string, or max_count is invalid,
+  # always return True
   @staticmethod
-  def allow(attribute, exclusion_list):
-    if attribute is None or attribute == '':
+  def attribute_allow(attribute, attribute_bag, max_count=1):
+    if attribute is None or attribute == '' or max_count <= 0:
       return True
-    elif attribute in exclusion_list:
+    elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
       return False
     else:
       return True
 
-  # make sure there is only one fallback per IPv4 address, and per IPv6 address
+  # add attribute to attribute_bag, incrementing the count if it is already
+  # present
+  # if attribute is None or the empty string, or count is invalid,
+  # do nothing
+  @staticmethod
+  def attribute_add(attribute, attribute_bag, count=1):
+    if attribute is None or attribute == '' or count <= 0:
+      pass
+    attribute_bag.setdefault(attribute, 0)
+    attribute_bag[attribute] += count
+
+  # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
+  # and per IPv6 address
   # there is only one IPv4 address on each fallback: the IPv4 DirPort address
   # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
   # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
   # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
-  # (clients assume the IPv6 DirPort is the same as the IPv4 DirPort, but
-  # typically only use the IPv6 ORPort)
+  # (clients only use the IPv6 ORPort)
   # if there is no IPv6 address, only the IPv4 address is checked
   # return the number of candidates we excluded
   def limit_fallbacks_same_ip(self):
     ip_limit_fallbacks = []
-    ip_list = []
+    ip_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if (CandidateList.allow(f.dirip, ip_list)
-          and CandidateList.allow(f.ipv6addr, ip_list)):
+      if (CandidateList.attribute_allow(f.dirip, ip_list,
+                                        MAX_FALLBACKS_PER_IPV4)
+          and CandidateList.attribute_allow(f.ipv6addr, ip_list,
+                                            MAX_FALLBACKS_PER_IPV6)):
         ip_limit_fallbacks.append(f)
-        ip_list.append(f.dirip)
+        CandidateList.attribute_add(f.dirip, ip_list)
         if f.has_ipv6():
-          ip_list.append(f.ipv6addr)
-      elif not CandidateList.allow(f.dirip, ip_list):
-        logging.info('Eliminated %s: already have fallback on IPv4 %s'%(
-                                                          f._fpr, f.dirip))
-      elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list):
-        logging.info('Eliminated %s: already have fallback on IPv6 %s'%(
-                                                          f._fpr, f.ipv6addr))
+          CandidateList.attribute_add(f.ipv6addr, ip_list)
+      elif not CandidateList.attribute_allow(f.dirip, ip_list,
+                                             MAX_FALLBACKS_PER_IPV4):
+        log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
+                     %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
+                       f.dirip))
+      elif (f.has_ipv6() and
+            not CandidateList.attribute_allow(f.ipv6addr, ip_list,
+                                              MAX_FALLBACKS_PER_IPV6)):
+        log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
+                     %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
+                                                             ip_list),
+                       f.ipv6addr))
     original_count = len(self.fallbacks)
     self.fallbacks = ip_limit_fallbacks
     return original_count - len(self.fallbacks)
 
-  # make sure there is only one fallback per ContactInfo
+  # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
+  # ContactInfo
   # if there is no ContactInfo, allow the fallback
   # this check can be gamed by providing no ContactInfo, or by setting the
   # ContactInfo to match another fallback
@@ -1520,41 +1741,96 @@ class CandidateList(dict):
   # go down at similar times, its usefulness outweighs the risk
   def limit_fallbacks_same_contact(self):
     contact_limit_fallbacks = []
-    contact_list = []
+    contact_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if CandidateList.allow(f._data['contact'], contact_list):
+      if CandidateList.attribute_allow(f._data['contact'], contact_list,
+                                       MAX_FALLBACKS_PER_CONTACT):
         contact_limit_fallbacks.append(f)
-        contact_list.append(f._data['contact'])
+        CandidateList.attribute_add(f._data['contact'], contact_list)
       else:
-        logging.info(('Eliminated %s: already have fallback on ' +
-                       'ContactInfo %s')%(f._fpr, f._data['contact']))
+        log_excluded(
+          'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
+          %(f._fpr, CandidateList.attribute_count(f._data['contact'],
+                                                  contact_list),
+            f._data['contact']))
     original_count = len(self.fallbacks)
     self.fallbacks = contact_limit_fallbacks
     return original_count - len(self.fallbacks)
 
-  # make sure there is only one fallback per effective family
+  # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
+  # family
   # if there is no family, allow the fallback
-  # this check can't be gamed, because we use effective family, which ensures
-  # mutual family declarations
+  # we use effective family, which ensures mutual family declarations
+  # but the check can be gamed by not declaring a family at all
   # if any indirect families exist, the result depends on the order in which
   # fallbacks are sorted in the list
   def limit_fallbacks_same_family(self):
     family_limit_fallbacks = []
-    fingerprint_list = []
+    fingerprint_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if CandidateList.allow(f._fpr, fingerprint_list):
+      if CandidateList.attribute_allow(f._fpr, fingerprint_list,
+                                       MAX_FALLBACKS_PER_FAMILY):
         family_limit_fallbacks.append(f)
-        fingerprint_list.append(f._fpr)
-        fingerprint_list.extend(f._data['effective_family'])
+        CandidateList.attribute_add(f._fpr, fingerprint_list)
+        for family_fingerprint in f._data['effective_family']:
+          CandidateList.attribute_add(family_fingerprint, fingerprint_list)
       else:
-        # technically, we already have a fallback with this fallback in its
-        # effective family
-        logging.info('Eliminated %s: already have fallback in effective ' +
-                      'family'%(f._fpr))
+        # we already have a fallback with this fallback in its effective
+        # family
+        log_excluded(
+          'Eliminated %s: already have %d fallback(s) in effective family'
+          %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
     original_count = len(self.fallbacks)
     self.fallbacks = family_limit_fallbacks
     return original_count - len(self.fallbacks)
 
+  # try once to get the descriptors for fingerprint_list using stem
+  # returns an empty list on exception
+  @staticmethod
+  def get_fallback_descriptors_once(fingerprint_list):
+    desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True)
+    return desc_list
+
+  # try up to max_retries times to get the descriptors for fingerprint_list
+  # using stem. Stops retrying when all descriptors have been retrieved.
+  # returns a list containing the descriptors that were retrieved
+  @staticmethod
+  def get_fallback_descriptors(fingerprint_list, max_retries=5):
+    # we can't use stem's retries=, because we want to support more than 96
+    # descriptors
+    #
+    # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list
+    max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS
+    remaining_list = fingerprint_list
+    desc_list = []
+    for _ in xrange(max_retries):
+      if len(remaining_list) == 0:
+        break
+      new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])
+      for d in new_desc_list:
+        try:
+          remaining_list.remove(d.fingerprint)
+        except ValueError:
+          # warn and ignore if a directory mirror returned a bad descriptor
+          logging.warning("Directory mirror returned unwanted descriptor %s, ignoring",
+                          d.fingerprint)
+          continue
+        desc_list.append(d)
+    return desc_list
+
+  # find the fallbacks that cache extra-info documents
+  # Onionoo doesn't know this, so we have to use stem
+  def mark_extra_info_caches(self):
+    fingerprint_list = [ f._fpr for f in self.fallbacks ]
+    logging.info("Downloading fallback descriptors to find extra-info caches")
+    desc_list = CandidateList.get_fallback_descriptors(fingerprint_list)
+    for d in desc_list:
+      self[d.fingerprint]._extra_info_cache = d.extra_info_cache
+    missing_descriptor_list = [ f._fpr for f in self.fallbacks
+                                if f._extra_info_cache is None ]
+    for f in missing_descriptor_list:
+      logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f))
+
   # try a download check on each fallback candidate in order
   # stop after max_count successful downloads
   # but don't remove any candidates from the array
@@ -1714,7 +1990,7 @@ class CandidateList(dict):
    # this doesn't actually tell us anything useful
    #self.describe_fallback_ipv4_netblock_mask(8)
    self.describe_fallback_ipv4_netblock_mask(16)
-   self.describe_fallback_ipv4_netblock_mask(24)
+   #self.describe_fallback_ipv4_netblock_mask(24)
 
   # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
   # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
@@ -1724,7 +2000,7 @@ class CandidateList(dict):
     #self.describe_fallback_ipv6_netblock_mask(12)
     #self.describe_fallback_ipv6_netblock_mask(23)
     self.describe_fallback_ipv6_netblock_mask(32)
-    self.describe_fallback_ipv6_netblock_mask(48)
+    #self.describe_fallback_ipv6_netblock_mask(48)
     self.describe_fallback_ipv6_netblock_mask(64)
 
   # log a message about the proportion of fallbacks in each IPv4 and IPv6
@@ -1802,6 +2078,18 @@ class CandidateList(dict):
                     CandidateList.describe_percentage(dir_count,
                                                       fallback_count)))
 
+  # return a list of fallbacks which cache extra-info documents
+  def fallbacks_with_extra_info_cache(self):
+    return filter(lambda x: x._extra_info_cache, self.fallbacks)
+
+  # log a message about the proportion of fallbacks that cache extra-info docs
+  def describe_fallback_extra_info_caches(self):
+    extra_info_falback_count = len(self.fallbacks_with_extra_info_cache())
+    fallback_count = len(self.fallbacks)
+    logging.warning('%s of fallbacks cache extra-info documents'%(
+                    CandidateList.describe_percentage(extra_info_falback_count,
+                                                      fallback_count)))
+
   # return a list of fallbacks which have the Exit flag
   def fallbacks_with_exit(self):
     return filter(lambda x: x.is_exit(), self.fallbacks)
@@ -1829,10 +2117,6 @@ class CandidateList(dict):
   def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
                           guard_count, target_count):
     s = ''
-    s += '/* To comment-out entries in this file, use C comments, and add *'
-    s += ' to the start of each line. (stem finds fallback entries using "'
-    s += ' at the start of a line.) */'
-    s += '\n'
     # Report:
     #  whether we checked consensus download times
     #  the number of fallback directories (and limits/exclusions, if relevant)
@@ -1878,8 +2162,8 @@ class CandidateList(dict):
     min_bw = min_fb._data['measured_bandwidth']
     max_fb = self.fallback_max()
     max_bw = max_fb._data['measured_bandwidth']
-    s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0),
-                                              max_bw/(1024.0*1024.0))
+    s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
+                                                 max_bw/(1024.0*1024.0))
     s += '\n'
     s += '*/'
     if fallback_count < MIN_FALLBACK_COUNT:
@@ -1892,12 +2176,57 @@ class CandidateList(dict):
       s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
     return s
 
+def process_existing():
+  logging.basicConfig(level=logging.INFO)
+  logging.getLogger('stem').setLevel(logging.INFO)
+  whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
+               'name': FALLBACK_FILE_NAME}
+  blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': BLACKLIST_FILE_NAME}
+  list_fallbacks(whitelist, blacklist)
+
+def process_default():
+  logging.basicConfig(level=logging.WARNING)
+  logging.getLogger('stem').setLevel(logging.WARNING)
+  whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': WHITELIST_FILE_NAME}
+  blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': BLACKLIST_FILE_NAME}
+  list_fallbacks(whitelist, blacklist)
+
 ## Main Function
+def main():
+  if get_command() == 'check_existing':
+    process_existing()
+  else:
+    process_default()
+
+def get_command():
+  if len(sys.argv) == 2:
+    return sys.argv[1]
+  else:
+    return None
+
+def log_excluded(msg, *args):
+  if get_command() == 'check_existing':
+    logging.warning(msg, *args)
+  else:
+    logging.info(msg, *args)
 
-def list_fallbacks():
+def list_fallbacks(whitelist, blacklist):
   """ Fetches required onionoo documents and evaluates the
       fallback directory criteria for each of the relays """
 
+  print "/* type=fallback */"
+  print ("/* version={} */"
+         .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
+  now = datetime.datetime.utcnow()
+  timestamp = now.strftime('%Y%m%d%H%M%S')
+  print ("/* timestamp={} */"
+         .format(cleanse_c_multiline_comment(timestamp)))
+  # end the header with a separator, to make it easier for parsers
+  print SECTION_SEPARATOR_COMMENT
+
   logging.warning('Downloading and parsing Onionoo data. ' +
                   'This may take some time.')
   # find relays that could be fallbacks
@@ -1927,7 +2256,7 @@ def list_fallbacks():
   # warning that the details have changed from those in the whitelist.
   # instead, there will be an info-level log during the eligibility check.
   initial_count = len(candidates.fallbacks)
-  excluded_count = candidates.apply_filter_lists()
+  excluded_count = candidates.apply_filter_lists(whitelist, blacklist)
   print candidates.summarise_filters(initial_count, excluded_count)
   eligible_count = len(candidates.fallbacks)
 
@@ -1963,6 +2292,9 @@ def list_fallbacks():
                     'This may take some time.')
   failed_count = candidates.perform_download_consensus_checks(max_count)
 
+  # work out which fallbacks cache extra-infos
+  candidates.mark_extra_info_caches()
+
   # analyse and log interesting diversity metrics
   # like netblock, ports, exit, IPv4-only
   # (we can't easily analyse AS, and it's hard to accurately analyse country)
@@ -1971,6 +2303,7 @@ def list_fallbacks():
   if HAVE_IPADDRESS:
     candidates.describe_fallback_netblocks()
   candidates.describe_fallback_ports()
+  candidates.describe_fallback_extra_info_caches()
   candidates.describe_fallback_exit_flag()
 
   # output C comments summarising the fallback selection process
@@ -1985,15 +2318,20 @@ def list_fallbacks():
   for s in fetch_source_list():
     print describe_fetch_source(s)
 
+  # start the list with a separator, to make it easy for parsers
+  print SECTION_SEPARATOR_COMMENT
+
+  # sort the list differently depending on why we've created it:
   # if we're outputting the final fallback list, sort by fingerprint
   # this makes diffs much more stable
-  # otherwise, leave sorted by bandwidth, which allows operators to be
-  # contacted in priority order
-  if not OUTPUT_CANDIDATES:
-    candidates.sort_fallbacks_by_fingerprint()
+  # otherwise, if we're trying to find a bandwidth cutoff, or we want to
+  # contact operators in priority order, sort by bandwidth (not yet
+  # implemented)
+  # otherwise, if we're contacting operators, sort by contact
+  candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
 
   for x in candidates.fallbacks:
     print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
 
 if __name__ == "__main__":
-  list_fallbacks()
+  main()
diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh
new file mode 100755
index 0000000000..a5a92579d3
--- /dev/null
+++ b/scripts/maint/updateRustDependencies.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Tor Project, Inc.
+# Copyright (c) 2018 isis agora lovecruft
+# See LICENSE for license information
+#
+# updateRustDependencies.sh
+# -------------------------
+# Update our vendored Rust dependencies, either adding/removing
+# dependencies and/or upgrading current dependencies to newer
+# versions.
+#
+# To use this script, first add your dependencies, exactly specifying
+# their versions, into the appropriate *crate-level* Cargo.toml in
+# src/rust/ (i.e. *not* /src/rust/Cargo.toml, but instead the one for
+# your crate).
+#
+# Next, run this script.  Then, go into src/ext/rust and commit the
+# changes to the tor-rust-dependencies repo.
+
+set -e
+
+HERE=`dirname $(realpath $0)`
+TOPLEVEL=`dirname $(dirname $HERE)`
+TOML="$TOPLEVEL/src/rust/Cargo.toml"
+VENDORED="$TOPLEVEL/src/ext/rust/crates"
+CARGO=`which cargo`
+
+if ! test -f "$TOML"  ; then
+    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML"
+fi
+
+if ! test -d "$VENDORED" ; then
+    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED"
+fi
+
+if test -z "$CARGO" ; then
+    printf "Error: cargo must be installed and in your \$PATH\n"
+fi
+
+if test -z `cargo --list | grep vendor` ; then
+    printf "Error: cargo-vendor not installed\n"
+fi
+
+$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED
diff --git a/scripts/test/appveyor-irc-notify.py b/scripts/test/appveyor-irc-notify.py
new file mode 100644
index 0000000000..cfe0afe7ae
--- /dev/null
+++ b/scripts/test/appveyor-irc-notify.py
@@ -0,0 +1,219 @@
+# coding=utf8
+# Copyright (C) 2015-2016 Christopher R. Wood
+# Copyright (c) 2018 The Tor Project
+# Copyright (c) 2018 isis agora lovecruft
+#
+# From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py
+# and: https://github.com/gridsync/gridsync
+#
+# Modified by nexB on October 2016:
+#  - rework the handling of environment variables.
+#  - made the script use functions
+#  - support only Appveyor loading its environment variable to craft IRC notices.
+#
+# Modified by isis agora lovecruft <isis@torproject.org> in 2018:
+#  - Make IRC server configurable.
+#  - Make bot IRC nick deterministic.
+#  - Make bot join the channel rather than sending NOTICE messages externally.
+#  - Fix a bug which always caused sys.exit() to be logged as a traceback.
+#  - Actually reset the IRC colour codes after printing.
+#
+# Modified by Marcin Cieślak in 2018:
+#  - Accept UTF-8
+#  - only guess github URLs
+#  - stop using ANSI colors
+#
+# Modified by teor in 2018:
+#  - fix github provider detection ('gitHub' or 'gitHubEnterprise', apparently)
+#  - make short commits 10 hexdigits long (that's what git does for tor)
+#  - generate correct branches and URLs for pull requests and tags
+#  - switch to one URL per line
+
+# This program is free software; you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software Foundation;
+# either version 2 of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with this
+# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street,
+# Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Simple AppVeyor IRC notification script.
+
+The first argument is an IRC server and port; the second is the channel. Other
+arguments passed to the script will be sent as notice messages content and any
+{var}-formatted environment variables will be expanded automatically, replaced
+with a corresponding Appveyor environment variable value. Use commas to
+delineate multiple messages.
+
+
+Example:
+export APPVEYOR_ACCOUNT_NAME=isislovecruft
+export APPVEYOR_BUILD_VERSION=1
+export APPVEYOR_PROJECT_NAME=tor
+export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number
+export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title
+export APPVEYOR_REPO_BRANCH=repo_branch
+export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8
+export APPVEYOR_REPO_COMMIT_AUTHOR=isislovecruft
+export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test"
+export APPVEYOR_REPO_COMMIT_TIMESTAMP=2018-04-23
+export APPVEYOR_REPO_NAME=isislovecruft/tor
+export APPVEYOR_REPO_PROVIDER=github
+export APPVEYOR_URL=https://ci.appveyor.com
+python ./appveyor-irc-notify.py irc.oftc.net:6697 tor-ci '{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}','Build #{build_version} passed. Details: {build_url} |  Commit: {commit_url}
+
+See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples
+in Appveyor's YAML:
+
+    on_success:
+      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success
+    on_failure:
+      - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure
+"""
+
+from __future__ import print_function
+from __future__ import absolute_import
+
+import os
+import random
+import socket
+import ssl
+import sys
+import time
+
+
+def appveyor_vars():
+    """
+    Return a dict of key value crafted from appveyor environment variables.
+    """
+
+    vars = dict([
+            (
+                v.replace('APPVEYOR_', '').lower(),
+                os.getenv(v, '').decode('utf-8')
+            ) for v in [
+                'APPVEYOR_ACCOUNT_NAME',
+                'APPVEYOR_BUILD_VERSION',
+                'APPVEYOR_PROJECT_NAME',
+                'APPVEYOR_PULL_REQUEST_HEAD_COMMIT',
+                'APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH',
+                'APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME',
+                'APPVEYOR_PULL_REQUEST_NUMBER',
+                'APPVEYOR_PULL_REQUEST_TITLE',
+                'APPVEYOR_REPO_BRANCH',
+                'APPVEYOR_REPO_COMMIT',
+                'APPVEYOR_REPO_COMMIT_AUTHOR',
+                'APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL',
+                'APPVEYOR_REPO_COMMIT_MESSAGE',
+                'APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED',
+                'APPVEYOR_REPO_COMMIT_TIMESTAMP',
+                'APPVEYOR_REPO_NAME',
+                'APPVEYOR_REPO_PROVIDER',
+                'APPVEYOR_REPO_TAG_NAME',
+                'APPVEYOR_URL',
+            ]
+    ])
+
+    BUILD_FMT = u'{url}/project/{account_name}/{project_name}/build/{build_version}'
+
+    if vars["repo_tag_name"]:
+        BRANCH_FMT = u'{repo_name} {repo_tag_name} {short_commit}'
+    else:
+        BRANCH_FMT = u'{repo_name} {repo_branch} {short_commit}'
+
+    vars.update(head_commit=vars["repo_commit"])
+
+    if vars["repo_provider"].lower().startswith('github'):
+        COMMIT_FMT = u'https://github.com/{repo_name}/commit/{repo_commit}'
+        if vars["pull_request_number"]:
+            vars.update(head_commit=vars["pull_request_head_commit"])
+            BRANCH_FMT = u'{repo_name} {repo_branch} pull {pull_request_head_repo_name} {pull_request_head_repo_branch} {short_commit}'
+            COMMIT_FMT = u'https://github.com/{pull_request_head_repo_name}/commit/{pull_request_head_commit}'
+            PULL_FMT = u'https://github.com/{repo_name}/pull/{pull_request_number}'
+            vars.update(pull_url=PULL_FMT.format(**vars))
+        vars.update(commit_url=COMMIT_FMT.format(**vars))
+
+    vars.update(short_commit=vars["head_commit"][:10])
+
+    vars.update(
+        build_url=BUILD_FMT.format(**vars),
+        branch_detail=BRANCH_FMT.format(**vars),
+    )
+    return vars
+
+
+def notify():
+    """
+    Send IRC notification
+    """
+    apvy_vars = appveyor_vars()
+
+    server, port = sys.argv[1].rsplit(":", 1)
+    channel = sys.argv[2]
+    success = sys.argv[3] == "success"
+    failure = sys.argv[3] == "failure"
+
+    if success or failure:
+        messages = []
+        messages.append(u"{branch_detail} - {repo_commit_author}: {repo_commit_message}")
+
+        if success:
+            messages.append(u"Build #{build_version} passed. Details: {build_url}")
+        if failure:
+            messages.append(u"Build #{build_version} failed. Details: {build_url}")
+
+        if "commit_url" in apvy_vars:
+            messages.append(u"Commit: {commit_url}")
+
+        if "pull_url" in apvy_vars:
+            messages.append(u"Pull: {pull_url}")
+
+    else:
+        messages = sys.argv[3:]
+        messages = ' '.join(messages)
+        messages = messages.decode("utf-8").split(',')
+
+    print(repr(apvy_vars))
+    messages = [msg.format(**apvy_vars).strip() for msg in messages]
+
+    irc_username = 'appveyor-ci'
+    irc_nick = irc_username
+
+    # establish connection
+    irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
+    irc_sock.connect((socket.gethostbyname(server), int(port)))
+    irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode())
+    irc_sock.send('JOIN #{0}\r\n'.format(channel).encode())
+    irc_file = irc_sock.makefile()
+
+    while irc_file:
+        line = irc_file.readline()
+        print(line.rstrip())
+        response = line.split()
+
+        if response[0] == 'PING':
+            irc_file.send('PONG {}\r\n'.format(response[1]).encode())
+
+        elif response[1] == '433':
+            irc_sock.send('NICK {}\r\n'.format(irc_nick).encode())
+
+        elif response[1] == '001':
+            time.sleep(5)
+            # send notification
+            for msg in messages:
+                print(u'PRIVMSG #{} :{}'.format(channel, msg).encode("utf-8"))
+                irc_sock.send(u'PRIVMSG #{} :{}\r\n'.format(channel, msg).encode("utf-8"))
+            time.sleep(5)
+            return
+
+
+if __name__ == '__main__':
+    try:
+        notify()
+    except:
+        import traceback
+        print('ERROR: Failed to send notification: \n' + traceback.format_exc())
diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff
index 7da7f0be9d..6179dff63e 100755
--- a/scripts/test/cov-diff
+++ b/scripts/test/cov-diff
@@ -7,11 +7,15 @@
 DIRA="$1"
 DIRB="$2"
 
-for A in $DIRA/*; do
-  B=$DIRB/`basename $A`
-  perl -pe 's/^\s*\!*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
-  perl -pe 's/^\s*\!*\d+:/        1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
-  diff -u "$A.tmp" "$B.tmp"
+for B in $DIRB/*; do
+  A=$DIRA/`basename $B`
+  if [ -f $A ]; then
+      perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
+  else
+      cat /dev/null > "$A.tmp"
+  fi
+  perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
+  diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;'
   rm "$A.tmp" "$B.tmp"
 done
 
diff --git a/scripts/test/cov-exclude b/scripts/test/cov-exclude
index 5117f11ec4..5cb9b1282d 100755
--- a/scripts/test/cov-exclude
+++ b/scripts/test/cov-exclude
@@ -26,3 +26,9 @@ if ($excluding or $exclude_this) {
   s{^\s*\#\#+:}{        x:};
   s{^   (\s*)(\d+):}{$1!!!$2:};
 }
+
+if (eof and $excluding) {
+    warn "Runaway LCOV_EXCL_START in $ARGV";
+    $excluding = 0;
+}
+
diff --git a/scripts/test/coverage b/scripts/test/coverage
index f4ae475828..f10c5afaae 100755
--- a/scripts/test/coverage
+++ b/scripts/test/coverage
@@ -29,7 +29,7 @@ for fn in src/or/*.c src/common/*.c; do
           gcov -o $on $fn
           if [ -e $GC ]
           then
-            if [ -n $dst ]
+            if [ -d "$dst" ]
             then
               mv $GC $dst/$GC
             fi
diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh
index 36e69e6d00..8d126cbcee 100644..100755
--- a/scripts/test/scan-build.sh
+++ b/scripts/test/scan-build.sh
@@ -5,37 +5,76 @@
 # This script is used for running a bunch of clang scan-build checkers
 # on Tor.
 
+# These don't seem to cause false positives in our code, so let's turn
+# them on.
 CHECKERS="\
-    -disable-checker deadcode.DeadStores \
-    -enable-checker alpha.core.CastSize \
+    -enable-checker alpha.core.CallAndMessageUnInitRefArg \
     -enable-checker alpha.core.CastToStruct \
+    -enable-checker alpha.core.Conversion \
+    -enable-checker alpha.core.FixedAddr \
     -enable-checker alpha.core.IdenticalExpr \
+    -enable-checker alpha.core.PointerArithm \
     -enable-checker alpha.core.SizeofPtr \
-    -enable-checker alpha.security.ArrayBoundV2 \
+    -enable-checker alpha.core.TestAfterDivZero \
     -enable-checker alpha.security.MallocOverflow \
     -enable-checker alpha.security.ReturnPtrRange \
-    -enable-checker alpha.unix.SimpleStream
+    -enable-checker alpha.unix.BlockInCriticalSection \
+    -enable-checker alpha.unix.Chroot \
+    -enable-checker alpha.unix.PthreadLock \
+    -enable-checker alpha.unix.PthreadLock \
+    -enable-checker alpha.unix.SimpleStream \
+    -enable-checker alpha.unix.Stream \
     -enable-checker alpha.unix.cstring.BufferOverlap \
     -enable-checker alpha.unix.cstring.NotNullTerminated \
-    -enable-checker alpha.unix.cstring.OutOfBounds \
-    -enable-checker alpha.core.FixedAddr \
+    -enable-checker valist.CopyToSelf \
+    -enable-checker valist.Uninitialized \
+    -enable-checker valist.Unterminated \
+    -enable-checker security.FloatLoopCounter \
     -enable-checker security.insecureAPI.strcpy \
-    -enable-checker alpha.unix.PthreadLock \
-    -enable-checker alpha.core.PointerArithm \
-    -enable-checker alpha.core.TestAfterDivZero \
 "
 
+# These have high false-positive rates.
+EXTRA_CHECKERS="\
+    -enable-checker alpha.security.ArrayBoundV2 \
+    -enable-checker alpha.unix.cstring.OutOfBounds \
+    -enable-checker alpha.core.CastSize \
+"
+
+# These don't seem to generate anything useful
+NOISY_CHECKERS="\
+    -enable-checker alpha.clone.CloneChecker \
+    -enable-checker alpha.deadcode.UnreachableCode \
+"
+
+if test "x$SCAN_BUILD_OUTPUT" != "x"; then
+   OUTPUTARG="-o $SCAN_BUILD_OUTPUT"
+else
+   OUTPUTARG=""
+fi
+
 scan-build \
     $CHECKERS \
     ./configure
 
 scan-build \
+    make clean
+
+# Make this not get scanned for dead assignments, since it has lots of
+# dead assignments we don't care about.
+scan-build \
     $CHECKERS \
-    make -j2 -k
+    -disable-checker deadcode.DeadStores \
+    make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a
 
+scan-build \
+    $CHECKERS $OUTPUTARG \
+    make -j5 -k
+
+CHECKERS="\
+"
 
 # This one gives a false positive on every strcmp.
 #    -enable-checker alpha.core.PointerSub
 
 # Needs work
-#  alpha.unix.MallocWithAnnotations ??
+#    -enable-checker alpha.unix.MallocWithAnnotations
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake
index b8c5dd4fea..a1c819fffa 100644
--- a/src/common/Makefile.nmake
+++ b/src/common/Makefile.nmake
@@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \
 	log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \
 	util_codedigest.obj
 
-LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \
-	crypto_curve25519.obj curve25519-donna.obj
+LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \
+	tortls.obj crypto_curve25519.obj curve25519-donna.obj
 
 LIBOR_EVENT_OBJECTS = compat_libevent.obj
 
diff --git a/src/common/address.c b/src/common/address.c
index 794345a138..9446675712 100644
--- a/src/common/address.c
+++ b/src/common/address.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,7 +33,7 @@
 #include <process.h>
 #include <windows.h>
 #include <iphlpapi.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include "compat.h"
 #include "util.h"
@@ -159,6 +159,8 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa,
   tor_assert(a);
   tor_assert(sa);
 
+  /* This memset is redundant; leaving it in to avoid any future accidents,
+     however. */
   memset(a, 0, sizeof(*a));
 
   if (sa->sa_family == AF_INET) {
@@ -196,7 +198,7 @@ tor_sockaddr_to_str(const struct sockaddr *sa)
     tor_asprintf(&result, "unix:%s", s_un->sun_path);
     return result;
   }
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
   if (sa->sa_family == AF_UNSPEC)
     return tor_strdup("unspec");
 
@@ -235,8 +237,8 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family)
  *
  * Return 0 on success, -1 on failure; 1 on transient failure.
  */
-int
-tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
 {
   /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
    * something.
@@ -303,7 +305,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
       return result;
     }
     return (err == EAI_AGAIN) ? 1 : -1;
-#else
+#else /* !(defined(HAVE_GETADDRINFO)) */
     struct hostent *ent;
     int err;
 #ifdef HAVE_GETHOSTBYNAME_R_6_ARG
@@ -328,7 +330,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
 #else
     err = h_errno;
 #endif
-#endif /* endif HAVE_GETHOSTBYNAME_R_6_ARG. */
+#endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
     if (ent) {
       if (ent->h_addrtype == AF_INET) {
         tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
@@ -344,7 +346,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
 #else
     return (err == TRY_AGAIN) ? 1 : -1;
 #endif
-#endif
+#endif /* defined(HAVE_GETADDRINFO) */
   }
 }
 
@@ -562,8 +564,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
 
 /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name,
  * and store the result in the <b>outlen</b>-byte buffer at
- * <b>out</b>.  Return the number of chars written to <b>out</b>, not
- * including the trailing \0, on success. Returns -1 on failure. */
+ * <b>out</b>.  Returns a non-negative integer on success.
+ * Returns -1 on failure. */
 int
 tor_addr_to_PTR_name(char *out, size_t outlen,
                      const tor_addr_t *addr)
@@ -905,8 +907,8 @@ tor_addr_is_loopback(const tor_addr_t *addr)
       return (tor_addr_to_ipv4h(addr) & 0xff000000) == 0x7f000000;
     case AF_UNSPEC:
       return 0;
-    default:
       /* LCOV_EXCL_START */
+    default:
       tor_fragile_assert();
       return 0;
       /* LCOV_EXCL_STOP */
@@ -1029,8 +1031,10 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src)
       memcpy(dest->addr.in6_addr.s6_addr, src->addr.in6_addr.s6_addr, 16);
     case AF_UNSPEC:
       break;
+      // LCOV_EXCL_START
     default:
-      tor_fragile_assert(); // LCOV_EXCL_LINE
+      tor_fragile_assert();
+      // LCOV_EXCL_STOP
     }
 }
 
@@ -1121,7 +1125,7 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
       case AF_UNIX:
         /* HACKHACKHACKHACKHACK:
          * tor_addr_t doesn't contain a copy of sun_path, so it's not
-         * possible to comapre this at all.
+         * possible to compare this at all.
          *
          * Since the only time we currently actually should be comparing
          * 2 AF_UNIX addresses is when dealing with ISO_CLIENTADDR (which
@@ -1136,8 +1140,8 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
           return 0;
         else
           return 1;
-      default:
         /* LCOV_EXCL_START */
+      default:
         tor_fragile_assert();
         return 0;
         /* LCOV_EXCL_STOP */
@@ -1195,8 +1199,8 @@ tor_addr_hash(const tor_addr_t *addr)
     return siphash24g(unspec_hash_input, sizeof(unspec_hash_input));
   case AF_INET6:
     return siphash24g(&addr->addr.in6_addr.s6_addr, 16);
-  default:
     /* LCOV_EXCL_START */
+  default:
     tor_fragile_assert();
     return 0;
     /* LCOV_EXCL_STOP */
@@ -1432,7 +1436,7 @@ get_interface_addresses_ifaddrs(int severity, sa_family_t family)
 
   return result;
 }
-#endif
+#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 
@@ -1526,7 +1530,7 @@ get_interface_addresses_win32(int severity, sa_family_t family)
   return result;
 }
 
-#endif
+#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 
@@ -1539,6 +1543,18 @@ get_interface_addresses_win32(int severity, sa_family_t family)
 #define _SIZEOF_ADDR_IFREQ sizeof
 #endif
 
+/* Free ifc->ifc_buf safely. */
+static void
+ifconf_free_ifc_buf(struct ifconf *ifc)
+{
+  /* On macOS, tor_free() takes the address of ifc.ifc_buf, which leads to
+   * undefined behaviour, because pointer-to-pointers are expected to be
+   * aligned at 8-bytes, but the ifconf structure is packed.  So we use
+   * raw_free() instead. */
+  raw_free(ifc->ifc_buf);
+  ifc->ifc_buf = NULL;
+}
+
 /** Convert <b>*buf</b>, an ifreq structure array of size <b>buflen</b>,
  * into smartlist of <b>tor_addr_t</b> structures.
  */
@@ -1625,10 +1641,10 @@ get_interface_addresses_ioctl(int severity, sa_family_t family)
  done:
   if (fd >= 0)
     close(fd);
-  tor_free(ifc.ifc_buf);
+  ifconf_free_ifc_buf(&ifc);
   return result;
 }
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 /** Try to ask our network interfaces what addresses they are bound to.
  * Return a new smartlist of tor_addr_t on success, and NULL on failure.
@@ -1684,7 +1700,7 @@ get_interface_address6_via_udp_socket_hack,(int severity,
                                             sa_family_t family,
                                             tor_addr_t *addr))
 {
-  struct sockaddr_storage my_addr, target_addr;
+  struct sockaddr_storage target_addr;
   int sock=-1, r=-1;
   socklen_t addr_len;
 
@@ -1727,21 +1743,19 @@ get_interface_address6_via_udp_socket_hack,(int severity,
     goto err;
   }
 
-  if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) {
+  if (tor_addr_from_getsockname(addr, sock) < 0) {
     int e = tor_socket_errno(sock);
     log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s",
            tor_socket_strerror(e));
     goto err;
   }
 
- if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) {
-    if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
-      log_fn(severity, LD_NET, "Address that we determined via UDP socket"
-                               " magic is unsuitable for public comms.");
-    } else {
-      r=0;
-    }
- }
+  if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) {
+    log_fn(severity, LD_NET, "Address that we determined via UDP socket"
+           " magic is unsuitable for public comms.");
+  } else {
+    r=0;
+  }
 
  err:
   if (sock >= 0)
@@ -1783,14 +1797,14 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr))
       break;
   } SMARTLIST_FOREACH_END(a);
 
-  free_interface_address6_list(addrs);
+  interface_address6_list_free(addrs);
   return rv;
 }
 
 /** Free a smartlist of IP addresses returned by get_interface_address6_list.
  */
 void
-free_interface_address6_list(smartlist_t *addrs)
+interface_address6_list_free_(smartlist_t *addrs)
 {
   if (addrs != NULL) {
     SMARTLIST_FOREACH(addrs, tor_addr_t *, a, tor_free(a));
@@ -1805,11 +1819,12 @@ free_interface_address6_list(smartlist_t *addrs)
  * An empty smartlist means that there are no addresses of the selected type
  * matching these criteria.
  * Returns NULL on failure.
- * Use free_interface_address6_list to free the returned list.
+ * Use interface_address6_list_free to free the returned list.
  */
-MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity,
-                                                     sa_family_t family,
-                                                     int include_internal))
+MOCK_IMPL(smartlist_t *,
+get_interface_address6_list,(int severity,
+                             sa_family_t family,
+                             int include_internal))
 {
   smartlist_t *addrs;
   tor_addr_t addr;
@@ -2077,7 +2092,8 @@ parse_port_range(const char *port, uint16_t *port_min_out,
 
 /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual),
  *  write it as a string into the <b>buf_len</b>-byte buffer in
- *  <b>buf</b>.
+ *  <b>buf</b>. Returns a non-negative integer on success.
+ *  Returns -1 on failure.
  */
 int
 tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len)
@@ -2128,7 +2144,8 @@ get_interface_address,(int severity, uint32_t *addr))
 }
 
 /** Return true if we can tell that <b>name</b> is a canonical name for the
- * loopback address. */
+ * loopback address.  Return true also for *.local hostnames, which are
+ * multicast DNS names for hosts on the local network. */
 int
 tor_addr_hostname_is_local(const char *name)
 {
@@ -2149,3 +2166,11 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port)
   return ap;
 }
 
+/** Return true iff <a>a</b> and <b>b</b> are the same address and port */
+int
+tor_addr_port_eq(const tor_addr_port_t *a,
+                 const tor_addr_port_t *b)
+{
+  return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port;
+}
+
diff --git a/src/common/address.h b/src/common/address.h
index d57abd0d9e..c9d9543dee 100644
--- a/src/common/address.h
+++ b/src/common/address.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,7 +44,7 @@
 #endif
 
 // TODO win32 specific includes
-#endif // ADDRESS_PRIVATE
+#endif /* defined(ADDRESS_PRIVATE) */
 
 /** The number of bits from an address to consider while doing a masked
  * comparison. */
@@ -190,7 +190,8 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u)
  */
 #define TOR_ADDR_BUF_LEN 48
 
-int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out);
+MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
+                                tor_addr_t *addr_out));
 char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC;
 
 /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6
@@ -205,7 +206,9 @@ const char * fmt_addr32(uint32_t addr);
 
 MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family,
 tor_addr_t *addr));
-void free_interface_address6_list(smartlist_t * addrs);
+void interface_address6_list_free_(smartlist_t * addrs);// XXXX
+#define interface_address6_list_free(addrs) \
+  FREE_AND_NULL(smartlist_t, interface_address6_list_free_, (addrs))
 MOCK_DECL(smartlist_t *,get_interface_address6_list,(int severity,
                                                      sa_family_t family,
                                                      int include_internal));
@@ -322,13 +325,8 @@ int addr_mask_get_bits(uint32_t mask);
 int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 char *tor_dup_ip(uint32_t addr) ATTR_MALLOC;
 MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr));
-/** Free a smartlist of IP addresses returned by get_interface_address_list.
- */
-static inline void
-free_interface_address_list(smartlist_t *addrs)
-{
-  free_interface_address6_list(addrs);
-}
+#define interface_address_list_free(lst)\
+  interface_address6_list_free(lst)
 /** Return a smartlist of the IPv4 addresses of all interfaces on the server.
  * Excludes loopback and multicast addresses. Only includes internal addresses
  * if include_internal is true. (Note that a relay behind NAT may use an
@@ -344,6 +342,8 @@ get_interface_address_list(int severity, int include_internal)
 }
 
 tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port);
+int tor_addr_port_eq(const tor_addr_port_t *a,
+                     const tor_addr_port_t *b);
 
 #ifdef ADDRESS_PRIVATE
 MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity,
@@ -357,23 +357,23 @@ STATIC smartlist_t *ifaddrs_to_smartlist(const struct ifaddrs *ifa,
                                          sa_family_t family);
 STATIC smartlist_t *get_interface_addresses_ifaddrs(int severity,
                                                     sa_family_t family);
-#endif
+#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 STATIC smartlist_t *ip_adapter_addresses_to_smartlist(
                                         const IP_ADAPTER_ADDRESSES *addresses);
 STATIC smartlist_t *get_interface_addresses_win32(int severity,
                                                   sa_family_t family);
-#endif
+#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 STATIC smartlist_t *ifreq_to_smartlist(char *ifr,
                                        size_t buflen);
 STATIC smartlist_t *get_interface_addresses_ioctl(int severity,
                                                   sa_family_t family);
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
-#endif // ADDRESS_PRIVATE
+#endif /* defined(ADDRESS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_ADDRESS_H) */
 
diff --git a/src/common/address_set.c b/src/common/address_set.c
index 4924cb65c2..b2f4bb4c95 100644
--- a/src/common/address_set.c
+++ b/src/common/address_set.c
@@ -15,7 +15,7 @@
 #include "address.h"
 #include "compat.h"
 #include "container.h"
-#include "crypto.h"
+#include "crypto_rand.h"
 #include "util.h"
 #include "siphash.h"
 
@@ -34,7 +34,7 @@
  * independent siphashes rather than messing around with bit-shifts.  The
  * approach here is probably more sound, and we should prefer it if&when we
  * unify the implementations.
- **/
+ */
 
 struct address_set_t {
   /** siphash keys to make N_HASHES independent hashes for each address. */
@@ -63,7 +63,7 @@ address_set_new(int max_addresses_guess)
 }
 
 /**
- * Release all storage associated with <b>set</b>
+ * Release all storage associated with <b>set</b>.
  */
 void
 address_set_free(address_set_t *set)
@@ -107,7 +107,7 @@ address_set_add_ipv4h(address_set_t *set, uint32_t addr)
 }
 
 /**
- * Return true if <b>addr</b> if a member of <b>set</b>.  (And probably,
+ * Return true if <b>addr</b> is a member of <b>set</b>.  (And probably,
  * return false if <b>addr</b> is not a member of set.)
  */
 int
diff --git a/src/common/address_set.h b/src/common/address_set.h
index aedf17fc66..28d29f3fdf 100644
--- a/src/common/address_set.h
+++ b/src/common/address_set.h
@@ -2,7 +2,7 @@
 /* See LICENSE for licensing information */
 
 /**
- * \file addressset.h
+ * \file address_set.h
  * \brief Types to handle sets of addresses.
  *
  * This module was first written on a semi-emergency basis to improve the
diff --git a/src/common/aes.c b/src/common/aes.c
index 8ab2d2fc6e..0c4f9b9bca 100644
--- a/src/common/aes.c
+++ b/src/common/aes.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,8 +16,9 @@
   #include <ws2tcpip.h>
 #endif
 
+#include "compat_openssl.h"
 #include <openssl/opensslv.h>
-#include "crypto.h"
+#include "crypto_openssl_mgt.h"
 
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0)
 #error "We require OpenSSL >= 1.0.0"
@@ -40,6 +41,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include "util.h"
 #include "torlog.h"
 #include "di_ops.h"
+#include "crypto_util.h"
 
 #ifdef ANDROID
 /* Android's OpenSSL seems to have removed all of its Engine support. */
@@ -66,11 +68,11 @@ ENABLE_GCC_WARNING(redundant-decls)
 #elif OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,0,1) &&               \
   (defined(__i386) || defined(__i386__) || defined(_M_IX86) ||          \
    defined(__x86_64) || defined(__x86_64__) ||                          \
-   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))          \
+   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))
 
 #define USE_EVP_AES_CTR
 
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,1,0) || ... */
 
 /* We have 2 strategies for getting the AES block cipher: Via OpenSSL's
  * AES_encrypt function, or via OpenSSL's EVP_EncryptUpdate function.
@@ -110,12 +112,16 @@ aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits)
   return (aes_cnt_cipher_t *) cipher;
 }
 void
-aes_cipher_free(aes_cnt_cipher_t *cipher_)
+aes_cipher_free_(aes_cnt_cipher_t *cipher_)
 {
   if (!cipher_)
     return;
   EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_;
+#ifdef OPENSSL_1_1_API
+  EVP_CIPHER_CTX_reset(cipher);
+#else
   EVP_CIPHER_CTX_cleanup(cipher);
+#endif
   EVP_CIPHER_CTX_free(cipher);
 }
 void
@@ -142,7 +148,7 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else
+#else /* !(defined(USE_EVP_AES_CTR)) */
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
@@ -163,7 +169,7 @@ struct aes_cnt_cipher {
   uint32_t counter2;
   uint32_t counter1;
   uint32_t counter0;
-#endif
+#endif /* !defined(WORDS_BIGENDIAN) */
 
   union {
     /** The counter, in big-endian order, as bytes. */
@@ -212,7 +218,7 @@ evaluate_evp_for_aes(int force_val)
     log_info(LD_CRYPTO, "No AES engine found; using AES_* functions.");
     should_use_EVP = 0;
   }
-#endif
+#endif /* defined(DISABLE_ENGINES) */
 
   return 0;
 }
@@ -254,7 +260,7 @@ evaluate_ctr_for_aes(void)
     /* LCOV_EXCL_START */
     log_err(LD_CRYPTO, "This OpenSSL has a buggy version of counter mode; "
                   "quitting tor.");
-    exit(1);
+    exit(1); // exit ok: openssl is broken.
     /* LCOV_EXCL_STOP */
   }
   return 0;
@@ -312,7 +318,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
   cipher->counter1 = 0;
   cipher->counter2 = 0;
   cipher->counter3 = 0;
-#endif
+#endif /* defined(USING_COUNTER_VARS) */
 
   memset(cipher->ctr_buf.buf, 0, sizeof(cipher->ctr_buf.buf));
 
@@ -324,7 +330,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const uint8_t *key, int key_bits)
 /** Release storage held by <b>cipher</b>
  */
 void
-aes_cipher_free(aes_cnt_cipher_t *cipher)
+aes_cipher_free_(aes_cnt_cipher_t *cipher)
 {
   if (!cipher)
     return;
@@ -341,7 +347,7 @@ aes_cipher_free(aes_cnt_cipher_t *cipher)
   STMT_END
 #else
 #define UPDATE_CTR_BUF(c, n)
-#endif
+#endif /* defined(USING_COUNTER_VARS) */
 
 /* Helper function to use EVP with openssl's counter-mode wrapper. */
 static void
@@ -396,9 +402,9 @@ aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv)
   cipher->counter2 = ntohl(get_uint32(iv+4));
   cipher->counter1 = ntohl(get_uint32(iv+8));
   cipher->counter0 = ntohl(get_uint32(iv+12));
-#endif
+#endif /* defined(USING_COUNTER_VARS) */
   cipher->pos = 0;
   memcpy(cipher->ctr_buf.buf, iv, 16);
 }
 
-#endif
+#endif /* defined(USE_EVP_AES_CTR) */
diff --git a/src/common/aes.h b/src/common/aes.h
index 1cda53f2fa..0b17cd55a4 100644
--- a/src/common/aes.h
+++ b/src/common/aes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Implements a minimal interface to counter-mode AES. */
@@ -17,11 +17,13 @@ typedef struct aes_cnt_cipher aes_cnt_cipher_t;
 
 aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv,
                                  int key_bits);
-void aes_cipher_free(aes_cnt_cipher_t *cipher);
+void aes_cipher_free_(aes_cnt_cipher_t *cipher);
+#define aes_cipher_free(cipher) \
+  FREE_AND_NULL(aes_cnt_cipher_t, aes_cipher_free_, (cipher))
 void aes_crypt_inplace(aes_cnt_cipher_t *cipher, char *data, size_t len);
 
 int evaluate_evp_for_aes(int force_value);
 int evaluate_ctr_for_aes(void);
 
-#endif
+#endif /* !defined(TOR_AES_H) */
 
diff --git a/src/common/backtrace.c b/src/common/backtrace.c
index 81e04e94eb..f2498b2aa6 100644
--- a/src/common/backtrace.c
+++ b/src/common/backtrace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -37,7 +37,7 @@
 #include <sys/ucontext.h>
 #elif defined(HAVE_UCONTEXT_H)
 #include <ucontext.h>
-#endif
+#endif /* defined(HAVE_CYGWIN_SIGNAL_H) || ... */
 
 #define EXPOSE_CLEAN_BACKTRACE
 #include "backtrace.h"
@@ -76,21 +76,21 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
 #ifdef PC_FROM_UCONTEXT
 #if defined(__linux__)
   const size_t n = 1;
-#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \
+#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \
   || defined(__FreeBSD__)
   const size_t n = 2;
 #else
   const size_t n = 1;
-#endif
+#endif /* defined(__linux__) || ... */
   if (depth <= n)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else
+#else /* !(defined(PC_FROM_UCONTEXT)) */
   (void) depth;
   (void) ctx;
   (void) stack;
-#endif
+#endif /* defined(PC_FROM_UCONTEXT) */
 }
 
 /** Log a message <b>msg</b> at <b>severity</b> in <b>domain</b>, and follow
@@ -202,7 +202,7 @@ remove_bt_handler(void)
 {
   tor_mutex_uninit(&cb_buf_mutex);
 }
-#endif
+#endif /* defined(USE_BACKTRACE) */
 
 #ifdef NO_BACKTRACE_IMPL
 void
@@ -221,7 +221,7 @@ static void
 remove_bt_handler(void)
 {
 }
-#endif
+#endif /* defined(NO_BACKTRACE_IMPL) */
 
 /** Set up code to handle generating error messages on crashes. */
 int
diff --git a/src/common/backtrace.h b/src/common/backtrace.h
index b53fd2c668..3d0ab8a90a 100644
--- a/src/common/backtrace.h
+++ b/src/common/backtrace.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BACKTRACE_H
@@ -15,7 +15,7 @@ void clean_up_backtrace_handler(void);
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx);
 #endif
-#endif
+#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */
 
-#endif
+#endif /* !defined(TOR_BACKTRACE_H) */
 
diff --git a/src/common/buffers.c b/src/common/buffers.c
new file mode 100644
index 0000000000..a01add9bef
--- /dev/null
+++ b/src/common/buffers.c
@@ -0,0 +1,1146 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.c
+ * \brief Implements a generic buffer interface.
+ *
+ * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
+ * from memory, sockets, file descriptors, TLS connections, or another buf_t.
+ * Buffers are implemented as linked lists of memory chunks.
+ *
+ * All socket-backed and TLS-based connection_t objects have a pair of
+ * buffers: one for incoming data, and one for outcoming data.  These are fed
+ * and drained from functions in connection.c, trigged by events that are
+ * monitored in main.c.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "buffers.h"
+#include "compat.h"
+#include "compress.h"
+#include "util.h"
+#include "torint.h"
+#include "torlog.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+//#define PARANOIA
+
+#ifdef PARANOIA
+/** Helper: If PARANOIA is defined, assert that the buffer in local variable
+ * <b>buf</b> is well-formed. */
+#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END
+#else
+#define check() STMT_NIL
+#endif /* defined(PARANOIA) */
+
+/* Implementation notes:
+ *
+ * After flirting with memmove, and dallying with ring-buffers, we're finally
+ * getting up to speed with the 1970s and implementing buffers as a linked
+ * list of small chunks.  Each buffer has such a list; data is removed from
+ * the head of the list, and added at the tail.  The list is singly linked,
+ * and the buffer keeps a pointer to the head and the tail.
+ *
+ * Every chunk, except the tail, contains at least one byte of data.  Data in
+ * each chunk is contiguous.
+ *
+ * When you need to treat the first N characters on a buffer as a contiguous
+ * string, use the buf_pullup function to make them so.  Don't do this more
+ * than necessary.
+ *
+ * The major free Unix kernels have handled buffers like this since, like,
+ * forever.
+ */
+
+/* Chunk manipulation functions */
+
+#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0])
+
+/* We leave this many NUL bytes at the end of the buffer. */
+#ifdef DISABLE_MEMORY_SENTINELS
+#define SENTINEL_LEN 0
+#else
+#define SENTINEL_LEN 4
+#endif
+
+/* Header size plus NUL bytes at the end */
+#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
+
+/** Return the number of bytes needed to allocate a chunk to hold
+ * <b>memlen</b> bytes. */
+#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
+/** Return the number of usable bytes in a chunk allocated with
+ * malloc(<b>memlen</b>). */
+#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
+
+#define DEBUG_SENTINEL
+
+#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS)
+#define DBG_S(s) s
+#else
+#define DBG_S(s) (void)0
+#endif
+
+#ifdef DISABLE_MEMORY_SENTINELS
+#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
+#else
+#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
+    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
+    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
+    DBG_S(tor_assert(a == b));                                          \
+    memset(a,0,SENTINEL_LEN);                                           \
+  } while (0)
+#endif /* defined(DISABLE_MEMORY_SENTINELS) */
+
+/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
+ * to free up space at the end. */
+static inline void
+chunk_repack(chunk_t *chunk)
+{
+  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
+    memmove(chunk->mem, chunk->data, chunk->datalen);
+  }
+  chunk->data = &chunk->mem[0];
+}
+
+/** Keep track of total size of allocated chunks for consistency asserts */
+static size_t total_bytes_allocated_in_chunks = 0;
+static void
+buf_chunk_free_unchecked(chunk_t *chunk)
+{
+  if (!chunk)
+    return;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
+#endif
+  tor_assert(total_bytes_allocated_in_chunks >=
+             CHUNK_ALLOC_SIZE(chunk->memlen));
+  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
+  tor_free(chunk);
+}
+static inline chunk_t *
+chunk_new_with_alloc_size(size_t alloc)
+{
+  chunk_t *ch;
+  ch = tor_malloc(alloc);
+  ch->next = NULL;
+  ch->datalen = 0;
+#ifdef DEBUG_CHUNK_ALLOC
+  ch->DBG_alloc = alloc;
+#endif
+  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
+  total_bytes_allocated_in_chunks += alloc;
+  ch->data = &ch->mem[0];
+  CHUNK_SET_SENTINEL(ch, alloc);
+  return ch;
+}
+
+/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
+ * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
+static inline chunk_t *
+chunk_grow(chunk_t *chunk, size_t sz)
+{
+  off_t offset;
+  const size_t memlen_orig = chunk->memlen;
+  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
+  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
+  tor_assert(sz > chunk->memlen);
+  offset = chunk->data - chunk->mem;
+  chunk = tor_realloc(chunk, new_alloc);
+  chunk->memlen = sz;
+  chunk->data = chunk->mem + offset;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(chunk->DBG_alloc == orig_alloc);
+  chunk->DBG_alloc = new_alloc;
+#endif
+  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
+  CHUNK_SET_SENTINEL(chunk, new_alloc);
+  return chunk;
+}
+
+/** Every chunk should take up at least this many bytes. */
+#define MIN_CHUNK_ALLOC 256
+/** No chunk should take up more than this many bytes. */
+#define MAX_CHUNK_ALLOC 65536
+
+/** Return the allocation size we'd like to use to hold <b>target</b>
+ * bytes. */
+size_t
+buf_preferred_chunk_size(size_t target)
+{
+  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
+  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
+    return CHUNK_ALLOC_SIZE(target);
+  size_t sz = MIN_CHUNK_ALLOC;
+  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
+    sz <<= 1;
+  }
+  return sz;
+}
+
+/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
+ * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
+ * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
+ *
+ * Set *<b>head_out</b> to point to the first byte of available data, and
+ * *<b>len_out</b> to the number of bytes of data available at
+ * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than
+ * <b>bytes</b>, depending on the number of bytes available.
+ */
+void
+buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out)
+{
+  chunk_t *dest, *src;
+  size_t capacity;
+  if (!buf->head) {
+    *head_out = NULL;
+    *len_out = 0;
+    return;
+  }
+
+  check();
+  if (buf->datalen < bytes)
+    bytes = buf->datalen;
+
+  capacity = bytes;
+  if (buf->head->datalen >= bytes) {
+    *head_out = buf->head->data;
+    *len_out = buf->head->datalen;
+    return;
+  }
+
+  if (buf->head->memlen >= capacity) {
+    /* We don't need to grow the first chunk, but we might need to repack it.*/
+    size_t needed = capacity - buf->head->datalen;
+    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
+      chunk_repack(buf->head);
+    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
+  } else {
+    chunk_t *newhead;
+    size_t newsize;
+    /* We need to grow the chunk. */
+    chunk_repack(buf->head);
+    newsize = CHUNK_SIZE_WITH_ALLOC(buf_preferred_chunk_size(capacity));
+    newhead = chunk_grow(buf->head, newsize);
+    tor_assert(newhead->memlen >= capacity);
+    if (newhead != buf->head) {
+      if (buf->tail == buf->head)
+        buf->tail = newhead;
+      buf->head = newhead;
+    }
+  }
+
+  dest = buf->head;
+  while (dest->datalen < bytes) {
+    size_t n = bytes - dest->datalen;
+    src = dest->next;
+    tor_assert(src);
+    if (n >= src->datalen) {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
+      dest->datalen += src->datalen;
+      dest->next = src->next;
+      if (buf->tail == src)
+        buf->tail = dest;
+      buf_chunk_free_unchecked(src);
+    } else {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
+      dest->datalen += n;
+      src->data += n;
+      src->datalen -= n;
+      tor_assert(dest->datalen == bytes);
+    }
+  }
+
+  check();
+  *head_out = buf->head->data;
+  *len_out = buf->head->datalen;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* Write sz bytes from cp into a newly allocated buffer buf.
+ * Returns NULL when passed a NULL cp or zero sz.
+ * Asserts on failure: only for use in unit tests.
+ * buf must be freed using buf_free(). */
+buf_t *
+buf_new_with_data(const char *cp, size_t sz)
+{
+  /* Validate arguments */
+  if (!cp || sz <= 0) {
+    return NULL;
+  }
+
+  tor_assert(sz < SSIZE_T_CEILING);
+
+  /* Allocate a buffer */
+  buf_t *buf = buf_new_with_capacity(sz);
+  tor_assert(buf);
+  buf_assert_ok(buf);
+  tor_assert(!buf->head);
+
+  /* Allocate a chunk that is sz bytes long */
+  buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz));
+  buf->tail = buf->head;
+  tor_assert(buf->head);
+  buf_assert_ok(buf);
+  tor_assert(buf_allocation(buf) >= sz);
+
+  /* Copy the data and size the buffers */
+  tor_assert(sz <= buf_slack(buf));
+  tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head));
+  memcpy(&buf->head->mem[0], cp, sz);
+  buf->datalen = sz;
+  buf->head->datalen = sz;
+  buf->head->data = &buf->head->mem[0];
+  buf_assert_ok(buf);
+
+  /* Make sure everything is large enough */
+  tor_assert(buf_allocation(buf) >= sz);
+  tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf));
+  /* Does the buffer implementation allocate more than the requested size?
+   * (for example, by rounding up). If so, these checks will fail. */
+  tor_assert(buf_datalen(buf) == sz);
+  tor_assert(buf_slack(buf) == 0);
+
+  return buf;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** Remove the first <b>n</b> bytes from buf. */
+void
+buf_drain(buf_t *buf, size_t n)
+{
+  tor_assert(buf->datalen >= n);
+  while (n) {
+    tor_assert(buf->head);
+    if (buf->head->datalen > n) {
+      buf->head->datalen -= n;
+      buf->head->data += n;
+      buf->datalen -= n;
+      return;
+    } else {
+      chunk_t *victim = buf->head;
+      n -= victim->datalen;
+      buf->datalen -= victim->datalen;
+      buf->head = victim->next;
+      if (buf->tail == victim)
+        buf->tail = NULL;
+      buf_chunk_free_unchecked(victim);
+    }
+  }
+  check();
+}
+
+/** Create and return a new buf with default chunk capacity <b>size</b>.
+ */
+buf_t *
+buf_new_with_capacity(size_t size)
+{
+  buf_t *b = buf_new();
+  b->default_chunk_size = buf_preferred_chunk_size(size);
+  return b;
+}
+
+/** Allocate and return a new buffer with default capacity. */
+buf_t *
+buf_new(void)
+{
+  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
+  buf->magic = BUFFER_MAGIC;
+  buf->default_chunk_size = 4096;
+  return buf;
+}
+
+size_t
+buf_get_default_chunk_size(const buf_t *buf)
+{
+  return buf->default_chunk_size;
+}
+
+/** Remove all data from <b>buf</b>. */
+void
+buf_clear(buf_t *buf)
+{
+  chunk_t *chunk, *next;
+  buf->datalen = 0;
+  for (chunk = buf->head; chunk; chunk = next) {
+    next = chunk->next;
+    buf_chunk_free_unchecked(chunk);
+  }
+  buf->head = buf->tail = NULL;
+}
+
+/** Return the number of bytes stored in <b>buf</b> */
+MOCK_IMPL(size_t,
+buf_datalen, (const buf_t *buf))
+{
+  return buf->datalen;
+}
+
+/** Return the total length of all chunks used in <b>buf</b>. */
+size_t
+buf_allocation(const buf_t *buf)
+{
+  size_t total = 0;
+  const chunk_t *chunk;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    total += CHUNK_ALLOC_SIZE(chunk->memlen);
+  }
+  return total;
+}
+
+/** Return the number of bytes that can be added to <b>buf</b> without
+ * performing any additional allocation. */
+size_t
+buf_slack(const buf_t *buf)
+{
+  if (!buf->tail)
+    return 0;
+  else
+    return CHUNK_REMAINING_CAPACITY(buf->tail);
+}
+
+/** Release storage held by <b>buf</b>. */
+void
+buf_free_(buf_t *buf)
+{
+  if (!buf)
+    return;
+
+  buf_clear(buf);
+  buf->magic = 0xdeadbeef;
+  tor_free(buf);
+}
+
+/** Return a new copy of <b>in_chunk</b> */
+static chunk_t *
+chunk_copy(const chunk_t *in_chunk)
+{
+  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
+  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#ifdef DEBUG_CHUNK_ALLOC
+  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#endif
+  newch->next = NULL;
+  if (in_chunk->data) {
+    off_t offset = in_chunk->data - in_chunk->mem;
+    newch->data = newch->mem + offset;
+  }
+  return newch;
+}
+
+/** Return a new copy of <b>buf</b> */
+buf_t *
+buf_copy(const buf_t *buf)
+{
+  chunk_t *ch;
+  buf_t *out = buf_new();
+  out->default_chunk_size = buf->default_chunk_size;
+  for (ch = buf->head; ch; ch = ch->next) {
+    chunk_t *newch = chunk_copy(ch);
+    if (out->tail) {
+      out->tail->next = newch;
+      out->tail = newch;
+    } else {
+      out->head = out->tail = newch;
+    }
+  }
+  out->datalen = buf->datalen;
+  return out;
+}
+
+/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
+ * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
+ * than MAX_CHUNK_ALLOC. */
+chunk_t *
+buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
+{
+  chunk_t *chunk;
+
+  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
+    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
+  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
+    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
+  } else {
+    chunk = chunk_new_with_alloc_size(buf_preferred_chunk_size(capacity));
+  }
+
+  chunk->inserted_time = monotime_coarse_get_stamp();
+
+  if (buf->tail) {
+    tor_assert(buf->head);
+    buf->tail->next = chunk;
+    buf->tail = chunk;
+  } else {
+    tor_assert(!buf->head);
+    buf->head = buf->tail = chunk;
+  }
+  check();
+  return chunk;
+}
+
+/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
+ * timestamp units.  Requires the current monotonic timestamp as its
+ * input <b>now</b>.
+ */
+uint32_t
+buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
+{
+  if (buf->head) {
+    return now - buf->head->inserted_time;
+  } else {
+    return 0;
+  }
+}
+
+size_t
+buf_get_total_allocation(void)
+{
+  return total_bytes_allocated_in_chunks;
+}
+
+/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+ * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
+ * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
+ * and the number of bytes read otherwise. */
+static inline int
+read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
+              int *reached_eof, int *socket_error)
+{
+  ssize_t read_result;
+  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
+    at_most = CHUNK_REMAINING_CAPACITY(chunk);
+  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (read_result < 0) {
+    int e = tor_socket_errno(fd);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      *socket_error = e;
+      return -1;
+    }
+    return 0; /* would block. */
+  } else if (read_result == 0) {
+    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
+    *reached_eof = 1;
+    return 0;
+  } else { /* actually got bytes. */
+    buf->datalen += read_result;
+    chunk->datalen += read_result;
+    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
+              (int)buf->datalen);
+    tor_assert(read_result < INT_MAX);
+    return (int)read_result;
+  }
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+/* XXXX indicate "read blocked" somehow? */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes read" are not mutually exclusive.
+   */
+  int r = 0;
+  size_t total_read = 0;
+
+  check();
+  tor_assert(reached_eof);
+  tor_assert(SOCKET_OK(s));
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    check();
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
+      break;
+    }
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
+ * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, 0 on blocking, -1 on failure.
+ */
+static inline int
+flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen)
+{
+  ssize_t write_result;
+
+  if (sz > chunk->datalen)
+    sz = chunk->datalen;
+  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (write_result < 0) {
+    int e = tor_socket_errno(s);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      return -1;
+    }
+    log_debug(LD_NET,"write() would block, returning.");
+    return 0;
+  } else {
+    *buf_flushlen -= write_result;
+    buf_drain(buf, write_result);
+    tor_assert(write_result < INT_MAX);
+    return (int)write_result;
+  }
+}
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes flushed" are not mutually exclusive.
+   */
+  int r;
+  size_t flushed = 0;
+  tor_assert(buf_flushlen);
+  tor_assert(SOCKET_OK(s));
+  if (BUG(*buf_flushlen > buf->datalen)) {
+    *buf_flushlen = buf->datalen;
+  }
+  if (BUG(sz > *buf_flushlen)) {
+    sz = *buf_flushlen;
+  }
+
+  check();
+  while (sz) {
+    size_t flushlen0;
+    tor_assert(buf->head);
+    if (buf->head->datalen >= sz)
+      flushlen0 = sz;
+    else
+      flushlen0 = buf->head->datalen;
+
+    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    check();
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
+      break;
+  }
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
+
+/** Append <b>string_len</b> bytes from <b>string</b> to the end of
+ * <b>buf</b>.
+ *
+ * Return the new length of the buffer on success, -1 on failure.
+ */
+int
+buf_add(buf_t *buf, const char *string, size_t string_len)
+{
+  if (!string_len)
+    return (int)buf->datalen;
+  check();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - string_len))
+    return -1;
+
+  while (string_len) {
+    size_t copy;
+    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
+      buf_add_chunk_with_capacity(buf, string_len, 1);
+
+    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
+    if (copy > string_len)
+      copy = string_len;
+    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
+    string_len -= copy;
+    string += copy;
+    buf->datalen += copy;
+    buf->tail->datalen += copy;
+  }
+
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Add a nul-terminated <b>string</b> to <b>buf</b>, not including the
+ * terminating NUL. */
+void
+buf_add_string(buf_t *buf, const char *string)
+{
+  buf_add(buf, string, strlen(string));
+}
+
+/** As tor_snprintf, but write the results into a buf_t */
+void
+buf_add_printf(buf_t *buf, const char *format, ...)
+{
+  va_list ap;
+  va_start(ap,format);
+  buf_add_vprintf(buf, format, ap);
+  va_end(ap);
+}
+
+/** As tor_vsnprintf, but write the results into a buf_t. */
+void
+buf_add_vprintf(buf_t *buf, const char *format, va_list args)
+{
+  /* XXXX Faster implementations are easy enough, but let's optimize later */
+  char *tmp;
+  tor_vasprintf(&tmp, format, args);
+  buf_add(buf, tmp, strlen(tmp));
+  tor_free(tmp);
+}
+
+/** Return a heap-allocated string containing the contents of <b>buf</b>, plus
+ * a NUL byte. If <b>sz_out</b> is provided, set *<b>sz_out</b> to the length
+ * of the returned string, not including the terminating NUL. */
+char *
+buf_extract(buf_t *buf, size_t *sz_out)
+{
+  tor_assert(buf);
+
+  size_t sz = buf_datalen(buf);
+  char *result;
+  result = tor_malloc(sz+1);
+  buf_peek(buf, result, sz);
+  result[sz] = 0;
+  if (sz_out)
+    *sz_out = sz;
+  return result;
+}
+
+/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
+ * onto <b>string</b>.
+ */
+void
+buf_peek(const buf_t *buf, char *string, size_t string_len)
+{
+  chunk_t *chunk;
+
+  tor_assert(string);
+  /* make sure we don't ask for too much */
+  tor_assert(string_len <= buf->datalen);
+  /* buf_assert_ok(buf); */
+
+  chunk = buf->head;
+  while (string_len) {
+    size_t copy = string_len;
+    tor_assert(chunk);
+    if (chunk->datalen < copy)
+      copy = chunk->datalen;
+    memcpy(string, chunk->data, copy);
+    string_len -= copy;
+    string += copy;
+    chunk = chunk->next;
+  }
+}
+
+/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
+ * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
+ * must be \<= the number of bytes on the buffer.
+ */
+int
+buf_get_bytes(buf_t *buf, char *string, size_t string_len)
+{
+  /* There must be string_len bytes in buf; write them onto string,
+   * then memmove buf back (that is, remove them from buf).
+   *
+   * Return the number of bytes still on the buffer. */
+
+  check();
+  buf_peek(buf, string, string_len);
+  buf_drain(buf, string_len);
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
+ * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
+ * Return the number of bytes actually copied.
+ */
+int
+buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
+{
+  /* We can do way better here, but this doesn't turn up in any profiles. */
+  char b[4096];
+  size_t cp, len;
+
+  if (BUG(buf_out->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+    return -1;
+
+  len = *buf_flushlen;
+  if (len > buf_in->datalen)
+    len = buf_in->datalen;
+
+  cp = len; /* Remember the number of bytes we intend to copy. */
+  tor_assert(cp < INT_MAX);
+  while (len) {
+    /* This isn't the most efficient implementation one could imagine, since
+     * it does two copies instead of 1, but I kinda doubt that this will be
+     * critical path. */
+    size_t n = len > sizeof(b) ? sizeof(b) : len;
+    buf_get_bytes(buf_in, b, n);
+    buf_add(buf_out, b, n);
+    len -= n;
+  }
+  *buf_flushlen -= cp;
+  return (int)cp;
+}
+
+/** Moves all data from <b>buf_in</b> to <b>buf_out</b>, without copying.
+ */
+void
+buf_move_all(buf_t *buf_out, buf_t *buf_in)
+{
+  tor_assert(buf_out);
+  if (!buf_in)
+    return;
+
+  if (buf_out->head == NULL) {
+    buf_out->head = buf_in->head;
+    buf_out->tail = buf_in->tail;
+  } else {
+    buf_out->tail->next = buf_in->head;
+    buf_out->tail = buf_in->tail;
+  }
+
+  buf_out->datalen += buf_in->datalen;
+  buf_in->head = buf_in->tail = NULL;
+  buf_in->datalen = 0;
+}
+
+/** Internal structure: represents a position in a buffer. */
+typedef struct buf_pos_t {
+  const chunk_t *chunk; /**< Which chunk are we pointing to? */
+  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  size_t chunk_pos; /**< Total length of all previous chunks. */
+} buf_pos_t;
+
+/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
+static void
+buf_pos_init(const buf_t *buf, buf_pos_t *out)
+{
+  out->chunk = buf->head;
+  out->pos = 0;
+  out->chunk_pos = 0;
+}
+
+/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
+ * position of <b>out</b>, or later.  Return -1 if no instances are found;
+ * otherwise returns the absolute position of the character. */
+static off_t
+buf_find_pos_of_char(char ch, buf_pos_t *out)
+{
+  const chunk_t *chunk;
+  int pos;
+  tor_assert(out);
+  if (out->chunk) {
+    if (out->chunk->datalen) {
+      tor_assert(out->pos < (off_t)out->chunk->datalen);
+    } else {
+      tor_assert(out->pos == 0);
+    }
+  }
+  pos = out->pos;
+  for (chunk = out->chunk; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
+    if (cp) {
+      out->chunk = chunk;
+      tor_assert(cp - chunk->data < INT_MAX);
+      out->pos = (int)(cp - chunk->data);
+      return out->chunk_pos + out->pos;
+    } else {
+      out->chunk_pos += chunk->datalen;
+      pos = 0;
+    }
+  }
+  return -1;
+}
+
+/** Advance <b>pos</b> by a single character, if there are any more characters
+ * in the buffer.  Returns 0 on success, -1 on failure. */
+static inline int
+buf_pos_inc(buf_pos_t *pos)
+{
+  ++pos->pos;
+  if (pos->pos == (off_t)pos->chunk->datalen) {
+    if (!pos->chunk->next)
+      return -1;
+    pos->chunk_pos += pos->chunk->datalen;
+    pos->chunk = pos->chunk->next;
+    pos->pos = 0;
+  }
+  return 0;
+}
+
+/** Return true iff the <b>n</b>-character string in <b>s</b> appears
+ * (verbatim) at <b>pos</b>. */
+static int
+buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
+{
+  buf_pos_t p;
+  if (!n)
+    return 1;
+
+  memcpy(&p, pos, sizeof(p));
+
+  while (1) {
+    char ch = p.chunk->data[p.pos];
+    if (ch != *s)
+      return 0;
+    ++s;
+    /* If we're out of characters that don't match, we match.  Check this
+     * _before_ we test incrementing pos, in case we're at the end of the
+     * string. */
+    if (--n == 0)
+      return 1;
+    if (buf_pos_inc(&p)<0)
+      return 0;
+  }
+}
+
+/** Return the first position in <b>buf</b> at which the <b>n</b>-character
+ * string <b>s</b> occurs, or -1 if it does not occur. */
+int
+buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
+{
+  buf_pos_t pos;
+  buf_pos_init(buf, &pos);
+  while (buf_find_pos_of_char(*s, &pos) >= 0) {
+    if (buf_matches_at_pos(&pos, s, n)) {
+      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
+      return (int)(pos.chunk_pos + pos.pos);
+    } else {
+      if (buf_pos_inc(&pos)<0)
+        return -1;
+    }
+  }
+  return -1;
+}
+
+/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null
+ * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */
+int
+buf_peek_startswith(const buf_t *buf, const char *cmd)
+{
+  char tmp[PEEK_BUF_STARTSWITH_MAX];
+  size_t clen = strlen(cmd);
+  if (clen == 0)
+    return 1;
+  if (BUG(clen > sizeof(tmp)))
+    return 0;
+  if (buf->datalen < clen)
+    return 0;
+  buf_peek(buf, tmp, clen);
+  return fast_memeq(tmp, cmd, clen);
+}
+
+/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
+ * or -1 if <b>ch</b> does not appear on buf. */
+static off_t
+buf_find_offset_of_char(buf_t *buf, char ch)
+{
+  chunk_t *chunk;
+  off_t offset = 0;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data, ch, chunk->datalen);
+    if (cp)
+      return offset + (cp - chunk->data);
+    else
+      offset += chunk->datalen;
+  }
+  return -1;
+}
+
+/** Try to read a single LF-terminated line from <b>buf</b>, and write it
+ * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
+ * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
+ * line, not counting the terminating NUL.  Return 1 if we read a whole line,
+ * return 0 if we don't have a whole line yet, and return -1 if the line
+ * length exceeds *<b>data_len</b>.
+ */
+int
+buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
+{
+  size_t sz;
+  off_t offset;
+
+  if (!buf->head)
+    return 0;
+
+  offset = buf_find_offset_of_char(buf, '\n');
+  if (offset < 0)
+    return 0;
+  sz = (size_t) offset;
+  if (sz+2 > *data_len) {
+    *data_len = sz + 2;
+    return -1;
+  }
+  buf_get_bytes(buf, data_out, sz+1);
+  data_out[sz+1] = '\0';
+  *data_len = sz+1;
+  return 1;
+}
+
+/** Compress or uncompress the <b>data_len</b> bytes in <b>data</b> using the
+ * compression state <b>state</b>, appending the result to <b>buf</b>.  If
+ * <b>done</b> is true, flush the data in the state and finish the
+ * compression/uncompression.  Return -1 on failure, 0 on success. */
+int
+buf_add_compress(buf_t *buf, tor_compress_state_t *state,
+                      const char *data, size_t data_len,
+                      const int done)
+{
+  char *next;
+  size_t old_avail, avail;
+  int over = 0;
+
+  do {
+    int need_new_chunk = 0;
+    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
+      size_t cap = data_len / 4;
+      buf_add_chunk_with_capacity(buf, cap, 1);
+    }
+    next = CHUNK_WRITE_PTR(buf->tail);
+    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
+    switch (tor_compress_process(state, &next, &avail,
+                                 &data, &data_len, done)) {
+      case TOR_COMPRESS_DONE:
+        over = 1;
+        break;
+      case TOR_COMPRESS_ERROR:
+        return -1;
+      case TOR_COMPRESS_OK:
+        if (data_len == 0) {
+          tor_assert_nonfatal(!done);
+          over = 1;
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL:
+        if (avail) {
+          /* The compression module says we need more room
+           * (TOR_COMPRESS_BUFFER_FULL).  Start a new chunk automatically,
+           * whether were going to or not. */
+          need_new_chunk = 1;
+        }
+        if (data_len == 0 && !done) {
+          /* We've consumed all the input data, though, so there's no
+           * point in forging ahead right now. */
+          over = 1;
+        }
+        break;
+    }
+    buf->datalen += old_avail - avail;
+    buf->tail->datalen += old_avail - avail;
+    if (need_new_chunk) {
+      buf_add_chunk_with_capacity(buf, data_len/4, 1);
+    }
+
+  } while (!over);
+  check();
+  return 0;
+}
+
+/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
+int
+buf_set_to_copy(buf_t **output,
+                const buf_t *input)
+{
+  if (*output)
+    buf_free(*output);
+  *output = buf_copy(input);
+  return 0;
+}
+
+/** Log an error and exit if <b>buf</b> is corrupted.
+ */
+void
+buf_assert_ok(buf_t *buf)
+{
+  tor_assert(buf);
+  tor_assert(buf->magic == BUFFER_MAGIC);
+
+  if (! buf->head) {
+    tor_assert(!buf->tail);
+    tor_assert(buf->datalen == 0);
+  } else {
+    chunk_t *ch;
+    size_t total = 0;
+    tor_assert(buf->tail);
+    for (ch = buf->head; ch; ch = ch->next) {
+      total += ch->datalen;
+      tor_assert(ch->datalen <= ch->memlen);
+      tor_assert(ch->data >= &ch->mem[0]);
+      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
+      if (ch->data == &ch->mem[0]+ch->memlen) {
+        /* LCOV_EXCL_START */
+        static int warned = 0;
+        if (! warned) {
+          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
+          warned = 1;
+        }
+        /* LCOV_EXCL_STOP */
+      }
+      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
+      if (!ch->next)
+        tor_assert(ch == buf->tail);
+    }
+    tor_assert(buf->datalen == total);
+  }
+}
+
diff --git a/src/common/buffers.h b/src/common/buffers.h
new file mode 100644
index 0000000000..4275152de2
--- /dev/null
+++ b/src/common/buffers.h
@@ -0,0 +1,131 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.h
+ * \brief Header file for buffers.c.
+ **/
+
+#ifndef TOR_BUFFERS_H
+#define TOR_BUFFERS_H
+
+#include "compat.h"
+#include "torint.h"
+#include "testsupport.h"
+
+typedef struct buf_t buf_t;
+
+struct tor_compress_state_t;
+
+buf_t *buf_new(void);
+buf_t *buf_new_with_capacity(size_t size);
+size_t buf_get_default_chunk_size(const buf_t *buf);
+void buf_free_(buf_t *buf);
+#define buf_free(b) FREE_AND_NULL(buf_t, buf_free_, (b))
+void buf_clear(buf_t *buf);
+buf_t *buf_copy(const buf_t *buf);
+
+MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
+size_t buf_allocation(const buf_t *buf);
+size_t buf_slack(const buf_t *buf);
+
+uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
+size_t buf_get_total_allocation(void);
+
+int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                         int *reached_eof,
+                         int *socket_error);
+
+int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                        size_t *buf_flushlen);
+
+int buf_add(buf_t *buf, const char *string, size_t string_len);
+void buf_add_string(buf_t *buf, const char *string);
+void buf_add_printf(buf_t *buf, const char *format, ...)
+  CHECK_PRINTF(2, 3);
+void buf_add_vprintf(buf_t *buf, const char *format, va_list args)
+  CHECK_PRINTF(2, 0);
+int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state,
+                          const char *data, size_t data_len, int done);
+int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
+void buf_move_all(buf_t *buf_out, buf_t *buf_in);
+void buf_peek(const buf_t *buf, char *string, size_t string_len);
+void buf_drain(buf_t *buf, size_t n);
+int buf_get_bytes(buf_t *buf, char *string, size_t string_len);
+int buf_get_line(buf_t *buf, char *data_out, size_t *data_len);
+
+#define PEEK_BUF_STARTSWITH_MAX 16
+int buf_peek_startswith(const buf_t *buf, const char *cmd);
+
+int buf_set_to_copy(buf_t **output,
+                    const buf_t *input);
+
+void buf_assert_ok(buf_t *buf);
+
+int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
+void buf_pullup(buf_t *buf, size_t bytes,
+                const char **head_out, size_t *len_out);
+char *buf_extract(buf_t *buf, size_t *sz_out);
+
+#ifdef BUFFERS_PRIVATE
+#ifdef TOR_UNIT_TESTS
+buf_t *buf_new_with_data(const char *cp, size_t sz);
+#endif
+size_t buf_preferred_chunk_size(size_t target);
+
+#define DEBUG_CHUNK_ALLOC
+/** A single chunk on a buffer. */
+typedef struct chunk_t {
+  struct chunk_t *next; /**< The next chunk on the buffer. */
+  size_t datalen; /**< The number of bytes stored in this chunk */
+  size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */
+#ifdef DEBUG_CHUNK_ALLOC
+  size_t DBG_alloc;
+#endif
+  char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */
+  uint32_t inserted_time; /**< Timestamp when this chunk was inserted. */
+  char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in
+                * this chunk. */
+} chunk_t;
+
+/** Magic value for buf_t.magic, to catch pointer errors. */
+#define BUFFER_MAGIC 0xB0FFF312u
+/** A resizeable buffer, optimized for reading and writing. */
+struct buf_t {
+  uint32_t magic; /**< Magic cookie for debugging: Must be set to
+                   *   BUFFER_MAGIC. */
+  size_t datalen; /**< How many bytes is this buffer holding right now? */
+  size_t default_chunk_size; /**< Don't allocate any chunks smaller than
+                              * this for this buffer. */
+  chunk_t *head; /**< First chunk in the list, or NULL for none. */
+  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
+};
+
+chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped);
+/** If a read onto the end of a chunk would be smaller than this number, then
+ * just start a new chunk. */
+#define MIN_READ_LEN 8
+
+/** Return the number of bytes that can be written onto <b>chunk</b> without
+ * running out of space. */
+static inline size_t
+CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
+{
+  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
+}
+
+/** Return the next character in <b>chunk</b> onto which data can be appended.
+ * If the chunk is full, this might be off the end of chunk->mem. */
+static inline char *
+CHUNK_WRITE_PTR(chunk_t *chunk)
+{
+  return chunk->data + chunk->datalen;
+}
+
+#endif /* defined(BUFFERS_PRIVATE) */
+
+#endif /* !defined(TOR_BUFFERS_H) */
+
diff --git a/src/common/buffers_tls.c b/src/common/buffers_tls.c
new file mode 100644
index 0000000000..041f78b818
--- /dev/null
+++ b/src/common/buffers_tls.c
@@ -0,0 +1,179 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "buffers.h"
+#include "buffers_tls.h"
+#include "compat.h"
+#include "compress.h"
+#include "util.h"
+#include "torint.h"
+#include "torlog.h"
+#include "tortls.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** As read_to_chunk(), but return (negative) error code on error, blocking,
+ * or TLS, and the number of bytes read otherwise. */
+static inline int
+read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
+                  size_t at_most)
+{
+  int read_result;
+
+  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
+  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
+  if (read_result < 0)
+    return read_result;
+  buf->datalen += read_result;
+  chunk->datalen += read_result;
+  return read_result;
+}
+
+/** As read_to_buf, but reads from a TLS connection, and returns a TLS
+ * status value rather than the number of bytes read.
+ *
+ * Using TLS on OR connections complicates matters in two ways.
+ *
+ * First, a TLS stream has its own read buffer independent of the
+ * connection's read buffer.  (TLS needs to read an entire frame from
+ * the network before it can decrypt any data.  Thus, trying to read 1
+ * byte from TLS can require that several KB be read from the network
+ * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
+ * Because the data hasn't been read by Tor (it's still inside the TLS),
+ * this means that sometimes a connection "has stuff to read" even when
+ * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
+ * used in connection.c to detect TLS objects with non-empty internal
+ * buffers and read from them again.
+ *
+ * Second, the TLS stream's events do not correspond directly to network
+ * events: sometimes, before a TLS stream can read, the network must be
+ * ready to write -- or vice versa.
+ */
+int
+buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
+{
+  int r = 0;
+  size_t total_read = 0;
+
+  check_no_tls_errors();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk_tls(buf, chunk, tls, readlen);
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) /* eof, block, or no more to read. */
+      break;
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
+ * more if there is a forced pending write size.)  On success, deduct the
+ * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, and a TOR_TLS error code on failure or blocking.
+ */
+static inline int
+flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
+                size_t sz, size_t *buf_flushlen)
+{
+  int r;
+  size_t forced;
+  char *data;
+
+  forced = tor_tls_get_forced_write_size(tls);
+  if (forced > sz)
+    sz = forced;
+  if (chunk) {
+    data = chunk->data;
+    tor_assert(sz <= chunk->datalen);
+  } else {
+    data = NULL;
+    tor_assert(sz == 0);
+  }
+  r = tor_tls_write(tls, data, sz);
+  if (r < 0)
+    return r;
+  if (*buf_flushlen > (size_t)r)
+    *buf_flushlen -= r;
+  else
+    *buf_flushlen = 0;
+  buf_drain(buf, r);
+  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
+            r,(int)*buf_flushlen,(int)buf->datalen);
+  return r;
+}
+
+/** As buf_flush_to_socket(), but writes data to a TLS connection.  Can write
+ * more than <b>flushlen</b> bytes.
+ */
+int
+buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
+              size_t *buf_flushlen)
+{
+  int r;
+  size_t flushed = 0;
+  ssize_t sz;
+  tor_assert(buf_flushlen);
+  if (BUG(*buf_flushlen > buf->datalen)) {
+    *buf_flushlen = buf->datalen;
+  }
+  if (BUG(flushlen > *buf_flushlen)) {
+    flushlen = *buf_flushlen;
+  }
+  sz = (ssize_t) flushlen;
+
+  /* we want to let tls write even if flushlen is zero, because it might
+   * have a partial record pending */
+  check_no_tls_errors();
+
+  do {
+    size_t flushlen0;
+    if (buf->head) {
+      if ((ssize_t)buf->head->datalen >= sz)
+        flushlen0 = sz;
+      else
+        flushlen0 = buf->head->datalen;
+    } else {
+      flushlen0 = 0;
+    }
+
+    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0) /* Can't flush any more now. */
+      break;
+  } while (sz > 0);
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
+
diff --git a/src/common/buffers_tls.h b/src/common/buffers_tls.h
new file mode 100644
index 0000000000..2f9fda45a0
--- /dev/null
+++ b/src/common/buffers_tls.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BUFFERS_TLS_H
+#define TOR_BUFFERS_TLS_H
+
+struct buf_t;
+struct tor_tls_t;
+
+int buf_read_from_tls(struct buf_t *buf,
+                      struct tor_tls_t *tls, size_t at_most);
+int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls,
+                     size_t sz, size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_TLS_H) */
+
diff --git a/src/common/ciphers.inc b/src/common/ciphers.inc
index 23f5fd2da4..0084b3e325 100644
--- a/src/common/ciphers.inc
+++ b/src/common/ciphers.inc
@@ -33,6 +33,26 @@
 #else
    XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
 #endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+    CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
+#else
+   XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
+    CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
+#else
+   XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
+#endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+    CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+#else
+   XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+    CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+#else
+   XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+#endif
 #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
     CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA)
 #else
@@ -53,88 +73,28 @@
 #else
    XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA)
 #endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA
-    CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
-#else
-   XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
-#endif
-#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA
-    CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
-#else
-   XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
-#endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA
-    CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
-#else
-   XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
-#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
     CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #endif
-#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA
-    CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
-#else
-   XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
-#endif
-#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA
-    CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#else
-   XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA
     CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #endif
-#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA
-    CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
-#else
-   XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
-#endif
-#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA
-    CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#else
-   XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#endif
-#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA
-    CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
-#else
-   XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
-#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA
     CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #endif
-#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA
-    CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#else
-   XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA
     CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #endif
-#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA
-    CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#else
-   XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#endif
 #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA
     CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #else
    XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #endif
-#ifdef SSL3_TXT_RSA_RC4_128_SHA
-    CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
-#else
-   XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
-#endif
-#ifdef SSL3_TXT_RSA_RC4_128_MD5
-    CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
-#else
-   XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
-#endif
diff --git a/src/common/compat.c b/src/common/compat.c
index 9758751122..f8c4bdb4d8 100644
--- a/src/common/compat.c
+++ b/src/common/compat.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -88,21 +88,20 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
   while (cnt--)
     *vcptr++ = 0;
 }
-#endif
+#endif /* defined(HAVE_DECL_SECUREZEROMEMORY) && !HAVE_DECL_SECUREZEROMEMORY */
 #elif defined(HAVE_READPASSPHRASE_H)
 #include <readpassphrase.h>
 #else
 #include "tor_readpassphrase.h"
-#endif
+#endif /* defined(_WIN32) || ... */
 
 /* Includes for the process attaching prevention */
 #if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
 /* Only use the linux prctl;  the IRIX prctl is totally different */
 #include <sys/prctl.h>
 #elif defined(__APPLE__)
-#include <sys/types.h>
 #include <sys/ptrace.h>
-#endif
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) || ... */
 
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
@@ -116,7 +115,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
-#ifdef HAVE_SYS_MMAN_H
+#ifdef HAVE_MMAP
 #include <sys/mman.h>
 #endif
 #ifdef HAVE_SYS_SYSLIMITS_H
@@ -161,7 +160,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode)
    * are running on one without. */
   if (errno != EINVAL)
     return -1;
-#endif
+#endif /* defined(O_CLOEXEC) */
 
   log_debug(LD_FS, "Opening %s with flags %x", p, flags);
   fd = open(p, flags, mode);
@@ -173,7 +172,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode)
       return -1;
     }
   }
-#endif
+#endif /* defined(FD_CLOEXEC) */
   return fd;
 }
 
@@ -191,7 +190,7 @@ tor_fopen_cloexec(const char *path, const char *mode)
       return NULL;
     }
   }
-#endif
+#endif /* defined(FD_CLOEXEC) */
   return result;
 }
 
@@ -204,16 +203,17 @@ tor_rename(const char *path_old, const char *path_new)
                 sandbox_intern_string(path_new));
 }
 
-#if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN)
+#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN)
 /** Try to create a memory mapping for <b>filename</b> and return it.  On
- * failure, return NULL.  Sets errno properly, using ERANGE to mean
- * "empty file". */
+ * failure, return NULL. Sets errno properly, using ERANGE to mean
+ * "empty file". Must only be called on trusted Tor-owned files, as changing
+ * the underlying file's size causes unspecified behavior. */
 tor_mmap_t *
 tor_mmap_file(const char *filename)
 {
   int fd; /* router file */
   char *string;
-  int page_size, result;
+  int result;
   tor_mmap_t *res;
   size_t size, filesize;
   struct stat st;
@@ -242,14 +242,13 @@ tor_mmap_file(const char *filename)
     return NULL;
   }
   size = filesize = (size_t)(st.st_size);
-  /*
-   * Should we check for weird crap like mmapping a named pipe here,
-   * or just wait for if (!size) below to fail?
-   */
-  /* ensure page alignment */
-  page_size = getpagesize();
-  size += (size%page_size) ? page_size-(size%page_size) : 0;
 
+  if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
+    log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
+    errno = EFBIG;
+    close(fd);
+    return NULL;
+  }
   if (!size) {
     /* Zero-length file. If we call mmap on it, it will succeed but
      * return NULL, and bad things will happen. So just fail. */
@@ -403,40 +402,8 @@ tor_munmap_file(tor_mmap_t *handle)
   return 0;
 }
 #else
-tor_mmap_t *
-tor_mmap_file(const char *filename)
-{
-  struct stat st;
-  char *res = read_file_to_str(filename, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
-  tor_mmap_t *handle;
-  if (! res)
-    return NULL;
-  handle = tor_malloc_zero(sizeof(tor_mmap_t));
-  handle->data = res;
-  handle->size = st.st_size;
-  return handle;
-}
-
-/** Unmap the file mapped with tor_mmap_file(), and return 0 for success
- * or -1 for failure.
- */
-
-int
-tor_munmap_file(tor_mmap_t *handle)
-{
-  char *d = NULL;
-  if (handle == NULL)
-    return 0;
-
-  d = (char*)handle->data;
-  tor_free(d);
-  memwipe(handle, 0, sizeof(tor_mmap_t));
-  tor_free(handle);
-
-  /* Can't fail in this mmap()/munmap()-free case */
-  return 0;
-}
-#endif
+#error "cannot implement tor_mmap_file"
+#endif /* defined(HAVE_MMAP) || ... || ... */
 
 /** Replacement for snprintf.  Differs from platform snprintf in two
  * ways: First, always NUL-terminates its output.  Second, always
@@ -576,7 +543,7 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
   }
   *strp = strp_tmp;
   return len;
-#endif
+#endif /* defined(HAVE_VASPRINTF) || ... */
 }
 
 /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at
@@ -622,7 +589,7 @@ tor_memmem(const void *_haystack, size_t hlen,
     }
   }
   return NULL;
-#endif
+#endif /* defined(HAVE_MEMMEM) && (!defined(__GNUC__) || __GNUC__ >= 2) */
 }
 
 /**
@@ -760,7 +727,7 @@ tor_fix_source_file(const char *fname)
   }
   return r;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /**
  * Read a 16-bit value beginning at <b>cp</b>.  Equivalent to
@@ -854,7 +821,7 @@ replace_file(const char *from, const char *to)
       return -1;
     }
   return tor_rename(from,to);
-#endif
+#endif /* !defined(_WIN32) */
 }
 
 /** Change <b>fname</b>'s modification time to now. */
@@ -940,7 +907,7 @@ tor_lockfile_lock(const char *filename, int blocking, int *locked_out)
       return NULL;
     }
   }
-#endif
+#endif /* defined(_WIN32) || ... */
 
   result = tor_malloc(sizeof(tor_lockfile_t));
   result->filename = tor_strdup(filename);
@@ -968,7 +935,7 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile)
   }
 #else
   /* Closing the lockfile is sufficient. */
-#endif
+#endif /* defined(_WIN32) || ... */
 
   close(lockfile->fd);
   lockfile->fd = -1;
@@ -1016,9 +983,9 @@ tor_fd_seekend(int fd)
    * no need to worry. */
   if (rc < 0 && errno == ESPIPE)
     rc = 0;
-#endif
+#endif /* defined(ESPIPE) */
   return (rc < 0) ? -1 : 0;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Move <b>fd</b> to position <b>pos</b> in the file. Return -1 on error, 0
@@ -1057,7 +1024,7 @@ tor_ftruncate(int fd)
 static bitarray_t *open_sockets = NULL;
 /** The size of <b>open_sockets</b>, in bits. */
 static int max_socket = -1;
-#endif
+#endif /* defined(DEBUG_SOCKET_COUNTING) */
 
 /** Count of number of sockets currently open.  (Undercounts sockets opened by
  * eventdns and libevent.) */
@@ -1127,7 +1094,7 @@ tor_close_socket,(tor_socket_t s))
     tor_assert(open_sockets && s <= max_socket);
     bitarray_clear(open_sockets, s);
   }
-#endif
+#endif /* defined(DEBUG_SOCKET_COUNTING) */
   if (r == 0) {
     --n_sockets_open;
   } else {
@@ -1137,7 +1104,7 @@ tor_close_socket,(tor_socket_t s))
 #else
     if (r != EBADF)
       --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force.
-#endif
+#endif /* defined(_WIN32) */
     r = -1;
   }
 
@@ -1170,9 +1137,9 @@ mark_socket_open(tor_socket_t s)
   }
   bitarray_set(open_sockets, s);
 }
-#else
-#define mark_socket_open(s) STMT_NIL
-#endif
+#else /* !(defined(DEBUG_SOCKET_COUNTING)) */
+#define mark_socket_open(s) ((void) (s))
+#endif /* defined(DEBUG_SOCKET_COUNTING) */
 /** @} */
 
 /** As socket(), but counts the number of open sockets. */
@@ -1230,7 +1197,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
    * support, we are running on one without. */
   if (errno != EINVAL)
     return s;
-#endif /* SOCK_CLOEXEC && SOCK_NONBLOCK */
+#endif /* defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) */
 
   s = socket(domain, type, protocol);
   if (! SOCKET_OK(s))
@@ -1244,9 +1211,9 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
       return TOR_INVALID_SOCKET;
     }
   }
-#else
+#else /* !(defined(FD_CLOEXEC)) */
   (void)cloexec;
-#endif
+#endif /* defined(FD_CLOEXEC) */
 
   if (nonblock) {
     if (set_socket_nonblocking(s) == -1) {
@@ -1258,11 +1225,22 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
   goto socket_ok; /* So that socket_ok will not be unused. */
 
  socket_ok:
+  tor_take_socket_ownership(s);
+  return s;
+}
+
+/**
+ * For socket accounting: remember that we are the owner of the socket
+ * <b>s</b>. This will prevent us from overallocating sockets, and prevent us
+ * from asserting later when we close the socket <b>s</b>.
+ */
+void
+tor_take_socket_ownership(tor_socket_t s)
+{
   socket_accounting_lock();
   ++n_sockets_open;
   mark_socket_open(s);
   socket_accounting_unlock();
-  return s;
 }
 
 /** As accept(), but counts the number of open sockets. */
@@ -1302,7 +1280,8 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
     return TOR_INVALID_SOCKET;
   }
 
-#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK)
+#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) \
+  && defined(SOCK_NONBLOCK)
   int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) |
                   (nonblock ? SOCK_NONBLOCK : 0);
   s = accept4(sockfd, addr, len, ext_flags);
@@ -1314,7 +1293,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
    * we are missing SOCK_CLOEXEC/SOCK_NONBLOCK support. */
   if (errno != EINVAL && errno != ENOSYS)
     return s;
-#endif
+#endif /* defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) ... */
 
   s = accept(sockfd, addr, len);
   if (!SOCKET_OK(s))
@@ -1328,9 +1307,9 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
       return TOR_INVALID_SOCKET;
     }
   }
-#else
+#else /* !(defined(FD_CLOEXEC)) */
   (void)cloexec;
-#endif
+#endif /* defined(FD_CLOEXEC) */
 
   if (nonblock) {
     if (set_socket_nonblocking(s) == -1) {
@@ -1342,10 +1321,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
   goto socket_ok; /* So that socket_ok will not be unused. */
 
  socket_ok:
-  socket_accounting_lock();
-  ++n_sockets_open;
-  mark_socket_open(s);
-  socket_accounting_unlock();
+  tor_take_socket_ownership(s);
   return s;
 }
 
@@ -1368,6 +1344,24 @@ tor_getsockname,(tor_socket_t sock, struct sockaddr *address,
    return getsockname(sock, address, address_len);
 }
 
+/**
+ * Find the local address associated with the socket <b>sock</b>, and
+ * place it in *<b>addr_out</b>.  Return 0 on success, -1 on failure.
+ *
+ * (As tor_getsockname, but instead places the result in a tor_addr_t.) */
+int
+tor_addr_from_getsockname(tor_addr_t *addr_out, tor_socket_t sock)
+{
+  struct sockaddr_storage ss;
+  socklen_t ss_len = sizeof(ss);
+  memset(&ss, 0, sizeof(ss));
+
+  if (tor_getsockname(sock, (struct sockaddr *) &ss, &ss_len) < 0)
+    return -1;
+
+  return tor_addr_from_sockaddr(addr_out, (struct sockaddr *)&ss, NULL);
+}
+
 /** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1
  * on failure.
  */
@@ -1390,7 +1384,7 @@ set_socket_nonblocking(tor_socket_t sock)
     log_warn(LD_NET, "Couldn't set file status flags: %s", strerror(errno));
     return -1;
   }
-#endif
+#endif /* defined(_WIN32) */
 
   return 0;
 }
@@ -1428,7 +1422,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
    * are running on one without. */
   if (errno != EINVAL)
     return -errno;
-#endif
+#endif /* defined(SOCK_CLOEXEC) */
 
   r = socketpair(family, type, protocol, fd);
   if (r < 0)
@@ -1451,7 +1445,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
       return -errno;
     }
   }
-#endif
+#endif /* defined(FD_CLOEXEC) */
   goto sockets_ok; /* So that sockets_ok will not be unused. */
 
  sockets_ok:
@@ -1467,9 +1461,9 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
   socket_accounting_unlock();
 
   return 0;
-#else
+#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */
   return tor_ersatz_socketpair(family, type, protocol, fd);
-#endif
+#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */
 }
 
 #ifdef NEED_ERSATZ_SOCKETPAIR
@@ -1626,7 +1620,7 @@ tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
 
 #undef SIZEOF_SOCKADDR
 
-#endif
+#endif /* defined(NEED_ERSATZ_SOCKETPAIR) */
 
 /* Return the maximum number of allowed sockets. */
 int
@@ -1680,7 +1674,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
 #else
   const char *platform = "unknown platforms with no getrlimit()";
   const unsigned long MAX_CONNECTIONS = 15000;
-#endif
+#endif /* defined(CYGWIN) || defined(__CYGWIN__) || ... */
   log_fn(LOG_INFO, LD_NET,
          "This platform is missing getrlimit(). Proceeding.");
   if (limit > MAX_CONNECTIONS) {
@@ -1691,7 +1685,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
     return -1;
   }
   limit = MAX_CONNECTIONS;
-#else /* HAVE_GETRLIMIT */
+#else /* !(!defined(HAVE_GETRLIMIT)) */
   struct rlimit rlim;
 
   if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
@@ -1741,7 +1735,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
         couldnt_set = 0;
       }
     }
-#endif /* OPEN_MAX */
+#endif /* defined(OPEN_MAX) */
     if (couldnt_set) {
       log_warn(LD_CONFIG,"Couldn't set maximum number of file descriptors: %s",
                strerror(setrlimit_errno));
@@ -1749,7 +1743,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
   }
   /* leave some overhead for logs, etc, */
   limit = rlim.rlim_cur;
-#endif /* HAVE_GETRLIMIT */
+#endif /* !defined(HAVE_GETRLIMIT) */
 
   if (limit > INT_MAX)
     limit = INT_MAX;
@@ -1787,7 +1781,7 @@ log_credential_status(void)
            "UID is %u (real), %u (effective), %u (saved)",
            (unsigned)ruid, (unsigned)euid, (unsigned)suid);
   }
-#else
+#else /* !(defined(HAVE_GETRESUID)) */
   /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
   ruid = getuid();
   euid = geteuid();
@@ -1796,7 +1790,7 @@ log_credential_status(void)
   log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
          "UID is %u (real), %u (effective), unknown (saved)",
          (unsigned)ruid, (unsigned)euid);
-#endif
+#endif /* defined(HAVE_GETRESUID) */
 
   /* log GIDs */
 #ifdef HAVE_GETRESGID
@@ -1808,7 +1802,7 @@ log_credential_status(void)
            "GID is %u (real), %u (effective), %u (saved)",
            (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
   }
-#else
+#else /* !(defined(HAVE_GETRESGID)) */
   /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
   rgid = getgid();
   egid = getegid();
@@ -1816,7 +1810,7 @@ log_credential_status(void)
   log_fn(CREDENTIAL_LOG_LEVEL, LD_GENERAL,
          "GID is %u (real), %u (effective), unknown (saved)",
          (unsigned)rgid, (unsigned)egid);
-#endif
+#endif /* defined(HAVE_GETRESGID) */
 
   /* log supplementary groups */
   sup_gids_size = 64;
@@ -1856,7 +1850,7 @@ log_credential_status(void)
 
   return 0;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 #ifndef _WIN32
 /** Cached struct from the last getpwname() call we did successfully. */
@@ -1881,9 +1875,12 @@ tor_passwd_dup(const struct passwd *pw)
   return new_pw;
 }
 
+#define tor_passwd_free(pw) \
+  FREE_AND_NULL(struct passwd, tor_passwd_free_, (pw))
+
 /** Helper: free one of our cached 'struct passwd' values. */
 static void
-tor_passwd_free(struct passwd *pw)
+tor_passwd_free_(struct passwd *pw)
 {
   if (!pw)
     return;
@@ -1956,7 +1953,7 @@ tor_getpwuid(uid_t uid)
 
   return NULL;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /** Return true iff we were compiled with capability support, and capabilities
  * seem to work. **/
@@ -1969,9 +1966,9 @@ have_capability_support(void)
     return 0;
   cap_free(caps);
   return 1;
-#else
+#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
   return 0;
-#endif
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
 }
 
 #ifdef HAVE_LINUX_CAPABILITIES
@@ -2030,7 +2027,7 @@ drop_capabilities(int pre_setuid)
 
   return 0;
 }
-#endif
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
 
 /** Call setuid and setgid to run as <b>user</b> and switch to their
  * primary group.  Return 0 on success.  On failure, log and return -1.
@@ -2080,13 +2077,13 @@ switch_id(const char *user, const unsigned flags)
     if (drop_capabilities(1))
       return -1;
   }
-#else
+#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
   (void) keep_bindlow;
   if (warn_if_no_caps) {
     log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
              "on this system.");
   }
-#endif
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
 
   /* Properly switch egid,gid,euid,uid here or bail out */
   if (setgroups(1, &pw->pw_gid)) {
@@ -2146,7 +2143,7 @@ switch_id(const char *user, const unsigned flags)
     if (drop_capabilities(0))
       return -1;
   }
-#endif
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
 
 #if !defined(CYGWIN) && !defined(__CYGWIN__)
   /* If we tried to drop privilege to a group/user other than root, attempt to
@@ -2170,7 +2167,7 @@ switch_id(const char *user, const unsigned flags)
       return -1;
     }
   }
-#endif
+#endif /* !defined(CYGWIN) && !defined(__CYGWIN__) */
 
   /* Check what really happened */
   if (log_credential_status()) {
@@ -2179,8 +2176,8 @@ switch_id(const char *user, const unsigned flags)
 
   have_already_switched_id = 1; /* mark success so we never try again */
 
-#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL)
-#ifdef PR_SET_DUMPABLE
+#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && \
+  defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
   if (pw->pw_uid) {
     /* Re-enable core dumps if we're not running as root. */
     log_info(LD_CONFIG, "Re-enabling coredumps");
@@ -2188,17 +2185,16 @@ switch_id(const char *user, const unsigned flags)
       log_warn(LD_CONFIG, "Unable to re-enable coredumps: %s",strerror(errno));
     }
   }
-#endif
-#endif
+#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */
   return 0;
 
-#else
+#else /* !(!defined(_WIN32)) */
   (void)user;
   (void)flags;
 
   log_warn(LD_CONFIG, "Switching users is unsupported on your OS.");
   return -1;
-#endif
+#endif /* !defined(_WIN32) */
 }
 
 /* We only use the linux prctl for now. There is no Win32 support; this may
@@ -2221,35 +2217,32 @@ switch_id(const char *user, const unsigned flags)
 int
 tor_disable_debugger_attach(void)
 {
-  int r, attempted;
-  r = -1;
-  attempted = 0;
+  int r = -1;
   log_debug(LD_CONFIG,
             "Attemping to disable debugger attachment to Tor for "
             "unprivileged users.");
-#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && defined(HAVE_PRCTL)
-#ifdef PR_SET_DUMPABLE
-  attempted = 1;
+#if defined(__linux__) && defined(HAVE_SYS_PRCTL_H) \
+  && defined(HAVE_PRCTL) && defined(PR_SET_DUMPABLE)
+#define TRIED_TO_DISABLE
   r = prctl(PR_SET_DUMPABLE, 0);
-#endif
-#endif
-#if defined(__APPLE__) && defined(PT_DENY_ATTACH)
-  if (r < 0) {
-    attempted = 1;
-    r = ptrace(PT_DENY_ATTACH, 0, 0, 0);
-  }
-#endif
+#elif defined(__APPLE__) && defined(PT_DENY_ATTACH)
+#define TRIED_TO_ATTACH
+  r = ptrace(PT_DENY_ATTACH, 0, 0, 0);
+#endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) ... || ... */
 
   // XXX: TODO - Mac OS X has dtrace and this may be disabled.
   // XXX: TODO - Windows probably has something similar
-  if (r == 0 && attempted) {
+#ifdef TRIED_TO_DISABLE
+  if (r == 0) {
     log_debug(LD_CONFIG,"Debugger attachment disabled for "
               "unprivileged users.");
     return 1;
-  } else if (attempted) {
+  } else {
     log_warn(LD_CONFIG, "Unable to disable debugger attaching: %s",
              strerror(errno));
   }
+#endif /* defined(TRIED_TO_DISABLE) */
+#undef TRIED_TO_DISABLE
   return r;
 }
 
@@ -2268,7 +2261,7 @@ get_user_homedir(const char *username)
   }
   return tor_strdup(pw->pw_dir);
 }
-#endif
+#endif /* defined(HAVE_PWD_H) */
 
 /** Modify <b>fname</b> to contain the name of its parent directory.  Doesn't
  * actually examine the filesystem; does a purely syntactic modification.
@@ -2296,7 +2289,7 @@ get_parent_directory(char *fname)
   if (fname[0] && fname[1] == ':') {
     fname += 2;
   }
-#endif
+#endif /* defined(_WIN32) */
   /* Now we want to remove all path-separators at the end of the string,
    * and to remove the end of the string starting with the path separator
    * before the last non-path-separator.  In perl, this would be
@@ -2335,17 +2328,36 @@ get_parent_directory(char *fname)
 static char *
 alloc_getcwd(void)
 {
-#ifdef PATH_MAX
-#define MAX_CWD PATH_MAX
-#else
-#define MAX_CWD 4096
-#endif
+#ifdef HAVE_GET_CURRENT_DIR_NAME
+  /* Glibc makes this nice and simple for us. */
+  char *cwd = get_current_dir_name();
+  char *result = NULL;
+  if (cwd) {
+    /* We make a copy here, in case tor_malloc() is not malloc(). */
+    result = tor_strdup(cwd);
+    raw_free(cwd); // alias for free to avoid tripping check-spaces.
+  }
+  return result;
+#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+  size_t size = 1024;
+  char *buf = NULL;
+  char *ptr = NULL;
+
+  while (ptr == NULL) {
+    buf = tor_realloc(buf, size);
+    ptr = getcwd(buf, size);
+
+    if (ptr == NULL && errno != ERANGE) {
+      tor_free(buf);
+      return NULL;
+    }
 
-  char path_buf[MAX_CWD];
-  char *path = getcwd(path_buf, sizeof(path_buf));
-  return path ? tor_strdup(path) : NULL;
+    size *= 2;
+  }
+  return buf;
+#endif /* defined(HAVE_GET_CURRENT_DIR_NAME) */
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /** Expand possibly relative path <b>fname</b> to an absolute path.
  * Return a newly allocated string, possibly equal to <b>fname</b>. */
@@ -2361,7 +2373,7 @@ make_path_absolute(char *fname)
   if (absfname_malloced) raw_free(absfname_malloced);
 
   return absfname;
-#else
+#else /* !(defined(_WIN32)) */
   char *absfname = NULL, *path = NULL;
 
   tor_assert(fname);
@@ -2384,7 +2396,7 @@ make_path_absolute(char *fname)
     }
   }
   return absfname;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 #ifndef HAVE__NSGETENVIRON
@@ -2393,8 +2405,8 @@ make_path_absolute(char *fname)
 #ifndef RUNNING_DOXYGEN
 extern char **environ;
 #endif
-#endif
-#endif
+#endif /* !defined(HAVE_EXTERN_ENVIRON_DECLARED) */
+#endif /* !defined(HAVE__NSGETENVIRON) */
 
 /** Return the current environment. This is a portable replacement for
  * 'environ'. */
@@ -2406,14 +2418,14 @@ get_environment(void)
    * when we do a mostly-static build on OSX 10.7, the resulting binary won't
    * work on OSX 10.6. */
   return *_NSGetEnviron();
-#else
+#else /* !(defined(HAVE__NSGETENVIRON)) */
   return environ;
-#endif
+#endif /* defined(HAVE__NSGETENVIRON) */
 }
 
 /** Get name of current host and write it to <b>name</b> array, whose
  * length is specified by <b>namelen</b> argument. Return 0 upon
- * successfull completion; otherwise return return -1. (Currently,
+ * successful completion; otherwise return return -1. (Currently,
  * this function is merely a mockable wrapper for POSIX gethostname().)
  */
 MOCK_IMPL(int,
@@ -2549,6 +2561,7 @@ tor_inet_pton(int af, const char *src, void *dst)
     int gapPos = -1, i, setWords=0;
     const char *dot = strchr(src, '.');
     const char *eow; /* end of words. */
+    memset(words, 0xf8, sizeof(words));
     if (dot == src)
       return 0;
     else if (!dot)
@@ -2586,7 +2599,7 @@ tor_inet_pton(int af, const char *src, void *dst)
         long r = strtol(src, &next, 16);
         if (next == NULL || next == src) {
           /* The 'next == src' error case can happen on versions of openbsd
-           * where treats "0xfoo" as an error, rather than as "0" followed by
+           * which treat "0xfoo" as an error, rather than as "0" followed by
            * "xfoo". */
           return 0;
         }
@@ -2673,7 +2686,8 @@ static int uname_result_is_set = 0;
 
 /** Return a pointer to a description of our platform.
  */
-MOCK_IMPL(const char *, get_uname, (void))
+MOCK_IMPL(const char *,
+get_uname,(void))
 {
 #ifdef HAVE_UNAME
   struct utsname u;
@@ -2684,7 +2698,7 @@ MOCK_IMPL(const char *, get_uname, (void))
       /* (Linux says 0 is success, Solaris says 1 is success) */
       strlcpy(uname_result, u.sysname, sizeof(uname_result));
     } else
-#endif
+#endif /* defined(HAVE_UNAME) */
       {
 #ifdef _WIN32
         OSVERSIONINFOEX info;
@@ -2780,11 +2794,11 @@ MOCK_IMPL(const char *, get_uname, (void))
         if (!is_server && !is_client) {
           strlcat(uname_result, " [client or server]", sizeof(uname_result));
         }
-#else
+#else /* !(defined(_WIN32)) */
         /* LCOV_EXCL_START -- can't provoke uname failure */
         strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
         /* LCOV_EXCL_STOP */
-#endif
+#endif /* defined(_WIN32) */
       }
     uname_result_is_set = 1;
   }
@@ -2840,7 +2854,7 @@ compute_num_cpus_impl(void)
     return -1;
 #else
   return -1;
-#endif
+#endif /* defined(_WIN32) || ... */
 }
 
 #define MAX_DETECTABLE_CPUS 16
@@ -2879,7 +2893,7 @@ compute_num_cpus(void)
 /** Helper: Deal with confused or out-of-bounds values from localtime_r and
  * friends.  (On some platforms, they can give out-of-bounds values or can
  * return NULL.)  If <b>islocal</b>, this is a localtime result; otherwise
- * it's from gmtime.  The function returned <b>r</b>, when given <b>timep</b>
+ * it's from gmtime.  The function returns <b>r</b>, when given <b>timep</b>
  * as its input. If we need to store new results, store them in
  * <b>resultbuf</b>. */
 static struct tm *
@@ -3003,7 +3017,7 @@ tor_localtime_r(const time_t *timep, struct tm *result)
     memcpy(result, r, sizeof(struct tm));
   return correct_tm(1, timep, result, r);
 }
-#endif
+#endif /* defined(HAVE_LOCALTIME_R) || ... */
 /** @} */
 
 /** @{ */
@@ -3046,9 +3060,13 @@ tor_gmtime_r(const time_t *timep, struct tm *result)
     memcpy(result, r, sizeof(struct tm));
   return correct_tm(0, timep, result, r);
 }
-#endif
+#endif /* defined(HAVE_GMTIME_R) || ... */
 
 #if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK)
+#define HAVE_UNIX_MLOCKALL
+#endif
+
+#ifdef HAVE_UNIX_MLOCKALL
 /** Attempt to raise the current and max rlimit to infinity for our process.
  * This only needs to be done once and can probably only be done when we have
  * not already dropped privileges.
@@ -3079,7 +3097,7 @@ tor_set_max_memlock(void)
 
   return 0;
 }
-#endif
+#endif /* defined(HAVE_UNIX_MLOCKALL) */
 
 /** Attempt to lock all current and all future memory pages.
  * This should only be called once and while we're privileged.
@@ -3104,7 +3122,7 @@ tor_mlockall(void)
    * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
    */
 
-#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK)
+#ifdef HAVE_UNIX_MLOCKALL
   if (tor_set_max_memlock() == 0) {
     log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY.");
   }
@@ -3125,10 +3143,10 @@ tor_mlockall(void)
                            "pages: %s", strerror(errno));
     return -1;
   }
-#else
+#else /* !(defined(HAVE_UNIX_MLOCKALL)) */
   log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
   return -1;
-#endif
+#endif /* defined(HAVE_UNIX_MLOCKALL) */
 }
 
 /**
@@ -3156,7 +3174,7 @@ tor_socket_errno(tor_socket_t sock)
   }
   return err;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 #if defined(_WIN32)
 #define E(code, s) { code, (s " [" #code " ]") }
@@ -3232,7 +3250,7 @@ tor_socket_strerror(int e)
   }
   return strerror(e);
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Called before we make any calls to network-related functions.
  * (Some operating systems require their network libraries to be
@@ -3258,7 +3276,7 @@ network_init(void)
   /* WSAData.iMaxSockets might show the max sockets we're allowed to use.
    * We might use it to complain if we're trying to be a server but have
    * too few sockets available. */
-#endif
+#endif /* defined(_WIN32) */
   return 0;
 }
 
@@ -3279,7 +3297,7 @@ format_win32_error(DWORD err)
                  FORMAT_MESSAGE_FROM_SYSTEM |
                  FORMAT_MESSAGE_IGNORE_INSERTS,
                  NULL, err,
-                 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                 MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
                  (LPVOID)&str,
                  0, NULL);
 
@@ -3294,9 +3312,9 @@ format_win32_error(DWORD err)
     result = tor_malloc(len);
     wcstombs(result,str,len);
     result[len-1] = '\0';
-#else
+#else /* !(defined(UNICODE)) */
     result = tor_strdup(str);
-#endif
+#endif /* defined(UNICODE) */
   } else {
     result = tor_strdup("<unformattable error>");
   }
@@ -3305,7 +3323,7 @@ format_win32_error(DWORD err)
   }
   return result;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 #if defined(HW_PHYSMEM64)
 /* This appears to be an OpenBSD thing */
@@ -3313,7 +3331,7 @@ format_win32_error(DWORD err)
 #elif defined(HW_MEMSIZE)
 /* OSX defines this one */
 #define INT64_HW_MEM HW_MEMSIZE
-#endif
+#endif /* defined(HW_PHYSMEM64) || ... */
 
 /**
  * Helper: try to detect the total system memory, and return it. On failure,
@@ -3346,8 +3364,8 @@ get_total_system_memory_impl(void)
   tor_free(s);
   return result * 1024;
 
- err:
   /* LCOV_EXCL_START Can't reach this unless proc is broken. */
+ err:
   tor_free(s);
   close(fd);
   return 0;
@@ -3387,15 +3405,15 @@ get_total_system_memory_impl(void)
 #else
   /* I have no clue. */
   return 0;
-#endif
+#endif /* defined(__linux__) || ... */
 }
 
 /**
  * Try to find out how much physical memory the system has. On success,
  * return 0 and set *<b>mem_out</b> to that value. On failure, return -1.
  */
-int
-get_total_system_memory(size_t *mem_out)
+MOCK_IMPL(int,
+get_total_system_memory, (size_t *mem_out))
 {
   static size_t mem_cached=0;
   uint64_t m = get_total_system_memory_impl();
@@ -3420,7 +3438,7 @@ get_total_system_memory(size_t *mem_out)
      * size_t. */
     m = SIZE_MAX;
   }
-#endif
+#endif /* SIZE_MAX != UINT64_MAX */
 
   *mem_out = mem_cached = (size_t) m;
 
@@ -3501,7 +3519,7 @@ tor_getpass(const char *prompt, char *output, size_t buflen)
   return r;
 #else
 #error "No implementation for tor_getpass found!"
-#endif
+#endif /* defined(HAVE_READPASSPHRASE) || ... */
 }
 
 /** Return the amount of free disk space we have permission to use, in
@@ -3541,6 +3559,6 @@ tor_get_avail_disk_space(const char *path)
   (void)path;
   errno = ENOSYS;
   return -1;
-#endif
+#endif /* defined(HAVE_STATVFS) || ... */
 }
 
diff --git a/src/common/compat.h b/src/common/compat.h
index ee1c9454de..c7e7f8d9ef 100644
--- a/src/common/compat.h
+++ b/src/common/compat.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_H
@@ -10,6 +10,9 @@
 #ifdef _WIN32
 #include <winsock2.h>
 #include <ws2tcpip.h>
+#ifndef SIO_IDEAL_SEND_BACKLOG_QUERY
+#define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
+#endif
 #endif
 #include "torint.h"
 #include "testsupport.h"
@@ -50,8 +53,8 @@
  * clang rejects because it is off the end of a less-than-3. Clang hates this,
  * even though those references never actually happen. */
 #    undef strcmp
-#  endif
-#endif
+#endif /* __has_feature(address_sanitizer) */
+#endif /* defined(__has_feature) */
 
 #include <stdio.h>
 #include <errno.h>
@@ -76,13 +79,13 @@
    __attribute__ ((format(printf, formatIdx, firstArg)))
 #else
 #define CHECK_PRINTF(formatIdx, firstArg)
-#endif
+#endif /* defined(__GNUC__) */
 #ifdef __GNUC__
 #define CHECK_SCANF(formatIdx, firstArg) \
    __attribute__ ((format(scanf, formatIdx, firstArg)))
 #else
 #define CHECK_SCANF(formatIdx, firstArg)
-#endif
+#endif /* defined(__GNUC__) */
 
 /* What GCC do we have? */
 #ifdef __GNUC__
@@ -109,18 +112,18 @@
           PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
 #    define ENABLE_GCC_WARNING(warningopt) \
           PRAGMA_DIAGNOSTIC_(pop)
-#  else
+#else /* !(defined(__clang__) || GCC_VERSION >= 406) */
 /* older version of gcc: no push/pop support. */
 #    define DISABLE_GCC_WARNING(warningopt) \
          PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
 #    define ENABLE_GCC_WARNING(warningopt) \
          PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
-#  endif
-#else /* ifdef __GNUC__ */
+#endif /* defined(__clang__) || GCC_VERSION >= 406 */
+#else /* !(defined(__GNUC__)) */
 /* not gcc at all */
 # define DISABLE_GCC_WARNING(warning)
 # define ENABLE_GCC_WARNING(warning)
-#endif
+#endif /* defined(__GNUC__) */
 
 /* inline is __inline on windows. */
 #ifdef _WIN32
@@ -142,9 +145,9 @@
 #define __func__ __FUNC__
 #else
 #define __func__ "???"
-#endif
-#endif /* ifndef MAVE_MACRO__func__ */
-#endif /* if not windows */
+#endif /* defined(HAVE_MACRO__FUNCTION__) || ... */
+#endif /* !defined(HAVE_MACRO__func__) */
+#endif /* defined(_MSC_VER) */
 
 #define U64_TO_DBL(x) ((double) (x))
 #define DBL_TO_U64(x) ((uint64_t) (x))
@@ -157,7 +160,7 @@
  * problems), but if enumerated types are unsigned, we must use unsigned,
  * so that the loss of precision doesn't make large values negative. */
 #define ENUM_BF(t) t
-#endif
+#endif /* defined(ENUM_VALS_ARE_SIGNED) */
 
 /* GCC has several useful attributes. */
 #if defined(__GNUC__) && __GNUC__ >= 3
@@ -194,7 +197,7 @@
  * taken.  This can generate slightly better code with some CPUs.
  */
 #define PREDICT_UNLIKELY(exp) __builtin_expect(!!(exp), 0)
-#else
+#else /* !(defined(__GNUC__) && __GNUC__ >= 3) */
 #define ATTR_NORETURN
 #define ATTR_CONST
 #define ATTR_MALLOC
@@ -204,7 +207,7 @@
 #define ATTR_WUR
 #define PREDICT_LIKELY(exp) (exp)
 #define PREDICT_UNLIKELY(exp) (exp)
-#endif
+#endif /* defined(__GNUC__) && __GNUC__ >= 3 */
 
 /** Expands to a syntactically valid empty statement.  */
 #define STMT_NIL (void)0
@@ -224,7 +227,7 @@
 #else
 #define STMT_BEGIN do {
 #define STMT_END } while (0)
-#endif
+#endif /* defined(__GNUC__) || ... */
 
 /* Some tools (like coccinelle) don't like to see operators as macro
  * arguments. */
@@ -251,7 +254,7 @@
  */
 #undef strlcat
 #undef strlcpy
-#endif
+#endif /* defined __APPLE__ */
 
 #ifndef HAVE_STRLCAT
 size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
@@ -272,24 +275,28 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
 #define I64_PRINTF_ARG(a) (a)
 #define I64_SCANF_ARG(a) (a)
 #define I64_LITERAL(n) (n ## i64)
-#else
+#else /* !(defined(_MSC_VER)) */
 #define U64_PRINTF_ARG(a) ((long long unsigned int)(a))
 #define U64_SCANF_ARG(a) ((long long unsigned int*)(a))
 #define U64_LITERAL(n) (n ## llu)
 #define I64_PRINTF_ARG(a) ((long long signed int)(a))
 #define I64_SCANF_ARG(a) ((long long signed int*)(a))
 #define I64_LITERAL(n) (n ## ll)
+#endif /* defined(_MSC_VER) */
+
+#if defined(__MINGW32__) || defined(__MINGW64__)
+#define MINGW_ANY
 #endif
 
-#if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__)
+#if defined(_MSC_VER) || defined(MINGW_ANY)
 /** The formatting string used to put a uint64_t value in a printf() or
  * scanf() function.  See also U64_PRINTF_ARG and U64_SCANF_ARG. */
 #define U64_FORMAT "%I64u"
 #define I64_FORMAT "%I64d"
-#else
+#else /* !(defined(_MSC_VER) || defined(MINGW_ANY)) */
 #define U64_FORMAT "%llu"
 #define I64_FORMAT "%lld"
-#endif
+#endif /* defined(_MSC_VER) || defined(MINGW_ANY) */
 
 #if (SIZEOF_INTPTR_T == SIZEOF_INT)
 #define INTPTR_T_FORMAT "%d"
@@ -302,7 +309,7 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2));
 #define INTPTR_PRINTF_ARG(x) I64_PRINTF_ARG(x)
 #else
 #error Unknown: SIZEOF_INTPTR_T
-#endif
+#endif /* (SIZEOF_INTPTR_T == SIZEOF_INT) || ... */
 
 /** Represents an mmaped file. Allocated via tor_mmap_file; freed with
  * tor_munmap_file. */
@@ -311,12 +318,12 @@ typedef struct tor_mmap_t {
   size_t size; /**< Size of the file. */
 
   /* None of the fields below should be accessed from outside compat.c */
-#ifdef HAVE_SYS_MMAN_H
+#ifdef HAVE_MMAP
   size_t mapping_size; /**< Size of the actual mapping. (This is this file
                         * size, rounded up to the nearest page.) */
 #elif defined _WIN32
   HANDLE mmap_handle;
-#endif
+#endif /* defined(HAVE_MMAP) || ... */
 
 } tor_mmap_t;
 
@@ -378,7 +385,7 @@ const char *tor_fix_source_file(const char *fname);
 #else
 #define SHORT_FILE__ (__FILE__)
 #define tor_fix_source_file(s) (s)
-#endif
+#endif /* defined(_WIN32) */
 
 /* ===== Time compatibility */
 
@@ -397,7 +404,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
       (tvout)->tv_sec++;                                \
     }                                                   \
   } while (0)
-#endif
+#endif /* !defined(timeradd) */
 
 #ifndef timersub
 /** Replacement for timersub on platforms that do not have it: sets tvout to
@@ -411,13 +418,13 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
       (tvout)->tv_sec--;                                \
     }                                                   \
   } while (0)
-#endif
+#endif /* !defined(timersub) */
 
 #ifndef timercmp
-/** Replacement for timersub on platforms that do not have it: returns true
+/** Replacement for timercmp on platforms that do not have it: returns true
  * iff the relational operator "op" makes the expression tv1 op tv2 true.
  *
- * Note that while this definition should work for all boolean opeators, some
+ * Note that while this definition should work for all boolean operators, some
  * platforms' native timercmp definitions do not support >=, <=, or ==.  So
  * don't use those.
  */
@@ -425,7 +432,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
   (((tv1)->tv_sec == (tv2)->tv_sec) ?           \
    ((tv1)->tv_usec op (tv2)->tv_usec) :         \
    ((tv1)->tv_sec op (tv2)->tv_sec))
-#endif
+#endif /* !defined(timercmp) */
 
 /* ===== File compatibility */
 int tor_open_cloexec(const char *path, int flags, unsigned mode);
@@ -467,7 +474,7 @@ typedef int socklen_t;
 #define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT
 #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
 #define TOR_INVALID_SOCKET INVALID_SOCKET
-#else
+#else /* !(defined(_WIN32)) */
 /** Type used for a network socket. */
 #define tor_socket_t int
 #define TOR_SOCKET_T_FORMAT "%d"
@@ -475,10 +482,11 @@ typedef int socklen_t;
 #define SOCKET_OK(s) ((s) >= 0)
 /** Error/uninitialized value for a tor_socket_t. */
 #define TOR_INVALID_SOCKET (-1)
-#endif
+#endif /* defined(_WIN32) */
 
 int tor_close_socket_simple(tor_socket_t s);
 MOCK_DECL(int, tor_close_socket, (tor_socket_t s));
+void tor_take_socket_ownership(tor_socket_t s);
 tor_socket_t tor_open_socket_with_extensions(
                                            int domain, int type, int protocol,
                                            int cloexec, int nonblock);
@@ -502,6 +510,8 @@ int get_n_open_sockets(void);
 MOCK_DECL(int,
 tor_getsockname,(tor_socket_t socket, struct sockaddr *address,
                  socklen_t *address_len));
+struct tor_addr_t;
+int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 
 #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags)
 #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags)
@@ -522,19 +532,19 @@ struct in6_addr
 #define s6_addr16 in6_u.u6_addr16
 #define s6_addr32 in6_u.u6_addr32
 };
-#endif
+#endif /* !defined(HAVE_STRUCT_IN6_ADDR) */
 
 /** @{ */
 /** Many BSD variants seem not to define these. */
-#if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \
-    || defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__APPLE__) || defined(__darwin__) || \
+  defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
 #ifndef s6_addr16
 #define s6_addr16 __u6_addr.__u6_addr16
 #endif
 #ifndef s6_addr32
 #define s6_addr32 __u6_addr.__u6_addr32
 #endif
-#endif
+#endif /* defined(__APPLE__) || defined(__darwin__) || ... */
 /** @} */
 
 #ifndef HAVE_SA_FAMILY_T
@@ -566,7 +576,7 @@ struct sockaddr_in6 {
   struct in6_addr sin6_addr;
   // uint32_t sin6_scope_id;
 };
-#endif
+#endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
 int tor_inet_aton(const char *cp, struct in_addr *addr) ATTR_NONNULL((1,2));
@@ -607,14 +617,14 @@ int network_init(void);
 #define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
 int tor_socket_errno(tor_socket_t sock);
 const char *tor_socket_strerror(int e);
-#else
+#else /* !(defined(_WIN32)) */
 #define SOCK_ERRNO(e) e
 #if EAGAIN == EWOULDBLOCK
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
 #define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || 0)
 #else
 #define ERRNO_IS_EAGAIN(e)           ((e) == EAGAIN || (e) == EWOULDBLOCK)
-#endif
+#endif /* EAGAIN == EWOULDBLOCK */
 #define ERRNO_IS_EINTR(e)            ((e) == EINTR || 0)
 #define ERRNO_IS_EINPROGRESS(e)      ((e) == EINPROGRESS || 0)
 #define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0)
@@ -625,7 +635,7 @@ const char *tor_socket_strerror(int e);
 #define ERRNO_IS_EADDRINUSE(e)       (((e) == EADDRINUSE) || 0)
 #define tor_socket_errno(sock)       (errno)
 #define tor_socket_strerror(e)       strerror(e)
-#endif
+#endif /* defined(_WIN32) */
 
 /** Specified SOCKS5 status codes. */
 typedef enum {
@@ -691,7 +701,7 @@ char *make_path_absolute(char *fname);
 
 char **get_environment(void);
 
-int get_total_system_memory(size_t *mem_out);
+MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
 
 int compute_num_cpus(void);
 
@@ -728,7 +738,7 @@ char *format_win32_error(DWORD err);
 #define VER_SUITE_SINGLEUSERTS 0x00000100
 #endif
 
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef COMPAT_PRIVATE
 #if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS)
@@ -736,12 +746,12 @@ char *format_win32_error(DWORD err);
 STATIC int tor_ersatz_socketpair(int family, int type, int protocol,
                                    tor_socket_t fd[2]);
 #endif
-#endif
+#endif /* defined(COMPAT_PRIVATE) */
 
 ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
 
 /* This needs some of the declarations above so we include it here. */
 #include "compat_threads.h"
 
-#endif
+#endif /* !defined(TOR_COMPAT_H) */
 
diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c
index 4a3b1af922..e60eb148d8 100644
--- a/src/common/compat_libevent.c
+++ b/src/common/compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 #define COMPAT_LIBEVENT_PRIVATE
 #include "compat_libevent.h"
 
-#include "crypto.h"
+#include "crypto_rand.h"
 
 #include "util.h"
 #include "torlog.h"
@@ -69,7 +69,7 @@ suppress_libevent_log_msg(const char *msg)
 
 /* Wrapper for event_free() that tolerates tor_event_free(NULL) */
 void
-tor_event_free(struct event *ev)
+tor_event_free_(struct event *ev)
 {
   if (ev == NULL)
     return;
@@ -79,6 +79,43 @@ tor_event_free(struct event *ev)
 /** Global event base for use by the main thread. */
 static struct event_base *the_event_base = NULL;
 
+/**
+ * @defgroup postloop post-loop event helpers
+ *
+ * If we're not careful, Libevent can susceptible to infinite event chains:
+ * one event can activate another, whose callback activates another, whose
+ * callback activates another, ad infinitum.  While this is happening,
+ * Libevent won't be checking timeouts, socket-based events, signals, and so
+ * on.
+ *
+ * We solve this problem by marking some events as "post-loop".  A post-loop
+ * event behaves like any ordinary event, but any events that _it_ activates
+ * cannot run until Libevent has checked for other events at least once.
+ *
+ * @{ */
+
+/**
+ * An event that stops Libevent from running any more events on the current
+ * iteration of its loop, until it has re-checked for socket events, signal
+ * events, timeouts, etc.
+ */
+static struct event *rescan_mainloop_ev = NULL;
+
+/**
+ * Callback to implement rescan_mainloop_ev: it simply exits the mainloop,
+ * and relies on Tor to re-enter the mainloop since no error has occurred.
+ */
+static void
+rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg)
+{
+  (void)fd;
+  (void)events;
+  struct event_base *the_base = arg;
+  event_base_loopbreak(the_base);
+}
+
+/** @} */
+
 /* This is what passes for version detection on OSX.  We set
  * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before
  * 10.4.0 (aka 1040). */
@@ -88,8 +125,8 @@ static struct event_base *the_event_base = NULL;
   (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040)
 #else
 #define MACOSX_KQUEUE_IS_BROKEN 0
-#endif
-#endif
+#endif /* defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) */
+#endif /* defined(__APPLE__) */
 
 /** Initialize the Libevent library and set up the event base. */
 void
@@ -126,7 +163,16 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg)
   if (!the_event_base) {
     /* LCOV_EXCL_START */
     log_err(LD_GENERAL, "Unable to initialize Libevent: cannot continue.");
-    exit(1);
+    exit(1); // exit ok: libevent is broken.
+    /* LCOV_EXCL_STOP */
+  }
+
+  rescan_mainloop_ev = event_new(the_event_base, -1, 0,
+                                 rescan_mainloop_cb, the_event_base);
+  if (!rescan_mainloop_ev) {
+    /* LCOV_EXCL_START */
+    log_err(LD_GENERAL, "Unable to create rescan event: cannot continue.");
+    exit(1); // exit ok: libevent is broken.
     /* LCOV_EXCL_STOP */
   }
 
@@ -207,13 +253,42 @@ periodic_timer_new(struct event_base *base,
   }
   timer->cb = cb;
   timer->data = data;
-  event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/
+  periodic_timer_launch(timer, tv);
   return timer;
 }
 
+/**
+ * Launch the timer <b>timer</b> to run at <b>tv</b> from now, and every
+ * <b>tv</b> thereafter.
+ *
+ * If the timer is already enabled, this function does nothing.
+ */
+void
+periodic_timer_launch(periodic_timer_t *timer, const struct timeval *tv)
+{
+  tor_assert(timer);
+  if (event_pending(timer->ev, EV_TIMEOUT, NULL))
+    return;
+  event_add(timer->ev, tv);
+}
+
+/**
+ * Disable the provided <b>timer</b>, but do not free it.
+ *
+ * You can reenable the same timer later with periodic_timer_launch.
+ *
+ * If the timer is already disabled, this function does nothing.
+ */
+void
+periodic_timer_disable(periodic_timer_t *timer)
+{
+  tor_assert(timer);
+  (void) event_del(timer->ev);
+}
+
 /** Stop and free a periodic timer */
 void
-periodic_timer_free(periodic_timer_t *timer)
+periodic_timer_free_(periodic_timer_t *timer)
 {
   if (!timer)
     return;
@@ -221,6 +296,173 @@ periodic_timer_free(periodic_timer_t *timer)
   tor_free(timer);
 }
 
+/**
+ * Type used to represent events that run directly from the main loop,
+ * either because they are activated from elsewhere in the code, or
+ * because they have a simple timeout.
+ *
+ * We use this type to avoid exposing Libevent's API throughout the rest
+ * of the codebase.
+ *
+ * This type can't be used for all events: it doesn't handle events that
+ * are triggered by signals or by sockets.
+ */
+struct mainloop_event_t {
+  struct event *ev;
+  void (*cb)(mainloop_event_t *, void *);
+  void *userdata;
+};
+
+/**
+ * Internal: Implements mainloop event using a libevent event.
+ */
+static void
+mainloop_event_cb(evutil_socket_t fd, short what, void *arg)
+{
+  (void)fd;
+  (void)what;
+  mainloop_event_t *mev = arg;
+  mev->cb(mev, mev->userdata);
+}
+
+/**
+ * As mainloop_event_cb, but implements a post-loop event.
+ */
+static void
+mainloop_event_postloop_cb(evutil_socket_t fd, short what, void *arg)
+{
+  (void)fd;
+  (void)what;
+
+  /* Note that if rescan_mainloop_ev is already activated,
+   * event_active() will do nothing: only the first post-loop event that
+   * happens each time through the event loop will cause it to be
+   * activated.
+   *
+   * Because event_active() puts events on a FIFO queue, every event
+   * that is made active _after_ rescan_mainloop_ev will get its
+   * callback run after rescan_mainloop_cb is called -- that is, on the
+   * next iteration of the loop.
+   */
+  event_active(rescan_mainloop_ev, EV_READ, 1);
+
+  mainloop_event_t *mev = arg;
+  mev->cb(mev, mev->userdata);
+}
+
+/**
+ * Helper for mainloop_event_new() and mainloop_event_postloop_new().
+ */
+static mainloop_event_t *
+mainloop_event_new_impl(int postloop,
+                        void (*cb)(mainloop_event_t *, void *),
+                        void *userdata)
+{
+  tor_assert(cb);
+
+  struct event_base *base = tor_libevent_get_base();
+  mainloop_event_t *mev = tor_malloc_zero(sizeof(mainloop_event_t));
+  mev->ev = tor_event_new(base, -1, 0,
+                  postloop ? mainloop_event_postloop_cb : mainloop_event_cb,
+                  mev);
+  tor_assert(mev->ev);
+  mev->cb = cb;
+  mev->userdata = userdata;
+  return mev;
+}
+
+/**
+ * Create and return a new mainloop_event_t to run the function <b>cb</b>.
+ *
+ * When run, the callback function will be passed the mainloop_event_t
+ * and <b>userdata</b> as its arguments.  The <b>userdata</b> pointer
+ * must remain valid for as long as the mainloop_event_t event exists:
+ * it is your responsibility to free it.
+ *
+ * The event is not scheduled by default: Use mainloop_event_activate()
+ * or mainloop_event_schedule() to make it run.
+ */
+mainloop_event_t *
+mainloop_event_new(void (*cb)(mainloop_event_t *, void *),
+                   void *userdata)
+{
+  return mainloop_event_new_impl(0, cb, userdata);
+}
+
+/**
+ * As mainloop_event_new(), but create a post-loop event.
+ *
+ * A post-loop event behaves like any ordinary event, but any events
+ * that _it_ activates cannot run until Libevent has checked for other
+ * events at least once.
+ */
+mainloop_event_t *
+mainloop_event_postloop_new(void (*cb)(mainloop_event_t *, void *),
+                            void *userdata)
+{
+  return mainloop_event_new_impl(1, cb, userdata);
+}
+
+/**
+ * Schedule <b>event</b> to run in the main loop, immediately.  If it is
+ * not scheduled, it will run anyway. If it is already scheduled to run
+ * later, it will run now instead.  This function will have no effect if
+ * the event is already scheduled to run.
+ *
+ * This function may only be called from the main thread.
+ */
+void
+mainloop_event_activate(mainloop_event_t *event)
+{
+  tor_assert(event);
+  event_active(event->ev, EV_READ, 1);
+}
+
+/** Schedule <b>event</b> to run in the main loop, after a delay of <b>tv</b>.
+ *
+ * If the event is scheduled for a different time, cancel it and run
+ * after this delay instead.  If the event is currently pending to run
+ * <em>now</b>, has no effect.
+ *
+ * Do not call this function with <b>tv</b> == NULL -- use
+ * mainloop_event_activate() instead.
+ *
+ * This function may only be called from the main thread.
+ */
+int
+mainloop_event_schedule(mainloop_event_t *event, const struct timeval *tv)
+{
+  tor_assert(event);
+  if (BUG(tv == NULL)) {
+    // LCOV_EXCL_START
+    mainloop_event_activate(event);
+    return 0;
+    // LCOV_EXCL_STOP
+  }
+  return event_add(event->ev, tv);
+}
+
+/** Cancel <b>event</b> if it is currently active or pending. (Do nothing if
+ * the event is not currently active or pending.) */
+void
+mainloop_event_cancel(mainloop_event_t *event)
+{
+  if (!event)
+    return;
+  (void) event_del(event->ev);
+}
+
+/** Cancel <b>event</b> and release all storage associated with it. */
+void
+mainloop_event_free_(mainloop_event_t *event)
+{
+  if (!event)
+    return;
+  tor_event_free(event->ev);
+  memset(event, 0xb8, sizeof(*event));
+  tor_free(event);
+}
+
 int
 tor_init_libevent_rng(void)
 {
@@ -237,49 +479,58 @@ tor_init_libevent_rng(void)
   return rv;
 }
 
-#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \
-  && !defined(TOR_UNIT_TESTS)
+/**
+ * Un-initialize libevent in preparation for an exit
+ */
 void
-tor_gettimeofday_cached(struct timeval *tv)
+tor_libevent_free_all(void)
 {
-  event_base_gettimeofday_cached(the_event_base, tv);
+  tor_event_free(rescan_mainloop_ev);
+  if (the_event_base)
+    event_base_free(the_event_base);
+  the_event_base = NULL;
 }
-void
-tor_gettimeofday_cache_clear(void)
+
+/**
+ * Run the event loop for the provided event_base, handling events until
+ * something stops it.  If <b>once</b> is set, then just poll-and-run
+ * once, then exit.  Return 0 on success, -1 if an error occurred, or 1
+ * if we exited because no events were pending or active.
+ *
+ * This isn't reentrant or multithreaded.
+ */
+int
+tor_libevent_run_event_loop(struct event_base *base, int once)
 {
-  event_base_update_cache_time(the_event_base);
+  const int flags = once ? EVLOOP_ONCE : 0;
+  return event_base_loop(base, flags);
 }
-#else
-/** Cache the current hi-res time; the cache gets reset when libevent
- * calls us. */
-static struct timeval cached_time_hires = {0, 0};
 
-/** Return a fairly recent view of the current time. */
+/** Tell the event loop to exit after <b>delay</b>.  If <b>delay</b> is NULL,
+ * instead exit after we're done running the currently active events. */
 void
-tor_gettimeofday_cached(struct timeval *tv)
+tor_libevent_exit_loop_after_delay(struct event_base *base,
+                                   const struct timeval *delay)
 {
-  if (cached_time_hires.tv_sec == 0) {
-    tor_gettimeofday(&cached_time_hires);
-  }
-  *tv = cached_time_hires;
+  event_base_loopexit(base, delay);
 }
 
-/** Reset the cached view of the current time, so that the next time we try
- * to learn it, we will get an up-to-date value. */
+/** Tell the event loop to exit after running whichever callback is currently
+ * active. */
 void
-tor_gettimeofday_cache_clear(void)
+tor_libevent_exit_loop_after_callback(struct event_base *base)
 {
-  cached_time_hires.tv_sec = 0;
+  event_base_loopbreak(base);
 }
 
-#ifdef TOR_UNIT_TESTS
-/** For testing: force-update the cached time to a given value. */
+#if defined(TOR_UNIT_TESTS)
+/** For testing: called post-fork to make libevent reinitialize
+ * kernel structures. */
 void
-tor_gettimeofday_cache_set(const struct timeval *tv)
+tor_libevent_postfork(void)
 {
-  tor_assert(tv);
-  memcpy(&cached_time_hires, tv, sizeof(*tv));
+  int r = event_reinit(tor_libevent_get_base());
+  tor_assert(r == 0);
 }
-#endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h
index c2e34764e4..286a268122 100644
--- a/src/common/compat_libevent.h
+++ b/src/common/compat_libevent.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_LIBEVENT_H
@@ -7,8 +7,6 @@
 #include "orconfig.h"
 #include "testsupport.h"
 
-#include <event2/event.h>
-
 void configure_libevent_logging(void);
 void suppress_libevent_log_msg(const char *msg);
 
@@ -19,7 +17,12 @@ void suppress_libevent_log_msg(const char *msg);
   evdns_add_server_port_with_base(tor_libevent_get_base(), \
   (sock),(tcp),(cb),(data));
 
-void tor_event_free(struct event *ev);
+struct event;
+struct event_base;
+
+void tor_event_free_(struct event *ev);
+#define tor_event_free(ev) \
+  FREE_AND_NULL(struct event, tor_event_free_, (ev))
 
 typedef struct periodic_timer_t periodic_timer_t;
 
@@ -27,9 +30,25 @@ periodic_timer_t *periodic_timer_new(struct event_base *base,
              const struct timeval *tv,
              void (*cb)(periodic_timer_t *timer, void *data),
              void *data);
-void periodic_timer_free(periodic_timer_t *);
-
-#define tor_event_base_loopexit event_base_loopexit
+void periodic_timer_free_(periodic_timer_t *);
+void periodic_timer_launch(periodic_timer_t *, const struct timeval *tv);
+void periodic_timer_disable(periodic_timer_t *);
+#define periodic_timer_free(t) \
+  FREE_AND_NULL(periodic_timer_t, periodic_timer_free_, (t))
+
+typedef struct mainloop_event_t mainloop_event_t;
+mainloop_event_t *mainloop_event_new(void (*cb)(mainloop_event_t *, void *),
+                                     void *userdata);
+mainloop_event_t * mainloop_event_postloop_new(
+                                     void (*cb)(mainloop_event_t *, void *),
+                                     void *userdata);
+void mainloop_event_activate(mainloop_event_t *event);
+int mainloop_event_schedule(mainloop_event_t *event,
+                            const struct timeval *delay);
+void mainloop_event_cancel(mainloop_event_t *event);
+void mainloop_event_free_(mainloop_event_t *event);
+#define mainloop_event_free(event) \
+  FREE_AND_NULL(mainloop_event_t, mainloop_event_free_, (event))
 
 /** Defines a configuration for using libevent with Tor: passed as an argument
  * to tor_libevent_initialize() to describe how we want to set up. */
@@ -47,15 +66,19 @@ const char *tor_libevent_get_method(void);
 void tor_check_libevent_header_compatibility(void);
 const char *tor_libevent_get_version_str(void);
 const char *tor_libevent_get_header_version_str(void);
+void tor_libevent_free_all(void);
 
 int tor_init_libevent_rng(void);
 
-void tor_gettimeofday_cached(struct timeval *tv);
-void tor_gettimeofday_cache_clear(void);
 #ifdef TOR_UNIT_TESTS
-void tor_gettimeofday_cache_set(const struct timeval *tv);
+void tor_libevent_postfork(void);
 #endif
 
+int tor_libevent_run_event_loop(struct event_base *base, int once);
+void tor_libevent_exit_loop_after_delay(struct event_base *base,
+                                        const struct timeval *delay);
+void tor_libevent_exit_loop_after_callback(struct event_base *base);
+
 #ifdef COMPAT_LIBEVENT_PRIVATE
 
 /** Macro: returns the number of a Libevent version as a 4-byte number,
@@ -69,7 +92,7 @@ void tor_gettimeofday_cache_set(const struct timeval *tv);
 
 STATIC void
 libevent_logging_callback(int severity, const char *msg);
-#endif
+#endif /* defined(COMPAT_LIBEVENT_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_COMPAT_LIBEVENT_H) */
 
diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h
index 1bfe188075..d1481fb46c 100644
--- a/src/common/compat_openssl.h
+++ b/src/common/compat_openssl.h
@@ -1,18 +1,19 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_OPENSSL_H
 #define TOR_COMPAT_OPENSSL_H
 
 #include <openssl/opensslv.h>
+#include "crypto_openssl_mgt.h"
 
 /**
  * \file compat_openssl.h
  *
- * \brief compatability definitions for working with different openssl forks
+ * \brief compatibility definitions for working with different openssl forks
  **/
 
 #if !defined(LIBRESSL_VERSION_NUMBER) && \
@@ -25,10 +26,13 @@
 /* We define this macro if we're trying to build with the majorly refactored
  * API in OpenSSL 1.1 */
 #define OPENSSL_1_1_API
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) && ... */
+
+#ifndef OPENSSL_VERSION
+#define OPENSSL_VERSION SSLEAY_VERSION
 #endif
 
 #ifndef OPENSSL_1_1_API
-#define OPENSSL_VERSION SSLEAY_VERSION
 #define OpenSSL_version(v) SSLeay_version(v)
 #define OpenSSL_version_num() SSLeay()
 #define RAND_OpenSSL() RAND_SSLeay()
@@ -37,11 +41,11 @@
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else
+#else /* !(!defined(OPENSSL_1_1_API)) */
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
-#endif
+#endif /* !defined(OPENSSL_1_1_API) */
 
-#endif
+#endif /* !defined(TOR_COMPAT_OPENSSL_H) */
 
diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c
index c1ae66c1d2..002274c469 100644
--- a/src/common/compat_pthreads.c
+++ b/src/common/compat_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -201,20 +201,21 @@ tor_cond_init(tor_cond_t *cond)
   }
 
 #if defined(HAVE_CLOCK_GETTIME)
-#if defined(CLOCK_MONOTONIC) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
+#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
+  defined(CLOCK_MONOTONIC)
   /* Use monotonic time so when we timedwait() on it, any clock adjustment
    * won't affect the timeout value. */
   if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
     return -1;
   }
 #define USE_COND_CLOCK CLOCK_MONOTONIC
-#else /* !defined HAVE_PTHREAD_CONDATTR_SETCLOCK */
+#else /* !(defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ...) */
   /* On OSX Sierra, there is no pthread_condattr_setclock, so we are stuck
    * with the realtime clock.
    */
 #define USE_COND_CLOCK CLOCK_REALTIME
-#endif /* which clock to use */
-#endif /* HAVE_CLOCK_GETTIME */
+#endif /* defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && ... */
+#endif /* defined(HAVE_CLOCK_GETTIME) */
   if (pthread_cond_init(&cond->cond, &condattr)) {
     return -1;
   }
@@ -266,11 +267,11 @@ tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv)
       tvnow.tv_sec = ts.tv_sec;
       tvnow.tv_usec = (int)(ts.tv_nsec / 1000);
       timeradd(tv, &tvnow, &tvsum);
-#else
+#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK)) */
       if (gettimeofday(&tvnow, NULL) < 0)
         return -1;
       timeradd(tv, &tvnow, &tvsum);
-#endif /* HAVE_CLOCK_GETTIME, CLOCK_MONOTONIC */
+#endif /* defined(HAVE_CLOCK_GETTIME) && defined(USE_COND_CLOCK) */
 
       ts.tv_sec = tvsum.tv_sec;
       ts.tv_nsec = tvsum.tv_usec * 1000;
diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c
index f4809060d6..9f64c06342 100644
--- a/src/common/compat_threads.c
+++ b/src/common/compat_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,7 +48,7 @@ tor_mutex_new_nonrecursive(void)
 }
 /** Release all storage and system resources held by <b>m</b>. */
 void
-tor_mutex_free(tor_mutex_t *m)
+tor_mutex_free_(tor_mutex_t *m)
 {
   if (!m)
     return;
@@ -68,7 +68,7 @@ tor_cond_new(void)
 
 /** Free all storage held in <b>c</b>. */
 void
-tor_cond_free(tor_cond_t *c)
+tor_cond_free_(tor_cond_t *c)
 {
   if (!c)
     return;
@@ -94,51 +94,73 @@ in_main_thread(void)
 }
 
 #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE)
-/* As write(), but retry on EINTR */
+/* As write(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 write_ni(int fd, const void *buf, size_t n)
 {
   int r;
  again:
   r = (int) write(fd, buf, n);
-  if (r < 0 && errno == EINTR)
-    goto again;
+  if (r < 0) {
+    if (errno == EINTR)
+      goto again;
+    else
+      return -errno;
+  }
   return r;
 }
-/* As read(), but retry on EINTR */
+/* As read(), but retry on EINTR, and return the negative error code on error.
+ */
 static int
 read_ni(int fd, void *buf, size_t n)
 {
   int r;
  again:
   r = (int) read(fd, buf, n);
-  if (r < 0 && errno == EINTR)
-    goto again;
+  if (r < 0) {
+    if (errno == EINTR)
+      goto again;
+    else
+      return -errno;
+  }
   return r;
 }
-#endif
+#endif /* defined(HAVE_EVENTFD) || defined(HAVE_PIPE) */
 
-/** As send(), but retry on EINTR. */
+/** As send(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 send_ni(int fd, const void *buf, size_t n, int flags)
 {
   int r;
  again:
   r = (int) send(fd, buf, n, flags);
-  if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd)))
-    goto again;
+  if (r < 0) {
+    int error = tor_socket_errno(fd);
+    if (ERRNO_IS_EINTR(error))
+      goto again;
+    else
+      return -error;
+  }
   return r;
 }
 
-/** As recv(), but retry on EINTR. */
+/** As recv(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 recv_ni(int fd, void *buf, size_t n, int flags)
 {
   int r;
  again:
   r = (int) recv(fd, buf, n, flags);
-  if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd)))
-    goto again;
+  if (r < 0) {
+    int error = tor_socket_errno(fd);
+    if (ERRNO_IS_EINTR(error))
+      goto again;
+    else
+      return -error;
+  }
   return r;
 }
 
@@ -149,7 +171,7 @@ eventfd_alert(int fd)
 {
   uint64_t u = 1;
   int r = write_ni(fd, (void*)&u, sizeof(u));
-  if (r < 0 && errno != EAGAIN)
+  if (r < 0 && -r != EAGAIN)
     return -1;
   return 0;
 }
@@ -160,11 +182,11 @@ eventfd_drain(int fd)
 {
   uint64_t u = 0;
   int r = read_ni(fd, (void*)&u, sizeof(u));
-  if (r < 0 && errno != EAGAIN)
-    return -1;
+  if (r < 0 && -r != EAGAIN)
+    return r;
   return 0;
 }
-#endif
+#endif /* defined(HAVE_EVENTFD) */
 
 #ifdef HAVE_PIPE
 /** Send a byte over a pipe. Return 0 on success or EAGAIN; -1 on error */
@@ -172,8 +194,8 @@ static int
 pipe_alert(int fd)
 {
   ssize_t r = write_ni(fd, "x", 1);
-  if (r < 0 && errno != EAGAIN)
-    return -1;
+  if (r < 0 && -r != EAGAIN)
+    return (int)r;
   return 0;
 }
 
@@ -188,11 +210,11 @@ pipe_drain(int fd)
     r = read_ni(fd, buf, sizeof(buf));
   } while (r > 0);
   if (r < 0 && errno != EAGAIN)
-    return -1;
+    return -errno;
   /* A value of r = 0 means EOF on the fd so successfully drained. */
   return 0;
 }
-#endif
+#endif /* defined(HAVE_PIPE) */
 
 /** Send a byte on socket <b>fd</b>t.  Return 0 on success or EAGAIN,
  * -1 on error. */
@@ -200,13 +222,13 @@ static int
 sock_alert(tor_socket_t fd)
 {
   ssize_t r = send_ni(fd, "x", 1, 0);
-  if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd)))
-    return -1;
+  if (r < 0 && !ERRNO_IS_EAGAIN(-r))
+    return (int)r;
   return 0;
 }
 
 /** Drain all the input from a socket <b>fd</b>, and ignore it.  Return 0 on
- * success, -1 on error. */
+ * success, -errno on error. */
 static int
 sock_drain(tor_socket_t fd)
 {
@@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd)
   do {
     r = recv_ni(fd, buf, sizeof(buf), 0);
   } while (r > 0);
-  if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd)))
-    return -1;
+  if (r < 0 && !ERRNO_IS_EAGAIN(-r))
+    return (int)r;
   /* A value of r = 0 means EOF on the fd so successfully drained. */
   return 0;
 }
@@ -254,7 +276,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags)
     socks_out->drain_fn = eventfd_drain;
     return 0;
   }
-#endif
+#endif /* defined(HAVE_EVENTFD) */
 
 #ifdef HAVE_PIPE2
   /* Now we're going to try pipes. First type the pipe2() syscall, if we
@@ -267,7 +289,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags)
     socks_out->drain_fn = pipe_drain;
     return 0;
   }
-#endif
+#endif /* defined(HAVE_PIPE2) */
 
 #ifdef HAVE_PIPE
   /* Now try the regular pipe() syscall.  Pipes have a bit lower overhead than
@@ -291,7 +313,7 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags)
     socks_out->drain_fn = pipe_drain;
     return 0;
   }
-#endif
+#endif /* defined(HAVE_PIPE) */
 
   /* If nothing else worked, fall back on socketpair(). */
   if (!(flags & ASOCKS_NOSOCKETPAIR) &&
@@ -330,3 +352,55 @@ alert_sockets_close(alert_sockets_t *socks)
   socks->read_fd = socks->write_fd = -1;
 }
 
+#ifndef HAVE_WORKING_STDATOMIC
+/** Initialize a new atomic counter with the value 0 */
+void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  memset(counter, 0, sizeof(*counter));
+  tor_mutex_init_nonrecursive(&counter->mutex);
+}
+/** Clean up all resources held by an atomic counter. */
+void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  tor_mutex_uninit(&counter->mutex);
+  memset(counter, 0, sizeof(*counter));
+}
+/** Add a value to an atomic counter. */
+void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  tor_mutex_acquire(&counter->mutex);
+  counter->val += add;
+  tor_mutex_release(&counter->mutex);
+}
+/** Subtract a value from an atomic counter. */
+void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  // this relies on unsigned overflow, but that's fine.
+  atomic_counter_add(counter, -sub);
+}
+/** Return the current value of an atomic counter */
+size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  size_t val;
+  tor_mutex_acquire(&counter->mutex);
+  val = counter->val;
+  tor_mutex_release(&counter->mutex);
+  return val;
+}
+/** Replace the value of an atomic counter; return the old one. */
+size_t
+atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
+{
+  size_t oldval;
+  tor_mutex_acquire(&counter->mutex);
+  oldval = counter->val;
+  counter->val = newval;
+  tor_mutex_release(&counter->mutex);
+  return oldval;
+}
+#endif /* !defined(HAVE_WORKING_STDATOMIC) */
diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h
index 171a9f93ff..8bf8225689 100644
--- a/src/common/compat_threads.h
+++ b/src/common/compat_threads.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_THREADS_H
@@ -14,13 +14,21 @@
 #include <pthread.h>
 #endif
 
+#if defined(HAVE_STDATOMIC_H) && defined(STDATOMIC_WORKS)
+#define HAVE_WORKING_STDATOMIC
+#endif
+
+#ifdef HAVE_WORKING_STDATOMIC
+#include <stdatomic.h>
+#endif
+
 #if defined(_WIN32)
 #define USE_WIN32_THREADS
 #elif defined(HAVE_PTHREAD_H) && defined(HAVE_PTHREAD_CREATE)
 #define USE_PTHREADS
 #else
 #error "No threading system was found"
-#endif
+#endif /* defined(_WIN32) || ... */
 
 int spawn_func(void (*func)(void *), void *data);
 void spawn_exit(void) ATTR_NORETURN;
@@ -41,7 +49,7 @@ typedef struct tor_mutex_t {
 #else
   /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
   int _unused;
-#endif
+#endif /* defined(USE_WIN32_THREADS) || ... */
 } tor_mutex_t;
 
 tor_mutex_t *tor_mutex_new(void);
@@ -50,7 +58,8 @@ void tor_mutex_init(tor_mutex_t *m);
 void tor_mutex_init_nonrecursive(tor_mutex_t *m);
 void tor_mutex_acquire(tor_mutex_t *m);
 void tor_mutex_release(tor_mutex_t *m);
-void tor_mutex_free(tor_mutex_t *m);
+void tor_mutex_free_(tor_mutex_t *m);
+#define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m))
 void tor_mutex_uninit(tor_mutex_t *m);
 unsigned long tor_get_thread_id(void);
 void tor_threads_init(void);
@@ -73,11 +82,12 @@ typedef struct tor_cond_t {
   int generation;
 #else
 #error no known condition implementation.
-#endif
+#endif /* defined(USE_PTHREADS) || ... */
 } tor_cond_t;
 
 tor_cond_t *tor_cond_new(void);
-void tor_cond_free(tor_cond_t *cond);
+void tor_cond_free_(tor_cond_t *cond);
+#define tor_cond_free(c) FREE_AND_NULL(tor_cond_t, tor_cond_free_, (c))
 int tor_cond_init(tor_cond_t *cond);
 void tor_cond_uninit(tor_cond_t *cond);
 int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex,
@@ -147,5 +157,70 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal);
  */
 void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
 
-#endif
-
+/**
+ * Atomic counter type; holds a size_t value.
+ */
+#ifdef HAVE_WORKING_STDATOMIC
+typedef struct atomic_counter_t {
+  atomic_size_t val;
+} atomic_counter_t;
+#define ATOMIC_LINKAGE static
+#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+typedef struct atomic_counter_t {
+  tor_mutex_t mutex;
+  size_t val;
+} atomic_counter_t;
+#define ATOMIC_LINKAGE
+#endif /* defined(HAVE_WORKING_STDATOMIC) */
+
+ATOMIC_LINKAGE void atomic_counter_init(atomic_counter_t *counter);
+ATOMIC_LINKAGE void atomic_counter_destroy(atomic_counter_t *counter);
+ATOMIC_LINKAGE void atomic_counter_add(atomic_counter_t *counter, size_t add);
+ATOMIC_LINKAGE void atomic_counter_sub(atomic_counter_t *counter, size_t sub);
+ATOMIC_LINKAGE size_t atomic_counter_get(atomic_counter_t *counter);
+ATOMIC_LINKAGE size_t atomic_counter_exchange(atomic_counter_t *counter,
+                                              size_t newval);
+#undef ATOMIC_LINKAGE
+
+#ifdef HAVE_WORKING_STDATOMIC
+/** Initialize a new atomic counter with the value 0 */
+static inline void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  atomic_init(&counter->val, 0);
+}
+/** Clean up all resources held by an atomic counter. */
+static inline void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  (void)counter;
+}
+/** Add a value to an atomic counter. */
+static inline void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  (void) atomic_fetch_add(&counter->val, add);
+}
+/** Subtract a value from an atomic counter. */
+static inline void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  (void) atomic_fetch_sub(&counter->val, sub);
+}
+/** Return the current value of an atomic counter */
+static inline size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  return atomic_load(&counter->val);
+}
+/** Replace the value of an atomic counter; return the old one. */
+static inline size_t
+atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
+{
+  return atomic_exchange(&counter->val, newval);
+}
+
+#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#endif /* defined(HAVE_WORKING_STDATOMIC) */
+
+#endif /* !defined(TOR_COMPAT_THREADS_H) */
diff --git a/src/common/compat_time.c b/src/common/compat_time.c
index d044bbe1d7..93b527def0 100644
--- a/src/common/compat_time.c
+++ b/src/common/compat_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,7 +28,7 @@
 /* as fallback implementation for tor_sleep_msec */
 #include <sys/select.h>
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef __APPLE__
 #include <mach/mach_time.h>
@@ -64,15 +64,15 @@ tor_sleep_msec(int msec)
   select(0, NULL, NULL, NULL, &tv);
 #else
   sleep(CEIL_DIV(msec, 1000));
-#endif
+#endif /* defined(_WIN32) || ... */
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Set *timeval to the current time of day.  On error, log and terminate.
  * (Same as gettimeofday(timeval,NULL), but never returns -1.)
  */
-void
-tor_gettimeofday(struct timeval *timeval)
+MOCK_IMPL(void,
+tor_gettimeofday, (struct timeval *timeval))
 {
 #ifdef _WIN32
   /* Epoch bias copied from perl: number of units between windows epoch and
@@ -90,7 +90,7 @@ tor_gettimeofday(struct timeval *timeval)
   if (ft.ft_64 < EPOCH_BIAS) {
     /* LCOV_EXCL_START */
     log_err(LD_GENERAL,"System time is before 1970; failing.");
-    exit(1);
+    exit(1); // exit ok: system clock is broken.
     /* LCOV_EXCL_STOP */
   }
   ft.ft_64 -= EPOCH_BIAS;
@@ -102,7 +102,7 @@ tor_gettimeofday(struct timeval *timeval)
     log_err(LD_GENERAL,"gettimeofday failed.");
     /* If gettimeofday dies, we have either given a bad timezone (we didn't),
        or segfaulted.*/
-    exit(1);
+    exit(1); // exit ok: gettimeofday failed.
     /* LCOV_EXCL_STOP */
   }
 #elif defined(HAVE_FTIME)
@@ -112,7 +112,7 @@ tor_gettimeofday(struct timeval *timeval)
   timeval->tv_usec = tb.millitm * 1000;
 #else
 #error "No way to get time."
-#endif
+#endif /* defined(_WIN32) || ... */
   return;
 }
 
@@ -187,8 +187,8 @@ monotime_coarse_set_mock_time_nsec(int64_t nsec)
   tor_assert_nonfatal(monotime_mocking_enabled == 1);
   mock_time_nsec_coarse = nsec;
 }
-#endif
-#endif
+#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* "ratchet" functions for monotonic time. */
 
@@ -235,7 +235,7 @@ ratchet_coarse_performance_counter(const int64_t count_raw)
   last_tick_count = count;
   return count;
 }
-#endif
+#endif /* defined(_WIN32) || defined(TOR_UNIT_TESTS) */
 
 #if defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS)
 static struct timeval last_timeofday = { 0, 0 };
@@ -251,7 +251,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
 {
   /* must hold lock */
   timeradd(timeval_raw, &timeofday_offset, out);
-  if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, <))) {
+  if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, OP_LT))) {
     /* time ran backwards. Instead, declare that no time occurred. */
     timersub(&last_timeofday, timeval_raw, &timeofday_offset);
     memcpy(out, &last_timeofday, sizeof(struct timeval));
@@ -259,7 +259,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
     memcpy(&last_timeofday, out, sizeof(struct timeval));
   }
 }
-#endif
+#endif /* defined(MONOTIME_USING_GETTIMEOFDAY) || defined(TOR_UNIT_TESTS) */
 
 #ifdef TOR_UNIT_TESTS
 /** For testing: reset all the ratchets */
@@ -271,7 +271,7 @@ monotime_reset_ratchets_for_testing(void)
   memset(&last_timeofday, 0, sizeof(struct timeval));
   memset(&timeofday_offset, 0, sizeof(struct timeval));
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef __APPLE__
 
@@ -279,6 +279,9 @@ monotime_reset_ratchets_for_testing(void)
  * nanoseconds.
  */
 static struct mach_timebase_info mach_time_info;
+static struct mach_timebase_info mach_time_info_msec_cvt;
+static int32_t mach_time_msec_cvt_threshold;
+static int monotime_shift = 0;
 
 static void
 monotime_init_internal(void)
@@ -287,6 +290,26 @@ monotime_init_internal(void)
   int r = mach_timebase_info(&mach_time_info);
   tor_assert(r == 0);
   tor_assert(mach_time_info.denom != 0);
+
+  {
+    // approximate only.
+    uint64_t ns_per_tick = mach_time_info.numer / mach_time_info.denom;
+    uint64_t ms_per_tick = ns_per_tick * ONE_MILLION;
+    // requires that tor_log2(0) == 0.
+    monotime_shift = tor_log2(ms_per_tick);
+  }
+  {
+    // For converting ticks to milliseconds in a 32-bit-friendly way, we
+    // will first right-shift by 20, and then multiply by 2048/1953, since
+    // (1<<20) * 1953/2048 is about 1e6.  We precompute a new numerator and
+    // denominator here to avoid multiple multiplies.
+    mach_time_info_msec_cvt.numer = mach_time_info.numer * 2048;
+    mach_time_info_msec_cvt.denom = mach_time_info.denom * 1953;
+    // For any value above this amount, we should divide before multiplying,
+    // to avoid overflow.  For a value below this, we should multiply
+    // before dividing, to improve accuracy.
+    mach_time_msec_cvt_threshold = INT32_MAX / mach_time_info_msec_cvt.numer;
+  }
 }
 
 /**
@@ -301,10 +324,25 @@ monotime_get(monotime_t *out)
       / mach_time_info.numer;
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
   out->abstime_ = mach_absolute_time();
 }
 
+#if defined(HAVE_MACH_APPROXIMATE_TIME)
+void
+monotime_coarse_get(monotime_coarse_t *out)
+{
+#ifdef TOR_UNIT_TESTS
+  if (monotime_mocking_enabled) {
+    out->abstime_ = (mock_time_nsec_coarse * mach_time_info.denom)
+      / mach_time_info.numer;
+    return;
+  }
+#endif /* defined(TOR_UNIT_TESTS) */
+  out->abstime_ = mach_approximate_time();
+}
+#endif
+
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
  */
@@ -321,6 +359,47 @@ monotime_diff_nsec(const monotime_t *start,
   return diff_nsec;
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  if (BUG(mach_time_info.denom == 0)) {
+    monotime_init();
+  }
+  const int64_t diff_ticks = end->abstime_ - start->abstime_;
+
+  /* We already require in di_ops.c that right-shift performs a sign-extend. */
+  const int32_t diff_microticks = (int32_t)(diff_ticks >> 20);
+
+  if (diff_microticks >= mach_time_msec_cvt_threshold) {
+    return (diff_microticks / mach_time_info_msec_cvt.denom) *
+      mach_time_info_msec_cvt.numer;
+  } else {
+    return (diff_microticks * mach_time_info_msec_cvt.numer) /
+      mach_time_info_msec_cvt.denom;
+  }
+}
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  return (uint32_t)(t->abstime_ >> monotime_shift);
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->abstime_ == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint64_t nsec = msec * ONE_MILLION;
+  const uint64_t ticks = (nsec * mach_time_info.denom) / mach_time_info.numer;
+  out->abstime_ = val->abstime_ + ticks;
+}
+
 /* end of "__APPLE__" */
 #elif defined(HAVE_CLOCK_GETTIME)
 
@@ -332,7 +411,7 @@ monotime_diff_nsec(const monotime_t *start,
  * an old Linux kernel. In that case, we will fall back to CLOCK_MONOTONIC.
  */
 static int clock_monotonic_coarse = CLOCK_MONOTONIC_COARSE;
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) */
 
 static void
 monotime_init_internal(void)
@@ -344,7 +423,7 @@ monotime_init_internal(void)
              "falling back to CLOCK_MONOTONIC.", strerror(errno));
     clock_monotonic_coarse = CLOCK_MONOTONIC;
   }
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) */
 }
 
 void
@@ -356,7 +435,7 @@ monotime_get(monotime_t *out)
     out->ts_.tv_nsec = (int) (mock_time_nsec % ONE_BILLION);
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
   int r = clock_gettime(CLOCK_MONOTONIC, &out->ts_);
   tor_assert(r == 0);
 }
@@ -371,7 +450,7 @@ monotime_coarse_get(monotime_coarse_t *out)
     out->ts_.tv_nsec = (int) (mock_time_nsec_coarse % ONE_BILLION);
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
   int r = clock_gettime(clock_monotonic_coarse, &out->ts_);
   if (PREDICT_UNLIKELY(r < 0) &&
       errno == EINVAL &&
@@ -386,7 +465,7 @@ monotime_coarse_get(monotime_coarse_t *out)
 
   tor_assert(r == 0);
 }
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) */
 
 int64_t
 monotime_diff_nsec(const monotime_t *start,
@@ -399,6 +478,46 @@ monotime_diff_nsec(const monotime_t *start,
   return diff_nsec;
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  const int32_t diff_sec = (int32_t)(end->ts_.tv_sec - start->ts_.tv_sec);
+  const int32_t diff_nsec = (int32_t)(end->ts_.tv_nsec - start->ts_.tv_nsec);
+  return diff_sec * 1000 + diff_nsec / ONE_MILLION;
+}
+
+/* This value is ONE_BILLION >> 20. */
+static const uint32_t STAMP_TICKS_PER_SECOND = 953;
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  uint32_t nsec = (uint32_t)t->ts_.tv_nsec;
+  uint32_t sec = (uint32_t)t->ts_.tv_sec;
+
+  return (sec * STAMP_TICKS_PER_SECOND) + (nsec >> 20);
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->ts_.tv_sec == 0 && val->ts_.tv_nsec == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint32_t sec = msec / 1000;
+  const uint32_t msec_remainder = msec % 1000;
+  out->ts_.tv_sec = val->ts_.tv_sec + sec;
+  out->ts_.tv_nsec = val->ts_.tv_nsec + (msec_remainder * ONE_MILLION);
+  if (out->ts_.tv_nsec > ONE_BILLION) {
+    out->ts_.tv_nsec -= ONE_BILLION;
+    out->ts_.tv_sec += 1;
+  }
+}
+
 /* end of "HAVE_CLOCK_GETTIME" */
 #elif defined (_WIN32)
 
@@ -462,7 +581,7 @@ monotime_get(monotime_t *out)
       / nsec_per_tick_numer;
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
   /* Alas, QueryPerformanceCounter is not always monotonic: see bug list at
 
@@ -486,7 +605,7 @@ monotime_coarse_get(monotime_coarse_t *out)
     out->tick_count_ = mock_time_nsec_coarse / ONE_MILLION;
     return;
   }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
   if (GetTickCount64_fn) {
     out->tick_count_ = (int64_t)GetTickCount64_fn();
@@ -517,6 +636,13 @@ monotime_coarse_diff_msec(const monotime_coarse_t *start,
   return diff_ticks;
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  return (int32_t)monotime_coarse_diff_msec(start, end);
+}
+
 int64_t
 monotime_coarse_diff_usec(const monotime_coarse_t *start,
                           const monotime_coarse_t *end)
@@ -531,6 +657,41 @@ monotime_coarse_diff_nsec(const monotime_coarse_t *start,
   return monotime_coarse_diff_msec(start, end) * ONE_MILLION;
 }
 
+static const uint32_t STAMP_TICKS_PER_SECOND = 1000;
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  return (uint32_t) t->tick_count_;
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->pcount_ == 0;
+}
+
+int
+monotime_coarse_is_zero(const monotime_coarse_t *val)
+{
+  return val->tick_count_ == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint64_t nsec = msec * ONE_MILLION;
+  const uint64_t ticks = (nsec * nsec_per_tick_denom) / nsec_per_tick_numer;
+  out->pcount_ = val->pcount_ + ticks;
+}
+
+void
+monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val,
+                         uint32_t msec)
+{
+  out->tick_count_ = val->tick_count_ + msec;
+}
+
 /* end of "_WIN32" */
 #elif defined(MONOTIME_USING_GETTIMEOFDAY)
 
@@ -567,10 +728,49 @@ monotime_diff_nsec(const monotime_t *start,
   return (diff.tv_sec * ONE_BILLION + diff.tv_usec * 1000);
 }
 
+int32_t
+monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                             const monotime_coarse_t *end)
+{
+  struct timeval diff;
+  timersub(&end->tv_, &start->tv_, &diff);
+  return diff.tv_sec * 1000 + diff.tv_usec / 1000;
+}
+
+/* This value is ONE_MILLION >> 10. */
+static const uint32_t STAMP_TICKS_PER_SECOND = 976;
+
+uint32_t
+monotime_coarse_to_stamp(const monotime_coarse_t *t)
+{
+  const uint32_t usec = (uint32_t)t->tv_.tv_usec;
+  const uint32_t sec = (uint32_t)t->tv_.tv_sec;
+  return (sec * STAMP_TICKS_PER_SECOND) | (nsec >> 10);
+}
+
+int
+monotime_is_zero(const monotime_t *val)
+{
+  return val->tv_.tv_sec == 0 && val->tv_.tv_usec == 0;
+}
+
+void
+monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec)
+{
+  const uint32_t sec = msec / 1000;
+  const uint32_t msec_remainder = msec % 1000;
+  out->tv_.tv_sec = val->tv_.tv_sec + sec;
+  out->tv_.tv_usec = val->tv_.tv_nsec + (msec_remainder * 1000);
+  if (out->tv_.tv_usec > ONE_MILLION) {
+    out->tv_.tv_usec -= ONE_MILLION;
+    out->tv_.tv_sec += 1;
+  }
+}
+
 /* end of "MONOTIME_USING_GETTIMEOFDAY" */
 #else
 #error "No way to implement monotonic timers."
-#endif
+#endif /* defined(__APPLE__) || ... */
 
 /**
  * Initialize the monotonic timer subsystem. Must be called before any
@@ -589,6 +789,19 @@ monotime_init(void)
   }
 }
 
+void
+monotime_zero(monotime_t *out)
+{
+  memset(out, 0, sizeof(*out));
+}
+#ifdef MONOTIME_COARSE_TYPE_IS_DIFFERENT
+void
+monotime_coarse_zero(monotime_coarse_t *out)
+{
+  memset(out, 0, sizeof(*out));
+}
+#endif
+
 int64_t
 monotime_diff_usec(const monotime_t *start,
                    const monotime_t *end)
@@ -653,5 +866,48 @@ monotime_coarse_absolute_msec(void)
 {
   return monotime_coarse_absolute_nsec() / ONE_MILLION;
 }
+#else
+#define initialized_at_coarse initialized_at
+#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
+
+/**
+ * Return the current time "stamp" as described by monotime_coarse_to_stamp.
+ */
+uint32_t
+monotime_coarse_get_stamp(void)
+{
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  return monotime_coarse_to_stamp(&now);
+}
+
+#ifdef __APPLE__
+uint64_t
+monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
+{
+  /* Recover as much precision as we can. */
+  uint64_t abstime_diff = (units << monotime_shift);
+  return (abstime_diff * mach_time_info.numer) /
+    (mach_time_info.denom * ONE_MILLION);
+}
+uint64_t
+monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
+{
+  uint64_t abstime_val =
+    (((uint64_t)msec) * ONE_MILLION * mach_time_info.denom) /
+    mach_time_info.numer;
+  return abstime_val >> monotime_shift;
+}
+#else
+uint64_t
+monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
+{
+  return (units * 1000) / STAMP_TICKS_PER_SECOND;
+}
+uint64_t
+monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
+{
+  return (msec * STAMP_TICKS_PER_SECOND) / 1000;
+}
 #endif
 
diff --git a/src/common/compat_time.h b/src/common/compat_time.h
index 2262446e57..f241aa5eba 100644
--- a/src/common/compat_time.h
+++ b/src/common/compat_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,13 +28,13 @@
 #include <time.h>
 #endif
 
-#if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC)
+#if !defined(HAVE_STRUCT_TIMEVAL_TV_SEC)
 /** Implementation of timeval for platforms that don't have it. */
 struct timeval {
   time_t tv_sec;
   unsigned int tv_usec;
 };
-#endif
+#endif /* !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) */
 
 /** Represents a monotonic timer in a platform-dependent way. */
 typedef struct monotime_t {
@@ -51,10 +51,11 @@ typedef struct monotime_t {
 #define MONOTIME_USING_GETTIMEOFDAY
   /* Otherwise, we will be stuck using gettimeofday. */
   struct timeval tv_;
-#endif
+#endif /* defined(__APPLE__) || ... */
 } monotime_t;
 
-#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC_COARSE)
+#if defined(CLOCK_MONOTONIC_COARSE) && \
+  defined(HAVE_CLOCK_GETTIME)
 #define MONOTIME_COARSE_FN_IS_DIFFERENT
 #define monotime_coarse_t monotime_t
 #elif defined(_WIN32)
@@ -64,9 +65,12 @@ typedef struct monotime_t {
 typedef struct monotime_coarse_t {
   uint64_t tick_count_;
 } monotime_coarse_t;
+#elif defined(__APPLE__) && defined(HAVE_MACH_APPROXIMATE_TIME)
+#define MONOTIME_COARSE_FN_IS_DIFFERENT
+#define monotime_coarse_t monotime_t
 #else
 #define monotime_coarse_t monotime_t
-#endif
+#endif /* defined(CLOCK_MONOTONIC_COARSE) && ... || ... */
 
 /**
  * Initialize the timing subsystem. This function is idempotent.
@@ -101,6 +105,21 @@ uint64_t monotime_absolute_usec(void);
  */
 uint64_t monotime_absolute_msec(void);
 
+/**
+ * Set <b>out</b> to zero.
+ */
+void monotime_zero(monotime_t *out);
+/**
+ * Return true iff <b>out</b> is zero
+ */
+int monotime_is_zero(const monotime_t *out);
+
+/**
+ * Set <b>out</b> to N milliseconds after <b>val</b>.
+ */
+/* XXXX We should add a more generic function here if we ever need to */
+void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec);
+
 #if defined(MONOTIME_COARSE_FN_IS_DIFFERENT)
 /**
  * Set <b>out</b> to the current coarse time.
@@ -109,12 +128,30 @@ void monotime_coarse_get(monotime_coarse_t *out);
 uint64_t monotime_coarse_absolute_nsec(void);
 uint64_t monotime_coarse_absolute_usec(void);
 uint64_t monotime_coarse_absolute_msec(void);
-#else
+#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */
 #define monotime_coarse_get monotime_get
 #define monotime_coarse_absolute_nsec monotime_absolute_nsec
 #define monotime_coarse_absolute_usec monotime_absolute_usec
 #define monotime_coarse_absolute_msec monotime_absolute_msec
-#endif
+#endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
+
+/**
+ * Return a "timestamp" approximation for a coarse monotonic timer.
+ * This timestamp is meant to be fast to calculate and easy to
+ * compare, and have a unit of something roughly around 1 msec.
+ *
+ * It will wrap over from time to time.
+ *
+ * It has no defined zero point.
+ */
+uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
+/**
+ * Convert a difference, expressed in the units of monotime_coarse_to_stamp,
+ * into an approximate number of milliseconds.
+ */
+uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
+uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
+uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
 int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start,
@@ -123,13 +160,48 @@ int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
-#else
+void monotime_coarse_zero(monotime_coarse_t *out);
+int monotime_coarse_is_zero(const monotime_coarse_t *val);
+void monotime_coarse_add_msec(monotime_coarse_t *out,
+                              const monotime_coarse_t *val, uint32_t msec);
+#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */
 #define monotime_coarse_diff_nsec monotime_diff_nsec
 #define monotime_coarse_diff_usec monotime_diff_usec
 #define monotime_coarse_diff_msec monotime_diff_msec
+#define monotime_coarse_zero monotime_zero
+#define monotime_coarse_is_zero monotime_is_zero
+#define monotime_coarse_add_msec monotime_add_msec
+#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
+
+/**
+ * As monotime_coarse_diff_msec, but avoid 64-bit division.
+ *
+ * Requires that the difference fit into an int32_t; not for use with
+ * large time differences.
+ */
+int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
+                                     const monotime_coarse_t *end);
+
+/**
+ * As monotime_coarse_diff_msec, but avoid 64-bit division if it is expensive.
+ *
+ * Requires that the difference fit into an int32_t; not for use with
+ * large time differences.
+ */
+static inline int32_t
+monotime_coarse_diff_msec32(const monotime_coarse_t *start,
+                            const monotime_coarse_t *end)
+{
+#if SIZEOF_VOID_P == 8
+  // on a 64-bit platform, let's assume 64/64 division is cheap.
+  return (int32_t) monotime_coarse_diff_msec(start, end);
+#else
+#define USING_32BIT_MSEC_HACK
+  return monotime_coarse_diff_msec32_(start, end);
 #endif
+}
 
-void tor_gettimeofday(struct timeval *timeval);
+MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
 
 #ifdef TOR_UNIT_TESTS
 void tor_sleep_msec(int msec);
@@ -142,7 +214,7 @@ void monotime_coarse_set_mock_time_nsec(int64_t);
 #else
 #define monotime_coarse_set_mock_time_nsec monotime_set_mock_time_nsec
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef COMPAT_TIME_PRIVATE
 #if defined(_WIN32) || defined(TOR_UNIT_TESTS)
@@ -156,7 +228,7 @@ STATIC void ratchet_timeval(const struct timeval *timeval_raw,
 #ifdef TOR_UNIT_TESTS
 void monotime_reset_ratchets_for_testing(void);
 #endif
-#endif
+#endif /* defined(COMPAT_TIME_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_COMPAT_TIME_H) */
 
diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c
index 735be4ad17..7021344f6e 100644
--- a/src/common/compat_winthreads.c
+++ b/src/common/compat_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,7 +18,6 @@
 #include "util.h"
 #include "container.h"
 #include "torlog.h"
-#include <process.h>
 
 /* This value is more or less total cargo-cult */
 #define SPIN_COUNT 2000
@@ -48,10 +47,12 @@ void
 spawn_exit(void)
 {
   _endthread();
+  // LCOV_EXCL_START
   //we should never get here. my compiler thinks that _endthread returns, this
   //is an attempt to fool it.
   tor_assert(0);
-  _exit(0);
+  _exit(0); // exit ok: unreachable.
+  // LCOV_EXCL_STOP
 }
 
 void
@@ -246,5 +247,5 @@ tor_threads_init(void)
   set_main_thread();
 }
 
-#endif
+#endif /* defined(_WIN32) */
 
diff --git a/src/common/compress.c b/src/common/compress.c
new file mode 100644
index 0000000000..cb1549f1aa
--- /dev/null
+++ b/src/common/compress.c
@@ -0,0 +1,675 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress.c
+ * \brief Common compression API.
+ **/
+
+#include "orconfig.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include "torint.h"
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_lzma.h"
+#include "compress_none.h"
+#include "compress_zlib.h"
+#include "compress_zstd.h"
+
+/** Total number of bytes allocated for compression state overhead. */
+static atomic_counter_t total_compress_allocation;
+
+/** @{ */
+/* These macros define the maximum allowable compression factor.  Anything of
+ * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to
+ * have an uncompression factor (uncompressed size:compressed size ratio) of
+ * any greater than MAX_UNCOMPRESSION_FACTOR.
+ *
+ * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to
+ * be small to limit the attack multiplier, but we also want it to be large
+ * enough so that no legitimate document --even ones we might invent in the
+ * future -- ever compresses by a factor of greater than
+ * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably
+ * large range of possible values. IMO, anything over 8 is probably safe; IMO
+ * anything under 50 is probably sufficient.
+ */
+#define MAX_UNCOMPRESSION_FACTOR 25
+#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64)
+/** @} */
+
+/** Return true if uncompressing an input of size <b>in_size</b> to an input of
+ * size at least <b>size_out</b> looks like a compression bomb. */
+MOCK_IMPL(int,
+tor_compress_is_compression_bomb,(size_t size_in, size_t size_out))
+{
+  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
+    return 0;
+
+  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
+}
+
+/** Guess the size that <b>in_len</b> will be after compression or
+ * decompression. */
+static size_t
+guess_compress_size(int compress, compress_method_t method,
+                    compression_level_t compression_level,
+                    size_t in_len)
+{
+  // ignore these for now.
+  (void)compression_level;
+  if (method == NO_METHOD) {
+    /* Guess that we'll need an extra byte, to avoid a needless realloc
+     * for nul-termination */
+    return (in_len < SIZE_MAX) ? in_len + 1 : in_len;
+  }
+
+  /* Always guess a factor of 2. */
+  if (compress) {
+    in_len /= 2;
+  } else {
+    if (in_len < SIZE_T_CEILING/2)
+      in_len *= 2;
+  }
+  return MAX(in_len, 1024);
+}
+
+/** Internal function to implement tor_compress/tor_uncompress, depending on
+ * whether <b>compress</b> is set.  All arguments are as for tor_compress or
+ * tor_uncompress. */
+static int
+tor_compress_impl(int compress,
+                  char **out, size_t *out_len,
+                  const char *in, size_t in_len,
+                  compress_method_t method,
+                  compression_level_t compression_level,
+                  int complete_only,
+                  int protocol_warn_level)
+{
+  tor_compress_state_t *stream;
+  int rv;
+
+  stream = tor_compress_new(compress, method, compression_level);
+
+  if (stream == NULL) {
+    log_warn(LD_GENERAL, "NULL stream while %scompressing",
+             compress?"":"de");
+    log_debug(LD_GENERAL, "method: %d level: %d at len: %lu",
+              method, compression_level, (unsigned long)in_len);
+    return -1;
+  }
+
+  size_t in_len_orig = in_len;
+  size_t out_remaining, out_alloc;
+  char *outptr;
+
+  out_remaining = out_alloc =
+    guess_compress_size(compress, method, compression_level, in_len);
+  *out = outptr = tor_malloc(out_remaining);
+
+  const int finish = complete_only || compress;
+
+  while (1) {
+    switch (tor_compress_process(stream,
+                                 &outptr, &out_remaining,
+                                 &in, &in_len, finish)) {
+      case TOR_COMPRESS_DONE:
+        if (in_len == 0 || compress) {
+          goto done;
+        } else {
+          // More data is present, and we're decompressing.  So we may need to
+          // reinitialize the stream if we are handling multiple concatenated
+          // inputs.
+          tor_compress_free(stream);
+          stream = tor_compress_new(compress, method, compression_level);
+          if (stream == NULL) {
+            log_warn(LD_GENERAL, "NULL stream while %scompressing",
+                     compress?"":"de");
+            goto err;
+          }
+        }
+        break;
+      case TOR_COMPRESS_OK:
+        if (compress || complete_only) {
+          log_fn(protocol_warn_level, LD_PROTOCOL,
+                 "Unexpected %s while %scompressing",
+                 complete_only?"end of input":"result",
+                 compress?"":"de");
+          log_debug(LD_GENERAL, "method: %d level: %d at len: %lu",
+                    method, compression_level, (unsigned long)in_len);
+          goto err;
+        } else {
+          if (in_len == 0) {
+            goto done;
+          }
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL: {
+        if (!compress && outptr < *out+out_alloc) {
+          // A buffer error in this case means that we have a problem
+          // with our input.
+          log_fn(protocol_warn_level, LD_PROTOCOL,
+                 "Possible truncated or corrupt compressed data");
+          goto err;
+        }
+        if (out_alloc >= SIZE_T_CEILING / 2) {
+          log_warn(LD_GENERAL, "While %scompressing data: ran out of space.",
+                   compress?"":"un");
+          goto err;
+        }
+        if (!compress &&
+            tor_compress_is_compression_bomb(in_len_orig, out_alloc)) {
+          // This should already have been caught down in the backend logic.
+          // LCOV_EXCL_START
+          tor_assert_nonfatal_unreached();
+          goto err;
+          // LCOV_EXCL_STOP
+        }
+        const size_t offset = outptr - *out;
+        out_alloc *= 2;
+        *out = tor_realloc(*out, out_alloc);
+        outptr = *out + offset;
+        out_remaining = out_alloc - offset;
+        break;
+      }
+      case TOR_COMPRESS_ERROR:
+        log_fn(protocol_warn_level, LD_GENERAL,
+               "Error while %scompressing data: bad input?",
+               compress?"":"un");
+        goto err; // bad data.
+
+        // LCOV_EXCL_START
+      default:
+        tor_assert_nonfatal_unreached();
+        goto err;
+        // LCOV_EXCL_STOP
+    }
+  }
+ done:
+  *out_len = outptr - *out;
+  if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) {
+    log_warn(LD_BUG, "We compressed something and got an insanely high "
+             "compression factor; other Tors would think this was a "
+             "compression bomb.");
+    goto err;
+  }
+  if (!compress) {
+    // NUL-terminate our output.
+    if (out_alloc == *out_len)
+      *out = tor_realloc(*out, out_alloc + 1);
+    (*out)[*out_len] = '\0';
+  }
+  rv = 0;
+  goto out;
+
+ err:
+  tor_free(*out);
+  *out_len = 0;
+  rv = -1;
+  goto out;
+
+ out:
+  tor_compress_free(stream);
+  return rv;
+}
+
+/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
+ * allocated buffer, using the method described in <b>method</b>.  Store the
+ * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
+ * Return 0 on success, -1 on failure.
+ */
+int
+tor_compress(char **out, size_t *out_len,
+             const char *in, size_t in_len,
+             compress_method_t method)
+{
+  return tor_compress_impl(1, out, out_len, in, in_len, method,
+                           BEST_COMPRESSION,
+                           1, LOG_WARN);
+}
+
+/** Given zero or more compressed strings of total length <b>in_len</b> bytes
+ * at <b>in</b>, uncompress them into a newly allocated buffer, using the
+ * method described in <b>method</b>.  Store the uncompressed string in
+ * *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on success, -1 on
+ * failure.
+ *
+ * If any bytes are written to <b>out</b>, an extra byte NUL is always
+ * written at the end, but not counted in <b>out_len</b>.  This is a
+ * safety feature to ensure that the output can be treated as a
+ * NUL-terminated string -- though of course, callers should check
+ * out_len anyway.
+ *
+ * If <b>complete_only</b> is true, we consider a truncated input as a
+ * failure; otherwise we decompress as much as we can.  Warn about truncated
+ * or corrupt inputs at <b>protocol_warn_level</b>.
+ */
+int
+tor_uncompress(char **out, size_t *out_len,
+               const char *in, size_t in_len,
+               compress_method_t method,
+               int complete_only,
+               int protocol_warn_level)
+{
+  return tor_compress_impl(0, out, out_len, in, in_len, method,
+                           BEST_COMPRESSION,
+                           complete_only, protocol_warn_level);
+}
+
+/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
+ * to be compressed or not.  If it is, return the likeliest compression method.
+ * Otherwise, return UNKNOWN_METHOD.
+ */
+compress_method_t
+detect_compression_method(const char *in, size_t in_len)
+{
+  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
+    return GZIP_METHOD;
+  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
+             (ntohs(get_uint16(in)) % 31) == 0) {
+    return ZLIB_METHOD;
+  } else if (in_len > 2 &&
+             fast_memeq(in, "\x5d\x00\x00", 3)) {
+    return LZMA_METHOD;
+  } else if (in_len > 3 &&
+             fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) {
+    return ZSTD_METHOD;
+  } else {
+    return UNKNOWN_METHOD;
+  }
+}
+
+/** Return 1 if a given <b>method</b> is supported; otherwise 0. */
+int
+tor_compress_supports_method(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_method_supported();
+    case LZMA_METHOD:
+      return tor_lzma_method_supported();
+    case ZSTD_METHOD:
+      return tor_zstd_method_supported();
+    case NO_METHOD:
+      return 1;
+    case UNKNOWN_METHOD:
+    default:
+      return 0;
+  }
+}
+
+/**
+ * Return a bitmask of the supported compression types, where 1&lt;&lt;m is
+ * set in the bitmask if and only if compression with method <b>m</b> is
+ * supported.
+ */
+unsigned
+tor_compress_get_supported_method_bitmask(void)
+{
+  static unsigned supported = 0;
+  if (supported == 0) {
+    compress_method_t m;
+    for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) {
+      if (tor_compress_supports_method(m)) {
+        supported |= (1u << m);
+      }
+    }
+  }
+  return supported;
+}
+
+/** Table of compression method names.  These should have an "x-" prefix,
+ * if they are not listed in the IANA content coding registry. */
+static const struct {
+  const char *name;
+  compress_method_t method;
+} compression_method_names[] = {
+  { "gzip", GZIP_METHOD },
+  { "deflate", ZLIB_METHOD },
+  // We call this "x-tor-lzma" rather than "x-lzma", because we impose a
+  // lower maximum memory usage on the decoding side.
+  { "x-tor-lzma", LZMA_METHOD },
+  { "x-zstd" , ZSTD_METHOD },
+  { "identity", NO_METHOD },
+
+  /* Later entries in this table are not canonical; these are recognized but
+   * not emitted. */
+  { "x-gzip", GZIP_METHOD },
+};
+
+/** Return the canonical string representation of the compression method
+ * <b>method</b>, or NULL if the method isn't recognized. */
+const char *
+compression_method_get_name(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) {
+    if (method == compression_method_names[i].method)
+      return compression_method_names[i].name;
+  }
+  return NULL;
+}
+
+/** Table of compression human readable method names. */
+static const struct {
+  compress_method_t method;
+  const char *name;
+} compression_method_human_names[] = {
+  { NO_METHOD, "uncompressed" },
+  { GZIP_METHOD, "gzipped" },
+  { ZLIB_METHOD, "deflated" },
+  { LZMA_METHOD, "LZMA compressed" },
+  { ZSTD_METHOD, "Zstandard compressed" },
+  { UNKNOWN_METHOD, "unknown encoding" },
+};
+
+/** Return a human readable string representation of the compression method
+ * <b>method</b>, or NULL if the method isn't recognized. */
+const char *
+compression_method_get_human_name(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) {
+    if (method == compression_method_human_names[i].method)
+      return compression_method_human_names[i].name;
+  }
+  return NULL;
+}
+
+/** Return the compression method represented by the string <b>name</b>, or
+ * UNKNOWN_METHOD if the string isn't recognized. */
+compress_method_t
+compression_method_get_by_name(const char *name)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) {
+    if (!strcmp(compression_method_names[i].name, name))
+      return compression_method_names[i].method;
+  }
+  return UNKNOWN_METHOD;
+}
+
+/** Return a string representation of the version of the library providing the
+ * compression method given in <b>method</b>. Returns NULL if <b>method</b> is
+ * unknown or unsupported. */
+const char *
+tor_compress_version_str(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_get_version_str();
+    case LZMA_METHOD:
+      return tor_lzma_get_version_str();
+    case ZSTD_METHOD:
+      return tor_zstd_get_version_str();
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+    default:
+      return NULL;
+  }
+}
+
+/** Return a string representation of the version of the library, found at
+ * compile time, providing the compression method given in <b>method</b>.
+ * Returns NULL if <b>method</b> is unknown or unsupported. */
+const char *
+tor_compress_header_version_str(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_get_header_version_str();
+    case LZMA_METHOD:
+      return tor_lzma_get_header_version_str();
+    case ZSTD_METHOD:
+      return tor_zstd_get_header_version_str();
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+    default:
+      return NULL;
+  }
+}
+
+/** Return the approximate number of bytes allocated for all
+ * supported compression schemas. */
+size_t
+tor_compress_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_compress_allocation) +
+         tor_zlib_get_total_allocation() +
+         tor_lzma_get_total_allocation() +
+         tor_zstd_get_total_allocation();
+}
+
+/** Internal state for an incremental compression/decompression.  The body of
+ * this struct is not exposed. */
+struct tor_compress_state_t {
+  compress_method_t method; /**< The compression method. */
+
+  union {
+    tor_zlib_compress_state_t *zlib_state;
+    tor_lzma_compress_state_t *lzma_state;
+    tor_zstd_compress_state_t *zstd_state;
+  } u; /**< Compression backend state. */
+};
+
+/** Construct and return a tor_compress_state_t object using <b>method</b>.  If
+ * <b>compress</b>, it's for compression; otherwise it's for decompression. */
+tor_compress_state_t *
+tor_compress_new(int compress, compress_method_t method,
+                 compression_level_t compression_level)
+{
+  tor_compress_state_t *state;
+
+  state = tor_malloc_zero(sizeof(tor_compress_state_t));
+  state->method = method;
+
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD: {
+      tor_zlib_compress_state_t *zlib_state =
+        tor_zlib_compress_new(compress, method, compression_level);
+
+      if (zlib_state == NULL)
+        goto err;
+
+      state->u.zlib_state = zlib_state;
+      break;
+    }
+    case LZMA_METHOD: {
+      tor_lzma_compress_state_t *lzma_state =
+        tor_lzma_compress_new(compress, method, compression_level);
+
+      if (lzma_state == NULL)
+        goto err;
+
+      state->u.lzma_state = lzma_state;
+      break;
+    }
+    case ZSTD_METHOD: {
+      tor_zstd_compress_state_t *zstd_state =
+        tor_zstd_compress_new(compress, method, compression_level);
+
+      if (zstd_state == NULL)
+        goto err;
+
+      state->u.zstd_state = zstd_state;
+      break;
+    }
+    case NO_METHOD: {
+      break;
+    }
+    case UNKNOWN_METHOD:
+      goto err;
+  }
+
+  atomic_counter_add(&total_compress_allocation,
+                     sizeof(tor_compress_state_t));
+  return state;
+
+ err:
+  tor_free(state);
+  return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_compress_process(tor_compress_state_t *state,
+                     char **out, size_t *out_len,
+                     const char **in, size_t *in_len,
+                     int finish)
+{
+  tor_assert(state != NULL);
+  const size_t in_len_orig = *in_len;
+  const size_t out_len_orig = *out_len;
+  tor_compress_output_t rv;
+
+  if (*out_len == 0 && (*in_len > 0 || finish)) {
+    // If we still have input data, but no space for output data, we might as
+    // well return early and let the caller do the reallocation of the out
+    // variable.
+    return TOR_COMPRESS_BUFFER_FULL;
+  }
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      rv = tor_zlib_compress_process(state->u.zlib_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case LZMA_METHOD:
+      rv = tor_lzma_compress_process(state->u.lzma_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case ZSTD_METHOD:
+      rv = tor_zstd_compress_process(state->u.zstd_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case NO_METHOD:
+      rv = tor_cnone_compress_process(out, out_len, in, in_len,
+                                      finish);
+      break;
+    default:
+    case UNKNOWN_METHOD:
+      goto err;
+  }
+  if (BUG((rv == TOR_COMPRESS_OK) &&
+          *in_len == in_len_orig &&
+          *out_len == out_len_orig)) {
+    log_warn(LD_GENERAL,
+             "More info on the bug: method == %s, finish == %d, "
+             " *in_len == in_len_orig == %lu, "
+             "*out_len == out_len_orig == %lu",
+             compression_method_get_human_name(state->method), finish,
+             (unsigned long)in_len_orig, (unsigned long)out_len_orig);
+    return TOR_COMPRESS_ERROR;
+  }
+
+  return rv;
+ err:
+  return TOR_COMPRESS_ERROR;
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_compress_free_(tor_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      tor_zlib_compress_free(state->u.zlib_state);
+      break;
+    case LZMA_METHOD:
+      tor_lzma_compress_free(state->u.lzma_state);
+      break;
+    case ZSTD_METHOD:
+      tor_zstd_compress_free(state->u.zstd_state);
+      break;
+    case NO_METHOD:
+      break;
+    case UNKNOWN_METHOD:
+      break;
+  }
+
+  atomic_counter_sub(&total_compress_allocation,
+                     sizeof(tor_compress_state_t));
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_compress_state_size(const tor_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+
+  size_t size = sizeof(tor_compress_state_t);
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      size += tor_zlib_compress_state_size(state->u.zlib_state);
+      break;
+    case LZMA_METHOD:
+      size += tor_lzma_compress_state_size(state->u.lzma_state);
+      break;
+    case ZSTD_METHOD:
+      size += tor_zstd_compress_state_size(state->u.zstd_state);
+      break;
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+      break;
+  }
+
+  return size;
+}
+
+/** Initialize all compression modules. */
+void
+tor_compress_init(void)
+{
+  atomic_counter_init(&total_compress_allocation);
+
+  tor_zlib_init();
+  tor_lzma_init();
+  tor_zstd_init();
+}
+
+/** Warn if we had any problems while setting up our compression libraries.
+ *
+ * (This isn't part of tor_compress_init, since the logs aren't set up yet.)
+ */
+void
+tor_compress_log_init_warnings(void)
+{
+  tor_zstd_warn_if_version_mismatched();
+}
+
diff --git a/src/common/compress.h b/src/common/compress.h
new file mode 100644
index 0000000000..65d63a4386
--- /dev/null
+++ b/src/common/compress.h
@@ -0,0 +1,93 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress.h
+ * \brief Headers for compress.c
+ **/
+
+#ifndef TOR_COMPRESS_H
+#define TOR_COMPRESS_H
+
+/** Enumeration of what kind of compression to use.  Only ZLIB_METHOD and
+ * GZIP_METHOD is guaranteed to be supported by the compress/uncompress
+ * functions here. Call tor_compress_supports_method() to check if a given
+ * compression schema is supported by Tor. */
+typedef enum {
+  NO_METHOD=0, // This method must be first.
+  GZIP_METHOD=1,
+  ZLIB_METHOD=2,
+  LZMA_METHOD=3,
+  ZSTD_METHOD=4,
+  UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle.
+} compress_method_t;
+
+/**
+ * Enumeration to define tradeoffs between memory usage and compression level.
+ * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
+ * memory.
+ **/
+typedef enum {
+  BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
+} compression_level_t;
+
+int tor_compress(char **out, size_t *out_len,
+                 const char *in, size_t in_len,
+                 compress_method_t method);
+
+int tor_uncompress(char **out, size_t *out_len,
+                   const char *in, size_t in_len,
+                   compress_method_t method,
+                   int complete_only,
+                   int protocol_warn_level);
+
+compress_method_t detect_compression_method(const char *in, size_t in_len);
+
+MOCK_DECL(int,tor_compress_is_compression_bomb,(size_t size_in,
+                                                size_t size_out));
+
+int tor_compress_supports_method(compress_method_t method);
+unsigned tor_compress_get_supported_method_bitmask(void);
+const char *compression_method_get_name(compress_method_t method);
+const char *compression_method_get_human_name(compress_method_t method);
+compress_method_t compression_method_get_by_name(const char *name);
+
+const char *tor_compress_version_str(compress_method_t method);
+
+const char *tor_compress_header_version_str(compress_method_t method);
+
+size_t tor_compress_get_total_allocation(void);
+
+/** Return values from tor_compress_process; see that function's documentation
+ * for details. */
+typedef enum {
+  TOR_COMPRESS_OK,
+  TOR_COMPRESS_DONE,
+  TOR_COMPRESS_BUFFER_FULL,
+  TOR_COMPRESS_ERROR
+} tor_compress_output_t;
+
+/** Internal state for an incremental compression/decompression. */
+typedef struct tor_compress_state_t tor_compress_state_t;
+
+tor_compress_state_t *tor_compress_new(int compress,
+                                       compress_method_t method,
+                                       compression_level_t level);
+
+tor_compress_output_t tor_compress_process(tor_compress_state_t *state,
+                                           char **out, size_t *out_len,
+                                           const char **in, size_t *in_len,
+                                           int finish);
+void tor_compress_free_(tor_compress_state_t *state);
+#define tor_compress_free(st) \
+  FREE_AND_NULL(tor_compress_state_t, tor_compress_free_, (st))
+
+size_t tor_compress_state_size(const tor_compress_state_t *state);
+
+void tor_compress_init(void);
+void tor_compress_log_init_warnings(void);
+
+#endif /* !defined(TOR_COMPRESS_H) */
+
diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c
new file mode 100644
index 0000000000..051c59ba2d
--- /dev/null
+++ b/src/common/compress_lzma.c
@@ -0,0 +1,361 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_lzma.c
+ * \brief Compression backend for LZMA.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_lzma.h"
+
+#ifdef HAVE_LZMA
+#include <lzma.h>
+#endif
+
+/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */
+#define MEMORY_LIMIT (16 * 1024 * 1024)
+
+/** Total number of bytes allocated for LZMA state. */
+static atomic_counter_t total_lzma_allocation;
+
+#ifdef HAVE_LZMA
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return 6;
+    case MEDIUM_COMPRESSION: return 4;
+    case LOW_COMPRESSION: return 2;
+  }
+}
+
+/** Convert a given <b>error</b> to a human readable error string. */
+static const char *
+lzma_error_str(lzma_ret error)
+{
+  switch (error) {
+    case LZMA_OK:
+      return "Operation completed successfully";
+    case LZMA_STREAM_END:
+      return "End of stream";
+    case LZMA_NO_CHECK:
+      return "Input stream lacks integrity check";
+    case LZMA_UNSUPPORTED_CHECK:
+      return "Unable to calculate integrity check";
+    case LZMA_GET_CHECK:
+      return "Integrity check available";
+    case LZMA_MEM_ERROR:
+      return "Unable to allocate memory";
+    case LZMA_MEMLIMIT_ERROR:
+      return "Memory limit reached";
+    case LZMA_FORMAT_ERROR:
+      return "Unknown file format";
+    case LZMA_OPTIONS_ERROR:
+      return "Unsupported options";
+    case LZMA_DATA_ERROR:
+      return "Corrupt input data";
+    case LZMA_BUF_ERROR:
+      return "Unable to progress";
+    case LZMA_PROG_ERROR:
+      return "Programming error";
+    default:
+      return "Unknown LZMA error";
+  }
+}
+#endif /* defined(HAVE_LZMA) */
+
+/** Return 1 if LZMA compression is supported; otherwise 0. */
+int
+tor_lzma_method_supported(void)
+{
+#ifdef HAVE_LZMA
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/** Return a string representation of the version of the currently running
+ * version of liblzma. Returns NULL if LZMA is unsupported. */
+const char *
+tor_lzma_get_version_str(void)
+{
+#ifdef HAVE_LZMA
+  return lzma_version_string();
+#else
+  return NULL;
+#endif
+}
+
+/** Return a string representation of the version of liblzma used at
+ * compilation time. Returns NULL if LZMA is unsupported. */
+const char *
+tor_lzma_get_header_version_str(void)
+{
+#ifdef HAVE_LZMA
+  return LZMA_VERSION_STRING;
+#else
+  return NULL;
+#endif
+}
+
+/** Internal LZMA state for incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_lzma_compress_state_t {
+#ifdef HAVE_LZMA
+  lzma_stream stream; /**< The LZMA stream. */
+#endif
+
+  int compress; /**< True if we are compressing; false if we are inflating */
+
+  /** Number of bytes read so far.  Used to detect compression bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect compression bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+#ifdef HAVE_LZMA
+/** Return an approximate number of bytes stored in memory to hold the LZMA
+ * encoder/decoder state. */
+static size_t
+tor_lzma_state_size_precalc(int compress, compression_level_t level)
+{
+  uint64_t memory_usage;
+
+  if (compress)
+    memory_usage = lzma_easy_encoder_memusage(memory_level(level));
+  else
+    memory_usage = lzma_easy_decoder_memusage(memory_level(level));
+
+  if (memory_usage == UINT64_MAX) {
+    // LCOV_EXCL_START
+    log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s",
+                         compress ? "encoder" : "decoder");
+    goto err;
+    // LCOV_EXCL_STOP
+  }
+
+  if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX)
+    memory_usage = SIZE_MAX;
+  else
+    memory_usage += sizeof(tor_lzma_compress_state_t);
+
+  return (size_t)memory_usage;
+
+ // LCOV_EXCL_START
+ err:
+  return 0;
+ // LCOV_EXCL_STOP
+}
+#endif /* defined(HAVE_LZMA) */
+
+/** Construct and return a tor_lzma_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_lzma_compress_state_t *
+tor_lzma_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t level)
+{
+  tor_assert(method == LZMA_METHOD);
+
+#ifdef HAVE_LZMA
+  tor_lzma_compress_state_t *result;
+  lzma_ret retval;
+  lzma_options_lzma stream_options;
+
+  // Note that we do not explicitly initialize the lzma_stream object here,
+  // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is
+  // also what `tor_malloc_zero()` does.
+  result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t));
+  result->compress = compress;
+  result->allocation = tor_lzma_state_size_precalc(compress, level);
+
+  if (compress) {
+    lzma_lzma_preset(&stream_options, memory_level(level));
+
+    retval = lzma_alone_encoder(&result->stream, &stream_options);
+
+    if (retval != LZMA_OK) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).",
+               lzma_error_str(retval), retval);
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  } else {
+    retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT);
+
+    if (retval != LZMA_OK) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
+               lzma_error_str(retval), retval);
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  atomic_counter_add(&total_lzma_allocation, result->allocation);
+  return result;
+
+ /* LCOV_EXCL_START */
+ err:
+  tor_free(result);
+  return NULL;
+ /* LCOV_EXCL_STOP */
+#else /* !(defined(HAVE_LZMA)) */
+  (void)compress;
+  (void)method;
+  (void)level;
+
+  return NULL;
+#endif /* defined(HAVE_LZMA) */
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_lzma_compress_process(tor_lzma_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+#ifdef HAVE_LZMA
+  lzma_ret retval;
+  lzma_action action;
+
+  tor_assert(state != NULL);
+  tor_assert(*in_len <= UINT_MAX);
+  tor_assert(*out_len <= UINT_MAX);
+
+  state->stream.next_in = (unsigned char *)*in;
+  state->stream.avail_in = *in_len;
+  state->stream.next_out = (unsigned char *)*out;
+  state->stream.avail_out = *out_len;
+
+  action = finish ? LZMA_FINISH : LZMA_RUN;
+
+  retval = lzma_code(&state->stream, action);
+
+  state->input_so_far += state->stream.next_in - ((unsigned char *)*in);
+  state->output_so_far += state->stream.next_out - ((unsigned char *)*out);
+
+  *out = (char *)state->stream.next_out;
+  *out_len = state->stream.avail_out;
+  *in = (const char *)state->stream.next_in;
+  *in_len = state->stream.avail_in;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  switch (retval) {
+    case LZMA_OK:
+      if (state->stream.avail_out == 0 || finish)
+        return TOR_COMPRESS_BUFFER_FULL;
+
+      return TOR_COMPRESS_OK;
+
+    case LZMA_BUF_ERROR:
+      if (state->stream.avail_in == 0 && !finish)
+        return TOR_COMPRESS_OK;
+
+      return TOR_COMPRESS_BUFFER_FULL;
+
+    case LZMA_STREAM_END:
+      return TOR_COMPRESS_DONE;
+
+    // We list all the possible values of `lzma_ret` here to silence the
+    // `switch-enum` warning and to detect if a new member was added.
+    case LZMA_NO_CHECK:
+    case LZMA_UNSUPPORTED_CHECK:
+    case LZMA_GET_CHECK:
+    case LZMA_MEM_ERROR:
+    case LZMA_MEMLIMIT_ERROR:
+    case LZMA_FORMAT_ERROR:
+    case LZMA_OPTIONS_ERROR:
+    case LZMA_DATA_ERROR:
+    case LZMA_PROG_ERROR:
+    default:
+      log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.",
+               state->compress ? "compression" : "decompression",
+               lzma_error_str(retval));
+      return TOR_COMPRESS_ERROR;
+  }
+#else /* !(defined(HAVE_LZMA)) */
+  (void)state;
+  (void)out;
+  (void)out_len;
+  (void)in;
+  (void)in_len;
+  (void)finish;
+  return TOR_COMPRESS_ERROR;
+#endif /* defined(HAVE_LZMA) */
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_lzma_compress_free_(tor_lzma_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_lzma_allocation, state->allocation);
+
+#ifdef HAVE_LZMA
+  lzma_end(&state->stream);
+#endif
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all LZMA states. */
+size_t
+tor_lzma_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_lzma_allocation);
+}
+
+/** Initialize the lzma module */
+void
+tor_lzma_init(void)
+{
+  atomic_counter_init(&total_lzma_allocation);
+}
+
diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h
new file mode 100644
index 0000000000..38a447c1f3
--- /dev/null
+++ b/src/common/compress_lzma.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_lzma.h
+ * \brief Header for compress_lzma.c
+ **/
+
+#ifndef TOR_COMPRESS_LZMA_H
+#define TOR_COMPRESS_LZMA_H
+
+int tor_lzma_method_supported(void);
+
+const char *tor_lzma_get_version_str(void);
+
+const char *tor_lzma_get_header_version_str(void);
+
+/** Internal state for an incremental LZMA compression/decompression. */
+typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t;
+
+tor_lzma_compress_state_t *
+tor_lzma_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_lzma_compress_process(tor_lzma_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_lzma_compress_free_(tor_lzma_compress_state_t *state);
+#define tor_lzma_compress_free(st)                      \
+  FREE_AND_NULL(tor_lzma_compress_state_t,   \
+                           tor_lzma_compress_free_, (st))
+
+size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state);
+
+size_t tor_lzma_get_total_allocation(void);
+
+void tor_lzma_init(void);
+
+#endif /* !defined(TOR_COMPRESS_LZMA_H) */
+
diff --git a/src/common/compress_none.c b/src/common/compress_none.c
new file mode 100644
index 0000000000..34314e4af7
--- /dev/null
+++ b/src/common/compress_none.c
@@ -0,0 +1,53 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_none.c
+ * \brief Compression backend for identity compression.
+ *
+ * We actually define this backend so that we can treat the identity transform
+ * as another case of compression.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_none.h"
+
+/** Transfer some bytes using the identity transformation.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_cnone_compress_process(char **out, size_t *out_len,
+                           const char **in, size_t *in_len,
+                           int finish)
+{
+  size_t n_to_copy = MIN(*in_len, *out_len);
+
+  memcpy(*out, *in, n_to_copy);
+  *out += n_to_copy;
+  *in += n_to_copy;
+  *out_len -= n_to_copy;
+  *in_len -= n_to_copy;
+  if (*in_len == 0) {
+    return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK;
+  } else {
+    return TOR_COMPRESS_BUFFER_FULL;
+  }
+}
+
diff --git a/src/common/compress_none.h b/src/common/compress_none.h
new file mode 100644
index 0000000000..77c3cef47b
--- /dev/null
+++ b/src/common/compress_none.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_none.h
+ * \brief Header for compress_none.c
+ **/
+
+#ifndef TOR_COMPRESS_NONE_H
+#define TOR_COMPRESS_NONE_H
+
+tor_compress_output_t
+tor_cnone_compress_process(char **out, size_t *out_len,
+                           const char **in, size_t *in_len,
+                           int finish);
+
+#endif /* !defined(TOR_COMPRESS_NONE_H) */
+
diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c
new file mode 100644
index 0000000000..23d71d27be
--- /dev/null
+++ b/src/common/compress_zlib.c
@@ -0,0 +1,304 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.c
+ * \brief Compression backend for gzip and zlib.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_zlib.h"
+
+/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
+   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
+   that nobody will care if the compile outputs a no-such-identifier warning.
+
+   Sorry, but we like -Werror over here, so I guess we need to define these.
+   I hope that zlib 1.2.6 doesn't break these too.
+*/
+#ifndef _LARGEFILE64_SOURCE
+#define _LARGEFILE64_SOURCE 0
+#endif
+#ifndef _LFS64_LARGEFILE
+#define _LFS64_LARGEFILE 0
+#endif
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 0
+#endif
+#ifndef off64_t
+#define off64_t int64_t
+#endif
+
+#include <zlib.h>
+
+#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
+#error "We require zlib version 1.2 or later."
+#endif
+
+static size_t tor_zlib_state_size_precalc(int inflate,
+                                          int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static atomic_counter_t total_zlib_allocation;
+
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION: return 9;
+    case HIGH_COMPRESSION: return 8;
+    case MEDIUM_COMPRESSION: return 7;
+    case LOW_COMPRESSION: return 6;
+  }
+}
+
+/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
+static inline int
+method_bits(compress_method_t method, compression_level_t level)
+{
+  /* Bits+16 means "use gzip" in zlib >= 1.2 */
+  const int flag = method == GZIP_METHOD ? 16 : 0;
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return flag + 15;
+    case MEDIUM_COMPRESSION: return flag + 13;
+    case LOW_COMPRESSION: return flag + 11;
+  }
+}
+
+/** Return 1 if zlib/gzip compression is supported; otherwise 0. */
+int
+tor_zlib_method_supported(void)
+{
+  /* We currently always support zlib/gzip, but we keep this function around in
+   * case we some day decide to deprecate zlib/gzip support.
+   */
+  return 1;
+}
+
+/** Return a string representation of the version of the currently running
+ * version of zlib. */
+const char *
+tor_zlib_get_version_str(void)
+{
+  return zlibVersion();
+}
+
+/** Return a string representation of the version of the version of zlib
+* used at compilation. */
+const char *
+tor_zlib_get_header_version_str(void)
+{
+  return ZLIB_VERSION;
+}
+
+/** Internal zlib state for an incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zlib_compress_state_t {
+  struct z_stream_s stream; /**< The zlib stream */
+  int compress; /**< True if we are compressing; false if we are inflating */
+
+  /** Number of bytes read so far.  Used to detect zlib bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect zlib bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
+{
+  windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+  if (inflate_) {
+    /* From zconf.h:
+
+       "The memory requirements for inflate are (in bytes) 1 << windowBits
+       that is, 32K for windowBits=15 (default value) plus a few kilobytes
+       for small objects."
+    */
+    return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+      (1 << 15) + A_FEW_KILOBYTES;
+  } else {
+    /* Also from zconf.h:
+
+       "The memory requirements for deflate are (in bytes):
+            (1 << (windowBits+2)) +  (1 << (memLevel+9))
+        ... plus a few kilobytes for small objects."
+    */
+    return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+  }
+#undef A_FEW_KILOBYTES
+}
+
+/** Construct and return a tor_zlib_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress_,
+                      compress_method_t method,
+                      compression_level_t compression_level)
+{
+  tor_zlib_compress_state_t *out;
+  int bits, memlevel;
+
+  if (! compress_) {
+    /* use this setting for decompression, since we might have the
+     * max number of window bits */
+    compression_level = BEST_COMPRESSION;
+  }
+
+  out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t));
+  out->stream.zalloc = Z_NULL;
+  out->stream.zfree = Z_NULL;
+  out->stream.opaque = NULL;
+  out->compress = compress_;
+  bits = method_bits(method, compression_level);
+  memlevel = memory_level(compression_level);
+  if (compress_) {
+    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
+                     bits, memlevel,
+                     Z_DEFAULT_STRATEGY) != Z_OK)
+    goto err; // LCOV_EXCL_LINE
+  } else {
+    if (inflateInit2(&out->stream, bits) != Z_OK)
+      goto err; // LCOV_EXCL_LINE
+  }
+  out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
+
+  atomic_counter_add(&total_zlib_allocation, out->allocation);
+
+  return out;
+
+ err:
+  tor_free(out);
+  return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+  int err;
+  tor_assert(state != NULL);
+  if (*in_len > UINT_MAX ||
+      *out_len > UINT_MAX) {
+    return TOR_COMPRESS_ERROR;
+  }
+
+  state->stream.next_in = (unsigned char*) *in;
+  state->stream.avail_in = (unsigned int)*in_len;
+  state->stream.next_out = (unsigned char*) *out;
+  state->stream.avail_out = (unsigned int)*out_len;
+
+  if (state->compress) {
+    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
+  } else {
+    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
+  }
+
+  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
+  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
+
+  *out = (char*) state->stream.next_out;
+  *out_len = state->stream.avail_out;
+  *in = (const char *) state->stream.next_in;
+  *in_len = state->stream.avail_in;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  switch (err)
+    {
+    case Z_STREAM_END:
+      return TOR_COMPRESS_DONE;
+    case Z_BUF_ERROR:
+      if (state->stream.avail_in == 0 && !finish)
+        return TOR_COMPRESS_OK;
+      return TOR_COMPRESS_BUFFER_FULL;
+    case Z_OK:
+      if (state->stream.avail_out == 0 || finish)
+        return TOR_COMPRESS_BUFFER_FULL;
+      return TOR_COMPRESS_OK;
+    default:
+      log_warn(LD_GENERAL, "Gzip returned an error: %s",
+               state->stream.msg ? state->stream.msg : "<no message>");
+      return TOR_COMPRESS_ERROR;
+    }
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zlib_compress_free_(tor_zlib_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_zlib_allocation, state->allocation);
+
+  if (state->compress)
+    deflateEnd(&state->stream);
+  else
+    inflateEnd(&state->stream);
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_zlib_allocation);
+}
+
+/** Set up global state for the zlib module */
+void
+tor_zlib_init(void)
+{
+  atomic_counter_init(&total_zlib_allocation);
+}
+
diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h
new file mode 100644
index 0000000000..e3c1a2b339
--- /dev/null
+++ b/src/common/compress_zlib.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.h
+ * \brief Header for compress_zlib.c
+ **/
+
+#ifndef TOR_COMPRESS_ZLIB_H
+#define TOR_COMPRESS_ZLIB_H
+
+int tor_zlib_method_supported(void);
+
+const char *tor_zlib_get_version_str(void);
+
+const char *tor_zlib_get_header_version_str(void);
+
+/** Internal state for an incremental zlib/gzip compression/decompression. */
+typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t;
+
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_zlib_compress_free_(tor_zlib_compress_state_t *state);
+#define tor_zlib_compress_free(st)                      \
+  FREE_AND_NULL(tor_zlib_compress_state_t,   \
+                           tor_zlib_compress_free_, (st))
+
+size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state);
+
+size_t tor_zlib_get_total_allocation(void);
+
+void tor_zlib_init(void);
+
+#endif /* !defined(TOR_COMPRESS_ZLIB_H) */
+
diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c
new file mode 100644
index 0000000000..dc8b4d621d
--- /dev/null
+++ b/src/common/compress_zstd.c
@@ -0,0 +1,540 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zstd.c
+ * \brief Compression backend for Zstandard.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_zstd.h"
+
+#ifdef ENABLE_ZSTD_ADVANCED_APIS
+/* This is a lie, but we make sure it doesn't get us in trouble by wrapping
+ * all invocations of zstd's static-only functions in a check to make sure
+ * that the compile-time version matches the run-time version. */
+#define ZSTD_STATIC_LINKING_ONLY
+#endif
+
+#ifdef HAVE_ZSTD
+#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
+DISABLE_GCC_WARNING(unused-const-variable)
+#endif
+#include <zstd.h>
+#ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
+ENABLE_GCC_WARNING(unused-const-variable)
+#endif
+#endif
+
+/** Total number of bytes allocated for Zstandard state. */
+static atomic_counter_t total_zstd_allocation;
+
+#ifdef HAVE_ZSTD
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return 9;
+    case MEDIUM_COMPRESSION: return 8;
+    case LOW_COMPRESSION: return 7;
+  }
+}
+#endif /* defined(HAVE_ZSTD) */
+
+/** Return 1 if Zstandard compression is supported; otherwise 0. */
+int
+tor_zstd_method_supported(void)
+{
+#ifdef HAVE_ZSTD
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+#ifdef HAVE_ZSTD
+/** Format a zstd version number as a string in <b>buf</b>. */
+static void
+tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
+{
+  tor_snprintf(buf, buflen,
+               "%u.%u.%u",
+               version_number / 10000 % 100,
+               version_number / 100 % 100,
+               version_number % 100);
+}
+#endif
+
+#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
+
+/** Return a string representation of the version of the currently running
+ * version of libzstd. Returns NULL if Zstandard is unsupported. */
+const char *
+tor_zstd_get_version_str(void)
+{
+#ifdef HAVE_ZSTD
+  static char version_str[VERSION_STR_MAX_LEN];
+
+  tor_zstd_format_version(version_str, sizeof(version_str),
+                          ZSTD_versionNumber());
+
+  return version_str;
+#else /* !(defined(HAVE_ZSTD)) */
+  return NULL;
+#endif /* defined(HAVE_ZSTD) */
+}
+
+/** Return a string representation of the version of the version of libzstd
+ * used at compilation time. Returns NULL if Zstandard is unsupported. */
+const char *
+tor_zstd_get_header_version_str(void)
+{
+#ifdef HAVE_ZSTD
+  return ZSTD_VERSION_STRING;
+#else
+  return NULL;
+#endif
+}
+
+#ifdef TOR_UNIT_TESTS
+static int static_apis_disable_for_testing = 0;
+#endif
+
+/** Return true iff we can use the "static-only" APIs. */
+int
+tor_zstd_can_use_static_apis(void)
+{
+#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)
+#ifdef TOR_UNIT_TESTS
+  if (static_apis_disable_for_testing) {
+    return 0;
+  }
+#endif
+  return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
+#else
+  return 0;
+#endif
+}
+
+/** Internal Zstandard state for incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zstd_compress_state_t {
+#ifdef HAVE_ZSTD
+  union {
+    /** Compression stream. Used when <b>compress</b> is true. */
+    ZSTD_CStream *compress_stream;
+    /** Decompression stream. Used when <b>compress</b> is false. */
+    ZSTD_DStream *decompress_stream;
+  } u; /**< Zstandard stream objects. */
+#endif /* defined(HAVE_ZSTD) */
+
+  int compress; /**< True if we are compressing; false if we are inflating */
+  int have_called_end; /**< True if we are compressing and we've called
+                        * ZSTD_endStream */
+
+  /** Number of bytes read so far.  Used to detect compression bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect compression bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+#ifdef HAVE_ZSTD
+/** Return an approximate number of bytes stored in memory to hold the
+ * Zstandard compression/decompression state. This is a fake estimate
+ * based on inspecting the zstd source: tor_zstd_state_size_precalc() is
+ * more accurate when it's allowed to use "static-only" functions */
+static size_t
+tor_zstd_state_size_precalc_fake(int compress, int preset)
+{
+  tor_assert(preset > 0);
+
+  size_t memory_usage = sizeof(tor_zstd_compress_state_t);
+
+  // The Zstandard library provides a number of functions that would be useful
+  // here, but they are, unfortunately, still considered experimental and are
+  // thus only available in libzstd if we link against the library statically.
+  //
+  // The code in this function tries to approximate the calculations without
+  // being able to use the following:
+  //
+  // - We do not have access to neither the internal members of ZSTD_CStream
+  //   and ZSTD_DStream and their internal context objects.
+  //
+  // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
+  //   are unexposed.
+  //
+  // In the future it might be useful to check if libzstd have started
+  // providing these functions in a stable manner and simplify this function.
+  if (compress) {
+    // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
+    memory_usage += 192;
+
+    // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
+    // variables that are not exposed via the public API. We use a _very_
+    // simplified function to calculate the estimated amount of bytes used in
+    // this struct.
+    // memory_usage += (preset - 0.5) * 1024 * 1024;
+    memory_usage += (preset * 1024 * 1024) - (512 * 1024);
+    // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
+    // - stream->outBuffSize: 128 KB:
+    memory_usage += 128 * 1024;
+    // - stream->inBuffSize: 2048 KB:
+    memory_usage += 2048 * 1024;
+  } else {
+    // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
+    memory_usage += 208;
+    // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
+    memory_usage += 150 * 1024;
+
+    // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
+    // - stream->inBuffSize: 0 KB.
+    // - stream->outBuffSize: 0 KB.
+  }
+
+  return memory_usage;
+}
+
+/** Return an approximate number of bytes stored in memory to hold the
+ * Zstandard compression/decompression state. */
+static size_t
+tor_zstd_state_size_precalc(int compress, int preset)
+{
+#ifdef ZSTD_STATIC_LINKING_ONLY
+  if (tor_zstd_can_use_static_apis()) {
+    if (compress) {
+#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE
+      return ZSTD_estimateCStreamSize(preset);
+#endif
+    } else {
+#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE
+      /* Could use DStream, but that takes a windowSize. */
+      return ZSTD_estimateDCtxSize();
+#endif
+    }
+  }
+#endif
+  return tor_zstd_state_size_precalc_fake(compress, preset);
+}
+#endif /* defined(HAVE_ZSTD) */
+
+/** Construct and return a tor_zstd_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zstd_compress_state_t *
+tor_zstd_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t level)
+{
+  tor_assert(method == ZSTD_METHOD);
+
+#ifdef HAVE_ZSTD
+  const int preset = memory_level(level);
+  tor_zstd_compress_state_t *result;
+  size_t retval;
+
+  result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
+  result->compress = compress;
+  result->allocation = tor_zstd_state_size_precalc(compress, preset);
+
+  if (compress) {
+    result->u.compress_stream = ZSTD_createCStream();
+
+    if (result->u.compress_stream == NULL) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error while creating Zstandard compression "
+               "stream");
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+
+    retval = ZSTD_initCStream(result->u.compress_stream, preset);
+
+    if (ZSTD_isError(retval)) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
+               ZSTD_getErrorName(retval));
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  } else {
+    result->u.decompress_stream = ZSTD_createDStream();
+
+    if (result->u.decompress_stream == NULL) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error while creating Zstandard decompression "
+               "stream");
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+
+    retval = ZSTD_initDStream(result->u.decompress_stream);
+
+    if (ZSTD_isError(retval)) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
+               ZSTD_getErrorName(retval));
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  atomic_counter_add(&total_zstd_allocation, result->allocation);
+  return result;
+
+ err:
+  // LCOV_EXCL_START
+  if (compress) {
+    ZSTD_freeCStream(result->u.compress_stream);
+  } else {
+    ZSTD_freeDStream(result->u.decompress_stream);
+  }
+
+  tor_free(result);
+  return NULL;
+  // LCOV_EXCL_STOP
+#else /* !(defined(HAVE_ZSTD)) */
+  (void)compress;
+  (void)method;
+  (void)level;
+
+  return NULL;
+#endif /* defined(HAVE_ZSTD) */
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zstd_compress_process(tor_zstd_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+#ifdef HAVE_ZSTD
+  size_t retval;
+
+  tor_assert(state != NULL);
+  tor_assert(*in_len <= UINT_MAX);
+  tor_assert(*out_len <= UINT_MAX);
+
+  ZSTD_inBuffer input = { *in, *in_len, 0 };
+  ZSTD_outBuffer output = { *out, *out_len, 0 };
+
+  if (BUG(finish == 0 && state->have_called_end)) {
+    finish = 1;
+  }
+
+  if (state->compress) {
+    if (! state->have_called_end)
+      retval = ZSTD_compressStream(state->u.compress_stream,
+                                   &output, &input);
+    else
+      retval = 0;
+  } else {
+    retval = ZSTD_decompressStream(state->u.decompress_stream,
+                                   &output, &input);
+  }
+
+  state->input_so_far += input.pos;
+  state->output_so_far += output.pos;
+
+  *out = (char *)output.dst + output.pos;
+  *out_len = output.size - output.pos;
+  *in = (char *)input.src + input.pos;
+  *in_len = input.size - input.pos;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  if (ZSTD_isError(retval)) {
+    log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
+             state->compress ? "compression" : "decompression",
+             ZSTD_getErrorName(retval));
+    return TOR_COMPRESS_ERROR;
+  }
+
+  if (state->compress && !state->have_called_end) {
+    retval = ZSTD_flushStream(state->u.compress_stream, &output);
+
+    *out = (char *)output.dst + output.pos;
+    *out_len = output.size - output.pos;
+
+    if (ZSTD_isError(retval)) {
+      log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
+               ZSTD_getErrorName(retval));
+      return TOR_COMPRESS_ERROR;
+    }
+
+    // ZSTD_flushStream returns 0 if the frame is done, or >0 if it
+    // is incomplete.
+    if (retval > 0) {
+      return TOR_COMPRESS_BUFFER_FULL;
+    }
+  }
+
+  if (!finish) {
+    // The caller says we're not done with the input, so no need to write an
+    // epilogue.
+    return TOR_COMPRESS_OK;
+  } else if (state->compress) {
+    if (*in_len) {
+      // We say that we're not done with the input, so we can't write an
+      // epilogue.
+      return TOR_COMPRESS_OK;
+    }
+
+    retval = ZSTD_endStream(state->u.compress_stream, &output);
+    state->have_called_end = 1;
+    *out = (char *)output.dst + output.pos;
+    *out_len = output.size - output.pos;
+
+    if (ZSTD_isError(retval)) {
+      log_warn(LD_GENERAL, "Zstandard compression unable to write "
+               "epilogue: %s.",
+               ZSTD_getErrorName(retval));
+      return TOR_COMPRESS_ERROR;
+    }
+
+    // endStream returns the number of bytes that is needed to write the
+    // epilogue.
+    if (retval > 0)
+      return TOR_COMPRESS_BUFFER_FULL;
+
+    return TOR_COMPRESS_DONE;
+  } else /* if (!state->compress) */ {
+    // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it
+    // is incomplete.
+    // We check this above.
+    tor_assert_nonfatal(!ZSTD_isError(retval));
+    // Start a new frame if this frame is done
+    if (retval == 0)
+      return TOR_COMPRESS_DONE;
+    // Don't check out_len, it might have some space left if the next output
+    // chunk is larger than the remaining space
+    else if (*in_len > 0)
+      return  TOR_COMPRESS_BUFFER_FULL;
+    else
+      return TOR_COMPRESS_OK;
+  }
+
+#else /* !(defined(HAVE_ZSTD)) */
+  (void)state;
+  (void)out;
+  (void)out_len;
+  (void)in;
+  (void)in_len;
+  (void)finish;
+
+  return TOR_COMPRESS_ERROR;
+#endif /* defined(HAVE_ZSTD) */
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zstd_compress_free_(tor_zstd_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_zstd_allocation, state->allocation);
+
+#ifdef HAVE_ZSTD
+  if (state->compress) {
+    ZSTD_freeCStream(state->u.compress_stream);
+  } else {
+    ZSTD_freeDStream(state->u.decompress_stream);
+  }
+#endif /* defined(HAVE_ZSTD) */
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all Zstandard
+ * states. */
+size_t
+tor_zstd_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_zstd_allocation);
+}
+
+/** Initialize the zstd module */
+void
+tor_zstd_init(void)
+{
+  atomic_counter_init(&total_zstd_allocation);
+}
+
+/** Warn if the header and library versions don't match. */
+void
+tor_zstd_warn_if_version_mismatched(void)
+{
+#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS)
+  if (! tor_zstd_can_use_static_apis()) {
+    char header_version[VERSION_STR_MAX_LEN];
+    char runtime_version[VERSION_STR_MAX_LEN];
+    tor_zstd_format_version(header_version, sizeof(header_version),
+                            ZSTD_VERSION_NUMBER);
+    tor_zstd_format_version(runtime_version, sizeof(runtime_version),
+                            ZSTD_versionNumber());
+
+    log_warn(LD_GENERAL,
+             "Tor was compiled with zstd %s, but is running with zstd %s. "
+             "For safety, we'll avoid using advanced zstd functionality.",
+             header_version, runtime_version);
+  }
+#endif
+}
+
+#ifdef TOR_UNIT_TESTS
+/** Testing only: disable usage of static-only APIs, so we can make sure that
+ * we still work without them. */
+void
+tor_zstd_set_static_apis_disabled_for_testing(int disabled)
+{
+  static_apis_disable_for_testing = disabled;
+}
+#endif
+
diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h
new file mode 100644
index 0000000000..bd42cf65ce
--- /dev/null
+++ b/src/common/compress_zstd.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zstd.h
+ * \brief Header for compress_zstd.c
+ **/
+
+#ifndef TOR_COMPRESS_ZSTD_H
+#define TOR_COMPRESS_ZSTD_H
+
+int tor_zstd_method_supported(void);
+
+const char *tor_zstd_get_version_str(void);
+
+const char *tor_zstd_get_header_version_str(void);
+
+int tor_zstd_can_use_static_apis(void);
+
+/** Internal state for an incremental Zstandard compression/decompression. */
+typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t;
+
+tor_zstd_compress_state_t *
+tor_zstd_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zstd_compress_process(tor_zstd_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_zstd_compress_free_(tor_zstd_compress_state_t *state);
+#define tor_zstd_compress_free(st)                      \
+  FREE_AND_NULL(tor_zstd_compress_state_t,   \
+                           tor_zstd_compress_free_, (st))
+
+size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state);
+
+size_t tor_zstd_get_total_allocation(void);
+
+void tor_zstd_init(void);
+void tor_zstd_warn_if_version_mismatched(void);
+
+#ifdef TOR_UNIT_TESTS
+void tor_zstd_set_static_apis_disabled_for_testing(int disabled);
+#endif
+
+#endif /* !defined(TOR_COMPRESS_ZSTD_H) */
+
diff --git a/src/common/confline.c b/src/common/confline.c
new file mode 100644
index 0000000000..bf613ab742
--- /dev/null
+++ b/src/common/confline.c
@@ -0,0 +1,538 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "compat.h"
+#include "confline.h"
+#include "torlog.h"
+#include "util.h"
+#include "container.h"
+
+static int config_get_lines_aux(const char *string, config_line_t **result,
+                                int extended, int allow_include,
+                                int *has_include, smartlist_t *opened_lst,
+                                int recursion_level, config_line_t **last);
+static smartlist_t *config_get_file_list(const char *path,
+                                         smartlist_t *opened_files);
+static int config_get_included_config(const char *path, int recursion_level,
+                                      int extended, config_line_t **config,
+                                      config_line_t **config_last,
+                                      smartlist_t *opened_lst);
+static int config_process_include(const char *path, int recursion_level,
+                                  int extended, config_line_t **list,
+                                  config_line_t **list_last,
+                                  smartlist_t *opened_lst);
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * append it to *<b>lst</b>. */
+void
+config_line_append(config_line_t **lst,
+                   const char *key,
+                   const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = NULL;
+  while (*lst)
+    lst = &((*lst)->next);
+
+  (*lst) = newline;
+}
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * and prepend it to *<b>lst</b> */
+void
+config_line_prepend(config_line_t **lst,
+                    const char *key,
+                    const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = *lst;
+  *lst = newline;
+}
+
+/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or
+ * NULL if no such key exists.
+ *
+ * (In options parsing, this is for handling commandline-only options only;
+ * other options should be looked up in the appropriate data structure.) */
+const config_line_t *
+config_line_find(const config_line_t *lines,
+                 const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
+/** Auxiliary function that does all the work of config_get_lines.
+ * <b>recursion_level</b> is the count of how many nested %includes we have.
+ * <b>opened_lst</b> will have a list of opened files if provided.
+ * Returns the a pointer to the last element of the <b>result</b> in
+ * <b>last</b>. */
+static int
+config_get_lines_aux(const char *string, config_line_t **result, int extended,
+                     int allow_include, int *has_include,
+                     smartlist_t *opened_lst, int recursion_level,
+                     config_line_t **last)
+{
+  config_line_t *list = NULL, **next, *list_last = NULL;
+  char *k, *v;
+  const char *parse_err;
+  int include_used = 0;
+
+  if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) {
+    log_warn(LD_CONFIG, "Error while parsing configuration: more than %d "
+             "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL);
+    return -1;
+  }
+
+  next = &list;
+  do {
+    k = v = NULL;
+    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
+    if (!string) {
+      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
+               parse_err?parse_err:"<unknown>");
+      config_free_lines(list);
+      tor_free(k);
+      tor_free(v);
+      return -1;
+    }
+    if (k && v) {
+      unsigned command = CONFIG_LINE_NORMAL;
+      if (extended) {
+        if (k[0] == '+') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          command = CONFIG_LINE_APPEND;
+        } else if (k[0] == '/') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          tor_free(v);
+          v = tor_strdup("");
+          command = CONFIG_LINE_CLEAR;
+        }
+      }
+
+      if (allow_include && !strcmp(k, "%include")) {
+        tor_free(k);
+        include_used = 1;
+
+        config_line_t *include_list;
+        if (config_process_include(v, recursion_level, extended, &include_list,
+                                   &list_last, opened_lst) < 0) {
+          log_warn(LD_CONFIG, "Error reading included configuration "
+                   "file or directory: \"%s\".", v);
+          config_free_lines(list);
+          tor_free(v);
+          return -1;
+        }
+        *next = include_list;
+        if (list_last)
+          next = &list_last->next;
+        tor_free(v);
+      } else {
+        /* This list can get long, so we keep a pointer to the end of it
+         * rather than using config_line_append over and over and getting
+         * n^2 performance. */
+        *next = tor_malloc_zero(sizeof(**next));
+        (*next)->key = k;
+        (*next)->value = v;
+        (*next)->next = NULL;
+        (*next)->command = command;
+        list_last = *next;
+        next = &((*next)->next);
+      }
+    } else {
+      tor_free(k);
+      tor_free(v);
+    }
+  } while (*string);
+
+  if (last) {
+    *last = list_last;
+  }
+  if (has_include) {
+    *has_include = include_used;
+  }
+  *result = list;
+  return 0;
+}
+
+/** Helper: parse the config string and strdup into key/value
+ * strings. Set *result to the list, or NULL if parsing the string
+ * failed. Set *has_include to 1 if <b>result</b> has values from
+ * %included files. <b>opened_lst</b> will have a list of opened files if
+ * provided. Return 0 on success, -1 on failure. Warn and ignore any
+ * misformatted lines.
+ *
+ * If <b>extended</b> is set, then treat keys beginning with / and with + as
+ * indicating "clear" and "append" respectively. */
+int
+config_get_lines_include(const char *string, config_line_t **result,
+                         int extended, int *has_include,
+                         smartlist_t *opened_lst)
+{
+  return config_get_lines_aux(string, result, extended, 1, has_include,
+                              opened_lst, 1, NULL);
+}
+
+/** Same as config_get_lines_include but does not allow %include */
+int
+config_get_lines(const char *string, config_line_t **result, int extended)
+{
+  return config_get_lines_aux(string, result, extended, 0, NULL, NULL, 1,
+                              NULL);
+}
+
+/** Adds a list of configuration files present on <b>path</b> to
+ * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
+ * only that file will be added to <b>file_list</b>. If it is a directory,
+ * all paths for files on that directory root (no recursion) except for files
+ * whose name starts with a dot will be added to <b>file_list</b>.
+ * <b>opened_files</b> will have a list of files opened by this function
+ * if provided. Return 0 on success, -1 on failure. Ignores empty files.
+ */
+static smartlist_t *
+config_get_file_list(const char *path, smartlist_t *opened_files)
+{
+  smartlist_t *file_list = smartlist_new();
+
+  if (opened_files) {
+    smartlist_add_strdup(opened_files, path);
+  }
+
+  file_status_t file_type = file_status(path);
+  if (file_type == FN_FILE) {
+    smartlist_add_strdup(file_list, path);
+    return file_list;
+  } else if (file_type == FN_DIR) {
+    smartlist_t *all_files = tor_listdir(path);
+    if (!all_files) {
+      smartlist_free(file_list);
+      return NULL;
+    }
+    smartlist_sort_strings(all_files);
+    SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
+      if (f[0] == '.') {
+        tor_free(f);
+        continue;
+      }
+
+      char *fullname;
+      tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
+      tor_free(f);
+
+      if (opened_files) {
+        smartlist_add_strdup(opened_files, fullname);
+      }
+
+      if (file_status(fullname) != FN_FILE) {
+        tor_free(fullname);
+        continue;
+      }
+      smartlist_add(file_list, fullname);
+    } SMARTLIST_FOREACH_END(f);
+    smartlist_free(all_files);
+    return file_list;
+  } else if (file_type == FN_EMPTY) {
+      return file_list;
+  } else {
+    smartlist_free(file_list);
+    return NULL;
+  }
+}
+
+/** Creates a list of config lines present on included <b>path</b>.
+ * Set <b>config</b> to the list and <b>config_last</b> to the last element of
+ * <b>config</b>. <b>opened_lst</b> will have a list of opened files if
+ * provided. Return 0 on success, -1 on failure. */
+static int
+config_get_included_config(const char *path, int recursion_level, int extended,
+                           config_line_t **config, config_line_t **config_last,
+                           smartlist_t *opened_lst)
+{
+  char *included_conf = read_file_to_str(path, 0, NULL);
+  if (!included_conf) {
+    return -1;
+  }
+
+  if (config_get_lines_aux(included_conf, config, extended, 1, NULL,
+                           opened_lst, recursion_level+1, config_last) < 0) {
+    tor_free(included_conf);
+    return -1;
+  }
+
+  tor_free(included_conf);
+  return 0;
+}
+
+/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
+ * list of configuration settings obtained and <b>list_last</b> to the last
+ * element of the same list. <b>opened_lst</b> will have a list of opened
+ * files if provided. Return 0 on success, -1 on failure. */
+static int
+config_process_include(const char *path, int recursion_level, int extended,
+                       config_line_t **list, config_line_t **list_last,
+                       smartlist_t *opened_lst)
+{
+  config_line_t *ret_list = NULL;
+  config_line_t **next = &ret_list;
+
+  smartlist_t *config_files = config_get_file_list(path, opened_lst);
+  if (!config_files) {
+    return -1;
+  }
+
+  int rv = -1;
+  SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
+    config_line_t *included_config = NULL;
+    if (config_get_included_config(config_file, recursion_level, extended,
+                                   &included_config, list_last,
+                                   opened_lst) < 0) {
+      goto done;
+    }
+
+    *next = included_config;
+    if (*list_last)
+      next = &(*list_last)->next;
+
+  } SMARTLIST_FOREACH_END(config_file);
+  *list = ret_list;
+  rv = 0;
+
+ done:
+  SMARTLIST_FOREACH(config_files, char *, f, tor_free(f));
+  smartlist_free(config_files);
+  return rv;
+}
+
+/**
+ * Free all the configuration lines on the linked list <b>front</b>.
+ */
+void
+config_free_lines_(config_line_t *front)
+{
+  config_line_t *tmp;
+
+  while (front) {
+    tmp = front;
+    front = tmp->next;
+
+    tor_free(tmp->key);
+    tor_free(tmp->value);
+    tor_free(tmp);
+  }
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
+config_line_t *
+config_lines_dup(const config_line_t *inp)
+{
+  return config_lines_dup_and_filter(inp, NULL);
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>,
+ * but only the ones whose keys begin with <b>key</b> (case-insensitive).
+ * If <b>key</b> is NULL, do not filter. */
+config_line_t *
+config_lines_dup_and_filter(const config_line_t *inp,
+                            const char *key)
+{
+  config_line_t *result = NULL;
+  config_line_t **next_out = &result;
+  while (inp) {
+    if (key && strcasecmpstart(inp->key, key)) {
+      inp = inp->next;
+      continue;
+    }
+    *next_out = tor_malloc_zero(sizeof(config_line_t));
+    (*next_out)->key = tor_strdup(inp->key);
+    (*next_out)->value = tor_strdup(inp->value);
+    inp = inp->next;
+    next_out = &((*next_out)->next);
+  }
+  (*next_out) = NULL;
+  return result;
+}
+
+/** Return true iff a and b contain identical keys and values in identical
+ * order. */
+int
+config_lines_eq(config_line_t *a, config_line_t *b)
+{
+  while (a && b) {
+    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
+      return 0;
+    a = a->next;
+    b = b->next;
+  }
+  if (a || b)
+    return 0;
+  return 1;
+}
+
+/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
+int
+config_count_key(const config_line_t *a, const char *key)
+{
+  int n = 0;
+  while (a) {
+    if (!strcasecmp(a->key, key)) {
+      ++n;
+    }
+    a = a->next;
+  }
+  return n;
+}
+
+/** Given a string containing part of a configuration file or similar format,
+ * advance past comments and whitespace and try to parse a single line.  If we
+ * parse a line successfully, set *<b>key_out</b> to a new string holding the
+ * key portion and *<b>value_out</b> to a new string holding the value portion
+ * of the line, and return a pointer to the start of the next line.  If we run
+ * out of data, return a pointer to the end of the string.  If we encounter an
+ * error, return NULL and set *<b>err_out</b> (if provided) to an error
+ * message.
+ */
+const char *
+parse_config_line_from_str_verbose(const char *line, char **key_out,
+                                   char **value_out,
+                                   const char **err_out)
+{
+  /*
+    See torrc_format.txt for a description of the (silly) format this parses.
+   */
+  const char *key, *val, *cp;
+  int continuation = 0;
+
+  tor_assert(key_out);
+  tor_assert(value_out);
+
+  *key_out = *value_out = NULL;
+  key = val = NULL;
+  /* Skip until the first keyword. */
+  while (1) {
+    while (TOR_ISSPACE(*line))
+      ++line;
+    if (*line == '#') {
+      while (*line && *line != '\n')
+        ++line;
+    } else {
+      break;
+    }
+  }
+
+  if (!*line) { /* End of string? */
+    *key_out = *value_out = NULL;
+    return line;
+  }
+
+  /* Skip until the next space or \ followed by newline. */
+  key = line;
+  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
+         ! (line[0] == '\\' && line[1] == '\n'))
+    ++line;
+  *key_out = tor_strndup(key, line-key);
+
+  /* Skip until the value. */
+  while (*line == ' ' || *line == '\t')
+    ++line;
+
+  val = line;
+
+  /* Find the end of the line. */
+  if (*line == '\"') { // XXX No continuation handling is done here
+    if (!(line = unescape_string(line, value_out, NULL))) {
+      if (err_out)
+        *err_out = "Invalid escape sequence in quoted string";
+      return NULL;
+    }
+    while (*line == ' ' || *line == '\t')
+      ++line;
+    if (*line == '\r' && *(++line) == '\n')
+      ++line;
+    if (*line && *line != '#' && *line != '\n') {
+      if (err_out)
+        *err_out = "Excess data after quoted string";
+      return NULL;
+    }
+  } else {
+    /* Look for the end of the line. */
+    while (*line && *line != '\n' && (*line != '#' || continuation)) {
+      if (*line == '\\' && line[1] == '\n') {
+        continuation = 1;
+        line += 2;
+      } else if (*line == '#') {
+        do {
+          ++line;
+        } while (*line && *line != '\n');
+        if (*line == '\n')
+          ++line;
+      } else {
+        ++line;
+      }
+    }
+
+    if (*line == '\n') {
+      cp = line++;
+    } else {
+      cp = line;
+    }
+    /* Now back cp up to be the last nonspace character */
+    while (cp>val && TOR_ISSPACE(*(cp-1)))
+      --cp;
+
+    tor_assert(cp >= val);
+
+    /* Now copy out and decode the value. */
+    *value_out = tor_strndup(val, cp-val);
+    if (continuation) {
+      char *v_out, *v_in;
+      v_out = v_in = *value_out;
+      while (*v_in) {
+        if (*v_in == '#') {
+          do {
+            ++v_in;
+          } while (*v_in && *v_in != '\n');
+          if (*v_in == '\n')
+            ++v_in;
+        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
+          v_in += 2;
+        } else {
+          *v_out++ = *v_in++;
+        }
+      }
+      *v_out = '\0';
+    }
+  }
+
+  if (*line == '#') {
+    do {
+      ++line;
+    } while (*line && *line != '\n');
+  }
+  while (TOR_ISSPACE(*line)) ++line;
+
+  return line;
+}
+
diff --git a/src/common/confline.h b/src/common/confline.h
new file mode 100644
index 0000000000..772a9bbbdc
--- /dev/null
+++ b/src/common/confline.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONFLINE_H
+#define TOR_CONFLINE_H
+
+#include "container.h"
+
+/** Ordinary configuration line. */
+#define CONFIG_LINE_NORMAL 0
+/** Appends to previous configuration for the same option, even if we
+ * would ordinary replace it. */
+#define CONFIG_LINE_APPEND 1
+/* Removes all previous configuration for an option. */
+#define CONFIG_LINE_CLEAR 2
+
+#define MAX_INCLUDE_RECURSION_LEVEL 31
+
+/** A linked list of lines in a config file, or elsewhere */
+typedef struct config_line_t {
+  char *key;
+  char *value;
+  struct config_line_t *next;
+
+  /** What special treatment (if any) does this line require? */
+  unsigned int command:2;
+  /** If true, subsequent assignments to this linelist should replace
+   * it, not extend it.  Set only on the first item in a linelist in an
+   * or_options_t. */
+  unsigned int fragile:1;
+} config_line_t;
+
+void config_line_append(config_line_t **lst,
+                        const char *key, const char *val);
+void config_line_prepend(config_line_t **lst,
+                         const char *key, const char *val);
+config_line_t *config_lines_dup(const config_line_t *inp);
+config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
+                                           const char *key);
+const config_line_t *config_line_find(const config_line_t *lines,
+                                      const char *key);
+int config_lines_eq(config_line_t *a, config_line_t *b);
+int config_count_key(const config_line_t *a, const char *key);
+int config_get_lines(const char *string, config_line_t **result, int extended);
+int config_get_lines_include(const char *string, config_line_t **result,
+                             int extended, int *has_include,
+                             smartlist_t *opened_lst);
+void config_free_lines_(config_line_t *front);
+#define config_free_lines(front) \
+  do {                           \
+    config_free_lines_(front);   \
+    (front) = NULL;              \
+  } while (0)
+const char *parse_config_line_from_str_verbose(const char *line,
+                                       char **key_out, char **value_out,
+                                       const char **err_out);
+#endif /* !defined(TOR_CONFLINE_H) */
+
diff --git a/src/common/container.c b/src/common/container.c
index ec59dccf62..72ad3a9258 100644
--- a/src/common/container.c
+++ b/src/common/container.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "util.h"
 #include "torlog.h"
 #include "container.h"
-#include "crypto.h"
+#include "crypto_digest.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -42,7 +42,7 @@ smartlist_new,(void))
  * list's elements.
  */
 MOCK_IMPL(void,
-smartlist_free,(smartlist_t *sl))
+smartlist_free_,(smartlist_t *sl))
 {
   if (!sl)
     return;
@@ -132,6 +132,24 @@ smartlist_remove(smartlist_t *sl, const void *element)
     }
 }
 
+/** As <b>smartlist_remove</b>, but do not change the order of
+ * any elements not removed */
+void
+smartlist_remove_keeporder(smartlist_t *sl, const void *element)
+{
+  int i, j, num_used_orig = sl->num_used;
+  if (element == NULL)
+    return;
+
+  for (i=j=0; j < num_used_orig; ++j) {
+    if (sl->list[j] == element) {
+      --sl->num_used;
+    } else {
+      sl->list[i++] = sl->list[j];
+    }
+  }
+}
+
 /** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
  * return NULL. */
 void *
@@ -610,7 +628,7 @@ smartlist_uniq(smartlist_t *sl,
  * less than member, and greater than 0 if key is greater then member.
  */
 void *
-smartlist_bsearch(smartlist_t *sl, const void *key,
+smartlist_bsearch(const smartlist_t *sl, const void *key,
                   int (*compare)(const void *key, const void **member))
 {
   int found, idx;
@@ -825,13 +843,13 @@ smartlist_sort_pointers(smartlist_t *sl)
  *   }
  *
  *   void timer_heap_insert(smartlist_t *heap, timer_t *timer) {
- *      smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index),
+ *      smartlist_pqueue_add(heap, compare, offsetof(timer_t, heap_index),
  *         timer);
  *   }
  *
  *   void timer_heap_pop(smartlist_t *heap) {
  *      return smartlist_pqueue_pop(heap, compare,
- *         STRUCT_OFFSET(timer_t, heap_index));
+ *         offsetof(timer_t, heap_index));
  *   }
  */
 
@@ -1145,19 +1163,26 @@ HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
              digest256map_entry_hash,
              digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
 
+#define strmap_entry_free(ent) \
+  FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent))
+#define digestmap_entry_free(ent) \
+  FREE_AND_NULL(digestmap_entry_t, digestmap_entry_free_, (ent))
+#define digest256map_entry_free(ent) \
+  FREE_AND_NULL(digest256map_entry_t, digest256map_entry_free_, (ent))
+
 static inline void
-strmap_entry_free(strmap_entry_t *ent)
+strmap_entry_free_(strmap_entry_t *ent)
 {
   tor_free(ent->key);
   tor_free(ent);
 }
 static inline void
-digestmap_entry_free(digestmap_entry_t *ent)
+digestmap_entry_free_(digestmap_entry_t *ent)
 {
   tor_free(ent);
 }
 static inline void
-digest256map_entry_free(digest256map_entry_t *ent)
+digest256map_entry_free_(digest256map_entry_t *ent)
 {
   tor_free(ent);
 }
@@ -1317,7 +1342,7 @@ digest256map_assign_key(digest256map_entry_t *ent, const uint8_t *key)
    * those entries.  If free_val is provided, invoked it every value in \
    * <b>map</b>. */                                                     \
   MOCK_IMPL(void,                                                       \
-  prefix##_free, (maptype *map, void (*free_val)(void*)))               \
+  prefix##_free_, (maptype *map, void (*free_val)(void*)))              \
   {                                                                     \
     prefix##_entry_t **ent, **next, *this;                              \
     if (!map)                                                           \
@@ -1507,7 +1532,7 @@ digestset_new(int max_elements)
 
 /** Free all storage held in <b>set</b>. */
 void
-digestset_free(digestset_t *set)
+digestset_free_(digestset_t *set)
 {
   if (!set)
     return;
diff --git a/src/common/container.h b/src/common/container.h
index 71495b660a..7457c1e918 100644
--- a/src/common/container.h
+++ b/src/common/container.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CONTAINER_H
@@ -28,11 +28,14 @@ typedef struct smartlist_t {
 } smartlist_t;
 
 MOCK_DECL(smartlist_t *, smartlist_new, (void));
-MOCK_DECL(void, smartlist_free, (smartlist_t *sl));
+MOCK_DECL(void, smartlist_free_, (smartlist_t *sl));
+#define smartlist_free(sl) FREE_AND_NULL(smartlist_t, smartlist_free_, (sl))
+
 void smartlist_clear(smartlist_t *sl);
 void smartlist_add(smartlist_t *sl, void *element);
 void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
 void smartlist_remove(smartlist_t *sl, const void *element);
+void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
 void *smartlist_pop_last(smartlist_t *sl);
 void smartlist_reverse(smartlist_t *sl);
 void smartlist_string_remove(smartlist_t *sl, const char *element);
@@ -73,11 +76,11 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
   tor_assert(sl->num_used > idx);
   sl->list[idx] = val;
 }
-#else
+#else /* !(defined(DEBUG_SMARTLIST)) */
 #define smartlist_len(sl) ((sl)->num_used)
 #define smartlist_get(sl, idx) ((sl)->list[idx])
 #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
-#endif
+#endif /* defined(DEBUG_SMARTLIST) */
 
 /** Exchange the elements at indices <b>idx1</b> and <b>idx2</b> of the
  * smartlist <b>sl</b>. */
@@ -117,7 +120,7 @@ const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl);
 void smartlist_uniq_strings(smartlist_t *sl);
 void smartlist_uniq_digests(smartlist_t *sl);
 void smartlist_uniq_digests256(smartlist_t *sl);
-void *smartlist_bsearch(smartlist_t *sl, const void *key,
+void *smartlist_bsearch(const smartlist_t *sl, const void *key,
                         int (*compare)(const void *key, const void **member));
 int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
                           int (*compare)(const void *key, const void **member),
@@ -223,6 +226,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
 
 #define SMARTLIST_FOREACH_END(var)              \
     var = NULL;                                 \
+    (void) var ## _sl_idx;                      \
   } STMT_END
 
 /**
@@ -348,7 +352,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
   void* prefix##set(maptype *map, keytype key, void *val);              \
   void* prefix##get(const maptype *map, keytype key);                   \
   void* prefix##remove(maptype *map, keytype key);                      \
-  MOCK_DECL(void, prefix##free, (maptype *map, void (*free_val)(void*))); \
+  MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \
   int prefix##isempty(const maptype *map);                              \
   int prefix##size(const maptype *map);                                 \
   prefix##iter_t *prefix##iter_init(maptype *map);                      \
@@ -366,6 +370,16 @@ DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
  * table. */
 DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
 
+#define MAP_FREE_AND_NULL(maptype, map, fn)     \
+  do {                                          \
+    maptype ## _free_((map), (fn));             \
+    (map) = NULL;                               \
+  } while (0)
+
+#define strmap_free(map, fn) MAP_FREE_AND_NULL(strmap, (map), (fn))
+#define digestmap_free(map, fn) MAP_FREE_AND_NULL(digestmap, (map), (fn))
+#define digest256map_free(map, fn) MAP_FREE_AND_NULL(digest256map, (map), (fn))
+
 #undef DECLARE_MAP_FNS
 
 /** Iterates over the key-value pairs in a map <b>map</b> in order.
@@ -526,9 +540,9 @@ void* strmap_remove_lc(strmap_t *map, const char *key);
     return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
   }                                                                     \
   ATTR_UNUSED static inline void                                        \
-  prefix##f##ree(maptype *map, void (*free_val)(void*))                 \
+  prefix##f##ree_(maptype *map, void (*free_val)(void*))                \
   {                                                                     \
-    digestmap_free((digestmap_t*)map, free_val);                        \
+    digestmap_free_((digestmap_t*)map, free_val);                       \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
   prefix##isempty(maptype *map)                                         \
@@ -578,7 +592,7 @@ void* strmap_remove_lc(strmap_t *map, const char *key);
 #define BITARRAY_SHIFT 6
 #else
 #error "int is neither 4 nor 8 bytes. I can't deal with that."
-#endif
+#endif /* SIZEOF_INT == 4 || ... */
 #define BITARRAY_MASK ((1u<<BITARRAY_SHIFT)-1)
 
 /** A random-access array of one-bit-wide elements. */
@@ -612,10 +626,12 @@ bitarray_expand(bitarray_t *ba,
 }
 /** Free the bit array <b>ba</b>. */
 static inline void
-bitarray_free(bitarray_t *ba)
+bitarray_free_(bitarray_t *ba)
 {
   tor_free(ba);
 }
+#define bitarray_free(ba) FREE_AND_NULL(bitarray_t, bitarray_free_, (ba))
+
 /** Set the <b>bit</b>th bit in <b>b</b> to 1. */
 static inline void
 bitarray_set(bitarray_t *b, int bit)
@@ -677,7 +693,8 @@ digestset_contains(const digestset_t *set, const char *digest)
 #undef BIT
 
 digestset_t *digestset_new(int max_elements);
-void digestset_free(digestset_t* set);
+void digestset_free_(digestset_t* set);
+#define digestset_free(set) FREE_AND_NULL(digestset_t, digestset_free_, (set))
 
 /* These functions, given an <b>array</b> of <b>n_elements</b>, return the
  * <b>nth</b> lowest element. <b>nth</b>=0 gives the lowest element;
@@ -721,5 +738,5 @@ third_quartile_uint32(uint32_t *array, int n_elements)
   return find_nth_uint32(array, n_elements, (n_elements*3)/4);
 }
 
-#endif
+#endif /* !defined(TOR_CONTAINER_H) */
 
diff --git a/src/common/crypto.c b/src/common/crypto.c
index f8495bb107..d5b7c96916 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,27 +20,29 @@
 /* Windows defines this; so does OpenSSL 0.9.8h and later. We don't actually
  * use either definition. */
 #undef OCSP_RESPONSE
-#endif
+#endif /* defined(_WIN32) */
 
 #define CRYPTO_PRIVATE
-#include "crypto.h"
 #include "compat_openssl.h"
+#include "crypto.h"
 #include "crypto_curve25519.h"
+#include "crypto_digest.h"
 #include "crypto_ed25519.h"
 #include "crypto_format.h"
+#include "crypto_rand.h"
+#include "crypto_rsa.h"
+#include "crypto_util.h"
 
 DISABLE_GCC_WARNING(redundant-decls)
 
 #include <openssl/err.h>
-#include <openssl/rsa.h>
-#include <openssl/pem.h>
 #include <openssl/evp.h>
 #include <openssl/engine.h>
-#include <openssl/rand.h>
 #include <openssl/bn.h>
 #include <openssl/dh.h>
 #include <openssl/conf.h>
 #include <openssl/hmac.h>
+#include <openssl/ssl.h>
 
 ENABLE_GCC_WARNING(redundant-decls)
 
@@ -50,7 +52,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 #else
 #pragma GCC diagnostic warning "-Wredundant-decls"
 #endif
-#endif
+#endif /* __GNUC__ && GCC_VERSION >= 402 */
 
 #ifdef HAVE_CTYPE_H
 #include <ctype.h>
@@ -58,18 +60,6 @@ ENABLE_GCC_WARNING(redundant-decls)
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_FCNTL_H
-#include <sys/fcntl.h>
-#endif
-#ifdef HAVE_SYS_SYSCALL_H
-#include <sys/syscall.h>
-#endif
-#ifdef HAVE_SYS_RANDOM_H
-#include <sys/random.h>
-#endif
 
 #include "torlog.h"
 #include "torint.h"
@@ -82,80 +72,14 @@ ENABLE_GCC_WARNING(redundant-decls)
 
 #include "keccak-tiny/keccak-tiny.h"
 
-#ifdef ANDROID
-/* Android's OpenSSL seems to have removed all of its Engine support. */
-#define DISABLE_ENGINES
-#endif
-
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \
-  !defined(LIBRESSL_VERSION_NUMBER)
-/* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require
- * seting up various callbacks.
- *
- * OpenSSL 1.1.0pre4 has a messed up `ERR_remove_thread_state()` prototype,
- * while the previous one was restored in pre5, and the function made a no-op
- * (along with a deprecated annotation, which produces a compiler warning).
- *
- * While it is possible to support all three versions of the thread API,
- * a version that existed only for one snapshot pre-release is kind of
- * pointless, so let's not.
- */
-#define NEW_THREAD_API
-#endif
-
-/** Longest recognized */
-#define MAX_DNS_LABEL_SIZE 63
-
-/** Largest strong entropy request */
-#define MAX_STRONGEST_RAND_SIZE 256
-
-#ifndef NEW_THREAD_API
-/** A number of preallocated mutexes for use by OpenSSL. */
-static tor_mutex_t **openssl_mutexes_ = NULL;
-/** How many mutexes have we allocated for use by OpenSSL? */
-static int n_openssl_mutexes_ = 0;
-#endif
-
-/** A public key, or a public/private key-pair. */
-struct crypto_pk_t
-{
-  int refs; /**< reference count, so we don't have to copy keys */
-  RSA *key; /**< The key itself */
-};
-
 /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake
  * while we're waiting for the second.*/
 struct crypto_dh_t {
   DH *dh; /**< The openssl DH object */
 };
 
-static int setup_openssl_threading(void);
 static int tor_check_dh_key(int severity, const BIGNUM *bn);
 
-/** Return the number of bytes added by padding method <b>padding</b>.
- */
-static inline int
-crypto_get_rsa_padding_overhead(int padding)
-{
-  switch (padding)
-    {
-    case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD;
-    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
-    }
-}
-
-/** Given a padding method <b>padding</b>, return the correct OpenSSL constant.
- */
-static inline int
-crypto_get_rsa_padding(int padding)
-{
-  switch (padding)
-    {
-    case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING;
-    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
-    }
-}
-
 /** Boolean: has OpenSSL's crypto been initialized? */
 static int crypto_early_initialized_ = 0;
 
@@ -198,7 +122,7 @@ log_engine(const char *fn, ENGINE *e)
     log_info(LD_CRYPTO, "Using default implementation for %s", fn);
   }
 }
-#endif
+#endif /* !defined(DISABLE_ENGINES) */
 
 #ifndef DISABLE_ENGINES
 /** Try to load an engine in a shared library via fully qualified path.
@@ -218,76 +142,14 @@ try_load_engine(const char *path, const char *engine)
   }
   return e;
 }
-#endif
-
-/* Returns a trimmed and human-readable version of an openssl version string
-* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10
-* May 2012' and this will parse them into a form similar to '1.0.0b' */
-static char *
-parse_openssl_version_str(const char *raw_version)
-{
-  const char *end_of_version = NULL;
-  /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's
-     trim that down. */
-  if (!strcmpstart(raw_version, "OpenSSL ")) {
-    raw_version += strlen("OpenSSL ");
-    end_of_version = strchr(raw_version, ' ');
-  }
-
-  if (end_of_version)
-    return tor_strndup(raw_version,
-                      end_of_version-raw_version);
-  else
-    return tor_strdup(raw_version);
-}
-
-static char *crypto_openssl_version_str = NULL;
-/* Return a human-readable version of the run-time openssl version number. */
-const char *
-crypto_openssl_get_version_str(void)
-{
-  if (crypto_openssl_version_str == NULL) {
-    const char *raw_version = OpenSSL_version(OPENSSL_VERSION);
-    crypto_openssl_version_str = parse_openssl_version_str(raw_version);
-  }
-  return crypto_openssl_version_str;
-}
-
-static char *crypto_openssl_header_version_str = NULL;
-/* Return a human-readable version of the compile-time openssl version
-* number. */
-const char *
-crypto_openssl_get_header_version_str(void)
-{
-  if (crypto_openssl_header_version_str == NULL) {
-    crypto_openssl_header_version_str =
-                        parse_openssl_version_str(OPENSSL_VERSION_TEXT);
-  }
-  return crypto_openssl_header_version_str;
-}
+#endif /* !defined(DISABLE_ENGINES) */
 
-/** Make sure that openssl is using its default PRNG. Return 1 if we had to
- * adjust it; 0 otherwise. */
-STATIC int
-crypto_force_rand_ssleay(void)
-{
-  RAND_METHOD *default_method;
-  default_method = RAND_OpenSSL();
-  if (RAND_get_rand_method() != default_method) {
-    log_notice(LD_CRYPTO, "It appears that one of our engines has provided "
-               "a replacement the OpenSSL RNG. Resetting it to the default "
-               "implementation.");
-    RAND_set_rand_method(default_method);
-    return 1;
-  }
-  return 0;
-}
+static int have_seeded_siphash = 0;
 
 /** Set up the siphash key if we haven't already done so. */
 int
 crypto_init_siphash_key(void)
 {
-  static int have_seeded_siphash = 0;
   struct sipkey key;
   if (have_seeded_siphash)
     return 0;
@@ -307,8 +169,15 @@ crypto_early_init(void)
 
     crypto_early_initialized_ = 1;
 
+#ifdef OPENSSL_1_1_API
+    OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS |
+                     OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
+                     OPENSSL_INIT_ADD_ALL_CIPHERS |
+                     OPENSSL_INIT_ADD_ALL_DIGESTS, NULL);
+#else
     ERR_load_crypto_strings();
     OpenSSL_add_all_algorithms();
+#endif
 
     setup_openssl_threading();
 
@@ -394,7 +263,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
 #else
       log_engine("ECDH", ENGINE_get_default_ECDH());
       log_engine("ECDSA", ENGINE_get_default_ECDSA());
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
       log_engine("RAND", ENGINE_get_default_RAND());
       log_engine("RAND (which we will not use)", ENGINE_get_default_RAND());
       log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1));
@@ -412,7 +281,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
       log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm));
 #endif
 
-#endif
+#endif /* defined(DISABLE_ENGINES) */
     } else {
       log_info(LD_CRYPTO, "NOT using OpenSSL engine support.");
     }
@@ -437,73 +306,6 @@ crypto_thread_cleanup(void)
 #endif
 }
 
-/** used internally: quicly validate a crypto_pk_t object as a private key.
- * Return 1 iff the public key is valid, 0 if obviously invalid.
- */
-static int
-crypto_pk_private_ok(const crypto_pk_t *k)
-{
-#ifdef OPENSSL_1_1_API
-  if (!k || !k->key)
-    return 0;
-
-  const BIGNUM *p, *q;
-  RSA_get0_factors(k->key, &p, &q);
-  return p != NULL; /* XXX/yawning: Should we check q? */
-#else
-  return k && k->key && k->key->p;
-#endif
-}
-
-/** used by tortls.c: wrap an RSA* in a crypto_pk_t. */
-crypto_pk_t *
-crypto_new_pk_from_rsa_(RSA *rsa)
-{
-  crypto_pk_t *env;
-  tor_assert(rsa);
-  env = tor_malloc(sizeof(crypto_pk_t));
-  env->refs = 1;
-  env->key = rsa;
-  return env;
-}
-
-/** Helper, used by tor-checkkey.c and tor-gencert.c.  Return the RSA from a
- * crypto_pk_t. */
-RSA *
-crypto_pk_get_rsa_(crypto_pk_t *env)
-{
-  return env->key;
-}
-
-/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t.  Iff
- * private is set, include the private-key portion of the key. Return a valid
- * pointer on success, and NULL on failure. */
-MOCK_IMPL(EVP_PKEY *,
-          crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private))
-{
-  RSA *key = NULL;
-  EVP_PKEY *pkey = NULL;
-  tor_assert(env->key);
-  if (private) {
-    if (!(key = RSAPrivateKey_dup(env->key)))
-      goto error;
-  } else {
-    if (!(key = RSAPublicKey_dup(env->key)))
-      goto error;
-  }
-  if (!(pkey = EVP_PKEY_new()))
-    goto error;
-  if (!(EVP_PKEY_assign_RSA(pkey, key)))
-    goto error;
-  return pkey;
- error:
-  if (pkey)
-    EVP_PKEY_free(pkey);
-  if (key)
-    RSA_free(key);
-  return NULL;
-}
-
 /** Used by tortls.c: Get the DH* from a crypto_dh_t.
  */
 DH *
@@ -512,38 +314,6 @@ crypto_dh_get_dh_(crypto_dh_t *dh)
   return dh->dh;
 }
 
-/** Allocate and return storage for a public key.  The key itself will not yet
- * be set.
- */
-MOCK_IMPL(crypto_pk_t *,
-          crypto_pk_new,(void))
-{
-  RSA *rsa;
-
-  rsa = RSA_new();
-  tor_assert(rsa);
-  return crypto_new_pk_from_rsa_(rsa);
-}
-
-/** Release a reference to an asymmetric key; when all the references
- * are released, free the key.
- */
-void
-crypto_pk_free(crypto_pk_t *env)
-{
-  if (!env)
-    return;
-
-  if (--env->refs > 0)
-    return;
-  tor_assert(env->refs == 0);
-
-  if (env->key)
-    RSA_free(env->key);
-
-  tor_free(env);
-}
-
 /** Allocate and return a new symmetric cipher using the provided key and iv.
  * The key is <b>bits</b> bits long; the IV is CIPHER_IV_LEN bytes.  Both
  * must be provided. Key length must be 128, 192, or 256 */
@@ -592,7 +362,7 @@ crypto_cipher_new(const char *key)
 /** Free a symmetric cipher.
  */
 void
-crypto_cipher_free(crypto_cipher_t *env)
+crypto_cipher_free_(crypto_cipher_t *env)
 {
   if (!env)
     return;
@@ -600,865 +370,6 @@ crypto_cipher_free(crypto_cipher_t *env)
   aes_cipher_free(env);
 }
 
-/* public key crypto */
-
-/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>.
- * Return 0 on success, -1 on failure.
- */
-MOCK_IMPL(int,
-          crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits))
-{
-  tor_assert(env);
-
-  if (env->key) {
-    RSA_free(env->key);
-    env->key = NULL;
-  }
-
-  {
-    BIGNUM *e = BN_new();
-    RSA *r = NULL;
-    if (!e)
-      goto done;
-    if (! BN_set_word(e, 65537))
-      goto done;
-    r = RSA_new();
-    if (!r)
-      goto done;
-    if (RSA_generate_key_ex(r, bits, e, NULL) == -1)
-      goto done;
-
-    env->key = r;
-    r = NULL;
-  done:
-    if (e)
-      BN_clear_free(e);
-    if (r)
-      RSA_free(r);
-  }
-
-  if (!env->key) {
-    crypto_log_errors(LOG_WARN, "generating RSA key");
-    return -1;
-  }
-
-  return 0;
-}
-
-/** A PEM callback that always reports a failure to get a password */
-static int
-pem_no_password_cb(char *buf, int size, int rwflag, void *u)
-{
-  (void)buf;
-  (void)size;
-  (void)rwflag;
-  (void)u;
-  /* The openssl documentation says that a callback "must" return 0 if an
-   * error occurred.  But during the 1.1.1 series (commit c82c3462267afdbbaa5
-   * they changed the interpretation so that 0 indicates an empty password and
-   * -1 indicates an error. We want to reject any encrypted PEM buffers, so we
-   * return -1.  This will work on older OpenSSL versions and LibreSSL too. */
-  return -1;
-}
-
-/** Read a PEM-encoded private key from the <b>len</b>-byte string <b>s</b>
- * into <b>env</b>.  Return 0 on success, -1 on failure.  If len is -1,
- * the string is nul-terminated.
- */
-int
-crypto_pk_read_private_key_from_string(crypto_pk_t *env,
-                                       const char *s, ssize_t len)
-{
-  BIO *b;
-
-  tor_assert(env);
-  tor_assert(s);
-  tor_assert(len < INT_MAX && len < SSIZE_T_CEILING);
-
-  /* Create a read-only memory BIO, backed by the string 's' */
-  b = BIO_new_mem_buf((char*)s, (int)len);
-  if (!b)
-    return -1;
-
-  if (env->key)
-    RSA_free(env->key);
-
-  env->key = PEM_read_bio_RSAPrivateKey(b,NULL,pem_no_password_cb,NULL);
-
-  BIO_free(b);
-
-  if (!env->key) {
-    crypto_log_errors(LOG_WARN, "Error parsing private key");
-    return -1;
-  }
-  return 0;
-}
-
-/** Read a PEM-encoded private key from the file named by
- * <b>keyfile</b> into <b>env</b>.  Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
-                                         const char *keyfile)
-{
-  char *contents;
-  int r;
-
-  /* Read the file into a string. */
-  contents = read_file_to_str(keyfile, 0, NULL);
-  if (!contents) {
-    log_warn(LD_CRYPTO, "Error reading private key from \"%s\"", keyfile);
-    return -1;
-  }
-
-  /* Try to parse it. */
-  r = crypto_pk_read_private_key_from_string(env, contents, -1);
-  memwipe(contents, 0, strlen(contents));
-  tor_free(contents);
-  if (r)
-    return -1; /* read_private_key_from_string already warned, so we don't.*/
-
-  /* Make sure it's valid. */
-  if (crypto_pk_check_key(env) <= 0)
-    return -1;
-
-  return 0;
-}
-
-/** Helper function to implement crypto_pk_write_*_key_to_string. Return 0 on
- * success, -1 on failure. */
-static int
-crypto_pk_write_key_to_string_impl(crypto_pk_t *env, char **dest,
-                                   size_t *len, int is_public)
-{
-  BUF_MEM *buf;
-  BIO *b;
-  int r;
-
-  tor_assert(env);
-  tor_assert(env->key);
-  tor_assert(dest);
-
-  b = BIO_new(BIO_s_mem()); /* Create a memory BIO */
-  if (!b)
-    return -1;
-
-  /* Now you can treat b as if it were a file.  Just use the
-   * PEM_*_bio_* functions instead of the non-bio variants.
-   */
-  if (is_public)
-    r = PEM_write_bio_RSAPublicKey(b, env->key);
-  else
-    r = PEM_write_bio_RSAPrivateKey(b, env->key, NULL,NULL,0,NULL,NULL);
-
-  if (!r) {
-    crypto_log_errors(LOG_WARN, "writing RSA key to string");
-    BIO_free(b);
-    return -1;
-  }
-
-  BIO_get_mem_ptr(b, &buf);
-
-  *dest = tor_malloc(buf->length+1);
-  memcpy(*dest, buf->data, buf->length);
-  (*dest)[buf->length] = 0; /* nul terminate it */
-  *len = buf->length;
-
-  BIO_free(b);
-
-  return 0;
-}
-
-/** PEM-encode the public key portion of <b>env</b> and write it to a
- * newly allocated string.  On success, set *<b>dest</b> to the new
- * string, *<b>len</b> to the string's length, and return 0.  On
- * failure, return -1.
- */
-int
-crypto_pk_write_public_key_to_string(crypto_pk_t *env, char **dest,
-                                     size_t *len)
-{
-  return crypto_pk_write_key_to_string_impl(env, dest, len, 1);
-}
-
-/** PEM-encode the private key portion of <b>env</b> and write it to a
- * newly allocated string.  On success, set *<b>dest</b> to the new
- * string, *<b>len</b> to the string's length, and return 0.  On
- * failure, return -1.
- */
-int
-crypto_pk_write_private_key_to_string(crypto_pk_t *env, char **dest,
-                                     size_t *len)
-{
-  return crypto_pk_write_key_to_string_impl(env, dest, len, 0);
-}
-
-/** Read a PEM-encoded public key from the first <b>len</b> characters of
- * <b>src</b>, and store the result in <b>env</b>.  Return 0 on success, -1 on
- * failure.
- */
-int
-crypto_pk_read_public_key_from_string(crypto_pk_t *env, const char *src,
-                                      size_t len)
-{
-  BIO *b;
-
-  tor_assert(env);
-  tor_assert(src);
-  tor_assert(len<INT_MAX);
-
-  b = BIO_new(BIO_s_mem()); /* Create a memory BIO */
-  if (!b)
-    return -1;
-
-  BIO_write(b, src, (int)len);
-
-  if (env->key)
-    RSA_free(env->key);
-  env->key = PEM_read_bio_RSAPublicKey(b, NULL, pem_no_password_cb, NULL);
-  BIO_free(b);
-  if (!env->key) {
-    crypto_log_errors(LOG_WARN, "reading public key from string");
-    return -1;
-  }
-
-  return 0;
-}
-
-/** Write the private key from <b>env</b> into the file named by <b>fname</b>,
- * PEM-encoded.  Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
-                                        const char *fname)
-{
-  BIO *bio;
-  char *cp;
-  long len;
-  char *s;
-  int r;
-
-  tor_assert(crypto_pk_private_ok(env));
-
-  if (!(bio = BIO_new(BIO_s_mem())))
-    return -1;
-  if (PEM_write_bio_RSAPrivateKey(bio, env->key, NULL,NULL,0,NULL,NULL)
-      == 0) {
-    crypto_log_errors(LOG_WARN, "writing private key");
-    BIO_free(bio);
-    return -1;
-  }
-  len = BIO_get_mem_data(bio, &cp);
-  tor_assert(len >= 0);
-  s = tor_malloc(len+1);
-  memcpy(s, cp, len);
-  s[len]='\0';
-  r = write_str_to_file(fname, s, 0);
-  BIO_free(bio);
-  memwipe(s, 0, strlen(s));
-  tor_free(s);
-  return r;
-}
-
-/** Return true iff <b>env</b> has a valid key.
- */
-int
-crypto_pk_check_key(crypto_pk_t *env)
-{
-  int r;
-  tor_assert(env);
-
-  r = RSA_check_key(env->key);
-  if (r <= 0)
-    crypto_log_errors(LOG_WARN,"checking RSA key");
-  return r;
-}
-
-/** Return true iff <b>key</b> contains the private-key portion of the RSA
- * key. */
-int
-crypto_pk_key_is_private(const crypto_pk_t *key)
-{
-  tor_assert(key);
-  return crypto_pk_private_ok(key);
-}
-
-/** Return true iff <b>env</b> contains a public key whose public exponent
- * equals 65537.
- */
-int
-crypto_pk_public_exponent_ok(crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-  const BIGNUM *e;
-
-#ifdef OPENSSL_1_1_API
-  const BIGNUM *n, *d;
-  RSA_get0_key(env->key, &n, &e, &d);
-#else
-  e = env->key->e;
-#endif
-  return BN_is_word(e, 65537);
-}
-
-/** Compare the public-key components of a and b.  Return less than 0
- * if a\<b, 0 if a==b, and greater than 0 if a\>b.  A NULL key is
- * considered to be less than all non-NULL keys, and equal to itself.
- *
- * Note that this may leak information about the keys through timing.
- */
-int
-crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b)
-{
-  int result;
-  char a_is_non_null = (a != NULL) && (a->key != NULL);
-  char b_is_non_null = (b != NULL) && (b->key != NULL);
-  char an_argument_is_null = !a_is_non_null | !b_is_non_null;
-
-  result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null));
-  if (an_argument_is_null)
-    return result;
-
-  const BIGNUM *a_n, *a_e;
-  const BIGNUM *b_n, *b_e;
-
-#ifdef OPENSSL_1_1_API
-  const BIGNUM *a_d, *b_d;
-  RSA_get0_key(a->key, &a_n, &a_e, &a_d);
-  RSA_get0_key(b->key, &b_n, &b_e, &b_d);
-#else
-  a_n = a->key->n;
-  a_e = a->key->e;
-  b_n = b->key->n;
-  b_e = b->key->e;
-#endif
-
-  tor_assert(a_n != NULL && a_e != NULL);
-  tor_assert(b_n != NULL && b_e != NULL);
-
-  result = BN_cmp(a_n, b_n);
-  if (result)
-    return result;
-  return BN_cmp(a_e, b_e);
-}
-
-/** Compare the public-key components of a and b.  Return non-zero iff
- * a==b.  A NULL key is considered to be distinct from all non-NULL
- * keys, and equal to itself.
- *
- *  Note that this may leak information about the keys through timing.
- */
-int
-crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b)
-{
-  return (crypto_pk_cmp_keys(a, b) == 0);
-}
-
-/** Return the size of the public key modulus in <b>env</b>, in bytes. */
-size_t
-crypto_pk_keysize(const crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-  return (size_t) RSA_size((RSA*)env->key);
-}
-
-/** Return the size of the public key modulus of <b>env</b>, in bits. */
-int
-crypto_pk_num_bits(crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-#ifdef OPENSSL_1_1_API
-  /* It's so stupid that there's no other way to check that n is valid
-   * before calling RSA_bits().
-   */
-  const BIGNUM *n, *e, *d;
-  RSA_get0_key(env->key, &n, &e, &d);
-  tor_assert(n != NULL);
-
-  return RSA_bits(env->key);
-#else
-  tor_assert(env->key->n);
-  return BN_num_bits(env->key->n);
-#endif
-}
-
-/** Increase the reference count of <b>env</b>, and return it.
- */
-crypto_pk_t *
-crypto_pk_dup_key(crypto_pk_t *env)
-{
-  tor_assert(env);
-  tor_assert(env->key);
-
-  env->refs++;
-  return env;
-}
-
-#ifdef TOR_UNIT_TESTS
-/** For testing: replace dest with src.  (Dest must have a refcount
- * of 1) */
-void
-crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src)
-{
-  tor_assert(dest);
-  tor_assert(dest->refs == 1);
-  tor_assert(src);
-  RSA_free(dest->key);
-  dest->key = RSAPrivateKey_dup(src->key);
-}
-#endif
-
-/** Make a real honest-to-goodness copy of <b>env</b>, and return it.
- * Returns NULL on failure. */
-crypto_pk_t *
-crypto_pk_copy_full(crypto_pk_t *env)
-{
-  RSA *new_key;
-  int privatekey = 0;
-  tor_assert(env);
-  tor_assert(env->key);
-
-  if (crypto_pk_private_ok(env)) {
-    new_key = RSAPrivateKey_dup(env->key);
-    privatekey = 1;
-  } else {
-    new_key = RSAPublicKey_dup(env->key);
-  }
-  if (!new_key) {
-    /* LCOV_EXCL_START
-     *
-     * We can't cause RSA*Key_dup() to fail, so we can't really test this.
-     */
-    log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.",
-            privatekey?"private":"public");
-    crypto_log_errors(LOG_ERR,
-                      privatekey ? "Duplicating a private key" :
-                      "Duplicating a public key");
-    tor_fragile_assert();
-    return NULL;
-    /* LCOV_EXCL_STOP */
-  }
-
-  return crypto_new_pk_from_rsa_(new_key);
-}
-
-/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key
- * in <b>env</b>, using the padding method <b>padding</b>.  On success,
- * write the result to <b>to</b>, and return the number of bytes
- * written.  On failure, return -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
-                         const char *from, size_t fromlen, int padding)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen<INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-
-  r = RSA_public_encrypt((int)fromlen,
-                         (unsigned char*)from, (unsigned char*)to,
-                         env->key, crypto_get_rsa_padding(padding));
-  if (r<0) {
-    crypto_log_errors(LOG_WARN, "performing RSA encryption");
-    return -1;
-  }
-  return r;
-}
-
-/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key
- * in <b>env</b>, using the padding method <b>padding</b>.  On success,
- * write the result to <b>to</b>, and return the number of bytes
- * written.  On failure, return -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_decrypt(crypto_pk_t *env, char *to,
-                          size_t tolen,
-                          const char *from, size_t fromlen,
-                          int padding, int warnOnFailure)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(env->key);
-  tor_assert(fromlen<INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-  if (!crypto_pk_key_is_private(env))
-    /* Not a private key */
-    return -1;
-
-  r = RSA_private_decrypt((int)fromlen,
-                          (unsigned char*)from, (unsigned char*)to,
-                          env->key, crypto_get_rsa_padding(padding));
-
-  if (r<0) {
-    crypto_log_errors(warnOnFailure?LOG_WARN:LOG_DEBUG,
-                      "performing RSA decryption");
-    return -1;
-  }
-  return r;
-}
-
-/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the
- * public key in <b>env</b>, using PKCS1 padding.  On success, write the
- * signed data to <b>to</b>, and return the number of bytes written.
- * On failure, return -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_public_checksig(const crypto_pk_t *env, char *to,
-                          size_t tolen,
-                          const char *from, size_t fromlen)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-  r = RSA_public_decrypt((int)fromlen,
-                         (unsigned char*)from, (unsigned char*)to,
-                         env->key, RSA_PKCS1_PADDING);
-
-  if (r<0) {
-    crypto_log_errors(LOG_INFO, "checking RSA signature");
-    return -1;
-  }
-  return r;
-}
-
-/** Check a siglen-byte long signature at <b>sig</b> against
- * <b>datalen</b> bytes of data at <b>data</b>, using the public key
- * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for
- * SHA1(data).  Else return -1.
- */
-int
-crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data,
-                               size_t datalen, const char *sig, size_t siglen)
-{
-  char digest[DIGEST_LEN];
-  char *buf;
-  size_t buflen;
-  int r;
-
-  tor_assert(env);
-  tor_assert(data);
-  tor_assert(sig);
-  tor_assert(datalen < SIZE_T_CEILING);
-  tor_assert(siglen < SIZE_T_CEILING);
-
-  if (crypto_digest(digest,data,datalen)<0) {
-    log_warn(LD_BUG, "couldn't compute digest");
-    return -1;
-  }
-  buflen = crypto_pk_keysize(env);
-  buf = tor_malloc(buflen);
-  r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen);
-  if (r != DIGEST_LEN) {
-    log_warn(LD_CRYPTO, "Invalid signature");
-    tor_free(buf);
-    return -1;
-  }
-  if (tor_memneq(buf, digest, DIGEST_LEN)) {
-    log_warn(LD_CRYPTO, "Signature mismatched with digest.");
-    tor_free(buf);
-    return -1;
-  }
-  tor_free(buf);
-
-  return 0;
-}
-
-/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in
- * <b>env</b>, using PKCS1 padding.  On success, write the signature to
- * <b>to</b>, and return the number of bytes written.  On failure, return
- * -1.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
-                       const char *from, size_t fromlen)
-{
-  int r;
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < INT_MAX);
-  tor_assert(tolen >= crypto_pk_keysize(env));
-  if (!crypto_pk_key_is_private(env))
-    /* Not a private key */
-    return -1;
-
-  r = RSA_private_encrypt((int)fromlen,
-                          (unsigned char*)from, (unsigned char*)to,
-                          (RSA*)env->key, RSA_PKCS1_PADDING);
-  if (r<0) {
-    crypto_log_errors(LOG_WARN, "generating RSA signature");
-    return -1;
-  }
-  return r;
-}
-
-/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at
- * <b>from</b>; sign the data with the private key in <b>env</b>, and
- * store it in <b>to</b>.  Return the number of bytes written on
- * success, and -1 on failure.
- *
- * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
- * at least the length of the modulus of <b>env</b>.
- */
-int
-crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen)
-{
-  int r;
-  char digest[DIGEST_LEN];
-  if (crypto_digest(digest,from,fromlen)<0)
-    return -1;
-  r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN);
-  memwipe(digest, 0, sizeof(digest));
-  return r;
-}
-
-/** Perform a hybrid (public/secret) encryption on <b>fromlen</b>
- * bytes of data from <b>from</b>, with padding type 'padding',
- * storing the results on <b>to</b>.
- *
- * Returns the number of bytes written on success, -1 on failure.
- *
- * The encrypted data consists of:
- *   - The source data, padded and encrypted with the public key, if the
- *     padded source data is no longer than the public key, and <b>force</b>
- *     is false, OR
- *   - The beginning of the source data prefixed with a 16-byte symmetric key,
- *     padded and encrypted with the public key; followed by the rest of
- *     the source data encrypted in AES-CTR mode with the symmetric key.
- */
-int
-crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
-                                char *to, size_t tolen,
-                                const char *from,
-                                size_t fromlen,
-                                int padding, int force)
-{
-  int overhead, outlen, r;
-  size_t pkeylen, symlen;
-  crypto_cipher_t *cipher = NULL;
-  char *buf = NULL;
-
-  tor_assert(env);
-  tor_assert(from);
-  tor_assert(to);
-  tor_assert(fromlen < SIZE_T_CEILING);
-
-  overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding));
-  pkeylen = crypto_pk_keysize(env);
-
-  if (!force && fromlen+overhead <= pkeylen) {
-    /* It all fits in a single encrypt. */
-    return crypto_pk_public_encrypt(env,to,
-                                    tolen,
-                                    from,fromlen,padding);
-  }
-  tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN);
-  tor_assert(tolen >= pkeylen);
-
-  char key[CIPHER_KEY_LEN];
-  crypto_rand(key, sizeof(key)); /* generate a new key. */
-  cipher = crypto_cipher_new(key);
-
-  buf = tor_malloc(pkeylen+1);
-  memcpy(buf, key, CIPHER_KEY_LEN);
-  memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN);
-
-  /* Length of symmetrically encrypted data. */
-  symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN);
-
-  outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding);
-  if (outlen!=(int)pkeylen) {
-    goto err;
-  }
-  r = crypto_cipher_encrypt(cipher, to+outlen,
-                            from+pkeylen-overhead-CIPHER_KEY_LEN, symlen);
-
-  if (r<0) goto err;
-  memwipe(buf, 0, pkeylen);
-  memwipe(key, 0, sizeof(key));
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  tor_assert(outlen+symlen < INT_MAX);
-  return (int)(outlen + symlen);
- err:
-
-  memwipe(buf, 0, pkeylen);
-  memwipe(key, 0, sizeof(key));
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  return -1;
-}
-
-/** Invert crypto_pk_public_hybrid_encrypt. Returns the number of bytes
- * written on success, -1 on failure. */
-int
-crypto_pk_private_hybrid_decrypt(crypto_pk_t *env,
-                                 char *to,
-                                 size_t tolen,
-                                 const char *from,
-                                 size_t fromlen,
-                                 int padding, int warnOnFailure)
-{
-  int outlen, r;
-  size_t pkeylen;
-  crypto_cipher_t *cipher = NULL;
-  char *buf = NULL;
-
-  tor_assert(fromlen < SIZE_T_CEILING);
-  pkeylen = crypto_pk_keysize(env);
-
-  if (fromlen <= pkeylen) {
-    return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding,
-                                     warnOnFailure);
-  }
-
-  buf = tor_malloc(pkeylen);
-  outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding,
-                                     warnOnFailure);
-  if (outlen<0) {
-    log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO,
-           "Error decrypting public-key data");
-    goto err;
-  }
-  if (outlen < CIPHER_KEY_LEN) {
-    log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO,
-           "No room for a symmetric key");
-    goto err;
-  }
-  cipher = crypto_cipher_new(buf);
-  if (!cipher) {
-    goto err;
-  }
-  memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN);
-  outlen -= CIPHER_KEY_LEN;
-  tor_assert(tolen - outlen >= fromlen - pkeylen);
-  r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen);
-  if (r<0)
-    goto err;
-  memwipe(buf,0,pkeylen);
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  tor_assert(outlen + fromlen < INT_MAX);
-  return (int)(outlen + (fromlen-pkeylen));
- err:
-  memwipe(buf,0,pkeylen);
-  tor_free(buf);
-  crypto_cipher_free(cipher);
-  return -1;
-}
-
-/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>.
- * Return -1 on error, or the number of characters used on success.
- */
-int
-crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len)
-{
-  int len;
-  unsigned char *buf = NULL;
-
-  len = i2d_RSAPublicKey(pk->key, &buf);
-  if (len < 0 || buf == NULL)
-    return -1;
-
-  if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) {
-    OPENSSL_free(buf);
-    return -1;
-  }
-  /* We don't encode directly into 'dest', because that would be illegal
-   * type-punning.  (C99 is smarter than me, C99 is smarter than me...)
-   */
-  memcpy(dest,buf,len);
-  OPENSSL_free(buf);
-  return len;
-}
-
-/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on
- * success and NULL on failure.
- */
-crypto_pk_t *
-crypto_pk_asn1_decode(const char *str, size_t len)
-{
-  RSA *rsa;
-  unsigned char *buf;
-  const unsigned char *cp;
-  cp = buf = tor_malloc(len);
-  memcpy(buf,str,len);
-  rsa = d2i_RSAPublicKey(NULL, &cp, len);
-  tor_free(buf);
-  if (!rsa) {
-    crypto_log_errors(LOG_WARN,"decoding public key");
-    return NULL;
-  }
-  return crypto_new_pk_from_rsa_(rsa);
-}
-
-/** Given a private or public key <b>pk</b>, put a SHA1 hash of the
- * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space).
- * Return 0 on success, -1 on failure.
- */
-int
-crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out)
-{
-  unsigned char *buf = NULL;
-  int len;
-
-  len = i2d_RSAPublicKey((RSA*)pk->key, &buf);
-  if (len < 0 || buf == NULL)
-    return -1;
-  if (crypto_digest(digest_out, (char*)buf, len) < 0) {
-    OPENSSL_free(buf);
-    return -1;
-  }
-  OPENSSL_free(buf);
-  return 0;
-}
-
-/** Compute all digests of the DER encoding of <b>pk</b>, and store them
- * in <b>digests_out</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out)
-{
-  unsigned char *buf = NULL;
-  int len;
-
-  len = i2d_RSAPublicKey(pk->key, &buf);
-  if (len < 0 || buf == NULL)
-    return -1;
-  if (crypto_common_digests(digests_out, (char*)buf, len) < 0) {
-    OPENSSL_free(buf);
-    return -1;
-  }
-  OPENSSL_free(buf);
-  return 0;
-}
-
 /** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces
  * every four characters. */
 void
@@ -1479,127 +390,6 @@ crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in)
   *out = '\0';
 }
 
-/** Given a private or public key <b>pk</b>, put a fingerprint of the
- * public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 bytes of
- * space).  Return 0 on success, -1 on failure.
- *
- * Fingerprints are computed as the SHA1 digest of the ASN.1 encoding
- * of the public key, converted to hexadecimal, in upper case, with a
- * space after every four digits.
- *
- * If <b>add_space</b> is false, omit the spaces.
- */
-int
-crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space)
-{
-  char digest[DIGEST_LEN];
-  char hexdigest[HEX_DIGEST_LEN+1];
-  if (crypto_pk_get_digest(pk, digest)) {
-    return -1;
-  }
-  base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN);
-  if (add_space) {
-    crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest);
-  } else {
-    strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1);
-  }
-  return 0;
-}
-
-/** Given a private or public key <b>pk</b>, put a hashed fingerprint of
- * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1
- * bytes of space).  Return 0 on success, -1 on failure.
- *
- * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest
- * of the ASN.1 encoding of the public key, converted to hexadecimal, in
- * upper case.
- */
-int
-crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out)
-{
-  char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN];
-  if (crypto_pk_get_digest(pk, digest)) {
-    return -1;
-  }
-  if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) {
-    return -1;
-  }
-  base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN);
-  return 0;
-}
-
-/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the
- * Base64 encoding of the DER representation of the private key as a NUL
- * terminated string, and return it via <b>priv_out</b>.  Return 0 on
- * sucess, -1 on failure.
- *
- * It is the caller's responsibility to sanitize and free the resulting buffer.
- */
-int
-crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out)
-{
-  unsigned char *der = NULL;
-  int der_len;
-  int ret = -1;
-
-  *priv_out = NULL;
-
-  der_len = i2d_RSAPrivateKey(pk->key, &der);
-  if (der_len < 0 || der == NULL)
-    return ret;
-
-  size_t priv_len = base64_encode_size(der_len, 0) + 1;
-  char *priv = tor_malloc_zero(priv_len);
-  if (base64_encode(priv, priv_len, (char *)der, der_len, 0) >= 0) {
-    *priv_out = priv;
-    ret = 0;
-  } else {
-    tor_free(priv);
-  }
-
-  memwipe(der, 0, der_len);
-  OPENSSL_free(der);
-  return ret;
-}
-
-/** Given a string containing the Base64 encoded DER representation of the
- * private key <b>str</b>, decode and return the result on success, or NULL
- * on failure.
- */
-crypto_pk_t *
-crypto_pk_base64_decode(const char *str, size_t len)
-{
-  crypto_pk_t *pk = NULL;
-
-  char *der = tor_malloc_zero(len + 1);
-  int der_len = base64_decode(der, len, str, len);
-  if (der_len <= 0) {
-    log_warn(LD_CRYPTO, "Stored RSA private key seems corrupted (base64).");
-    goto out;
-  }
-
-  const unsigned char *dp = (unsigned char*)der; /* Shut the compiler up. */
-  RSA *rsa = d2i_RSAPrivateKey(NULL, &dp, der_len);
-  if (!rsa) {
-    crypto_log_errors(LOG_WARN, "decoding private key");
-    goto out;
-  }
-
-  pk = crypto_new_pk_from_rsa_(rsa);
-
-  /* Make sure it's valid. */
-  if (crypto_pk_check_key(pk) <= 0) {
-    crypto_pk_free(pk);
-    pk = NULL;
-    goto out;
-  }
-
- out:
-  memwipe(der, 0, len + 1);
-  tor_free(der);
-  return pk;
-}
-
 /* symmetric crypto */
 
 /** Encrypt <b>fromlen</b> bytes from <b>from</b> using the cipher
@@ -1711,469 +501,6 @@ crypto_cipher_decrypt_with_iv(const char *key,
   return (int)(fromlen - CIPHER_IV_LEN);
 }
 
-/* SHA-1 */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, 1 on failure.
- */
-int
-crypto_digest(char *digest, const char *m, size_t len)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, 1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  if (algorithm == DIGEST_SHA256)
-    return (SHA256((const uint8_t*)m,len,(uint8_t*)digest) == NULL);
-  else
-    return (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-            == -1);
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, 1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  if (algorithm == DIGEST_SHA512)
-    return (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-            == NULL);
-  else
-    return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-            == -1);
-}
-
-/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
- * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
- * success, -1 on failure. */
-int
-crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len)
-{
-  tor_assert(ds_out);
-  memset(ds_out, 0, sizeof(*ds_out));
-  if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0)
-    return -1;
-  if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0)
-    return -1;
-
-  return 0;
-}
-
-/** Return the name of an algorithm, as used in directory documents. */
-const char *
-crypto_digest_algorithm_get_name(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1:
-      return "sha1";
-    case DIGEST_SHA256:
-      return "sha256";
-    case DIGEST_SHA512:
-      return "sha512";
-    case DIGEST_SHA3_256:
-      return "sha3-256";
-    case DIGEST_SHA3_512:
-      return "sha3-512";
-    default:
-      // LCOV_EXCL_START
-      tor_fragile_assert();
-      return "??unknown_digest??";
-      // LCOV_EXCL_STOP
-  }
-}
-
-/** Given the name of a digest algorithm, return its integer value, or -1 if
- * the name is not recognized. */
-int
-crypto_digest_algorithm_parse_name(const char *name)
-{
-  if (!strcmp(name, "sha1"))
-    return DIGEST_SHA1;
-  else if (!strcmp(name, "sha256"))
-    return DIGEST_SHA256;
-  else if (!strcmp(name, "sha512"))
-    return DIGEST_SHA512;
-  else if (!strcmp(name, "sha3-256"))
-    return DIGEST_SHA3_256;
-  else if (!strcmp(name, "sha3-512"))
-    return DIGEST_SHA3_512;
-  else
-    return -1;
-}
-
-/** Given an algorithm, return the digest length in bytes. */
-size_t
-crypto_digest_algorithm_get_length(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1:
-      return DIGEST_LEN;
-    case DIGEST_SHA256:
-      return DIGEST256_LEN;
-    case DIGEST_SHA512:
-      return DIGEST512_LEN;
-    case DIGEST_SHA3_256:
-      return DIGEST256_LEN;
-    case DIGEST_SHA3_512:
-      return DIGEST512_LEN;
-    default:
-      tor_assert(0);              // LCOV_EXCL_LINE
-      return 0; /* Unreachable */ // LCOV_EXCL_LINE
-  }
-}
-
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (STRUCT_OFFSET(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-    case DIGEST_SHA3_256: /* FALLSTHROUGH */
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  crypto_digest_t tmpenv;
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: /* FALLSTHROUGH */
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  return tor_memdup(digest, alloc_bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  unsigned char *rv = NULL;
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-}
-
-/** Internal state for a eXtendable-Output Function (XOF). */
-struct crypto_xof_t {
-  keccak_state s;
-};
-
-/** Allocate a new XOF object backed by SHAKE-256.  The security level
- * provided is a function of the length of the output used.  Read and
- * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output
- * Functions" before using this construct.
- */
-crypto_xof_t *
-crypto_xof_new(void)
-{
-  crypto_xof_t *xof;
-  xof = tor_malloc(sizeof(crypto_xof_t));
-  keccak_xof_init(&xof->s, 256);
-  return xof;
-}
-
-/** Absorb bytes into a XOF object.  Must not be called after a call to
- * crypto_xof_squeeze_bytes() for the same instance, and will assert
- * if attempted.
- */
-void
-crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
-{
-  int i = keccak_xof_absorb(&xof->s, data, len);
-  tor_assert(i == 0);
-}
-
-/** Squeeze bytes out of a XOF object.  Calling this routine will render
- * the XOF instance ineligible to absorb further data.
- */
-void
-crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
-{
-  int i = keccak_xof_squeeze(&xof->s, out, len);
-  tor_assert(i == 0);
-}
-
-/** Cleanse and deallocate a XOF object. */
-void
-crypto_xof_free(crypto_xof_t *xof)
-{
-  if (!xof)
-    return;
-  memwipe(xof, 0, sizeof(crypto_xof_t));
-  tor_free(xof);
-}
-
 /* DH */
 
 /** Our DH 'g' parameter */
@@ -2207,12 +534,12 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     goto out;
   if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
     goto out;
-#else
+#else /* !(defined(OPENSSL_1_1_API)) */
   if (!(dh->p = BN_dup(p)))
     goto out;
   if (!(dh->g = BN_dup(g)))
     goto out;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
   /* Perform the validation. */
   int codes = 0;
@@ -2383,7 +710,7 @@ crypto_dh_new(int dh_type)
 
   if (!DH_set_length(res->dh, DH_PRIVATE_KEY_BITS))
     goto err;
-#else
+#else /* !(defined(OPENSSL_1_1_API)) */
   if (dh_type == DH_TYPE_TLS) {
     if (!(res->dh->p = BN_dup(dh_param_p_tls)))
       goto err;
@@ -2396,12 +723,13 @@ crypto_dh_new(int dh_type)
     goto err;
 
   res->dh->length = DH_PRIVATE_KEY_BITS;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
   return res;
- err:
+
   /* LCOV_EXCL_START
    * This error condition is only reached when an allocation fails */
+ err:
   crypto_log_errors(LOG_WARN, "creating DH object");
   if (res->dh) DH_free(res->dh); /* frees p and g too */
   tor_free(res);
@@ -2458,7 +786,7 @@ crypto_dh_generate_public(crypto_dh_t *dh)
              "the-universe chances really do happen.  Treating as a failure.");
     return -1;
   }
-#else
+#else /* !(defined(OPENSSL_1_1_API)) */
   if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
     /* LCOV_EXCL_START
      * If this happens, then openssl's DH implementation is busted. */
@@ -2471,7 +799,7 @@ crypto_dh_generate_public(crypto_dh_t *dh)
     goto again;
     /* LCOV_EXCL_STOP */
   }
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
   return 0;
 }
 
@@ -2492,7 +820,7 @@ crypto_dh_get_public(crypto_dh_t *dh, char *pubkey, size_t pubkey_len)
   DH_get0_key(dh->dh, &dh_pub, &dh_priv);
 #else
   dh_pub = dh->dh->pub_key;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
   if (!dh_pub) {
     if (crypto_dh_generate_public(dh)<0)
@@ -2643,7 +971,7 @@ crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len,
   for (cp = key_out, i=0; cp < key_out+key_out_len;
        ++i, cp += DIGEST_LEN) {
     tmp[key_in_len] = i;
-    if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1))
+    if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1) < 0)
       goto exit;
     memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out)));
   }
@@ -2718,7 +1046,7 @@ crypto_expand_key_material_rfc5869_sha256(
 /** Free a DH key exchange object.
  */
 void
-crypto_dh_free(crypto_dh_t *dh)
+crypto_dh_free_(crypto_dh_t *dh)
 {
   if (!dh)
     return;
@@ -2727,673 +1055,22 @@ crypto_dh_free(crypto_dh_t *dh)
   tor_free(dh);
 }
 
-/* random numbers */
-
-/** How many bytes of entropy we add at once.
- *
- * This is how much entropy OpenSSL likes to add right now, so maybe it will
- * work for us too. */
-#define ADD_ENTROPY 32
-
-/** Set the seed of the weak RNG to a random value. */
-void
-crypto_seed_weak_rng(tor_weak_rng_t *rng)
-{
-  unsigned seed;
-  crypto_rand((void*)&seed, sizeof(seed));
-  tor_init_weak_random(rng, seed);
-}
-
-#ifdef TOR_UNIT_TESTS
-int break_strongest_rng_syscall = 0;
-int break_strongest_rng_fallback = 0;
-#endif
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on
- * failure.  A maximum request size of 256 bytes is imposed.
- */
-static int
-crypto_strongest_rand_syscall(uint8_t *out, size_t out_len)
-{
-  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
-
-#ifdef TOR_UNIT_TESTS
-  if (break_strongest_rng_syscall)
-    return -1;
-#endif
-
-#if defined(_WIN32)
-  static int provider_set = 0;
-  static HCRYPTPROV provider;
-
-  if (!provider_set) {
-    if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
-                             CRYPT_VERIFYCONTEXT)) {
-      log_warn(LD_CRYPTO, "Can't get CryptoAPI provider [1]");
-      return -1;
-    }
-    provider_set = 1;
-  }
-  if (!CryptGenRandom(provider, out_len, out)) {
-    log_warn(LD_CRYPTO, "Can't get entropy from CryptoAPI.");
-    return -1;
-  }
-
-  return 0;
-#elif defined(__linux__) && defined(SYS_getrandom)
-  static int getrandom_works = 1; /* Be optimitic about our chances... */
-
-  /* getrandom() isn't as straight foward as getentropy(), and has
-   * no glibc wrapper.
-   *
-   * As far as I can tell from getrandom(2) and the source code, the
-   * requests we issue will always succeed (though it will block on the
-   * call if /dev/urandom isn't seeded yet), since we are NOT specifying
-   * GRND_NONBLOCK and the request is <= 256 bytes.
-   *
-   * The manpage is unclear on what happens if a signal interrupts the call
-   * while the request is blocked due to lack of entropy....
-   *
-   * We optimistically assume that getrandom() is available and functional
-   * because it is the way of the future, and 2 branch mispredicts pale in
-   * comparision to the overheads involved with failing to open
-   * /dev/srandom followed by opening and reading from /dev/urandom.
-   */
-  if (PREDICT_LIKELY(getrandom_works)) {
-    long ret;
-    /* A flag of '0' here means to read from '/dev/urandom', and to
-     * block if insufficient entropy is available to service the
-     * request.
-     */
-    const unsigned int flags = 0;
-    do {
-      ret = syscall(SYS_getrandom, out, out_len, flags);
-    } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN)));
-
-    if (PREDICT_UNLIKELY(ret == -1)) {
-      /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */
-      tor_assert(errno != EAGAIN);
-      tor_assert(errno != EINTR);
-
-      /* Probably ENOSYS. */
-      log_warn(LD_CRYPTO, "Can't get entropy from getrandom().");
-      getrandom_works = 0; /* Don't bother trying again. */
-      return -1;
-      /* LCOV_EXCL_STOP */
-    }
-
-    tor_assert(ret == (long)out_len);
-    return 0;
-  }
-
-  return -1; /* getrandom() previously failed unexpectedly. */
-#elif defined(HAVE_GETENTROPY)
-  /* getentropy() is what Linux's getrandom() wants to be when it grows up.
-   * the only gotcha is that requests are limited to 256 bytes.
-   */
-  return getentropy(out, out_len);
-#else
-  (void) out;
-#endif
-
-  /* This platform doesn't have a supported syscall based random. */
-  return -1;
-}
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * via the per-platform fallback mechanism, storing it into <b>out</b>.
- * Return 0 on success, -1 on failure.  A maximum request size of 256 bytes
- * is imposed.
- */
-static int
-crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
-{
-#ifdef TOR_UNIT_TESTS
-  if (break_strongest_rng_fallback)
-    return -1;
-#endif
-
-#ifdef _WIN32
-  /* Windows exclusively uses crypto_strongest_rand_syscall(). */
-  (void)out;
-  (void)out_len;
-  return -1;
-#else
-  static const char *filenames[] = {
-    "/dev/srandom", "/dev/urandom", "/dev/random", NULL
-  };
-  int fd, i;
-  size_t n;
-
-  for (i = 0; filenames[i]; ++i) {
-    log_debug(LD_FS, "Opening %s for entropy", filenames[i]);
-    fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0);
-    if (fd<0) continue;
-    log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]);
-    n = read_all(fd, (char*)out, out_len, 0);
-    close(fd);
-    if (n != out_len) {
-      /* LCOV_EXCL_START
-       * We can't make /dev/foorandom actually fail. */
-      log_warn(LD_CRYPTO,
-               "Error reading from entropy source (read only %lu bytes).",
-               (unsigned long)n);
-      return -1;
-      /* LCOV_EXCL_STOP */
-    }
-
-    return 0;
-  }
-
-  return -1;
-#endif
-}
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * storing it into <b>out</b>. Return 0 on success, -1 on failure.  A maximum
- * request size of 256 bytes is imposed.
- */
-STATIC int
-crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
-{
-  static const size_t sanity_min_size = 16;
-  static const int max_attempts = 3;
-  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
-
-  /* For buffers >= 16 bytes (128 bits), we sanity check the output by
-   * zero filling the buffer and ensuring that it actually was at least
-   * partially modified.
-   *
-   * Checking that any individual byte is non-zero seems like it would
-   * fail too often (p = out_len * 1/256) for comfort, but this is an
-   * "adjust according to taste" sort of check.
-   */
-  memwipe(out, 0, out_len);
-  for (int i = 0; i < max_attempts; i++) {
-    /* Try to use the syscall/OS favored mechanism to get strong entropy. */
-    if (crypto_strongest_rand_syscall(out, out_len) != 0) {
-      /* Try to use the less-favored mechanism to get strong entropy. */
-      if (crypto_strongest_rand_fallback(out, out_len) != 0) {
-        /* Welp, we tried.  Hopefully the calling code terminates the process
-         * since we're basically boned without good entropy.
-         */
-        log_warn(LD_CRYPTO,
-                 "Cannot get strong entropy: no entropy source found.");
-        return -1;
-      }
-    }
-
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
-      return 0;
-  }
-
-  /* LCOV_EXCL_START
-   *
-   * We tried max_attempts times to fill a buffer >= 128 bits long,
-   * and each time it returned all '0's.  Either the system entropy
-   * source is busted, or the user should go out and buy a ticket to
-   * every lottery on the planet.
-   */
-  log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer.");
-
-  return -1;
-  /* LCOV_EXCL_STOP */
-}
-
-/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
- * storing it into <b>out</b>.
- */
-void
-crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-#define DLEN SHA512_DIGEST_LENGTH
-  /* We're going to hash DLEN bytes from the system RNG together with some
-   * bytes from the openssl PRNG, in order to yield DLEN bytes.
-   */
-  uint8_t inp[DLEN*2];
-  uint8_t tmp[DLEN];
-  tor_assert(out);
-  while (out_len) {
-    crypto_rand((char*) inp, DLEN);
-    if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) {
-      // LCOV_EXCL_START
-      log_err(LD_CRYPTO, "Failed to load strong entropy when generating an "
-              "important key. Exiting.");
-      /* Die with an assertion so we get a stack trace. */
-      tor_assert(0);
-      // LCOV_EXCL_STOP
-    }
-    if (out_len >= DLEN) {
-      SHA512(inp, sizeof(inp), out);
-      out += DLEN;
-      out_len -= DLEN;
-    } else {
-      SHA512(inp, sizeof(inp), tmp);
-      memcpy(out, tmp, out_len);
-      break;
-    }
-  }
-  memwipe(tmp, 0, sizeof(tmp));
-  memwipe(inp, 0, sizeof(inp));
-#undef DLEN
-}
-
-/** Seed OpenSSL's random number generator with bytes from the operating
- * system.  Return 0 on success, -1 on failure.
- */
-int
-crypto_seed_rng(void)
-{
-  int rand_poll_ok = 0, load_entropy_ok = 0;
-  uint8_t buf[ADD_ENTROPY];
-
-  /* OpenSSL has a RAND_poll function that knows about more kinds of
-   * entropy than we do.  We'll try calling that, *and* calling our own entropy
-   * functions.  If one succeeds, we'll accept the RNG as seeded. */
-  rand_poll_ok = RAND_poll();
-  if (rand_poll_ok == 0)
-    log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE
-
-  load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf));
-  if (load_entropy_ok) {
-    RAND_seed(buf, sizeof(buf));
-  }
-
-  memwipe(buf, 0, sizeof(buf));
-
-  if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1)
-    return 0;
-  else
-    return -1;
-}
-
-/** Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking
- * for unit tests.
- *
- * This function is not allowed to fail; if it would fail to generate strong
- * entropy, it must terminate the process instead.
- */
-MOCK_IMPL(void,
-crypto_rand, (char *to, size_t n))
-{
-  crypto_rand_unmocked(to, n);
-}
-
-/** Write <b>n</b> bytes of strong random data to <b>to</b>.  Most callers
- * will want crypto_rand instead.
- *
- * This function is not allowed to fail; if it would fail to generate strong
- * entropy, it must terminate the process instead.
- */
-void
-crypto_rand_unmocked(char *to, size_t n)
-{
-  int r;
-  if (n == 0)
-    return;
-
-  tor_assert(n < INT_MAX);
-  tor_assert(to);
-  r = RAND_bytes((unsigned char*)to, (int)n);
-  /* We consider a PRNG failure non-survivable. Let's assert so that we get a
-   * stack trace about where it happened.
-   */
-  tor_assert(r >= 0);
-}
-
-/** Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive. */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/** Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- */
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/** As crypto_rand_int_range, but supports uint64_t. */
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
-{
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/** As crypto_rand_int_range, but supports time_t. */
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/** Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive. */
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/** Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- */
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
-}
-
-/** Generate and return a new random hostname starting with <b>prefix</b>,
- * ending with <b>suffix</b>, and containing no fewer than
- * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32
- * characters. Does not check for failure.
- *
- * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE.
- **/
-char *
-crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix,
-                       const char *suffix)
-{
-  char *result, *rand_bytes;
-  int randlen, rand_bytes_len;
-  size_t resultlen, prefixlen;
-
-  if (max_rand_len > MAX_DNS_LABEL_SIZE)
-    max_rand_len = MAX_DNS_LABEL_SIZE;
-  if (min_rand_len > max_rand_len)
-    min_rand_len = max_rand_len;
-
-  randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1);
-
-  prefixlen = strlen(prefix);
-  resultlen = prefixlen + strlen(suffix) + randlen + 16;
-
-  rand_bytes_len = ((randlen*5)+7)/8;
-  if (rand_bytes_len % 5)
-    rand_bytes_len += 5 - (rand_bytes_len%5);
-  rand_bytes = tor_malloc(rand_bytes_len);
-  crypto_rand(rand_bytes, rand_bytes_len);
-
-  result = tor_malloc(resultlen);
-  memcpy(result, prefix, prefixlen);
-  base32_encode(result+prefixlen, resultlen-prefixlen,
-                rand_bytes, rand_bytes_len);
-  tor_free(rand_bytes);
-  strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen));
-
-  return result;
-}
-
-/** Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b>
- * is empty. */
-void *
-smartlist_choose(const smartlist_t *sl)
-{
-  int len = smartlist_len(sl);
-  if (len)
-    return smartlist_get(sl,crypto_rand_int(len));
-  return NULL; /* no elements to choose from */
-}
-
-/** Scramble the elements of <b>sl</b> into a random order. */
-void
-smartlist_shuffle(smartlist_t *sl)
-{
-  int i;
-  /* From the end of the list to the front, choose at random from the
-     positions we haven't looked at yet, and swap that position into the
-     current position.  Remember to give "no swap" the same probability as
-     any other swap. */
-  for (i = smartlist_len(sl)-1; i > 0; --i) {
-    int j = crypto_rand_int(i+1);
-    smartlist_swap(sl, i, j);
-  }
-}
-
-/**
- * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to
- * the value <b>byte</b>.
- * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens.
- *
- * This function is preferable to memset, since many compilers will happily
- * optimize out memset() when they can convince themselves that the data being
- * cleared will never be read.
- *
- * Right now, our convention is to use this function when we are wiping data
- * that's about to become inaccessible, such as stack buffers that are about
- * to go out of scope or structures that are about to get freed.  (In
- * practice, it appears that the compilers we're currently using will optimize
- * out the memset()s for stack-allocated buffers, but not those for
- * about-to-be-freed structures. That could change, though, so we're being
- * wary.)  If there are live reads for the data, then you can just use
- * memset().
- */
-void
-memwipe(void *mem, uint8_t byte, size_t sz)
-{
-  if (sz == 0) {
-    return;
-  }
-  /* If sz is nonzero, then mem must not be NULL. */
-  tor_assert(mem != NULL);
-
-  /* Data this large is likely to be an underflow. */
-  tor_assert(sz < SIZE_T_CEILING);
-
-  /* Because whole-program-optimization exists, we may not be able to just
-   * have this function call "memset".  A smart compiler could inline it, then
-   * eliminate dead memsets, and declare itself to be clever. */
-
-#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY)
-  /* Here's what you do on windows. */
-  SecureZeroMemory(mem,sz);
-#elif defined(HAVE_RTLSECUREZEROMEMORY)
-  RtlSecureZeroMemory(mem,sz);
-#elif defined(HAVE_EXPLICIT_BZERO)
-  /* The BSDs provide this. */
-  explicit_bzero(mem, sz);
-#elif defined(HAVE_MEMSET_S)
-  /* This is in the C99 standard. */
-  memset_s(mem, sz, 0, sz);
-#else
-  /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk
-   * based on the pointer value, then uses that junk to update a global
-   * variable.  It's an elaborate ruse to trick the compiler into not
-   * optimizing out the "wipe this memory" code.  Read it if you like zany
-   * programming tricks! In later versions of Tor, we should look for better
-   * not-optimized-out memory wiping stuff...
-   *
-   * ...or maybe not.  In practice, there are pure-asm implementations of
-   * OPENSSL_cleanse() on most platforms, which ought to do the job.
-   **/
-
-  OPENSSL_cleanse(mem, sz);
-#endif
-
-  /* Just in case some caller of memwipe() is relying on getting a buffer
-   * filled with a particular value, fill the buffer.
-   *
-   * If this function gets inlined, this memset might get eliminated, but
-   * that's okay: We only care about this particular memset in the case where
-   * the caller should have been using memset(), and the memset() wouldn't get
-   * eliminated.  In other words, this is here so that we won't break anything
-   * if somebody accidentally calls memwipe() instead of memset().
-   **/
-  memset(mem, byte, sz);
-}
-
-#ifndef OPENSSL_THREADS
-#error OpenSSL has been built without thread support. Tor requires an \
- OpenSSL library with thread support enabled.
-#endif
-
-#ifndef NEW_THREAD_API
-/** Helper: OpenSSL uses this callback to manipulate mutexes. */
-static void
-openssl_locking_cb_(int mode, int n, const char *file, int line)
-{
-  (void)file;
-  (void)line;
-  if (!openssl_mutexes_)
-    /* This is not a really good fix for the
-     * "release-freed-lock-from-separate-thread-on-shutdown" problem, but
-     * it can't hurt. */
-    return;
-  if (mode & CRYPTO_LOCK)
-    tor_mutex_acquire(openssl_mutexes_[n]);
-  else
-    tor_mutex_release(openssl_mutexes_[n]);
-}
-
-static void
-tor_set_openssl_thread_id(CRYPTO_THREADID *threadid)
-{
-  CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id());
-}
-#endif
-
-#if 0
-/* This code is disabled, because OpenSSL never actually uses these callbacks.
- */
-
-/** OpenSSL helper type: wraps a Tor mutex so that OpenSSL can use it
- * as a lock. */
-struct CRYPTO_dynlock_value {
-  tor_mutex_t *lock;
-};
-
-/** OpenSSL callback function to allocate a lock: see CRYPTO_set_dynlock_*
- * documentation in OpenSSL's docs for more info. */
-static struct CRYPTO_dynlock_value *
-openssl_dynlock_create_cb_(const char *file, int line)
-{
-  struct CRYPTO_dynlock_value *v;
-  (void)file;
-  (void)line;
-  v = tor_malloc(sizeof(struct CRYPTO_dynlock_value));
-  v->lock = tor_mutex_new();
-  return v;
-}
-
-/** OpenSSL callback function to acquire or release a lock: see
- * CRYPTO_set_dynlock_* documentation in OpenSSL's docs for more info. */
-static void
-openssl_dynlock_lock_cb_(int mode, struct CRYPTO_dynlock_value *v,
-                         const char *file, int line)
-{
-  (void)file;
-  (void)line;
-  if (mode & CRYPTO_LOCK)
-    tor_mutex_acquire(v->lock);
-  else
-    tor_mutex_release(v->lock);
-}
-
-/** OpenSSL callback function to free a lock: see CRYPTO_set_dynlock_*
- * documentation in OpenSSL's docs for more info. */
-static void
-openssl_dynlock_destroy_cb_(struct CRYPTO_dynlock_value *v,
-                            const char *file, int line)
-{
-  (void)file;
-  (void)line;
-  tor_mutex_free(v->lock);
-  tor_free(v);
-}
-#endif
-
 /** @{ */
-/** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being
- * multithreaded. Returns 0. */
-static int
-setup_openssl_threading(void)
-{
-#ifndef NEW_THREAD_API
-  int i;
-  int n = CRYPTO_num_locks();
-  n_openssl_mutexes_ = n;
-  openssl_mutexes_ = tor_calloc(n, sizeof(tor_mutex_t *));
-  for (i=0; i < n; ++i)
-    openssl_mutexes_[i] = tor_mutex_new();
-  CRYPTO_set_locking_callback(openssl_locking_cb_);
-  CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id);
-#endif
-#if 0
-  CRYPTO_set_dynlock_create_callback(openssl_dynlock_create_cb_);
-  CRYPTO_set_dynlock_lock_callback(openssl_dynlock_lock_cb_);
-  CRYPTO_set_dynlock_destroy_callback(openssl_dynlock_destroy_cb_);
-#endif
-  return 0;
-}
-
 /** Uninitialize the crypto library. Return 0 on success. Does not detect
  * failure.
  */
 int
 crypto_global_cleanup(void)
 {
+#ifndef OPENSSL_1_1_API
   EVP_cleanup();
+#endif
 #ifndef NEW_THREAD_API
   ERR_remove_thread_state(NULL);
 #endif
+#ifndef OPENSSL_1_1_API
   ERR_free_strings();
+#endif
 
   if (dh_param_p)
     BN_clear_free(dh_param_p);
@@ -3402,31 +1079,40 @@ crypto_global_cleanup(void)
   if (dh_param_g)
     BN_clear_free(dh_param_g);
 
+  dh_param_p = dh_param_p_tls = dh_param_g = NULL;
+
 #ifndef DISABLE_ENGINES
+#ifndef OPENSSL_1_1_API
   ENGINE_cleanup();
 #endif
+#endif
 
   CONF_modules_unload(1);
+#ifndef OPENSSL_1_1_API
   CRYPTO_cleanup_all_ex_data();
-
-#ifndef NEW_THREAD_API
-  if (n_openssl_mutexes_) {
-    int n = n_openssl_mutexes_;
-    tor_mutex_t **ms = openssl_mutexes_;
-    int i;
-    openssl_mutexes_ = NULL;
-    n_openssl_mutexes_ = 0;
-    for (i=0;i<n;++i) {
-      tor_mutex_free(ms[i]);
-    }
-    tor_free(ms);
-  }
 #endif
 
-  tor_free(crypto_openssl_version_str);
-  tor_free(crypto_openssl_header_version_str);
+  crypto_openssl_free_all();
+
+  crypto_early_initialized_ = 0;
+  crypto_global_initialized_ = 0;
+  have_seeded_siphash = 0;
+  siphash_unset_global_key();
+
   return 0;
 }
 
 /** @} */
 
+#ifdef USE_DMALLOC
+/** Tell the crypto library to use Tor's allocation functions rather than
+ * calling libc's allocation functions directly. Return 0 on success, -1
+ * on failure. */
+int
+crypto_use_tor_alloc_functions(void)
+{
+  int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
+  return r ? 0 : -1;
+}
+#endif /* defined(USE_DMALLOC) */
+
diff --git a/src/common/crypto.h b/src/common/crypto.h
index 116e0a62fd..c773557310 100644
--- a/src/common/crypto.h
+++ b/src/common/crypto.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,125 +17,40 @@
 
 #include <stdio.h>
 #include "torint.h"
-#include "testsupport.h"
 #include "compat.h"
+#include "util.h"
+#include "crypto_rsa.h"
 
-/*
-  Macro to create an arbitrary OpenSSL version number as used by
-  OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard
-  to read.
-
-  Don't use this directly, instead use one of the other OPENSSL_V macros
-  below.
-
-  The format is: 4 bits major, 8 bits minor, 8 bits fix, 8 bits patch, 4 bit
-  status.
- */
-#define OPENSSL_VER(a,b,c,d,e)                                \
-  (((a)<<28) |                                                \
-   ((b)<<20) |                                                \
-   ((c)<<12) |                                                \
-   ((d)<< 4) |                                                \
-    (e))
-/** An openssl release number.  For example, OPENSSL_V(0,9,8,'j') is the
- * version for the released version of 0.9.8j */
-#define OPENSSL_V(a,b,c,d) \
-  OPENSSL_VER((a),(b),(c),(d)-'a'+1,0xf)
-/** An openssl release number for the first release in the series.  For
- * example, OPENSSL_V_NOPATCH(1,0,0) is the first released version of OpenSSL
- * 1.0.0. */
-#define OPENSSL_V_NOPATCH(a,b,c) \
-  OPENSSL_VER((a),(b),(c),0,0xf)
-/** The first version that would occur for any alpha or beta in an openssl
- * series. For example, OPENSSL_V_SERIES(0,9,8) is greater than any released
- * 0.9.7, and less than any released 0.9.8. */
-#define OPENSSL_V_SERIES(a,b,c) \
-  OPENSSL_VER((a),(b),(c),0,0)
-
-/** Length of the output of our message digest. */
-#define DIGEST_LEN 20
-/** Length of the output of our second (improved) message digests.  (For now
- * this is just sha256, but it could be any other 256-bit digest.) */
-#define DIGEST256_LEN 32
-/** Length of the output of our 64-bit optimized message digests (SHA512). */
-#define DIGEST512_LEN 64
-/** Length of our symmetric cipher's keys. */
+/** Length of our symmetric cipher's keys of 128-bit. */
 #define CIPHER_KEY_LEN 16
-/** Length of our symmetric cipher's IV. */
+/** Length of our symmetric cipher's IV of 128-bit. */
 #define CIPHER_IV_LEN 16
-/** Length of our public keys. */
-#define PK_BYTES (1024/8)
+/** Length of our symmetric cipher's keys of 256-bit. */
+#define CIPHER256_KEY_LEN 32
 /** Length of our DH keys. */
 #define DH_BYTES (1024/8)
 
-/** Length of a sha1 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST_LEN 27
-/** Length of a sha256 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST256_LEN 43
-/** Length of a sha512 message digest when encoded in base64 with trailing =
- * signs removed. */
-#define BASE64_DIGEST512_LEN 86
-
-/** Constant used to indicate OAEP padding for public-key encryption */
-#define PK_PKCS1_OAEP_PADDING 60002
-
-/** Number of bytes added for PKCS1-OAEP padding. */
-#define PKCS1_OAEP_PADDING_OVERHEAD 42
-
 /** Length of encoded public key fingerprints, including space; but not
  * including terminating NUL. */
 #define FINGERPRINT_LEN 49
-/** Length of hex encoding of SHA1 digest, not including final NUL. */
-#define HEX_DIGEST_LEN 40
-/** Length of hex encoding of SHA256 digest, not including final NUL. */
-#define HEX_DIGEST256_LEN 64
-/** Length of hex encoding of SHA512 digest, not including final NUL. */
-#define HEX_DIGEST512_LEN 128
-
-typedef enum {
-  DIGEST_SHA1 = 0,
-  DIGEST_SHA256 = 1,
-  DIGEST_SHA512 = 2,
-  DIGEST_SHA3_256 = 3,
-  DIGEST_SHA3_512 = 4,
-} digest_algorithm_t;
-#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
-#define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
 
-/** A set of all the digests we commonly compute, taken on a single
- * string.  Any digests that are shorter than 512 bits are right-padded
- * with 0 bits.
- *
- * Note that this representation wastes 44 bytes for the SHA1 case, so
- * don't use it for anything where we need to allocate a whole bunch at
- * once.
- **/
-typedef struct {
-  char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
-} common_digests_t;
-
-typedef struct crypto_pk_t crypto_pk_t;
 typedef struct aes_cnt_cipher crypto_cipher_t;
-typedef struct crypto_digest_t crypto_digest_t;
-typedef struct crypto_xof_t crypto_xof_t;
 typedef struct crypto_dh_t crypto_dh_t;
 
 /* global state */
-const char * crypto_openssl_get_version_str(void);
-const char * crypto_openssl_get_header_version_str(void);
+int crypto_init_siphash_key(void);
 int crypto_early_init(void) ATTR_WUR;
 int crypto_global_init(int hardwareAccel,
                        const char *accelName,
                        const char *accelPath) ATTR_WUR;
+#ifdef USE_DMALLOC
+int crypto_use_tor_alloc_functions(void);
+#endif
+
 void crypto_thread_cleanup(void);
 int crypto_global_cleanup(void);
 
 /* environment setup */
-MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void));
-void crypto_pk_free(crypto_pk_t *env);
-
 void crypto_set_tls_dh_prime(void);
 crypto_cipher_t *crypto_cipher_new(const char *key);
 crypto_cipher_t *crypto_cipher_new_with_bits(const char *key, int bits);
@@ -143,68 +58,9 @@ crypto_cipher_t *crypto_cipher_new_with_iv(const char *key, const char *iv);
 crypto_cipher_t *crypto_cipher_new_with_iv_and_bits(const uint8_t *key,
                                                     const uint8_t *iv,
                                                     int bits);
-void crypto_cipher_free(crypto_cipher_t *env);
-
-/* public key crypto */
-MOCK_DECL(int, crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits));
-#define crypto_pk_generate_key(env)                     \
-  crypto_pk_generate_key_with_bits((env), (PK_BYTES*8))
-
-int crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
-                                             const char *keyfile);
-int crypto_pk_write_public_key_to_string(crypto_pk_t *env,
-                                         char **dest, size_t *len);
-int crypto_pk_write_private_key_to_string(crypto_pk_t *env,
-                                          char **dest, size_t *len);
-int crypto_pk_read_public_key_from_string(crypto_pk_t *env,
-                                          const char *src, size_t len);
-int crypto_pk_read_private_key_from_string(crypto_pk_t *env,
-                                           const char *s, ssize_t len);
-int crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
-                                            const char *fname);
-
-int crypto_pk_check_key(crypto_pk_t *env);
-int crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b);
-int crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b);
-size_t crypto_pk_keysize(const crypto_pk_t *env);
-int crypto_pk_num_bits(crypto_pk_t *env);
-crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig);
-crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig);
-int crypto_pk_key_is_private(const crypto_pk_t *key);
-int crypto_pk_public_exponent_ok(crypto_pk_t *env);
-
-int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
-                             const char *from, size_t fromlen, int padding);
-int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen,
-                              int padding, int warnOnFailure);
-int crypto_pk_public_checksig(const crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen);
-int crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data,
-                               size_t datalen, const char *sig, size_t siglen);
-int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
-                           const char *from, size_t fromlen);
-int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
-                                  const char *from, size_t fromlen);
-int crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, char *to,
-                                    size_t tolen,
-                                    const char *from, size_t fromlen,
-                                    int padding, int force);
-int crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, char *to,
-                                     size_t tolen,
-                                     const char *from, size_t fromlen,
-                                     int padding, int warnOnFailure);
-
-int crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len);
-crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len);
-int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out);
-int crypto_pk_get_common_digests(crypto_pk_t *pk,
-                                 common_digests_t *digests_out);
-int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space);
-int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out);
-
-int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out);
-crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len);
+void crypto_cipher_free_(crypto_cipher_t *env);
+#define crypto_cipher_free(c) \
+  FREE_AND_NULL(crypto_cipher_t, crypto_cipher_free_, (c))
 
 /* symmetric crypto */
 const char *crypto_cipher_get_key(crypto_cipher_t *env);
@@ -222,44 +78,6 @@ int crypto_cipher_decrypt_with_iv(const char *key,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
-/* SHA-1 and other digests. */
-int crypto_digest(char *digest, const char *m, size_t len);
-int crypto_digest256(char *digest, const char *m, size_t len,
-                     digest_algorithm_t algorithm);
-int crypto_digest512(char *digest, const char *m, size_t len,
-                     digest_algorithm_t algorithm);
-int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len);
-struct smartlist_t;
-void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                                    const char *prepend,
-                                    const struct smartlist_t *lst,
-                                    const char *append,
-                                    digest_algorithm_t alg);
-void crypto_digest_smartlist(char *digest_out, size_t len_out,
-                             const struct smartlist_t *lst, const char *append,
-                             digest_algorithm_t alg);
-const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
-size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg);
-int crypto_digest_algorithm_parse_name(const char *name);
-crypto_digest_t *crypto_digest_new(void);
-crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
-crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
-void crypto_digest_free(crypto_digest_t *digest);
-void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                             size_t len);
-void crypto_digest_get_digest(crypto_digest_t *digest,
-                              char *out, size_t out_len);
-crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest);
-void crypto_digest_assign(crypto_digest_t *into,
-                          const crypto_digest_t *from);
-void crypto_hmac_sha256(char *hmac_out,
-                        const char *key, size_t key_len,
-                        const char *msg, size_t msg_len);
-crypto_xof_t *crypto_xof_new(void);
-void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
-void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
-void crypto_xof_free(crypto_xof_t *xof);
-
 /* Key negotiation */
 #define DH_TYPE_CIRCUIT 1
 #define DH_TYPE_REND 2
@@ -273,7 +91,8 @@ int crypto_dh_get_public(crypto_dh_t *dh, char *pubkey_out,
 ssize_t crypto_dh_compute_secret(int severity, crypto_dh_t *dh,
                              const char *pubkey, size_t pubkey_len,
                              char *secret_out, size_t secret_out_len);
-void crypto_dh_free(crypto_dh_t *dh);
+void crypto_dh_free_(crypto_dh_t *dh);
+#define crypto_dh_free(dh) FREE_AND_NULL(crypto_dh_t, crypto_dh_free_, (dh))
 
 int crypto_expand_key_material_TAP(const uint8_t *key_in,
                                    size_t key_in_len,
@@ -284,57 +103,12 @@ int crypto_expand_key_material_rfc5869_sha256(
                                     const uint8_t *info_in, size_t info_in_len,
                                     uint8_t *key_out, size_t key_out_len);
 
-/* random numbers */
-int crypto_seed_rng(void) ATTR_WUR;
-MOCK_DECL(void,crypto_rand,(char *to, size_t n));
-void crypto_rand_unmocked(char *to, size_t n);
-void crypto_strongest_rand(uint8_t *out, size_t out_len);
-int crypto_rand_int(unsigned int max);
-int crypto_rand_int_range(unsigned int min, unsigned int max);
-uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
-time_t crypto_rand_time_range(time_t min, time_t max);
-uint64_t crypto_rand_uint64(uint64_t max);
-double crypto_rand_double(void);
-struct tor_weak_rng_t;
-void crypto_seed_weak_rng(struct tor_weak_rng_t *rng);
-int crypto_init_siphash_key(void);
-
-char *crypto_random_hostname(int min_rand_len, int max_rand_len,
-                             const char *prefix, const char *suffix);
-
-struct smartlist_t;
-void *smartlist_choose(const struct smartlist_t *sl);
-void smartlist_shuffle(struct smartlist_t *sl);
-
-/** OpenSSL-based utility functions. */
-void memwipe(void *mem, uint8_t byte, size_t sz);
-
 /* Prototypes for private functions only used by tortls.c, crypto.c, and the
  * unit tests. */
-struct rsa_st;
-struct evp_pkey_st;
 struct dh_st;
-struct rsa_st *crypto_pk_get_rsa_(crypto_pk_t *env);
-crypto_pk_t *crypto_new_pk_from_rsa_(struct rsa_st *rsa);
-MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_evp_pkey_,(crypto_pk_t *env,
-                                                         int private));
 struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh);
 
 void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in);
 
-#ifdef CRYPTO_PRIVATE
-STATIC int crypto_force_rand_ssleay(void);
-STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
-
-#ifdef TOR_UNIT_TESTS
-extern int break_strongest_rng_syscall;
-extern int break_strongest_rng_fallback;
-#endif
-#endif
-
-#ifdef TOR_UNIT_TESTS
-void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src);
-#endif
-
-#endif
+#endif /* !defined(TOR_CRYPTO_H) */
 
diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c
index fcbee3aba2..996d94c6e2 100644
--- a/src/common/crypto_curve25519.c
+++ b/src/common/crypto_curve25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,9 +21,11 @@
 #include <sys/stat.h>
 #endif
 #include "container.h"
-#include "crypto.h"
 #include "crypto_curve25519.h"
+#include "crypto_digest.h"
 #include "crypto_format.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "util.h"
 #include "torlog.h"
 
@@ -43,7 +45,7 @@ int curve25519_donna(uint8_t *mypublic,
 #elif defined(HAVE_NACL_CRYPTO_SCALARMULT_CURVE25519_H)
 #include <nacl/crypto_scalarmult_curve25519.h>
 #endif
-#endif
+#endif /* defined(USE_CURVE25519_NACL) */
 
 static void pick_curve25519_basepoint_impl(void);
 
@@ -72,7 +74,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret,
   r = crypto_scalarmult_curve25519(output, secret, bp);
 #else
 #error "No implementation of curve25519 is available."
-#endif
+#endif /* defined(USE_CURVE25519_DONNA) || ... */
   memwipe(bp, 0, sizeof(bp));
   return r;
 }
@@ -80,7 +82,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret,
 /**
  * Helper function: Multiply the scalar "secret" by the Curve25519
  * basepoint (X=9), and store the result in "output".  Return 0 on
- * success, -1 on false.
+ * success, -1 on failure.
  */
 STATIC int
 curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret)
@@ -318,8 +320,11 @@ curve25519_basepoint_spot_check(void)
   }
 
   goto end;
+ // LCOV_EXCL_START -- we can only hit this code if there is a bug in our
+ // curve25519-basepoint implementation.
  fail:
   r = -1;
+ // LCOV_EXCL_STOP
  end:
   curve25519_use_ed = save_use_ed;
   return r;
diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h
index 4011820949..4834fa0836 100644
--- a/src/common/crypto_curve25519.h
+++ b/src/common/crypto_curve25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_CURVE25519_H
@@ -6,6 +6,8 @@
 
 #include "testsupport.h"
 #include "torint.h"
+#include "crypto_digest.h"
+#include "crypto_openssl_mgt.h"
 
 /** Length of a curve25519 public key when encoded. */
 #define CURVE25519_PUBKEY_LEN 32
@@ -71,7 +73,7 @@ STATIC int curve25519_impl(uint8_t *output, const uint8_t *secret,
                            const uint8_t *basepoint);
 
 STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret);
-#endif
+#endif /* defined(CRYPTO_CURVE25519_PRIVATE) */
 
 #define CURVE25519_BASE64_PADDED_LEN 44
 
@@ -83,5 +85,5 @@ int curve25519_public_to_base64(char *output,
 void curve25519_set_impl_params(int use_ed);
 void curve25519_init(void);
 
-#endif
+#endif /* !defined(TOR_CRYPTO_CURVE25519_H) */
 
diff --git a/src/common/crypto_digest.c b/src/common/crypto_digest.c
new file mode 100644
index 0000000000..9f9a1a1e2c
--- /dev/null
+++ b/src/common/crypto_digest.c
@@ -0,0 +1,583 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations.
+ **/
+
+#include "container.h"
+#include "crypto_digest.h"
+#include "crypto_openssl_mgt.h"
+#include "crypto_util.h"
+#include "torlog.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+int
+crypto_digest(char *digest, const char *m, size_t len)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL)
+    return -1;
+  return 0;
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256)
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
+  else
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512)
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
+  else
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
+ * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
+ * success, -1 on failure. */
+int
+crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len)
+{
+  tor_assert(ds_out);
+  memset(ds_out, 0, sizeof(*ds_out));
+  if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0)
+    return -1;
+  if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0)
+    return -1;
+
+  return 0;
+}
+
+/** Return the name of an algorithm, as used in directory documents. */
+const char *
+crypto_digest_algorithm_get_name(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1:
+      return "sha1";
+    case DIGEST_SHA256:
+      return "sha256";
+    case DIGEST_SHA512:
+      return "sha512";
+    case DIGEST_SHA3_256:
+      return "sha3-256";
+    case DIGEST_SHA3_512:
+      return "sha3-512";
+      // LCOV_EXCL_START
+    default:
+      tor_fragile_assert();
+      return "??unknown_digest??";
+      // LCOV_EXCL_STOP
+  }
+}
+
+/** Given the name of a digest algorithm, return its integer value, or -1 if
+ * the name is not recognized. */
+int
+crypto_digest_algorithm_parse_name(const char *name)
+{
+  if (!strcmp(name, "sha1"))
+    return DIGEST_SHA1;
+  else if (!strcmp(name, "sha256"))
+    return DIGEST_SHA256;
+  else if (!strcmp(name, "sha512"))
+    return DIGEST_SHA512;
+  else if (!strcmp(name, "sha3-256"))
+    return DIGEST_SHA3_256;
+  else if (!strcmp(name, "sha3-512"))
+    return DIGEST_SHA3_512;
+  else
+    return -1;
+}
+
+/** Given an algorithm, return the digest length in bytes. */
+size_t
+crypto_digest_algorithm_get_length(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1:
+      return DIGEST_LEN;
+    case DIGEST_SHA256:
+      return DIGEST256_LEN;
+    case DIGEST_SHA512:
+      return DIGEST512_LEN;
+    case DIGEST_SHA3_256:
+      return DIGEST256_LEN;
+    case DIGEST_SHA3_512:
+      return DIGEST512_LEN;
+    default:
+      tor_assert(0);              // LCOV_EXCL_LINE
+      return 0; /* Unreachable */ // LCOV_EXCL_LINE
+  }
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    SHA_CTX sha1; /**< state for SHA1 */
+    SHA256_CTX sha2; /**< state for SHA256 */
+    SHA512_CTX sha512; /**< state for SHA512 */
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1:
+      return END_OF_FIELD(d.sha1);
+    case DIGEST_SHA256:
+      return END_OF_FIELD(d.sha2);
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.sha512);
+    case DIGEST_SHA3_256:
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1:
+      SHA1_Init(&r->d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Init(&r->d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Init(&r->d.sha512);
+      break;
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Update(&digest->d.sha1, (void*)data, len);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Update(&digest->d.sha2, (void*)data, len);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Update(&digest->d.sha512, (void*)data, len);
+      break;
+    case DIGEST_SHA3_256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  crypto_digest_t tmpenv;
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
+  memcpy(&tmpenv, digest, alloc_bytes);
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Final(r, &tmpenv.d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Final(r, &tmpenv.d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Final(r, &tmpenv.d.sha512);
+      break;
+//LCOV_EXCL_START
+    case DIGEST_SHA3_256: /* FALLSTHROUGH */
+    case DIGEST_SHA3_512:
+    default:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      /* This is fatal, because it should never happen. */
+      tor_assert_unreached();
+      break;
+//LCOV_EXCL_STOP
+  }
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  return tor_memdup(digest, alloc_bytes);
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  unsigned char *rv = NULL;
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
+            (unsigned char*)hmac_out, NULL);
+  tor_assert(rv);
+}
+
+/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
+ * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
+ * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
+ * <b>mac_out</b>. This function can't fail. */
+void
+crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                    const uint8_t *key, size_t key_len,
+                    const uint8_t *msg, size_t msg_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t key_len_netorder = tor_htonll(key_len);
+
+  tor_assert(mac_out);
+  tor_assert(key);
+  tor_assert(msg);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+
+  /* Order matters here that is any subsystem using this function should
+   * expect this very precise ordering in the MAC construction. */
+  crypto_digest_add_bytes(digest, (const char *) &key_len_netorder,
+                          sizeof(key_len_netorder));
+  crypto_digest_add_bytes(digest, (const char *) key, key_len);
+  crypto_digest_add_bytes(digest, (const char *) msg, msg_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, len_out);
+  crypto_digest_free(digest);
+}
+
+/* xof functions  */
+
+/** Internal state for a eXtendable-Output Function (XOF). */
+struct crypto_xof_t {
+  keccak_state s;
+};
+
+/** Allocate a new XOF object backed by SHAKE-256.  The security level
+ * provided is a function of the length of the output used.  Read and
+ * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output
+ * Functions" before using this construct.
+ */
+crypto_xof_t *
+crypto_xof_new(void)
+{
+  crypto_xof_t *xof;
+  xof = tor_malloc(sizeof(crypto_xof_t));
+  keccak_xof_init(&xof->s, 256);
+  return xof;
+}
+
+/** Absorb bytes into a XOF object.  Must not be called after a call to
+ * crypto_xof_squeeze_bytes() for the same instance, and will assert
+ * if attempted.
+ */
+void
+crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
+{
+  int i = keccak_xof_absorb(&xof->s, data, len);
+  tor_assert(i == 0);
+}
+
+/** Squeeze bytes out of a XOF object.  Calling this routine will render
+ * the XOF instance ineligible to absorb further data.
+ */
+void
+crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
+{
+  int i = keccak_xof_squeeze(&xof->s, out, len);
+  tor_assert(i == 0);
+}
+
+/** Cleanse and deallocate a XOF object. */
+void
+crypto_xof_free_(crypto_xof_t *xof)
+{
+  if (!xof)
+    return;
+  memwipe(xof, 0, sizeof(crypto_xof_t));
+  tor_free(xof);
+}
+
diff --git a/src/common/crypto_digest.h b/src/common/crypto_digest.h
new file mode 100644
index 0000000000..3bd74acdfa
--- /dev/null
+++ b/src/common/crypto_digest.h
@@ -0,0 +1,136 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest.h
+ *
+ * \brief Headers for crypto_digest.c
+ **/
+
+#ifndef TOR_CRYPTO_DIGEST_H
+#define TOR_CRYPTO_DIGEST_H
+
+#include <stdio.h>
+
+#include "container.h"
+#include "torint.h"
+
+/** Length of the output of our message digest. */
+#define DIGEST_LEN 20
+/** Length of the output of our second (improved) message digests.  (For now
+ * this is just sha256, but it could be any other 256-bit digest.) */
+#define DIGEST256_LEN 32
+/** Length of the output of our 64-bit optimized message digests (SHA512). */
+#define DIGEST512_LEN 64
+
+/** Length of a sha1 message digest when encoded in base32 with trailing =
+ * signs removed. */
+#define BASE32_DIGEST_LEN 32
+/** Length of a sha1 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST_LEN 27
+/** Length of a sha256 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST256_LEN 43
+/** Length of a sha512 message digest when encoded in base64 with trailing =
+ * signs removed. */
+#define BASE64_DIGEST512_LEN 86
+
+/** Length of hex encoding of SHA1 digest, not including final NUL. */
+#define HEX_DIGEST_LEN 40
+/** Length of hex encoding of SHA256 digest, not including final NUL. */
+#define HEX_DIGEST256_LEN 64
+/** Length of hex encoding of SHA512 digest, not including final NUL. */
+#define HEX_DIGEST512_LEN 128
+
+typedef enum {
+  DIGEST_SHA1 = 0,
+  DIGEST_SHA256 = 1,
+  DIGEST_SHA512 = 2,
+  DIGEST_SHA3_256 = 3,
+  DIGEST_SHA3_512 = 4,
+} digest_algorithm_t;
+#define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
+#define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
+
+#define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512)
+/** Structure used to temporarily save the a digest object. Only implemented
+ * for SHA1 digest for now. */
+typedef struct crypto_digest_checkpoint_t {
+  uint8_t mem[DIGEST_CHECKPOINT_BYTES];
+} crypto_digest_checkpoint_t;
+
+/** A set of all the digests we commonly compute, taken on a single
+ * string.  Any digests that are shorter than 512 bits are right-padded
+ * with 0 bits.
+ *
+ * Note that this representation wastes 44 bytes for the SHA1 case, so
+ * don't use it for anything where we need to allocate a whole bunch at
+ * once.
+ **/
+typedef struct {
+  char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
+} common_digests_t;
+
+typedef struct crypto_digest_t crypto_digest_t;
+typedef struct crypto_xof_t crypto_xof_t;
+
+/* SHA-1 and other digests */
+int crypto_digest(char *digest, const char *m, size_t len);
+int crypto_digest256(char *digest, const char *m, size_t len,
+                     digest_algorithm_t algorithm);
+int crypto_digest512(char *digest, const char *m, size_t len,
+                     digest_algorithm_t algorithm);
+int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len);
+void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                                    const char *prepend,
+                                    const struct smartlist_t *lst,
+                                    const char *append,
+                                    digest_algorithm_t alg);
+void crypto_digest_smartlist(char *digest_out, size_t len_out,
+                             const struct smartlist_t *lst, const char *append,
+                             digest_algorithm_t alg);
+const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg);
+size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg);
+int crypto_digest_algorithm_parse_name(const char *name);
+crypto_digest_t *crypto_digest_new(void);
+crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
+crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
+void crypto_digest_free_(crypto_digest_t *digest);
+#define crypto_digest_free(d) \
+  FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d))
+void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                             size_t len);
+void crypto_digest_get_digest(crypto_digest_t *digest,
+                              char *out, size_t out_len);
+crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest);
+void crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                              const crypto_digest_t *digest);
+void crypto_digest_restore(crypto_digest_t *digest,
+                           const crypto_digest_checkpoint_t *checkpoint);
+void crypto_digest_assign(crypto_digest_t *into,
+                          const crypto_digest_t *from);
+void crypto_hmac_sha256(char *hmac_out,
+                        const char *key, size_t key_len,
+                        const char *msg, size_t msg_len);
+void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                         const uint8_t *key, size_t key_len,
+                         const uint8_t *msg, size_t msg_len);
+
+/* xof functions*/
+crypto_xof_t *crypto_xof_new(void);
+void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
+void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
+void crypto_xof_free_(crypto_xof_t *xof);
+#define crypto_xof_free(xof) \
+  FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+
+#ifdef TOR_UNIT_TESTS
+digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
+#endif
+
+#endif /* !defined(TOR_CRYPTO_DIGEST_H) */
+
diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c
index 30ed772274..9c13e3bdf0 100644
--- a/src/common/crypto_ed25519.c
+++ b/src/common/crypto_ed25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,26 +15,26 @@
  * keys to and from the corresponding Curve25519 keys.
  */
 
+#define CRYPTO_ED25519_PRIVATE
 #include "orconfig.h"
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
 
-#include "crypto.h"
-
 #include "crypto_curve25519.h"
+#include "crypto_digest.h"
 #include "crypto_ed25519.h"
 #include "crypto_format.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "torlog.h"
 #include "util.h"
+#include "util_format.h"
 
 #include "ed25519/ref10/ed25519_ref10.h"
 #include "ed25519/donna/ed25519_donna_tor.h"
 
-#include <openssl/sha.h>
-
 static void pick_ed25519_impl(void);
-static int ed25519_impl_spot_check(void);
 
 /** An Ed25519 implementation, as a set of function pointers. */
 typedef struct {
@@ -59,6 +59,9 @@ typedef struct {
 
   int (*pubkey_from_curve25519_pubkey)(unsigned char *, const unsigned char *,
                                        int);
+
+  int (*ed25519_scalarmult_with_group_order)(unsigned char *,
+                                             const unsigned char *);
 } ed25519_impl_t;
 
 /** The Ref10 Ed25519 implementation. This one is pure C and lightly
@@ -79,6 +82,7 @@ static const ed25519_impl_t impl_ref10 = {
   ed25519_ref10_blind_public_key,
 
   ed25519_ref10_pubkey_from_curve25519_pubkey,
+  ed25519_ref10_scalarmult_with_group_order,
 };
 
 /** The Ref10 Ed25519 implementation. This one is heavily optimized, but still
@@ -99,6 +103,7 @@ static const ed25519_impl_t impl_donna = {
   ed25519_donna_blind_public_key,
 
   ed25519_donna_pubkey_from_curve25519_pubkey,
+  ed25519_donna_scalarmult_with_group_order,
 };
 
 /** Which Ed25519 implementation are we using?  NULL if we haven't decided
@@ -147,7 +152,7 @@ crypto_ed25519_testing_restore_impl(void)
   ed25519_impl = saved_ed25519_impl;
   saved_ed25519_impl = NULL;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /**
  * Initialize a new ed25519 secret key in <b>seckey_out</b>.  If
@@ -211,10 +216,18 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
   return 0;
 }
 
+/** Return true iff 'pubkey' is set to zero (eg to indicate that it is not
+ * set). */
+int
+ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
+{
+  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+}
+
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
  * string <b>prefix_str</b>. Set <b>final_msg_len_out</b> to the size of the
- * final array. If an error occured, return NULL. It's the resonsibility of the
- * caller to free the returned array. */
+ * final array. If an error occurred, return NULL. It's the responsibility of
+ * the caller to free the returned array. */
 static uint8_t *
 get_prefixed_msg(const uint8_t *msg, size_t msg_len,
                  const char *prefix_str,
@@ -247,7 +260,7 @@ get_prefixed_msg(const uint8_t *msg, size_t msg_len,
  * Set <b>signature_out</b> to a signature of the <b>len</b>-byte message
  * <b>msg</b>, using the secret and public key in <b>keypair</b>.
  *
- * Return 0 if we successfuly signed the message, otherwise return -1.
+ * Return 0 if we successfully signed the message, otherwise return -1.
  */
 int
 ed25519_sign(ed25519_signature_t *signature_out,
@@ -267,11 +280,11 @@ ed25519_sign(ed25519_signature_t *signature_out,
  * Like ed25519_sign(), but also prefix <b>msg</b> with <b>prefix_str</b>
  * before signing. <b>prefix_str</b> must be a NUL-terminated string.
  */
-int
-ed25519_sign_prefixed(ed25519_signature_t *signature_out,
-                      const uint8_t *msg, size_t msg_len,
-                      const char *prefix_str,
-                      const ed25519_keypair_t *keypair)
+MOCK_IMPL(int,
+ed25519_sign_prefixed,(ed25519_signature_t *signature_out,
+                       const uint8_t *msg, size_t msg_len,
+                       const char *prefix_str,
+                       const ed25519_keypair_t *keypair))
 {
   int retval;
   size_t prefixed_msg_len;
@@ -281,9 +294,12 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out,
 
   prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str,
                                   &prefixed_msg_len);
-  if (!prefixed_msg) {
+  if (BUG(!prefixed_msg)) {
+    /* LCOV_EXCL_START -- only possible when the message and prefix are
+     * ridiculously huge */
     log_warn(LD_GENERAL, "Failed to get prefixed msg.");
     return -1;
+    /* LCOV_EXCL_STOP */
   }
 
   retval = ed25519_sign(signature_out,
@@ -300,10 +316,10 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out,
  *
  * Return 0 if the signature is valid; -1 if it isn't.
  */
-int
-ed25519_checksig(const ed25519_signature_t *signature,
-                 const uint8_t *msg, size_t len,
-                 const ed25519_public_key_t *pubkey)
+MOCK_IMPL(int,
+ed25519_checksig,(const ed25519_signature_t *signature,
+                  const uint8_t *msg, size_t len,
+                  const ed25519_public_key_t *pubkey))
 {
   return
     get_ed_impl()->open(signature->sig, msg, len, pubkey->pubkey) < 0 ? -1 : 0;
@@ -326,9 +342,12 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature,
 
   prefixed_msg = get_prefixed_msg(msg, msg_len, prefix_str,
                                   &prefixed_msg_len);
-  if (!prefixed_msg) {
+  if (BUG(!prefixed_msg)) {
+    /* LCOV_EXCL_START -- only possible when the message and prefix are
+     * ridiculously huge */
     log_warn(LD_GENERAL, "Failed to get prefixed msg.");
     return -1;
+    /* LCOV_EXCL_STOP */
   }
 
   retval = ed25519_checksig(signature,
@@ -346,10 +365,10 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature,
  * was valid. Otherwise return -N, where N is the number of invalid
  * signatures.
  */
-int
-ed25519_checksig_batch(int *okay_out,
-                       const ed25519_checkable_t *checkable,
-                       int n_checkable)
+MOCK_IMPL(int,
+ed25519_checksig_batch,(int *okay_out,
+                        const ed25519_checkable_t *checkable,
+                        int n_checkable))
 {
   int i, res;
   const ed25519_impl_t *impl = get_ed_impl();
@@ -434,14 +453,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
 {
   const char string[] = "Derive high part of ed25519 key from curve25519 key";
   ed25519_public_key_t pubkey_check;
-  SHA512_CTX ctx;
-  uint8_t sha512_output[64];
+  crypto_digest_t *ctx;
+  uint8_t sha512_output[DIGEST512_LEN];
 
   memcpy(out->seckey.seckey, inp->seckey.secret_key, 32);
-  SHA512_Init(&ctx);
-  SHA512_Update(&ctx, out->seckey.seckey, 32);
-  SHA512_Update(&ctx, string, sizeof(string));
-  SHA512_Final(sha512_output, &ctx);
+
+  ctx = crypto_digest512_new(DIGEST_SHA512);
+  crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32);
+  crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string));
+  crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output));
+  crypto_digest_free(ctx);
   memcpy(out->seckey.seckey + 32, sha512_output, 32);
 
   ed25519_public_key_generate(&out->pubkey, &out->seckey);
@@ -454,7 +475,6 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
   tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32));
 
   memwipe(&pubkey_check, 0, sizeof(pubkey_check));
-  memwipe(&ctx, 0, sizeof(ctx));
   memwipe(sha512_output, 0, sizeof(sha512_output));
 
   return 0;
@@ -483,7 +503,8 @@ ed25519_public_key_from_curve25519_public_key(ed25519_public_key_t *pubkey,
  * service descriptors are encrypted with a key derived from the service's
  * long-term public key, and then signed with (and stored at a position
  * indexed by) a short-term key derived by blinding the long-term keys.
- */
+ *
+ * Return 0 if blinding was successful, else return -1. */
 int
 ed25519_keypair_blind(ed25519_keypair_t *out,
                       const ed25519_keypair_t *inp,
@@ -494,7 +515,9 @@ ed25519_keypair_blind(ed25519_keypair_t *out,
   get_ed_impl()->blind_secret_key(out->seckey.seckey,
                                   inp->seckey.seckey, param);
 
-  ed25519_public_blind(&pubkey_check, &inp->pubkey, param);
+  if (ed25519_public_blind(&pubkey_check, &inp->pubkey, param) < 0) {
+    return -1;
+  }
   ed25519_public_key_generate(&out->pubkey, &out->seckey);
 
   tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32));
@@ -507,15 +530,14 @@ ed25519_keypair_blind(ed25519_keypair_t *out,
 /**
  * Given an ed25519 public key in <b>inp</b>, generate a corresponding blinded
  * public key in <b>out</b>, blinded with the 32-byte parameter in
- * <b>param</b>.  Return 0 on sucess, -1 on railure.
+ * <b>param</b>.  Return 0 on success, -1 on railure.
  */
 int
 ed25519_public_blind(ed25519_public_key_t *out,
                      const ed25519_public_key_t *inp,
                      const uint8_t *param)
 {
-  get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param);
-  return 0;
+  return get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param);
 }
 
 /**
@@ -601,7 +623,7 @@ ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out,
 
 /** Release all storage held for <b>kp</b>. */
 void
-ed25519_keypair_free(ed25519_keypair_t *kp)
+ed25519_keypair_free_(ed25519_keypair_t *kp)
 {
   if (! kp)
     return;
@@ -620,10 +642,22 @@ ed25519_pubkey_eq(const ed25519_public_key_t *key1,
   return tor_memeq(key1->pubkey, key2->pubkey, ED25519_PUBKEY_LEN);
 }
 
+/**
+ * Set <b>dest</b> to contain the same key as <b>src</b>.
+ */
+void
+ed25519_pubkey_copy(ed25519_public_key_t *dest,
+                    const ed25519_public_key_t *src)
+{
+  tor_assert(dest);
+  tor_assert(src);
+  memcpy(dest, src, sizeof(ed25519_public_key_t));
+}
+
 /** Check whether the given Ed25519 implementation seems to be working.
  * If so, return 0; otherwise return -1. */
-static int
-ed25519_impl_spot_check(void)
+MOCK_IMPL(STATIC int,
+ed25519_impl_spot_check,(void))
 {
   static const uint8_t alicesk[32] = {
     0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b,
@@ -686,13 +720,16 @@ ed25519_impl_spot_check(void)
     return -1;
 
   /* XXX/yawning: Someone that's more paranoid than I am, can write "Assume
-   * ref0 is cannonical, and fuzz impl against it" if they want, but I doubt
+   * ref0 is canonical, and fuzz impl against it" if they want, but I doubt
    * that will catch anything that the known answer tests won't.
    */
   goto end;
 
+ // LCOV_EXCL_START -- We can only reach this if our ed25519 implementation is
+ // broken.
  fail:
   r = -1;
+ // LCOV_EXCL_STOP
  end:
   return r;
 }
@@ -726,7 +763,7 @@ pick_ed25519_impl(void)
   /* LCOV_EXCL_STOP */
 }
 
-/* Initialize the Ed25519 implementation. This is neccessary if you're
+/* Initialize the Ed25519 implementation. This is necessary if you're
  * going to use them in a multithreaded setting, and not otherwise. */
 void
 ed25519_init(void)
@@ -734,3 +771,47 @@ ed25519_init(void)
   pick_ed25519_impl();
 }
 
+/* Return true if <b>point</b> is the identity element of the ed25519 group. */
+static int
+ed25519_point_is_identity_element(const uint8_t *point)
+{
+  /* The identity element in ed25159 is the point with coordinates (0,1). */
+  static const uint8_t ed25519_identity[32] = {
+    0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+  tor_assert(sizeof(ed25519_identity) == ED25519_PUBKEY_LEN);
+  return tor_memeq(point, ed25519_identity, sizeof(ed25519_identity));
+}
+
+/** Validate <b>pubkey</b> to ensure that it has no torsion component.
+ *  Return 0 if <b>pubkey</b> is valid, else return -1. */
+int
+ed25519_validate_pubkey(const ed25519_public_key_t *pubkey)
+{
+  uint8_t result[32] = {9};
+
+  /* First check that we were not given the identity element */
+  if (ed25519_point_is_identity_element(pubkey->pubkey)) {
+    log_warn(LD_CRYPTO, "ed25519 pubkey is the identity");
+    return -1;
+  }
+
+  /* For any point on the curve, doing l*point should give the identity element
+   * (where l is the group order). Do the computation and check that the
+   * identity element is returned. */
+  if (get_ed_impl()->ed25519_scalarmult_with_group_order(result,
+                                                         pubkey->pubkey) < 0) {
+    log_warn(LD_CRYPTO, "ed25519 group order scalarmult failed");
+    return -1;
+  }
+
+  if (!ed25519_point_is_identity_element(result)) {
+    log_warn(LD_CRYPTO, "ed25519 validation failed");
+    return -1;
+  }
+
+  return 0;
+}
+
diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h
index 31afc49ccc..74269ccffd 100644
--- a/src/common/crypto_ed25519.h
+++ b/src/common/crypto_ed25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_ED25519_H
@@ -7,6 +7,7 @@
 #include "testsupport.h"
 #include "torint.h"
 #include "crypto_curve25519.h"
+#include "util.h"
 
 #define ED25519_PUBKEY_LEN 32
 #define ED25519_SECKEY_LEN 64
@@ -51,21 +52,24 @@ int ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong);
 int ed25519_sign(ed25519_signature_t *signature_out,
                  const uint8_t *msg, size_t len,
                  const ed25519_keypair_t *key);
-int ed25519_checksig(const ed25519_signature_t *signature,
-                     const uint8_t *msg, size_t len,
-                     const ed25519_public_key_t *pubkey);
+MOCK_DECL(int,ed25519_checksig,(const ed25519_signature_t *signature,
+                                const uint8_t *msg, size_t len,
+                                const ed25519_public_key_t *pubkey));
+
+MOCK_DECL(int,
+ed25519_sign_prefixed,(ed25519_signature_t *signature_out,
+                       const uint8_t *msg, size_t len,
+                       const char *prefix_str,
+                       const ed25519_keypair_t *keypair));
 
-int
-ed25519_sign_prefixed(ed25519_signature_t *signature_out,
-                      const uint8_t *msg, size_t len,
-                      const char *prefix_str,
-                      const ed25519_keypair_t *keypair);
 int
 ed25519_checksig_prefixed(const ed25519_signature_t *signature,
                           const uint8_t *msg, size_t len,
                           const char *prefix_str,
                           const ed25519_public_key_t *pubkey);
 
+int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey);
+
 /**
  * A collection of information necessary to check an Ed25519 signature. Used
  * for batch verification.
@@ -81,9 +85,9 @@ typedef struct {
   size_t len;
 } ed25519_checkable_t;
 
-int ed25519_checksig_batch(int *okay_out,
-                           const ed25519_checkable_t *checkable,
-                           int n_checkable);
+MOCK_DECL(int, ed25519_checksig_batch,(int *okay_out,
+                                       const ed25519_checkable_t *checkable,
+                                       int n_checkable));
 
 int ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
                                             int *signbit_out,
@@ -114,18 +118,28 @@ int ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out,
                                   char **tag_out,
                                   const char *filename);
 
-void ed25519_keypair_free(ed25519_keypair_t *kp);
+void ed25519_keypair_free_(ed25519_keypair_t *kp);
+#define ed25519_keypair_free(kp) \
+  FREE_AND_NULL(ed25519_keypair_t, ed25519_keypair_free_, (kp))
 
 int ed25519_pubkey_eq(const ed25519_public_key_t *key1,
                       const ed25519_public_key_t *key2);
+void ed25519_pubkey_copy(ed25519_public_key_t *dest,
+                         const ed25519_public_key_t *src);
 
 void ed25519_set_impl_params(int use_donna);
 void ed25519_init(void);
 
+int ed25519_validate_pubkey(const ed25519_public_key_t *pubkey);
+
 #ifdef TOR_UNIT_TESTS
 void crypto_ed25519_testing_force_impl(const char *name);
 void crypto_ed25519_testing_restore_impl(void);
 #endif
 
+#ifdef CRYPTO_ED25519_PRIVATE
+MOCK_DECL(STATIC int, ed25519_impl_spot_check, (void));
 #endif
 
+#endif /* !defined(TOR_CRYPTO_ED25519_H) */
+
diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c
index 2f6d847c83..460e85bac1 100644
--- a/src/common/crypto_format.c
+++ b/src/common/crypto_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,10 +15,11 @@
 #include <sys/stat.h>
 #endif
 #include "container.h"
-#include "crypto.h"
 #include "crypto_curve25519.h"
+#include "crypto_digest.h"
 #include "crypto_ed25519.h"
 #include "crypto_format.h"
+#include "crypto_util.h"
 #include "util.h"
 #include "util_format.h"
 #include "torlog.h"
@@ -161,6 +162,27 @@ curve25519_public_from_base64(curve25519_public_key_t *pkey,
   }
 }
 
+/** For logging convenience: Convert <b>pkey</b> to a statically allocated
+ * base64 string and return it. Not threadsafe. Format not meant to be
+ * computer-readable; it may change in the future. Subsequent calls invalidate
+ * previous returns. */
+const char *
+ed25519_fmt(const ed25519_public_key_t *pkey)
+{
+  static char formatted[ED25519_BASE64_LEN+1];
+  if (pkey) {
+    if (ed25519_public_key_is_zero(pkey)) {
+      strlcpy(formatted, "<unset>", sizeof(formatted));
+    } else {
+      int r = ed25519_public_to_base64(formatted, pkey);
+      tor_assert(!r);
+    }
+  } else {
+    strlcpy(formatted, "<null>", sizeof(formatted));
+  }
+  return formatted;
+}
+
 /** Try to decode the string <b>input</b> into an ed25519 public key. On
  * success, store the value in <b>pkey</b> and return 0. Otherwise return
  * -1. */
diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h
index 012e228cc4..bbd85dc720 100644
--- a/src/common/crypto_format.h
+++ b/src/common/crypto_format.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_FORMAT_H
@@ -28,6 +28,7 @@ int ed25519_public_from_base64(ed25519_public_key_t *pkey,
                                const char *input);
 int ed25519_public_to_base64(char *output,
                              const ed25519_public_key_t *pkey);
+const char *ed25519_fmt(const ed25519_public_key_t *pkey);
 
 /* XXXX move these to crypto_format.h */
 #define ED25519_SIG_BASE64_LEN 86
@@ -42,5 +43,5 @@ int digest_from_base64(char *digest, const char *d64);
 int digest256_to_base64(char *d64, const char *digest);
 int digest256_from_base64(char *digest, const char *d64);
 
-#endif
+#endif /* !defined(TOR_CRYPTO_FORMAT_H) */
 
diff --git a/src/common/crypto_openssl_mgt.c b/src/common/crypto_openssl_mgt.c
new file mode 100644
index 0000000000..ea3519efa2
--- /dev/null
+++ b/src/common/crypto_openssl_mgt.c
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_openssl.c
+ *
+ * \brief Block of functions related to operations from OpenSSL.
+ **/
+
+#include "compat_openssl.h"
+#include "crypto_openssl_mgt.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/engine.h>
+#include <openssl/rand.h>
+#include <openssl/bn.h>
+#include <openssl/dh.h>
+#include <openssl/conf.h>
+#include <openssl/hmac.h>
+#include <openssl/crypto.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#ifndef NEW_THREAD_API
+/** A number of preallocated mutexes for use by OpenSSL. */
+static tor_mutex_t **openssl_mutexes_ = NULL;
+/** How many mutexes have we allocated for use by OpenSSL? */
+static int n_openssl_mutexes_ = 0;
+#endif /* !defined(NEW_THREAD_API) */
+
+/** Declare STATIC functions */
+STATIC char * parse_openssl_version_str(const char *raw_version);
+#ifndef NEW_THREAD_API
+STATIC void openssl_locking_cb_(int mode, int n, const char *file, int line);
+STATIC void tor_set_openssl_thread_id(CRYPTO_THREADID *threadid);
+#endif
+
+/* Returns a trimmed and human-readable version of an openssl version string
+* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10
+* May 2012' and this will parse them into a form similar to '1.0.0b' */
+STATIC char *
+parse_openssl_version_str(const char *raw_version)
+{
+  const char *end_of_version = NULL;
+  /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's
+     trim that down. */
+  if (!strcmpstart(raw_version, "OpenSSL ")) {
+    raw_version += strlen("OpenSSL ");
+    end_of_version = strchr(raw_version, ' ');
+  }
+
+  if (end_of_version)
+    return tor_strndup(raw_version,
+                      end_of_version-raw_version);
+  else
+    return tor_strdup(raw_version);
+}
+
+static char *crypto_openssl_version_str = NULL;
+/* Return a human-readable version of the run-time openssl version number. */
+const char *
+crypto_openssl_get_version_str(void)
+{
+  if (crypto_openssl_version_str == NULL) {
+    const char *raw_version = OpenSSL_version(OPENSSL_VERSION);
+    crypto_openssl_version_str = parse_openssl_version_str(raw_version);
+  }
+  return crypto_openssl_version_str;
+}
+
+static char *crypto_openssl_header_version_str = NULL;
+/* Return a human-readable version of the compile-time openssl version
+* number. */
+const char *
+crypto_openssl_get_header_version_str(void)
+{
+  if (crypto_openssl_header_version_str == NULL) {
+    crypto_openssl_header_version_str =
+                        parse_openssl_version_str(OPENSSL_VERSION_TEXT);
+  }
+  return crypto_openssl_header_version_str;
+}
+
+#ifndef OPENSSL_THREADS
+#error OpenSSL has been built without thread support. Tor requires an \
+ OpenSSL library with thread support enabled.
+#endif
+
+#ifndef NEW_THREAD_API
+/** Helper: OpenSSL uses this callback to manipulate mutexes. */
+STATIC void
+openssl_locking_cb_(int mode, int n, const char *file, int line)
+{
+  (void)file;
+  (void)line;
+  if (!openssl_mutexes_)
+    /* This is not a really good fix for the
+     * "release-freed-lock-from-separate-thread-on-shutdown" problem, but
+     * it can't hurt. */
+    return;
+  if (mode & CRYPTO_LOCK)
+    tor_mutex_acquire(openssl_mutexes_[n]);
+  else
+    tor_mutex_release(openssl_mutexes_[n]);
+}
+
+STATIC void
+tor_set_openssl_thread_id(CRYPTO_THREADID *threadid)
+{
+  CRYPTO_THREADID_set_numeric(threadid, tor_get_thread_id());
+}
+#endif /* !defined(NEW_THREAD_API) */
+
+/** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being
+ * multithreaded. Returns 0. */
+int
+setup_openssl_threading(void)
+{
+#ifndef NEW_THREAD_API
+  int i;
+  int n = CRYPTO_num_locks();
+  n_openssl_mutexes_ = n;
+  openssl_mutexes_ = tor_calloc(n, sizeof(tor_mutex_t *));
+  for (i=0; i < n; ++i)
+    openssl_mutexes_[i] = tor_mutex_new();
+  CRYPTO_set_locking_callback(openssl_locking_cb_);
+  CRYPTO_THREADID_set_callback(tor_set_openssl_thread_id);
+#endif /* !defined(NEW_THREAD_API) */
+  return 0;
+}
+
+/** free OpenSSL variables */
+void
+crypto_openssl_free_all(void)
+{
+  tor_free(crypto_openssl_version_str);
+  tor_free(crypto_openssl_header_version_str);
+
+#ifndef NEW_THREAD_API
+  if (n_openssl_mutexes_) {
+    int n = n_openssl_mutexes_;
+    tor_mutex_t **ms = openssl_mutexes_;
+    int i;
+    openssl_mutexes_ = NULL;
+    n_openssl_mutexes_ = 0;
+    for (i=0;i<n;++i) {
+      tor_mutex_free(ms[i]);
+    }
+    tor_free(ms);
+  }
+#endif /* !defined(NEW_THREAD_API) */
+}
+
diff --git a/src/common/crypto_openssl_mgt.h b/src/common/crypto_openssl_mgt.h
new file mode 100644
index 0000000000..09b6737962
--- /dev/null
+++ b/src/common/crypto_openssl_mgt.h
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_openssl.h
+ *
+ * \brief Headers for crypto_openssl.c
+ **/
+
+#ifndef TOR_CRYPTO_OPENSSL_H
+#define TOR_CRYPTO_OPENSSL_H
+
+#include <stdio.h>
+#include "util.h"
+
+#include <openssl/engine.h>
+
+/*
+  Macro to create an arbitrary OpenSSL version number as used by
+  OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard
+  to read.
+
+  Don't use this directly, instead use one of the other OPENSSL_V macros
+  below.
+
+  The format is: 4 bits major, 8 bits minor, 8 bits fix, 8 bits patch, 4 bit
+  status.
+ */
+#define OPENSSL_VER(a,b,c,d,e)                                \
+  (((a)<<28) |                                                \
+   ((b)<<20) |                                                \
+   ((c)<<12) |                                                \
+   ((d)<< 4) |                                                \
+    (e))
+/** An openssl release number.  For example, OPENSSL_V(0,9,8,'j') is the
+ * version for the released version of 0.9.8j */
+#define OPENSSL_V(a,b,c,d) \
+  OPENSSL_VER((a),(b),(c),(d)-'a'+1,0xf)
+/** An openssl release number for the first release in the series.  For
+ * example, OPENSSL_V_NOPATCH(1,0,0) is the first released version of OpenSSL
+ * 1.0.0. */
+#define OPENSSL_V_NOPATCH(a,b,c) \
+  OPENSSL_VER((a),(b),(c),0,0xf)
+/** The first version that would occur for any alpha or beta in an openssl
+ * series. For example, OPENSSL_V_SERIES(0,9,8) is greater than any released
+ * 0.9.7, and less than any released 0.9.8. */
+#define OPENSSL_V_SERIES(a,b,c) \
+  OPENSSL_VER((a),(b),(c),0,0)
+
+#ifdef ANDROID
+/* Android's OpenSSL seems to have removed all of its Engine support. */
+#define DISABLE_ENGINES
+#endif
+
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && \
+  !defined(LIBRESSL_VERSION_NUMBER)
+/* OpenSSL as of 1.1.0pre4 has an "new" thread API, which doesn't require
+ * seting up various callbacks.
+ *
+ * OpenSSL 1.1.0pre4 has a messed up `ERR_remove_thread_state()` prototype,
+ * while the previous one was restored in pre5, and the function made a no-op
+ * (along with a deprecated annotation, which produces a compiler warning).
+ *
+ * While it is possible to support all three versions of the thread API,
+ * a version that existed only for one snapshot pre-release is kind of
+ * pointless, so let's not.
+ */
+#define NEW_THREAD_API
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) && ... */
+
+/* global openssl state */
+const char * crypto_openssl_get_version_str(void);
+const char * crypto_openssl_get_header_version_str(void);
+
+/* OpenSSL threading setup function */
+int setup_openssl_threading(void);
+
+/* Tor OpenSSL utility functions */
+void crypto_openssl_free_all(void);
+
+#endif /* !defined(TOR_CRYPTO_OPENSSL_H) */
+
diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c
index 31e37c007d..c2bd1d26cb 100644
--- a/src/common/crypto_pwbox.c
+++ b/src/common/crypto_pwbox.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,8 +9,11 @@
  */
 
 #include "crypto.h"
-#include "crypto_s2k.h"
+#include "crypto_digest.h"
 #include "crypto_pwbox.h"
+#include "crypto_rand.h"
+#include "crypto_s2k.h"
+#include "crypto_util.h"
 #include "di_ops.h"
 #include "util.h"
 #include "pwbox.h"
@@ -107,7 +110,6 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out,
   rv = 0;
   goto out;
 
- err:
   /* LCOV_EXCL_START
 
      This error case is often unreachable if we're correctly coded, unless
@@ -123,6 +125,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out,
        - pwbox_encoded_encode can't fail unless we're using trunnel wrong,
          or it's buggy.
    */
+ err:
   tor_free(result);
   rv = -1;
   /* LCOV_EXCL_STOP */
diff --git a/src/common/crypto_pwbox.h b/src/common/crypto_pwbox.h
index aadd477078..a26b6d2c17 100644
--- a/src/common/crypto_pwbox.h
+++ b/src/common/crypto_pwbox.h
@@ -1,3 +1,6 @@
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #ifndef CRYPTO_PWBOX_H_INCLUDED_
 #define CRYPTO_PWBOX_H_INCLUDED_
 
@@ -16,5 +19,5 @@ int crypto_unpwbox(uint8_t **out, size_t *outlen_out,
                    const uint8_t *inp, size_t input_len,
                    const char *secret, size_t secret_len);
 
-#endif
+#endif /* !defined(CRYPTO_PWBOX_H_INCLUDED_) */
 
diff --git a/src/common/crypto_rand.c b/src/common/crypto_rand.c
new file mode 100644
index 0000000000..df2e2f65d3
--- /dev/null
+++ b/src/common/crypto_rand.c
@@ -0,0 +1,615 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand.c
+ *
+ * \brief Functions for initialising and seeding (pseudo-)random
+ * number generators, and working with randomness.
+ **/
+
+#ifndef CRYPTO_RAND_PRIVATE
+#define CRYPTO_RAND_PRIVATE
+
+#include "crypto_rand.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#include <wincrypt.h>
+#endif /* defined(_WIN32) */
+
+#include "container.h"
+#include "compat.h"
+#include "compat_openssl.h"
+#include "crypto_util.h"
+#include "sandbox.h"
+#include "testsupport.h"
+#include "torlog.h"
+#include "util.h"
+#include "util_format.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+#include <openssl/rand.h>
+ENABLE_GCC_WARNING(redundant-decls)
+
+#if __GNUC__ && GCC_VERSION >= 402
+#if GCC_VERSION >= 406
+#pragma GCC diagnostic pop
+#else
+#pragma GCC diagnostic warning "-Wredundant-decls"
+#endif
+#endif /* __GNUC__ && GCC_VERSION >= 402 */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+#ifdef HAVE_SYS_FCNTL_H
+#include <sys/fcntl.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_SYSCALL_H
+#include <sys/syscall.h>
+#endif
+#ifdef HAVE_SYS_RANDOM_H
+#include <sys/random.h>
+#endif
+
+/**
+ * How many bytes of entropy we add at once.
+ *
+ * This is how much entropy OpenSSL likes to add right now, so maybe it will
+ * work for us too.
+ **/
+#define ADD_ENTROPY 32
+
+/**
+ * Longest recognized DNS query.
+ **/
+#define MAX_DNS_LABEL_SIZE 63
+
+/**
+ * Largest strong entropy request permitted.
+ **/
+#define MAX_STRONGEST_RAND_SIZE 256
+
+/**
+ * Set the seed of the weak RNG to a random value.
+ **/
+void
+crypto_seed_weak_rng(tor_weak_rng_t *rng)
+{
+  unsigned seed;
+  crypto_rand((void*)&seed, sizeof(seed));
+  tor_init_weak_random(rng, seed);
+}
+
+#ifdef TOR_UNIT_TESTS
+int break_strongest_rng_syscall = 0;
+int break_strongest_rng_fallback = 0;
+#endif
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on
+ * failure.  A maximum request size of 256 bytes is imposed.
+ **/
+static int
+crypto_strongest_rand_syscall(uint8_t *out, size_t out_len)
+{
+  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
+
+  /* We only log at notice-level here because in the case that this function
+   * fails the crypto_strongest_rand_raw() caller will log with a warning-level
+   * message and let crypto_strongest_rand() error out and finally terminating
+   * Tor with an assertion error.
+   */
+
+#ifdef TOR_UNIT_TESTS
+  if (break_strongest_rng_syscall)
+    return -1;
+#endif
+
+#if defined(_WIN32)
+  static int provider_set = 0;
+  static HCRYPTPROV provider;
+
+  if (!provider_set) {
+    if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL,
+                             CRYPT_VERIFYCONTEXT)) {
+      log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1].");
+      return -1;
+    }
+    provider_set = 1;
+  }
+  if (!CryptGenRandom(provider, out_len, out)) {
+    log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI.");
+    return -1;
+  }
+
+  return 0;
+#elif defined(__linux__) && defined(SYS_getrandom)
+  static int getrandom_works = 1; /* Be optimistic about our chances... */
+
+  /* getrandom() isn't as straightforward as getentropy(), and has
+   * no glibc wrapper.
+   *
+   * As far as I can tell from getrandom(2) and the source code, the
+   * requests we issue will always succeed (though it will block on the
+   * call if /dev/urandom isn't seeded yet), since we are NOT specifying
+   * GRND_NONBLOCK and the request is <= 256 bytes.
+   *
+   * The manpage is unclear on what happens if a signal interrupts the call
+   * while the request is blocked due to lack of entropy....
+   *
+   * We optimistically assume that getrandom() is available and functional
+   * because it is the way of the future, and 2 branch mispredicts pale in
+   * comparison to the overheads involved with failing to open
+   * /dev/srandom followed by opening and reading from /dev/urandom.
+   */
+  if (PREDICT_LIKELY(getrandom_works)) {
+    long ret;
+    /* A flag of '0' here means to read from '/dev/urandom', and to
+     * block if insufficient entropy is available to service the
+     * request.
+     */
+    const unsigned int flags = 0;
+    do {
+      ret = syscall(SYS_getrandom, out, out_len, flags);
+    } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN)));
+
+    if (PREDICT_UNLIKELY(ret == -1)) {
+      /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */
+      tor_assert(errno != EAGAIN);
+      tor_assert(errno != EINTR);
+
+      /* Useful log message for errno. */
+      if (errno == ENOSYS) {
+        log_notice(LD_CRYPTO, "Can't get entropy from getrandom()."
+                   " You are running a version of Tor built to support"
+                   " getrandom(), but the kernel doesn't implement this"
+                   " function--probably because it is too old?"
+                   " Trying fallback method instead.");
+      } else {
+        log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s."
+                              " Trying fallback method instead.",
+                   strerror(errno));
+      }
+
+      getrandom_works = 0; /* Don't bother trying again. */
+      return -1;
+      /* LCOV_EXCL_STOP */
+    }
+
+    tor_assert(ret == (long)out_len);
+    return 0;
+  }
+
+  return -1; /* getrandom() previously failed unexpectedly. */
+#elif defined(HAVE_GETENTROPY)
+  /* getentropy() is what Linux's getrandom() wants to be when it grows up.
+   * the only gotcha is that requests are limited to 256 bytes.
+   */
+  return getentropy(out, out_len);
+#else
+  (void) out;
+#endif /* defined(_WIN32) || ... */
+
+  /* This platform doesn't have a supported syscall based random. */
+  return -1;
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * via the per-platform fallback mechanism, storing it into <b>out</b>.
+ * Return 0 on success, -1 on failure.  A maximum request size of 256 bytes
+ * is imposed.
+ **/
+static int
+crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
+{
+#ifdef TOR_UNIT_TESTS
+  if (break_strongest_rng_fallback)
+    return -1;
+#endif
+
+#ifdef _WIN32
+  /* Windows exclusively uses crypto_strongest_rand_syscall(). */
+  (void)out;
+  (void)out_len;
+  return -1;
+#else /* !(defined(_WIN32)) */
+  static const char *filenames[] = {
+    "/dev/srandom", "/dev/urandom", "/dev/random", NULL
+  };
+  int fd, i;
+  size_t n;
+
+  for (i = 0; filenames[i]; ++i) {
+    log_debug(LD_FS, "Considering %s as entropy source", filenames[i]);
+    fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0);
+    if (fd<0) continue;
+    log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]);
+    n = read_all(fd, (char*)out, out_len, 0);
+    close(fd);
+    if (n != out_len) {
+      /* LCOV_EXCL_START
+       * We can't make /dev/foorandom actually fail. */
+      log_notice(LD_CRYPTO,
+                 "Error reading from entropy source %s (read only %lu bytes).",
+                 filenames[i],
+                 (unsigned long)n);
+      return -1;
+      /* LCOV_EXCL_STOP */
+    }
+
+    return 0;
+  }
+
+  return -1;
+#endif /* defined(_WIN32) */
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * storing it into <b>out</b>. Return 0 on success, -1 on failure.  A maximum
+ * request size of 256 bytes is imposed.
+ **/
+STATIC int
+crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
+{
+  static const size_t sanity_min_size = 16;
+  static const int max_attempts = 3;
+  tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE);
+
+  /* For buffers >= 16 bytes (128 bits), we sanity check the output by
+   * zero filling the buffer and ensuring that it actually was at least
+   * partially modified.
+   *
+   * Checking that any individual byte is non-zero seems like it would
+   * fail too often (p = out_len * 1/256) for comfort, but this is an
+   * "adjust according to taste" sort of check.
+   */
+  memwipe(out, 0, out_len);
+  for (int i = 0; i < max_attempts; i++) {
+    /* Try to use the syscall/OS favored mechanism to get strong entropy. */
+    if (crypto_strongest_rand_syscall(out, out_len) != 0) {
+      /* Try to use the less-favored mechanism to get strong entropy. */
+      if (crypto_strongest_rand_fallback(out, out_len) != 0) {
+        /* Welp, we tried.  Hopefully the calling code terminates the process
+         * since we're basically boned without good entropy.
+         */
+        log_warn(LD_CRYPTO,
+                 "Cannot get strong entropy: no entropy source found.");
+        return -1;
+      }
+    }
+
+    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
+      return 0;
+  }
+
+  /* LCOV_EXCL_START
+   *
+   * We tried max_attempts times to fill a buffer >= 128 bits long,
+   * and each time it returned all '0's.  Either the system entropy
+   * source is busted, or the user should go out and buy a ticket to
+   * every lottery on the planet.
+   */
+  log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer.");
+
+  return -1;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * Try to get <b>out_len</b> bytes of the strongest entropy we can generate,
+ * storing it into <b>out</b>.
+ **/
+void
+crypto_strongest_rand(uint8_t *out, size_t out_len)
+{
+#define DLEN SHA512_DIGEST_LENGTH
+  /* We're going to hash DLEN bytes from the system RNG together with some
+   * bytes from the openssl PRNG, in order to yield DLEN bytes.
+   */
+  uint8_t inp[DLEN*2];
+  uint8_t tmp[DLEN];
+  tor_assert(out);
+  while (out_len) {
+    crypto_rand((char*) inp, DLEN);
+    if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_CRYPTO, "Failed to load strong entropy when generating an "
+              "important key. Exiting.");
+      /* Die with an assertion so we get a stack trace. */
+      tor_assert(0);
+      // LCOV_EXCL_STOP
+    }
+    if (out_len >= DLEN) {
+      SHA512(inp, sizeof(inp), out);
+      out += DLEN;
+      out_len -= DLEN;
+    } else {
+      SHA512(inp, sizeof(inp), tmp);
+      memcpy(out, tmp, out_len);
+      break;
+    }
+  }
+  memwipe(tmp, 0, sizeof(tmp));
+  memwipe(inp, 0, sizeof(inp));
+#undef DLEN
+}
+
+/**
+ * Seed OpenSSL's random number generator with bytes from the operating
+ * system.  Return 0 on success, -1 on failure.
+ **/
+int
+crypto_seed_rng(void)
+{
+  int rand_poll_ok = 0, load_entropy_ok = 0;
+  uint8_t buf[ADD_ENTROPY];
+
+  /* OpenSSL has a RAND_poll function that knows about more kinds of
+   * entropy than we do.  We'll try calling that, *and* calling our own entropy
+   * functions.  If one succeeds, we'll accept the RNG as seeded. */
+  rand_poll_ok = RAND_poll();
+  if (rand_poll_ok == 0)
+    log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE
+
+  load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf));
+  if (load_entropy_ok) {
+    RAND_seed(buf, sizeof(buf));
+  }
+
+  memwipe(buf, 0, sizeof(buf));
+
+  if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking
+ * for unit tests.
+ *
+ * This function is not allowed to fail; if it would fail to generate strong
+ * entropy, it must terminate the process instead.
+ **/
+MOCK_IMPL(void,
+crypto_rand, (char *to, size_t n))
+{
+  crypto_rand_unmocked(to, n);
+}
+
+/**
+ * Write <b>n</b> bytes of strong random data to <b>to</b>.  Most callers
+ * will want crypto_rand instead.
+ *
+ * This function is not allowed to fail; if it would fail to generate strong
+ * entropy, it must terminate the process instead.
+ **/
+void
+crypto_rand_unmocked(char *to, size_t n)
+{
+  int r;
+  if (n == 0)
+    return;
+
+  tor_assert(n < INT_MAX);
+  tor_assert(to);
+  r = RAND_bytes((unsigned char*)to, (int)n);
+  /* We consider a PRNG failure non-survivable. Let's assert so that we get a
+   * stack trace about where it happened.
+   */
+  tor_assert(r >= 0);
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
+ * INT_MAX+1, inclusive.
+ */
+int
+crypto_rand_int(unsigned int max)
+{
+  unsigned int val;
+  unsigned int cutoff;
+  tor_assert(max <= ((unsigned int)INT_MAX)+1);
+  tor_assert(max > 0); /* don't div by 0 */
+
+  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
+   * distribution with clipping at the upper end of unsigned int's
+   * range.
+   */
+  cutoff = UINT_MAX - (UINT_MAX%max);
+  while (1) {
+    crypto_rand((char*)&val, sizeof(val));
+    if (val < cutoff)
+      return val % max;
+  }
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values i such
+ * that min <= i < max.
+ *
+ * <b>min</b> MUST be in range [0, <b>max</b>).
+ * <b>max</b> MUST be in range (min, INT_MAX].
+ **/
+int
+crypto_rand_int_range(unsigned int min, unsigned int max)
+{
+  tor_assert(min < max);
+  tor_assert(max <= INT_MAX);
+
+  /* The overflow is avoided here because crypto_rand_int() returns a value
+   * between 0 and (max - min) inclusive. */
+  return min + crypto_rand_int(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports uint64_t.
+ **/
+uint64_t
+crypto_rand_uint64_range(uint64_t min, uint64_t max)
+{
+  tor_assert(min < max);
+  return min + crypto_rand_uint64(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports time_t.
+ **/
+time_t
+crypto_rand_time_range(time_t min, time_t max)
+{
+  tor_assert(min < max);
+  return min + (time_t)crypto_rand_uint64(max - min);
+}
+
+/**
+ * Return a pseudorandom 64-bit integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.
+ **/
+uint64_t
+crypto_rand_uint64(uint64_t max)
+{
+  uint64_t val;
+  uint64_t cutoff;
+  tor_assert(max < UINT64_MAX);
+  tor_assert(max > 0); /* don't div by 0 */
+
+  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
+   * distribution with clipping at the upper end of unsigned int's
+   * range.
+   */
+  cutoff = UINT64_MAX - (UINT64_MAX%max);
+  while (1) {
+    crypto_rand((char*)&val, sizeof(val));
+    if (val < cutoff)
+      return val % max;
+  }
+}
+
+/**
+ * Return a pseudorandom double d, chosen uniformly from the range
+ * 0.0 <= d < 1.0.
+ **/
+double
+crypto_rand_double(void)
+{
+  /* We just use an unsigned int here; we don't really care about getting
+   * more than 32 bits of resolution */
+  unsigned int u;
+  crypto_rand((char*)&u, sizeof(u));
+#if SIZEOF_INT == 4
+#define UINT_MAX_AS_DOUBLE 4294967296.0
+#elif SIZEOF_INT == 8
+#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
+#else
+#error SIZEOF_INT is neither 4 nor 8
+#endif /* SIZEOF_INT == 4 || ... */
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
+/**
+ * Generate and return a new random hostname starting with <b>prefix</b>,
+ * ending with <b>suffix</b>, and containing no fewer than
+ * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32
+ * characters. Does not check for failure.
+ *
+ * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE.
+ **/
+char *
+crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix,
+                       const char *suffix)
+{
+  char *result, *rand_bytes;
+  int randlen, rand_bytes_len;
+  size_t resultlen, prefixlen;
+
+  if (max_rand_len > MAX_DNS_LABEL_SIZE)
+    max_rand_len = MAX_DNS_LABEL_SIZE;
+  if (min_rand_len > max_rand_len)
+    min_rand_len = max_rand_len;
+
+  randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1);
+
+  prefixlen = strlen(prefix);
+  resultlen = prefixlen + strlen(suffix) + randlen + 16;
+
+  rand_bytes_len = ((randlen*5)+7)/8;
+  if (rand_bytes_len % 5)
+    rand_bytes_len += 5 - (rand_bytes_len%5);
+  rand_bytes = tor_malloc(rand_bytes_len);
+  crypto_rand(rand_bytes, rand_bytes_len);
+
+  result = tor_malloc(resultlen);
+  memcpy(result, prefix, prefixlen);
+  base32_encode(result+prefixlen, resultlen-prefixlen,
+                rand_bytes, rand_bytes_len);
+  tor_free(rand_bytes);
+  strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen));
+
+  return result;
+}
+
+/**
+ * Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b>
+ * is empty.
+ **/
+void *
+smartlist_choose(const smartlist_t *sl)
+{
+  int len = smartlist_len(sl);
+  if (len)
+    return smartlist_get(sl,crypto_rand_int(len));
+  return NULL; /* no elements to choose from */
+}
+
+/**
+ * Scramble the elements of <b>sl</b> into a random order.
+ **/
+void
+smartlist_shuffle(smartlist_t *sl)
+{
+  int i;
+  /* From the end of the list to the front, choose at random from the
+     positions we haven't looked at yet, and swap that position into the
+     current position.  Remember to give "no swap" the same probability as
+     any other swap. */
+  for (i = smartlist_len(sl)-1; i > 0; --i) {
+    int j = crypto_rand_int(i+1);
+    smartlist_swap(sl, i, j);
+  }
+}
+
+/** Make sure that openssl is using its default PRNG. Return 1 if we had to
+ * adjust it; 0 otherwise. */
+int
+crypto_force_rand_ssleay(void)
+{
+  RAND_METHOD *default_method;
+  default_method = RAND_OpenSSL();
+  if (RAND_get_rand_method() != default_method) {
+    log_notice(LD_CRYPTO, "It appears that one of our engines has provided "
+               "a replacement the OpenSSL RNG. Resetting it to the default "
+               "implementation.");
+    RAND_set_rand_method(default_method);
+    return 1;
+  }
+  return 0;
+}
+
+#endif /* !defined(CRYPTO_RAND_PRIVATE) */
+
diff --git a/src/common/crypto_rand.h b/src/common/crypto_rand.h
new file mode 100644
index 0000000000..bb02e51001
--- /dev/null
+++ b/src/common/crypto_rand.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand.h
+ *
+ * \brief Common functions for using (pseudo-)random number generators.
+ **/
+
+#ifndef TOR_CRYPTO_RAND_H
+#define TOR_CRYPTO_RAND_H
+
+#include "torint.h"
+#include "util.h"
+
+/* random numbers */
+int crypto_seed_rng(void) ATTR_WUR;
+MOCK_DECL(void,crypto_rand,(char *to, size_t n));
+void crypto_rand_unmocked(char *to, size_t n);
+void crypto_strongest_rand(uint8_t *out, size_t out_len);
+int crypto_rand_int(unsigned int max);
+int crypto_rand_int_range(unsigned int min, unsigned int max);
+uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
+time_t crypto_rand_time_range(time_t min, time_t max);
+uint64_t crypto_rand_uint64(uint64_t max);
+double crypto_rand_double(void);
+struct tor_weak_rng_t;
+void crypto_seed_weak_rng(struct tor_weak_rng_t *rng);
+
+char *crypto_random_hostname(int min_rand_len, int max_rand_len,
+                             const char *prefix, const char *suffix);
+
+struct smartlist_t;
+void *smartlist_choose(const struct smartlist_t *sl);
+void smartlist_shuffle(struct smartlist_t *sl);
+int crypto_force_rand_ssleay(void);
+
+#ifdef CRYPTO_RAND_PRIVATE
+
+STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
+
+#ifdef TOR_UNIT_TESTS
+extern int break_strongest_rng_syscall;
+extern int break_strongest_rng_fallback;
+#endif
+#endif /* defined(CRYPTO_RAND_PRIVATE) */
+
+#endif /* !defined(TOR_CRYPTO_RAND_H) */
+
diff --git a/src/common/crypto_rsa.c b/src/common/crypto_rsa.c
new file mode 100644
index 0000000000..f66cdef3c5
--- /dev/null
+++ b/src/common/crypto_rsa.c
@@ -0,0 +1,1183 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rsa.c
+ * \brief Block of functions related with RSA utilities and operations.
+ **/
+
+#include "crypto.h"
+#include "crypto_curve25519.h"
+#include "crypto_digest.h"
+#include "crypto_format.h"
+#include "compat_openssl.h"
+#include "crypto_rand.h"
+#include "crypto_rsa.h"
+#include "crypto_util.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/engine.h>
+#include <openssl/rand.h>
+#include <openssl/bn.h>
+#include <openssl/dh.h>
+#include <openssl/conf.h>
+#include <openssl/hmac.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+#include "torlog.h"
+#include "util.h"
+#include "util_format.h"
+
+/** Declaration for crypto_pk_t structure. */
+struct crypto_pk_t
+{
+  int refs; /**< reference count, so we don't have to copy keys */
+  RSA *key; /**< The key itself */
+};
+
+/** Log all pending crypto errors at level <b>severity</b>.  Use
+ * <b>doing</b> to describe our current activities.
+ */
+static void
+crypto_log_errors(int severity, const char *doing)
+{
+  unsigned long err;
+  const char *msg, *lib, *func;
+  while ((err = ERR_get_error()) != 0) {
+    msg = (const char*)ERR_reason_error_string(err);
+    lib = (const char*)ERR_lib_error_string(err);
+    func = (const char*)ERR_func_error_string(err);
+    if (!msg) msg = "(null)";
+    if (!lib) lib = "(null)";
+    if (!func) func = "(null)";
+    if (BUG(!doing)) doing = "(null)";
+    tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)",
+              doing, msg, lib, func);
+  }
+}
+
+/** Return the number of bytes added by padding method <b>padding</b>.
+ */
+int
+crypto_get_rsa_padding_overhead(int padding)
+{
+  switch (padding)
+    {
+    case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD;
+    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
+    }
+}
+
+/** Given a padding method <b>padding</b>, return the correct OpenSSL constant.
+ */
+int
+crypto_get_rsa_padding(int padding)
+{
+  switch (padding)
+    {
+    case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING;
+    default: tor_assert(0); return -1; // LCOV_EXCL_LINE
+    }
+}
+
+/** used internally: quicly validate a crypto_pk_t object as a private key.
+ * Return 1 iff the public key is valid, 0 if obviously invalid.
+ */
+static int
+crypto_pk_private_ok(const crypto_pk_t *k)
+{
+#ifdef OPENSSL_1_1_API
+  if (!k || !k->key)
+    return 0;
+
+  const BIGNUM *p, *q;
+  RSA_get0_factors(k->key, &p, &q);
+  return p != NULL; /* XXX/yawning: Should we check q? */
+#else /* !(defined(OPENSSL_1_1_API)) */
+  return k && k->key && k->key->p;
+#endif /* defined(OPENSSL_1_1_API) */
+}
+
+/** used by tortls.c: wrap an RSA* in a crypto_pk_t. */
+crypto_pk_t *
+crypto_new_pk_from_rsa_(RSA *rsa)
+{
+  crypto_pk_t *env;
+  tor_assert(rsa);
+  env = tor_malloc(sizeof(crypto_pk_t));
+  env->refs = 1;
+  env->key = rsa;
+  return env;
+}
+
+/** Helper, used by tor-gencert.c.  Return the RSA from a
+ * crypto_pk_t. */
+RSA *
+crypto_pk_get_rsa_(crypto_pk_t *env)
+{
+  return env->key;
+}
+
+/** used by tortls.c: get an equivalent EVP_PKEY* for a crypto_pk_t.  Iff
+ * private is set, include the private-key portion of the key. Return a valid
+ * pointer on success, and NULL on failure. */
+MOCK_IMPL(EVP_PKEY *,
+crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private))
+{
+  RSA *key = NULL;
+  EVP_PKEY *pkey = NULL;
+  tor_assert(env->key);
+  if (private) {
+    if (!(key = RSAPrivateKey_dup(env->key)))
+      goto error;
+  } else {
+    if (!(key = RSAPublicKey_dup(env->key)))
+      goto error;
+  }
+  if (!(pkey = EVP_PKEY_new()))
+    goto error;
+  if (!(EVP_PKEY_assign_RSA(pkey, key)))
+    goto error;
+  return pkey;
+ error:
+  if (pkey)
+    EVP_PKEY_free(pkey);
+  if (key)
+    RSA_free(key);
+  return NULL;
+}
+
+/** Allocate and return storage for a public key.  The key itself will not yet
+ * be set.
+ */
+MOCK_IMPL(crypto_pk_t *,
+crypto_pk_new,(void))
+{
+  RSA *rsa;
+
+  rsa = RSA_new();
+  tor_assert(rsa);
+  return crypto_new_pk_from_rsa_(rsa);
+}
+
+/** Release a reference to an asymmetric key; when all the references
+ * are released, free the key.
+ */
+void
+crypto_pk_free_(crypto_pk_t *env)
+{
+  if (!env)
+    return;
+
+  if (--env->refs > 0)
+    return;
+  tor_assert(env->refs == 0);
+
+  if (env->key)
+    RSA_free(env->key);
+
+  tor_free(env);
+}
+
+/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits))
+{
+  tor_assert(env);
+
+  if (env->key) {
+    RSA_free(env->key);
+    env->key = NULL;
+  }
+
+  {
+    BIGNUM *e = BN_new();
+    RSA *r = NULL;
+    if (!e)
+      goto done;
+    if (! BN_set_word(e, 65537))
+      goto done;
+    r = RSA_new();
+    if (!r)
+      goto done;
+    if (RSA_generate_key_ex(r, bits, e, NULL) == -1)
+      goto done;
+
+    env->key = r;
+    r = NULL;
+  done:
+    if (e)
+      BN_clear_free(e);
+    if (r)
+      RSA_free(r);
+  }
+
+  if (!env->key) {
+    crypto_log_errors(LOG_WARN, "generating RSA key");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** A PEM callback that always reports a failure to get a password */
+static int
+pem_no_password_cb(char *buf, int size, int rwflag, void *u)
+{
+  (void)buf;
+  (void)size;
+  (void)rwflag;
+  (void)u;
+  return -1;
+}
+
+/** Read a PEM-encoded private key from the <b>len</b>-byte string <b>s</b>
+ * into <b>env</b>.  Return 0 on success, -1 on failure.  If len is -1,
+ * the string is nul-terminated.
+ */
+int
+crypto_pk_read_private_key_from_string(crypto_pk_t *env,
+                                       const char *s, ssize_t len)
+{
+  BIO *b;
+
+  tor_assert(env);
+  tor_assert(s);
+  tor_assert(len < INT_MAX && len < SSIZE_T_CEILING);
+
+  /* Create a read-only memory BIO, backed by the string 's' */
+  b = BIO_new_mem_buf((char*)s, (int)len);
+  if (!b)
+    return -1;
+
+  if (env->key)
+    RSA_free(env->key);
+
+  env->key = PEM_read_bio_RSAPrivateKey(b,NULL,pem_no_password_cb,NULL);
+
+  BIO_free(b);
+
+  if (!env->key) {
+    crypto_log_errors(LOG_WARN, "Error parsing private key");
+    return -1;
+  }
+  return 0;
+}
+
+/** Read a PEM-encoded private key from the file named by
+ * <b>keyfile</b> into <b>env</b>.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
+                                         const char *keyfile)
+{
+  char *contents;
+  int r;
+
+  /* Read the file into a string. */
+  contents = read_file_to_str(keyfile, 0, NULL);
+  if (!contents) {
+    log_warn(LD_CRYPTO, "Error reading private key from \"%s\"", keyfile);
+    return -1;
+  }
+
+  /* Try to parse it. */
+  r = crypto_pk_read_private_key_from_string(env, contents, -1);
+  memwipe(contents, 0, strlen(contents));
+  tor_free(contents);
+  if (r)
+    return -1; /* read_private_key_from_string already warned, so we don't.*/
+
+  /* Make sure it's valid. */
+  if (crypto_pk_check_key(env) <= 0)
+    return -1;
+
+  return 0;
+}
+
+/** Helper function to implement crypto_pk_write_*_key_to_string. Return 0 on
+ * success, -1 on failure. */
+static int
+crypto_pk_write_key_to_string_impl(crypto_pk_t *env, char **dest,
+                                   size_t *len, int is_public)
+{
+  BUF_MEM *buf;
+  BIO *b;
+  int r;
+
+  tor_assert(env);
+  tor_assert(env->key);
+  tor_assert(dest);
+
+  b = BIO_new(BIO_s_mem()); /* Create a memory BIO */
+  if (!b)
+    return -1;
+
+  /* Now you can treat b as if it were a file.  Just use the
+   * PEM_*_bio_* functions instead of the non-bio variants.
+   */
+  if (is_public)
+    r = PEM_write_bio_RSAPublicKey(b, env->key);
+  else
+    r = PEM_write_bio_RSAPrivateKey(b, env->key, NULL,NULL,0,NULL,NULL);
+
+  if (!r) {
+    crypto_log_errors(LOG_WARN, "writing RSA key to string");
+    BIO_free(b);
+    return -1;
+  }
+
+  BIO_get_mem_ptr(b, &buf);
+
+  *dest = tor_malloc(buf->length+1);
+  memcpy(*dest, buf->data, buf->length);
+  (*dest)[buf->length] = 0; /* nul terminate it */
+  *len = buf->length;
+
+  BIO_free(b);
+
+  return 0;
+}
+
+/** PEM-encode the public key portion of <b>env</b> and write it to a
+ * newly allocated string.  On success, set *<b>dest</b> to the new
+ * string, *<b>len</b> to the string's length, and return 0.  On
+ * failure, return -1.
+ */
+int
+crypto_pk_write_public_key_to_string(crypto_pk_t *env, char **dest,
+                                     size_t *len)
+{
+  return crypto_pk_write_key_to_string_impl(env, dest, len, 1);
+}
+
+/** PEM-encode the private key portion of <b>env</b> and write it to a
+ * newly allocated string.  On success, set *<b>dest</b> to the new
+ * string, *<b>len</b> to the string's length, and return 0.  On
+ * failure, return -1.
+ */
+int
+crypto_pk_write_private_key_to_string(crypto_pk_t *env, char **dest,
+                                     size_t *len)
+{
+  return crypto_pk_write_key_to_string_impl(env, dest, len, 0);
+}
+
+/** Read a PEM-encoded public key from the first <b>len</b> characters of
+ * <b>src</b>, and store the result in <b>env</b>.  Return 0 on success, -1 on
+ * failure.
+ */
+int
+crypto_pk_read_public_key_from_string(crypto_pk_t *env, const char *src,
+                                      size_t len)
+{
+  BIO *b;
+
+  tor_assert(env);
+  tor_assert(src);
+  tor_assert(len<INT_MAX);
+
+  b = BIO_new(BIO_s_mem()); /* Create a memory BIO */
+  if (!b)
+    return -1;
+
+  BIO_write(b, src, (int)len);
+
+  if (env->key)
+    RSA_free(env->key);
+  env->key = PEM_read_bio_RSAPublicKey(b, NULL, pem_no_password_cb, NULL);
+  BIO_free(b);
+  if (!env->key) {
+    crypto_log_errors(LOG_WARN, "reading public key from string");
+    return -1;
+  }
+
+  return 0;
+}
+
+/** Write the private key from <b>env</b> into the file named by <b>fname</b>,
+ * PEM-encoded.  Return 0 on success, -1 on failure.
+ */
+int
+crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
+                                        const char *fname)
+{
+  BIO *bio;
+  char *cp;
+  long len;
+  char *s;
+  int r;
+
+  tor_assert(crypto_pk_private_ok(env));
+
+  if (!(bio = BIO_new(BIO_s_mem())))
+    return -1;
+  if (PEM_write_bio_RSAPrivateKey(bio, env->key, NULL,NULL,0,NULL,NULL)
+      == 0) {
+    crypto_log_errors(LOG_WARN, "writing private key");
+    BIO_free(bio);
+    return -1;
+  }
+  len = BIO_get_mem_data(bio, &cp);
+  tor_assert(len >= 0);
+  s = tor_malloc(len+1);
+  memcpy(s, cp, len);
+  s[len]='\0';
+  r = write_str_to_file(fname, s, 0);
+  BIO_free(bio);
+  memwipe(s, 0, strlen(s));
+  tor_free(s);
+  return r;
+}
+
+/** Return true iff <b>env</b> has a valid key.
+ */
+int
+crypto_pk_check_key(crypto_pk_t *env)
+{
+  int r;
+  tor_assert(env);
+
+  r = RSA_check_key(env->key);
+  if (r <= 0)
+    crypto_log_errors(LOG_WARN,"checking RSA key");
+  return r;
+}
+
+/** Return true iff <b>key</b> contains the private-key portion of the RSA
+ * key. */
+int
+crypto_pk_key_is_private(const crypto_pk_t *key)
+{
+  tor_assert(key);
+  return crypto_pk_private_ok(key);
+}
+
+/** Return true iff <b>env</b> contains a public key whose public exponent
+ * equals 65537.
+ */
+int
+crypto_pk_public_exponent_ok(crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+  const BIGNUM *e;
+
+#ifdef OPENSSL_1_1_API
+  const BIGNUM *n, *d;
+  RSA_get0_key(env->key, &n, &e, &d);
+#else
+  e = env->key->e;
+#endif /* defined(OPENSSL_1_1_API) */
+  return BN_is_word(e, 65537);
+}
+
+/** Compare the public-key components of a and b.  Return less than 0
+ * if a\<b, 0 if a==b, and greater than 0 if a\>b.  A NULL key is
+ * considered to be less than all non-NULL keys, and equal to itself.
+ *
+ * Note that this may leak information about the keys through timing.
+ */
+int
+crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b)
+{
+  int result;
+  char a_is_non_null = (a != NULL) && (a->key != NULL);
+  char b_is_non_null = (b != NULL) && (b->key != NULL);
+  char an_argument_is_null = !a_is_non_null | !b_is_non_null;
+
+  result = tor_memcmp(&a_is_non_null, &b_is_non_null, sizeof(a_is_non_null));
+  if (an_argument_is_null)
+    return result;
+
+  const BIGNUM *a_n, *a_e;
+  const BIGNUM *b_n, *b_e;
+
+#ifdef OPENSSL_1_1_API
+  const BIGNUM *a_d, *b_d;
+  RSA_get0_key(a->key, &a_n, &a_e, &a_d);
+  RSA_get0_key(b->key, &b_n, &b_e, &b_d);
+#else
+  a_n = a->key->n;
+  a_e = a->key->e;
+  b_n = b->key->n;
+  b_e = b->key->e;
+#endif /* defined(OPENSSL_1_1_API) */
+
+  tor_assert(a_n != NULL && a_e != NULL);
+  tor_assert(b_n != NULL && b_e != NULL);
+
+  result = BN_cmp(a_n, b_n);
+  if (result)
+    return result;
+  return BN_cmp(a_e, b_e);
+}
+
+/** Compare the public-key components of a and b.  Return non-zero iff
+ * a==b.  A NULL key is considered to be distinct from all non-NULL
+ * keys, and equal to itself.
+ *
+ *  Note that this may leak information about the keys through timing.
+ */
+int
+crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b)
+{
+  return (crypto_pk_cmp_keys(a, b) == 0);
+}
+
+/** Return the size of the public key modulus in <b>env</b>, in bytes. */
+size_t
+crypto_pk_keysize(const crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+  return (size_t) RSA_size((RSA*)env->key);
+}
+
+/** Return the size of the public key modulus of <b>env</b>, in bits. */
+int
+crypto_pk_num_bits(crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+#ifdef OPENSSL_1_1_API
+  /* It's so stupid that there's no other way to check that n is valid
+   * before calling RSA_bits().
+   */
+  const BIGNUM *n, *e, *d;
+  RSA_get0_key(env->key, &n, &e, &d);
+  tor_assert(n != NULL);
+
+  return RSA_bits(env->key);
+#else /* !(defined(OPENSSL_1_1_API)) */
+  tor_assert(env->key->n);
+  return BN_num_bits(env->key->n);
+#endif /* defined(OPENSSL_1_1_API) */
+}
+
+/** Increase the reference count of <b>env</b>, and return it.
+ */
+crypto_pk_t *
+crypto_pk_dup_key(crypto_pk_t *env)
+{
+  tor_assert(env);
+  tor_assert(env->key);
+
+  env->refs++;
+  return env;
+}
+
+#ifdef TOR_UNIT_TESTS
+/** For testing: replace dest with src.  (Dest must have a refcount
+ * of 1) */
+void
+crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src)
+{
+  tor_assert(dest);
+  tor_assert(dest->refs == 1);
+  tor_assert(src);
+  RSA_free(dest->key);
+  dest->key = RSAPrivateKey_dup(src->key);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** Make a real honest-to-goodness copy of <b>env</b>, and return it.
+ * Returns NULL on failure. */
+crypto_pk_t *
+crypto_pk_copy_full(crypto_pk_t *env)
+{
+  RSA *new_key;
+  int privatekey = 0;
+  tor_assert(env);
+  tor_assert(env->key);
+
+  if (crypto_pk_private_ok(env)) {
+    new_key = RSAPrivateKey_dup(env->key);
+    privatekey = 1;
+  } else {
+    new_key = RSAPublicKey_dup(env->key);
+  }
+  if (!new_key) {
+    /* LCOV_EXCL_START
+     *
+     * We can't cause RSA*Key_dup() to fail, so we can't really test this.
+     */
+    log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.",
+            privatekey?"private":"public");
+    crypto_log_errors(LOG_ERR,
+                      privatekey ? "Duplicating a private key" :
+                      "Duplicating a public key");
+    tor_fragile_assert();
+    return NULL;
+    /* LCOV_EXCL_STOP */
+  }
+
+  return crypto_new_pk_from_rsa_(new_key);
+}
+
+/** Perform a hybrid (public/secret) encryption on <b>fromlen</b>
+ * bytes of data from <b>from</b>, with padding type 'padding',
+ * storing the results on <b>to</b>.
+ *
+ * Returns the number of bytes written on success, -1 on failure.
+ *
+ * The encrypted data consists of:
+ *   - The source data, padded and encrypted with the public key, if the
+ *     padded source data is no longer than the public key, and <b>force</b>
+ *     is false, OR
+ *   - The beginning of the source data prefixed with a 16-byte symmetric key,
+ *     padded and encrypted with the public key; followed by the rest of
+ *     the source data encrypted in AES-CTR mode with the symmetric key.
+ *
+ * NOTE that this format does not authenticate the symmetrically encrypted
+ * part of the data, and SHOULD NOT BE USED for new protocols.
+ */
+int
+crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env,
+                                char *to, size_t tolen,
+                                const char *from,
+                                size_t fromlen,
+                                int padding, int force)
+{
+  int overhead, outlen, r;
+  size_t pkeylen, symlen;
+  crypto_cipher_t *cipher = NULL;
+  char *buf = NULL;
+
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < SIZE_T_CEILING);
+
+  overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding));
+  pkeylen = crypto_pk_keysize(env);
+
+  if (!force && fromlen+overhead <= pkeylen) {
+    /* It all fits in a single encrypt. */
+    return crypto_pk_public_encrypt(env,to,
+                                    tolen,
+                                    from,fromlen,padding);
+  }
+  tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN);
+  tor_assert(tolen >= pkeylen);
+
+  char key[CIPHER_KEY_LEN];
+  crypto_rand(key, sizeof(key)); /* generate a new key. */
+  cipher = crypto_cipher_new(key);
+
+  buf = tor_malloc(pkeylen+1);
+  memcpy(buf, key, CIPHER_KEY_LEN);
+  memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN);
+
+  /* Length of symmetrically encrypted data. */
+  symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN);
+
+  outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding);
+  if (outlen!=(int)pkeylen) {
+    goto err;
+  }
+  r = crypto_cipher_encrypt(cipher, to+outlen,
+                            from+pkeylen-overhead-CIPHER_KEY_LEN, symlen);
+
+  if (r<0) goto err;
+  memwipe(buf, 0, pkeylen);
+  memwipe(key, 0, sizeof(key));
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  tor_assert(outlen+symlen < INT_MAX);
+  return (int)(outlen + symlen);
+ err:
+
+  memwipe(buf, 0, pkeylen);
+  memwipe(key, 0, sizeof(key));
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  return -1;
+}
+
+/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of
+ * bytes written on success, -1 on failure.
+ *
+ * NOTE that this format does not authenticate the symmetrically encrypted
+ * part of the data, and SHOULD NOT BE USED for new protocols.
+ */
+int
+crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env,
+                                 char *to,
+                                 size_t tolen,
+                                 const char *from,
+                                 size_t fromlen,
+                                 int padding, int warnOnFailure)
+{
+  int outlen, r;
+  size_t pkeylen;
+  crypto_cipher_t *cipher = NULL;
+  char *buf = NULL;
+
+  tor_assert(fromlen < SIZE_T_CEILING);
+  pkeylen = crypto_pk_keysize(env);
+
+  if (fromlen <= pkeylen) {
+    return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding,
+                                     warnOnFailure);
+  }
+
+  buf = tor_malloc(pkeylen);
+  outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding,
+                                     warnOnFailure);
+  if (outlen<0) {
+    log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO,
+           "Error decrypting public-key data");
+    goto err;
+  }
+  if (outlen < CIPHER_KEY_LEN) {
+    log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO,
+           "No room for a symmetric key");
+    goto err;
+  }
+  cipher = crypto_cipher_new(buf);
+  if (!cipher) {
+    goto err;
+  }
+  memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN);
+  outlen -= CIPHER_KEY_LEN;
+  tor_assert(tolen - outlen >= fromlen - pkeylen);
+  r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen);
+  if (r<0)
+    goto err;
+  memwipe(buf,0,pkeylen);
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  tor_assert(outlen + fromlen < INT_MAX);
+  return (int)(outlen + (fromlen-pkeylen));
+ err:
+  memwipe(buf,0,pkeylen);
+  tor_free(buf);
+  crypto_cipher_free(cipher);
+  return -1;
+}
+
+/** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key
+ * in <b>env</b>, using the padding method <b>padding</b>.  On success,
+ * write the result to <b>to</b>, and return the number of bytes
+ * written.  On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
+                         const char *from, size_t fromlen, int padding)
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen<INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+
+  r = RSA_public_encrypt((int)fromlen,
+                         (unsigned char*)from, (unsigned char*)to,
+                         env->key, crypto_get_rsa_padding(padding));
+  if (r<0) {
+    crypto_log_errors(LOG_WARN, "performing RSA encryption");
+    return -1;
+  }
+  return r;
+}
+
+/** Decrypt <b>fromlen</b> bytes from <b>from</b> with the private key
+ * in <b>env</b>, using the padding method <b>padding</b>.  On success,
+ * write the result to <b>to</b>, and return the number of bytes
+ * written.  On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_decrypt(crypto_pk_t *env, char *to,
+                          size_t tolen,
+                          const char *from, size_t fromlen,
+                          int padding, int warnOnFailure)
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(env->key);
+  tor_assert(fromlen<INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  if (!crypto_pk_key_is_private(env))
+    /* Not a private key */
+    return -1;
+
+  r = RSA_private_decrypt((int)fromlen,
+                          (unsigned char*)from, (unsigned char*)to,
+                          env->key, crypto_get_rsa_padding(padding));
+
+  if (r<0) {
+    crypto_log_errors(warnOnFailure?LOG_WARN:LOG_DEBUG,
+                      "performing RSA decryption");
+    return -1;
+  }
+  return r;
+}
+
+/** Check the signature in <b>from</b> (<b>fromlen</b> bytes long) with the
+ * public key in <b>env</b>, using PKCS1 padding.  On success, write the
+ * signed data to <b>to</b>, and return the number of bytes written.
+ * On failure, return -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+MOCK_IMPL(int,
+crypto_pk_public_checksig,(const crypto_pk_t *env, char *to,
+                           size_t tolen,
+                           const char *from, size_t fromlen))
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  r = RSA_public_decrypt((int)fromlen,
+                         (unsigned char*)from, (unsigned char*)to,
+                         env->key, RSA_PKCS1_PADDING);
+
+  if (r<0) {
+    crypto_log_errors(LOG_INFO, "checking RSA signature");
+    return -1;
+  }
+  return r;
+}
+
+/** Sign <b>fromlen</b> bytes of data from <b>from</b> with the private key in
+ * <b>env</b>, using PKCS1 padding.  On success, write the signature to
+ * <b>to</b>, and return the number of bytes written.  On failure, return
+ * -1.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
+                       const char *from, size_t fromlen)
+{
+  int r;
+  tor_assert(env);
+  tor_assert(from);
+  tor_assert(to);
+  tor_assert(fromlen < INT_MAX);
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  if (!crypto_pk_key_is_private(env))
+    /* Not a private key */
+    return -1;
+
+  r = RSA_private_encrypt((int)fromlen,
+                          (unsigned char*)from, (unsigned char*)to,
+                          (RSA*)env->key, RSA_PKCS1_PADDING);
+  if (r<0) {
+    crypto_log_errors(LOG_WARN, "generating RSA signature");
+    return -1;
+  }
+  return r;
+}
+
+/** ASN.1-encode the public portion of <b>pk</b> into <b>dest</b>.
+ * Return -1 on error, or the number of characters used on success.
+ */
+int
+crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len)
+{
+  int len;
+  unsigned char *buf = NULL;
+
+  len = i2d_RSAPublicKey(pk->key, &buf);
+  if (len < 0 || buf == NULL)
+    return -1;
+
+  if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) {
+    OPENSSL_free(buf);
+    return -1;
+  }
+  /* We don't encode directly into 'dest', because that would be illegal
+   * type-punning.  (C99 is smarter than me, C99 is smarter than me...)
+   */
+  memcpy(dest,buf,len);
+  OPENSSL_free(buf);
+  return len;
+}
+
+/** Decode an ASN.1-encoded public key from <b>str</b>; return the result on
+ * success and NULL on failure.
+ */
+crypto_pk_t *
+crypto_pk_asn1_decode(const char *str, size_t len)
+{
+  RSA *rsa;
+  unsigned char *buf;
+  const unsigned char *cp;
+  cp = buf = tor_malloc(len);
+  memcpy(buf,str,len);
+  rsa = d2i_RSAPublicKey(NULL, &cp, len);
+  tor_free(buf);
+  if (!rsa) {
+    crypto_log_errors(LOG_WARN,"decoding public key");
+    return NULL;
+  }
+  return crypto_new_pk_from_rsa_(rsa);
+}
+
+/** Given a private or public key <b>pk</b>, put a fingerprint of the
+ * public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 bytes of
+ * space).  Return 0 on success, -1 on failure.
+ *
+ * Fingerprints are computed as the SHA1 digest of the ASN.1 encoding
+ * of the public key, converted to hexadecimal, in upper case, with a
+ * space after every four digits.
+ *
+ * If <b>add_space</b> is false, omit the spaces.
+ */
+int
+crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space)
+{
+  char digest[DIGEST_LEN];
+  char hexdigest[HEX_DIGEST_LEN+1];
+  if (crypto_pk_get_digest(pk, digest)) {
+    return -1;
+  }
+  base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN);
+  if (add_space) {
+    crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest);
+  } else {
+    strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1);
+  }
+  return 0;
+}
+
+/** Given a private or public key <b>pk</b>, put a hashed fingerprint of
+ * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1
+ * bytes of space).  Return 0 on success, -1 on failure.
+ *
+ * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest
+ * of the ASN.1 encoding of the public key, converted to hexadecimal, in
+ * upper case.
+ */
+int
+crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out)
+{
+  char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN];
+  if (crypto_pk_get_digest(pk, digest)) {
+    return -1;
+  }
+  if (crypto_digest(hashed_digest, digest, DIGEST_LEN) < 0) {
+    return -1;
+  }
+  base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN);
+  return 0;
+}
+
+/** Check a siglen-byte long signature at <b>sig</b> against
+ * <b>datalen</b> bytes of data at <b>data</b>, using the public key
+ * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for
+ * SHA1(data).  Else return -1.
+ */
+MOCK_IMPL(int,
+crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data,
+                                  size_t datalen, const char *sig,
+                                  size_t siglen))
+{
+  char digest[DIGEST_LEN];
+  char *buf;
+  size_t buflen;
+  int r;
+
+  tor_assert(env);
+  tor_assert(data);
+  tor_assert(sig);
+  tor_assert(datalen < SIZE_T_CEILING);
+  tor_assert(siglen < SIZE_T_CEILING);
+
+  if (crypto_digest(digest,data,datalen)<0) {
+    log_warn(LD_BUG, "couldn't compute digest");
+    return -1;
+  }
+  buflen = crypto_pk_keysize(env);
+  buf = tor_malloc(buflen);
+  r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen);
+  if (r != DIGEST_LEN) {
+    log_warn(LD_CRYPTO, "Invalid signature");
+    tor_free(buf);
+    return -1;
+  }
+  if (tor_memneq(buf, digest, DIGEST_LEN)) {
+    log_warn(LD_CRYPTO, "Signature mismatched with digest.");
+    tor_free(buf);
+    return -1;
+  }
+  tor_free(buf);
+
+  return 0;
+}
+
+/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at
+ * <b>from</b>; sign the data with the private key in <b>env</b>, and
+ * store it in <b>to</b>.  Return the number of bytes written on
+ * success, and -1 on failure.
+ *
+ * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
+ * at least the length of the modulus of <b>env</b>.
+ */
+int
+crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
+                              const char *from, size_t fromlen)
+{
+  int r;
+  char digest[DIGEST_LEN];
+  if (crypto_digest(digest,from,fromlen)<0)
+    return -1;
+  r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN);
+  memwipe(digest, 0, sizeof(digest));
+  return r;
+}
+
+/** Given a private or public key <b>pk</b>, put a SHA1 hash of the
+ * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space).
+ * Return 0 on success, -1 on failure.
+ */
+int
+crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out)
+{
+  char *buf;
+  size_t buflen;
+  int len;
+  int rv = -1;
+
+  buflen = crypto_pk_keysize(pk)*2;
+  buf = tor_malloc(buflen);
+  len = crypto_pk_asn1_encode(pk, buf, buflen);
+  if (len < 0)
+    goto done;
+
+  if (crypto_digest(digest_out, buf, len) < 0)
+    goto done;
+
+  rv = 0;
+  done:
+  tor_free(buf);
+  return rv;
+}
+
+/** Compute all digests of the DER encoding of <b>pk</b>, and store them
+ * in <b>digests_out</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out)
+{
+  char *buf;
+  size_t buflen;
+  int len;
+  int rv = -1;
+
+  buflen = crypto_pk_keysize(pk)*2;
+  buf = tor_malloc(buflen);
+  len = crypto_pk_asn1_encode(pk, buf, buflen);
+  if (len < 0)
+    goto done;
+
+  if (crypto_common_digests(digests_out, (char*)buf, len) < 0)
+    goto done;
+
+  rv = 0;
+ done:
+  tor_free(buf);
+  return rv;
+}
+
+/** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the
+ * Base64 encoding of the DER representation of the private key as a NUL
+ * terminated string, and return it via <b>priv_out</b>.  Return 0 on
+ * success, -1 on failure.
+ *
+ * It is the caller's responsibility to sanitize and free the resulting buffer.
+ */
+int
+crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out)
+{
+  unsigned char *der = NULL;
+  int der_len;
+  int ret = -1;
+
+  *priv_out = NULL;
+
+  der_len = i2d_RSAPrivateKey(pk->key, &der);
+  if (der_len < 0 || der == NULL)
+    return ret;
+
+  size_t priv_len = base64_encode_size(der_len, 0) + 1;
+  char *priv = tor_malloc_zero(priv_len);
+  if (base64_encode(priv, priv_len, (char *)der, der_len, 0) >= 0) {
+    *priv_out = priv;
+    ret = 0;
+  } else {
+    tor_free(priv);
+  }
+
+  memwipe(der, 0, der_len);
+  OPENSSL_free(der);
+  return ret;
+}
+
+/** Given a string containing the Base64 encoded DER representation of the
+ * private key <b>str</b>, decode and return the result on success, or NULL
+ * on failure.
+ */
+crypto_pk_t *
+crypto_pk_base64_decode(const char *str, size_t len)
+{
+  crypto_pk_t *pk = NULL;
+
+  char *der = tor_malloc_zero(len + 1);
+  int der_len = base64_decode(der, len, str, len);
+  if (der_len <= 0) {
+    log_warn(LD_CRYPTO, "Stored RSA private key seems corrupted (base64).");
+    goto out;
+  }
+
+  const unsigned char *dp = (unsigned char*)der; /* Shut the compiler up. */
+  RSA *rsa = d2i_RSAPrivateKey(NULL, &dp, der_len);
+  if (!rsa) {
+    crypto_log_errors(LOG_WARN, "decoding private key");
+    goto out;
+  }
+
+  pk = crypto_new_pk_from_rsa_(rsa);
+
+  /* Make sure it's valid. */
+  if (crypto_pk_check_key(pk) <= 0) {
+    crypto_pk_free(pk);
+    pk = NULL;
+    goto out;
+  }
+
+ out:
+  memwipe(der, 0, len + 1);
+  tor_free(der);
+  return pk;
+}
+
diff --git a/src/common/crypto_rsa.h b/src/common/crypto_rsa.h
new file mode 100644
index 0000000000..e952089318
--- /dev/null
+++ b/src/common/crypto_rsa.h
@@ -0,0 +1,119 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rsa.h
+ *
+ * \brief Headers for crypto_rsa.c
+ **/
+
+#ifndef TOR_CRYPTO_RSA_H
+#define TOR_CRYPTO_RSA_H
+
+#include "orconfig.h"
+
+#include "crypto_digest.h"
+#include <stdio.h>
+#include "torint.h"
+#include "testsupport.h"
+#include "compat.h"
+#include "util.h"
+#include "torlog.h"
+
+/** Length of our public keys. */
+#define PK_BYTES (1024/8)
+
+/** Constant used to indicate OAEP padding for public-key encryption */
+#define PK_PKCS1_OAEP_PADDING 60002
+
+/** Number of bytes added for PKCS1-OAEP padding. */
+#define PKCS1_OAEP_PADDING_OVERHEAD 42
+
+/** A public key, or a public/private key-pair. */
+typedef struct crypto_pk_t crypto_pk_t;
+
+/* RSA environment setup */
+MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void));
+void crypto_pk_free_(crypto_pk_t *env);
+#define crypto_pk_free(pk) FREE_AND_NULL(crypto_pk_t, crypto_pk_free_, (pk))
+int crypto_get_rsa_padding_overhead(int padding);
+int crypto_get_rsa_padding(int padding);
+
+/* public key crypto */
+MOCK_DECL(int, crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits));
+#define crypto_pk_generate_key(env)                     \
+  crypto_pk_generate_key_with_bits((env), (PK_BYTES*8))
+
+int crypto_pk_read_private_key_from_filename(crypto_pk_t *env,
+                                             const char *keyfile);
+int crypto_pk_write_public_key_to_string(crypto_pk_t *env,
+                                         char **dest, size_t *len);
+int crypto_pk_write_private_key_to_string(crypto_pk_t *env,
+                                          char **dest, size_t *len);
+int crypto_pk_read_public_key_from_string(crypto_pk_t *env,
+                                          const char *src, size_t len);
+int crypto_pk_read_private_key_from_string(crypto_pk_t *env,
+                                           const char *s, ssize_t len);
+int crypto_pk_write_private_key_to_filename(crypto_pk_t *env,
+                                            const char *fname);
+
+int crypto_pk_check_key(crypto_pk_t *env);
+int crypto_pk_cmp_keys(const crypto_pk_t *a, const crypto_pk_t *b);
+int crypto_pk_eq_keys(const crypto_pk_t *a, const crypto_pk_t *b);
+size_t crypto_pk_keysize(const crypto_pk_t *env);
+int crypto_pk_num_bits(crypto_pk_t *env);
+crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig);
+crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig);
+int crypto_pk_key_is_private(const crypto_pk_t *key);
+int crypto_pk_public_exponent_ok(crypto_pk_t *env);
+int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to,
+                                    size_t tolen,
+                                    const char *from, size_t fromlen,
+                                    int padding, int force);
+int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to,
+                                     size_t tolen,
+                                     const char *from, size_t fromlen,
+                                     int padding, int warnOnFailure);
+int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
+                             const char *from, size_t fromlen, int padding);
+int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen,
+                              const char *from, size_t fromlen,
+                              int padding, int warnOnFailure);
+MOCK_DECL(int, crypto_pk_public_checksig,(const crypto_pk_t *env,
+                                          char *to, size_t tolen,
+                                          const char *from, size_t fromlen));
+int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
+                           const char *from, size_t fromlen);
+int crypto_pk_asn1_encode(const crypto_pk_t *pk, char *dest, size_t dest_len);
+crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len);
+int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space);
+int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out);
+
+MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env,
+          const char *data, size_t datalen, const char *sig, size_t siglen));
+int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
+                                  const char *from, size_t fromlen);
+int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out);
+int crypto_pk_get_common_digests(crypto_pk_t *pk,
+                                 common_digests_t *digests_out);
+int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out);
+crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len);
+
+/* Prototypes for private functions only used by tortls.c, crypto.c, and the
+ * unit tests. */
+struct rsa_st;
+struct rsa_st *crypto_pk_get_rsa_(crypto_pk_t *env);
+crypto_pk_t *crypto_new_pk_from_rsa_(struct rsa_st *rsa);
+MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_evp_pkey_,(crypto_pk_t *env,
+                                                         int private));
+struct evp_pkey_st;
+
+#ifdef TOR_UNIT_TESTS
+void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src);
+#endif
+
+#endif
+
diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c
index 5dbd2ad91f..8543760ec5 100644
--- a/src/common/crypto_s2k.c
+++ b/src/common/crypto_s2k.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,10 +12,13 @@
 
 #define CRYPTO_S2K_PRIVATE
 
-#include "crypto.h"
-#include "util.h"
 #include "compat.h"
+#include "crypto.h"
+#include "crypto_digest.h"
+#include "crypto_rand.h"
 #include "crypto_s2k.h"
+#include "crypto_util.h"
+#include "util.h"
 
 #include <openssl/evp.h>
 
@@ -86,9 +89,11 @@ secret_to_key_key_len(uint8_t type)
       return DIGEST_LEN;
     case S2K_TYPE_SCRYPT:
       return DIGEST256_LEN;
+    // LCOV_EXCL_START
     default:
-      tor_fragile_assert(); // LCOV_EXCL_LINE
-      return -1;            // LCOV_EXCL_LINE
+      tor_fragile_assert();
+      return -1;
+    // LCOV_EXCL_STOP
   }
 }
 
@@ -169,9 +174,11 @@ make_specifier(uint8_t *spec_out, uint8_t type, unsigned flags)
       /* r = 8; p = 2. */
       spec_out[SCRYPT_SPEC_LEN-1] = (3u << 4) | (1u << 0);
       break;
+    // LCOV_EXCL_START - we should have returned above.
     default:
-      tor_fragile_assert(); // LCOV_EXCL_LINE - we should have returned above.
+      tor_fragile_assert();
       return S2K_BAD_ALGORITHM;
+    // LCOV_EXCL_STOP
   }
 
   return speclen;
@@ -290,9 +297,9 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv != 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else
+#else /* !(defined(HAVE_SCRYPT)) */
       return S2K_NO_SCRYPT_SUPPORT;
-#endif
+#endif /* defined(HAVE_SCRYPT) */
     }
     default:
       return S2K_BAD_ALGORITHM;
diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h
index 9b186450b1..849ff59ce8 100644
--- a/src/common/crypto_s2k.h
+++ b/src/common/crypto_s2k.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_S2K_H_INCLUDED
@@ -67,7 +67,7 @@ STATIC int secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
                                      const uint8_t *spec, size_t spec_len,
                                      const char *secret, size_t secret_len,
                                      int type);
-#endif
+#endif /* defined(CRYPTO_S2K_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CRYPTO_S2K_H_INCLUDED) */
 
diff --git a/src/common/crypto_util.c b/src/common/crypto_util.c
new file mode 100644
index 0000000000..b0d5b6b2f7
--- /dev/null
+++ b/src/common/crypto_util.c
@@ -0,0 +1,107 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_util.c
+ *
+ * \brief Common cryptographic utilities.
+ **/
+
+#ifndef CRYPTO_UTIL_PRIVATE
+#define CRYPTO_UTIL_PRIVATE
+
+#include "crypto_util.h"
+
+#include <string.h>
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#include <wincrypt.h>
+#endif /* defined(_WIN32) */
+
+#include "util.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/crypto.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+/**
+ * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to
+ * the value <b>byte</b>.
+ * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens.
+ *
+ * This function is preferable to memset, since many compilers will happily
+ * optimize out memset() when they can convince themselves that the data being
+ * cleared will never be read.
+ *
+ * Right now, our convention is to use this function when we are wiping data
+ * that's about to become inaccessible, such as stack buffers that are about
+ * to go out of scope or structures that are about to get freed.  (In
+ * practice, it appears that the compilers we're currently using will optimize
+ * out the memset()s for stack-allocated buffers, but not those for
+ * about-to-be-freed structures. That could change, though, so we're being
+ * wary.)  If there are live reads for the data, then you can just use
+ * memset().
+ */
+void
+memwipe(void *mem, uint8_t byte, size_t sz)
+{
+  if (sz == 0) {
+    return;
+  }
+  /* If sz is nonzero, then mem must not be NULL. */
+  tor_assert(mem != NULL);
+
+  /* Data this large is likely to be an underflow. */
+  tor_assert(sz < SIZE_T_CEILING);
+
+  /* Because whole-program-optimization exists, we may not be able to just
+   * have this function call "memset".  A smart compiler could inline it, then
+   * eliminate dead memsets, and declare itself to be clever. */
+
+#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY)
+  /* Here's what you do on windows. */
+  SecureZeroMemory(mem,sz);
+#elif defined(HAVE_RTLSECUREZEROMEMORY)
+  RtlSecureZeroMemory(mem,sz);
+#elif defined(HAVE_EXPLICIT_BZERO)
+  /* The BSDs provide this. */
+  explicit_bzero(mem, sz);
+#elif defined(HAVE_MEMSET_S)
+  /* This is in the C99 standard. */
+  memset_s(mem, sz, 0, sz);
+#else
+  /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk
+   * based on the pointer value, then uses that junk to update a global
+   * variable.  It's an elaborate ruse to trick the compiler into not
+   * optimizing out the "wipe this memory" code.  Read it if you like zany
+   * programming tricks! In later versions of Tor, we should look for better
+   * not-optimized-out memory wiping stuff...
+   *
+   * ...or maybe not.  In practice, there are pure-asm implementations of
+   * OPENSSL_cleanse() on most platforms, which ought to do the job.
+   **/
+
+  OPENSSL_cleanse(mem, sz);
+#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */
+
+  /* Just in case some caller of memwipe() is relying on getting a buffer
+   * filled with a particular value, fill the buffer.
+   *
+   * If this function gets inlined, this memset might get eliminated, but
+   * that's okay: We only care about this particular memset in the case where
+   * the caller should have been using memset(), and the memset() wouldn't get
+   * eliminated.  In other words, this is here so that we won't break anything
+   * if somebody accidentally calls memwipe() instead of memset().
+   **/
+  memset(mem, byte, sz);
+}
+
+#endif /* !defined(CRYPTO_UTIL_PRIVATE) */
+
diff --git a/src/common/crypto_util.h b/src/common/crypto_util.h
new file mode 100644
index 0000000000..922942b371
--- /dev/null
+++ b/src/common/crypto_util.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_util.h
+ *
+ * \brief Common functions for cryptographic routines.
+ **/
+
+#ifndef TOR_CRYPTO_UTIL_H
+#define TOR_CRYPTO_UTIL_H
+
+#include "torint.h"
+
+/** OpenSSL-based utility functions. */
+void memwipe(void *mem, uint8_t byte, size_t sz);
+
+#ifdef CRYPTO_UTIL_PRIVATE
+#ifdef TOR_UNIT_TESTS
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(CRYPTO_UTIL_PRIVATE) */
+
+#endif /* !defined(TOR_CRYPTO_UTIL_H) */
+
diff --git a/src/common/di_ops.c b/src/common/di_ops.c
index 4ed49e1164..90e9357c8e 100644
--- a/src/common/di_ops.c
+++ b/src/common/di_ops.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -86,7 +86,7 @@ tor_memcmp(const void *a, const void *b, size_t len)
   }
 
   return retval;
-#endif /* timingsafe_memcmp */
+#endif /* defined(HAVE_TIMINGSAFE_MEMCMP) */
 }
 
 /**
@@ -148,7 +148,7 @@ struct di_digest256_map_t {
 /** Release all storage held in <b>map</b>, calling free_fn on each value
  * as we go. */
 void
-dimap_free(di_digest256_map_t *map, dimap_free_fn free_fn)
+dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn)
 {
   while (map) {
     di_digest256_map_t *victim = map;
@@ -238,7 +238,7 @@ gt_i64_timei(uint64_t a, uint64_t b)
   int res = diff >> 63;
   return res & 1;
 }
-#endif
+#endif /* SIZEOF_VOID_P == 8 */
 
 /**
  * Given an array of list of <b>n_entries</b> uint64_t values, whose sum is
diff --git a/src/common/di_ops.h b/src/common/di_ops.h
index 0a154302bf..67d9c9f0df 100644
--- a/src/common/di_ops.h
+++ b/src/common/di_ops.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -37,7 +37,12 @@ int safe_mem_is_zero(const void *mem, size_t sz);
 typedef struct di_digest256_map_t di_digest256_map_t;
 typedef void (*dimap_free_fn)(void *);
 
-void dimap_free(di_digest256_map_t *map, dimap_free_fn free_fn);
+void dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn);
+#define dimap_free(map, free_fn)                \
+  do {                                          \
+    dimap_free_((map), (free_fn));              \
+    (map) = NULL;                               \
+  } while (0)
 void dimap_add_entry(di_digest256_map_t **map,
                      const uint8_t *key, void *val);
 void *dimap_search(const di_digest256_map_t *map, const uint8_t *key,
@@ -46,5 +51,5 @@ int select_array_member_cumulative_timei(const uint64_t *entries,
                                          int n_entries,
                                          uint64_t total, uint64_t rand_val);
 
-#endif
+#endif /* !defined(TOR_DI_OPS_H) */
 
diff --git a/src/common/handles.h b/src/common/handles.h
index 1ee2322579..aef8cd89ef 100644
--- a/src/common/handles.h
+++ b/src/common/handles.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -59,7 +59,7 @@
 #define HANDLE_DECL(name, structname, linkage)                          \
   typedef struct name ## _handle_t name ## _handle_t;                   \
   linkage  name ## _handle_t *name ## _handle_new(struct structname *object); \
-  linkage void name ## _handle_free(name ## _handle_t *);               \
+  linkage void name ## _handle_free_(name ## _handle_t *);              \
   linkage struct structname *name ## _handle_get(name ## _handle_t *);  \
   linkage void name ## _handles_clear(struct structname *object);
 
@@ -113,7 +113,7 @@
   }                                                                     \
                                                                         \
   linkage void                                                          \
-  name ## _handle_free(struct name ## _handle_t *ref)                   \
+  name ## _handle_free_(struct name ## _handle_t *ref)                   \
   {                                                                     \
     if (! ref) return;                                                  \
     name ## _handle_head_t *head = ref->head;                           \
@@ -149,5 +149,5 @@
     }                                                                   \
   }
 
-#endif /* TOR_HANDLE_H */
+#endif /* !defined(TOR_HANDLE_H) */
 
diff --git a/src/common/include.am b/src/common/include.am
index cb307e9d5f..cfaf993674 100644
--- a/src/common/include.am
+++ b/src/common/include.am
@@ -74,7 +74,11 @@ LIBOR_CTIME_A_SRC = \
    src/common/di_ops.c
 
 src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC)
+if UNITTESTS_ENABLED
 src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC)
+else
+src_common_libor_ctime_testing_a_SOURCES =
+endif
 src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
 src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 
@@ -82,9 +86,11 @@ LIBOR_A_SRC = \
   src/common/address.c					\
   src/common/address_set.c				\
   src/common/backtrace.c				\
+  src/common/buffers.c					\
   src/common/compat.c					\
   src/common/compat_threads.c				\
   src/common/compat_time.c				\
+  src/common/confline.c					\
   src/common/container.c				\
   src/common/log.c					\
   src/common/memarea.c					\
@@ -94,6 +100,8 @@ LIBOR_A_SRC = \
   src/common/util_format.c				\
   src/common/util_process.c				\
   src/common/sandbox.c					\
+  src/common/storagedir.c				\
+  src/common/token_bucket.c				\
   src/common/workqueue.c				\
   $(libor_extra_source)					\
   $(threads_impl_source)				\
@@ -104,11 +112,21 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
 
 LIBOR_CRYPTO_A_SRC = \
   src/common/aes.c		\
+  src/common/buffers_tls.c	\
+  src/common/compress.c	\
+  src/common/compress_lzma.c	\
+  src/common/compress_none.c	\
+  src/common/compress_zlib.c	\
+  src/common/compress_zstd.c	\
   src/common/crypto.c		\
+  src/common/crypto_digest.c     \
+  src/common/crypto_format.c	\
+  src/common/crypto_openssl_mgt.c    \
   src/common/crypto_pwbox.c     \
+  src/common/crypto_rand.c     \
+  src/common/crypto_rsa.c     \
   src/common/crypto_s2k.c	\
-  src/common/crypto_format.c	\
-  src/common/torgzip.c		\
+  src/common/crypto_util.c	\
   src/common/tortls.c		\
   src/common/crypto_curve25519.c \
   src/common/crypto_ed25519.c
@@ -123,9 +141,15 @@ src_common_libor_a_SOURCES = $(LIBOR_A_SRC)
 src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC)
 src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC)
 
+if UNITTESTS_ENABLED
 src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC)
 src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC)
 src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC)
+else
+src_common_libor_testing_a_SOURCES =
+src_common_libor_crypto_testing_a_SOURCES =
+src_common_libor_event_testing_a_SOURCES =
+endif
 
 src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
@@ -138,6 +162,8 @@ COMMONHEADERS = \
   src/common/address.h				\
   src/common/address_set.h			\
   src/common/backtrace.h			\
+  src/common/buffers.h				\
+  src/common/buffers_tls.h			\
   src/common/aes.h				\
   src/common/ciphers.inc			\
   src/common/compat.h				\
@@ -145,13 +171,24 @@ COMMONHEADERS = \
   src/common/compat_openssl.h			\
   src/common/compat_threads.h			\
   src/common/compat_time.h			\
+  src/common/compress.h				\
+  src/common/compress_lzma.h			\
+  src/common/compress_none.h			\
+  src/common/compress_zlib.h			\
+  src/common/compress_zstd.h			\
+  src/common/confline.h				\
   src/common/container.h			\
   src/common/crypto.h				\
+  src/common/crypto_digest.h            \
   src/common/crypto_curve25519.h		\
   src/common/crypto_ed25519.h			\
   src/common/crypto_format.h			\
+  src/common/crypto_openssl_mgt.h         \
   src/common/crypto_pwbox.h			\
+  src/common/crypto_rand.h          \
+  src/common/crypto_rsa.h           \
   src/common/crypto_s2k.h			\
+  src/common/crypto_util.h			\
   src/common/di_ops.h				\
   src/common/handles.h				\
   src/common/memarea.h				\
@@ -159,9 +196,10 @@ COMMONHEADERS = \
   src/common/procmon.h				\
   src/common/pubsub.h				\
   src/common/sandbox.h				\
+  src/common/storagedir.h			\
   src/common/testsupport.h			\
   src/common/timers.h				\
-  src/common/torgzip.h				\
+  src/common/token_bucket.h			\
   src/common/torint.h				\
   src/common/torlog.h				\
   src/common/tortls.h				\
diff --git a/src/common/log.c b/src/common/log.c
index 4db1c9f0d0..ebd50f62d3 100644
--- a/src/common/log.c
+++ b/src/common/log.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,6 +35,9 @@
 #define LOG_PRIVATE
 #include "torlog.h"
 #include "container.h"
+#ifdef HAVE_ANDROID_LOG_H
+#include <android/log.h>
+#endif // HAVE_ANDROID_LOG_H.
 
 /** Given a severity, yields an index into log_severity_list_t.masks to use
  * for that severity. */
@@ -49,6 +52,13 @@
 
 #define raw_assert(x) assert(x) // assert OK
 
+/** Defining compile-time constants for Tor log levels (used by the Rust
+ * log wrapper at src/rust/tor_log) */
+const int LOG_WARN_ = LOG_WARN;
+const int LOG_NOTICE_ = LOG_NOTICE;
+const log_domain_mask_t LD_GENERAL_ = LD_GENERAL;
+const log_domain_mask_t LD_NET_ = LD_NET;
+
 /** Information for a single logfile; only used in log.c */
 typedef struct logfile_t {
   struct logfile_t *next; /**< Next logfile_t in the linked list. */
@@ -58,12 +68,16 @@ typedef struct logfile_t {
   int needs_close; /**< Boolean: true if the stream gets closed on shutdown. */
   int is_temporary; /**< Boolean: close after initializing logging subsystem.*/
   int is_syslog; /**< Boolean: send messages to syslog. */
+  int is_android; /**< Boolean: send messages to Android's log subsystem. */
+  char *android_tag; /**< Identity Tag used in Android's log subsystem. */
   log_callback callback; /**< If not NULL, send messages to this function. */
   log_severity_list_t *severities; /**< Which severity of messages should we
                                     * log for each log domain? */
 } logfile_t;
 
-static void log_free(logfile_t *victim);
+static void log_free_(logfile_t *victim);
+#define log_free(lg)    \
+  FREE_AND_NULL(logfile_t, log_free_, (lg))
 
 /** Helper: map a log severity to descriptive string. */
 static inline const char *
@@ -101,6 +115,33 @@ should_log_function_name(log_domain_mask_t domain, int severity)
   }
 }
 
+#ifdef HAVE_ANDROID_LOG_H
+/** Helper function to convert Tor's log severity into the matching
+ * Android log priority.
+ */
+static int
+severity_to_android_log_priority(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG:
+      return ANDROID_LOG_VERBOSE;
+    case LOG_INFO:
+      return ANDROID_LOG_DEBUG;
+    case LOG_NOTICE:
+      return ANDROID_LOG_INFO;
+    case LOG_WARN:
+      return ANDROID_LOG_WARN;
+    case LOG_ERR:
+      return ANDROID_LOG_ERROR;
+    default:
+      // LCOV_EXCL_START
+      raw_assert(0);
+      return 0;
+      // LCOV_EXCL_STOP
+  }
+}
+#endif // HAVE_ANDROID_LOG_H.
+
 /** A mutex to guard changes to logfiles and logging. */
 static tor_mutex_t log_mutex;
 /** True iff we have initialized log_mutex */
@@ -129,6 +170,9 @@ typedef struct pending_log_message_t {
 /** Log messages waiting to be replayed onto callback-based logs */
 static smartlist_t *pending_cb_messages = NULL;
 
+/** Callback to invoke when pending_cb_messages becomes nonempty. */
+static pending_callback_callback pending_cb_cb = NULL;
+
 /** Log messages waiting to be replayed once the logging system is initialized.
  */
 static smartlist_t *pending_startup_messages = NULL;
@@ -191,6 +235,30 @@ log_set_application_name(const char *name)
   appname = name ? tor_strdup(name) : NULL;
 }
 
+/** Return true if some of the running logs might be interested in a log
+ * message of the given severity in the given domains. If this function
+ * returns true, the log message might be ignored anyway, but if it returns
+ * false, it is definitely_ safe not to log the message. */
+int
+log_message_is_interesting(int severity, log_domain_mask_t domain)
+{
+  (void) domain;
+  return (severity <= log_global_min_severity_);
+}
+
+/**
+ * As tor_log, but takes an optional function name, and does not treat its
+ * <b>string</b> as a printf format.
+ *
+ * For use by Rust integration.
+ */
+void
+tor_log_string(int severity, log_domain_mask_t domain,
+               const char *function, const char *string)
+{
+  log_fn_(severity, domain, function, "%s", string);
+}
+
 /** Log time granularity in milliseconds. */
 static int log_time_granularity = 1;
 
@@ -385,9 +453,12 @@ pending_log_message_new(int severity, log_domain_mask_t domain,
   return m;
 }
 
+#define pending_log_message_free(msg) \
+  FREE_AND_NULL(pending_log_message_t, pending_log_message_free_, (msg))
+
 /** Release all storage held by <b>msg</b>. */
 static void
-pending_log_message_free(pending_log_message_t *msg)
+pending_log_message_free_(pending_log_message_t *msg)
 {
   if (!msg)
     return;
@@ -396,6 +467,16 @@ pending_log_message_free(pending_log_message_t *msg)
   tor_free(msg);
 }
 
+/** Helper function: returns true iff the log file, given in <b>lf</b>, is
+ * handled externally via the system log API, the Android logging API, or is an
+ * external callback function. */
+static inline int
+logfile_is_external(const logfile_t *lf)
+{
+  raw_assert(lf);
+  return lf->is_syslog || lf->is_android || lf->callback;
+}
+
 /** Return true iff <b>lf</b> would like to receive a message with the
  * specified <b>severity</b> in the specified <b>domain</b>.
  */
@@ -406,7 +487,7 @@ logfile_wants_message(const logfile_t *lf, int severity,
   if (! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) {
     return 0;
   }
-  if (! (lf->fd >= 0 || lf->is_syslog || lf->callback)) {
+  if (! (lf->fd >= 0 || logfile_is_external(lf))) {
     return 0;
   }
   if (lf->seems_dead) {
@@ -444,17 +525,25 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
     if (m != msg_after_prefix) {
       tor_free(m);
     }
-#else
+#else /* !(defined(MAXLINE)) */
     /* We have syslog but not MAXLINE.  That's promising! */
     syslog(severity, "%s", msg_after_prefix);
-#endif
-#endif
+#endif /* defined(MAXLINE) */
+#endif /* defined(HAVE_SYSLOG_H) */
+  } else if (lf->is_android) {
+#ifdef HAVE_ANDROID_LOG_H
+    int priority = severity_to_android_log_priority(severity);
+    __android_log_write(priority, lf->android_tag, msg_after_prefix);
+#endif // HAVE_ANDROID_LOG_H.
   } else if (lf->callback) {
     if (domain & LD_NOCB) {
       if (!*callbacks_deferred && pending_cb_messages) {
         smartlist_add(pending_cb_messages,
             pending_log_message_new(severity,domain,NULL,msg_after_prefix));
         *callbacks_deferred = 1;
+        if (smartlist_len(pending_cb_messages) == 1 && pending_cb_cb) {
+          pending_cb_cb();
+        }
       }
     } else {
       lf->callback(severity, domain, msg_after_prefix);
@@ -641,8 +730,8 @@ tor_log_update_sigsafe_err_fds(void)
      /* Don't try callback to the control port, or syslogs: We can't
       * do them from a signal handler. Don't try stdout: we always do stderr.
       */
-    if (lf->is_temporary || lf->is_syslog ||
-        lf->callback || lf->seems_dead || lf->fd < 0)
+    if (lf->is_temporary || logfile_is_external(lf)
+        || lf->seems_dead || lf->fd < 0)
       continue;
     if (lf->severities->masks[SEVERITY_MASK_IDX(LOG_ERR)] &
         (LD_BUG|LD_GENERAL)) {
@@ -678,11 +767,11 @@ tor_log_get_logfile_names(smartlist_t *out)
   LOCK_LOGS();
 
   for (lf = logfiles; lf; lf = lf->next) {
-    if (lf->is_temporary || lf->is_syslog || lf->callback)
+    if (lf->is_temporary || logfile_is_external(lf))
       continue;
     if (lf->filename == NULL)
       continue;
-    smartlist_add(out, tor_strdup(lf->filename));
+    smartlist_add_strdup(out, lf->filename);
   }
 
   UNLOCK_LOGS();
@@ -721,12 +810,13 @@ log_fn_ratelim_(ratelim_t *ratelim, int severity, log_domain_mask_t domain,
 
 /** Free all storage held by <b>victim</b>. */
 static void
-log_free(logfile_t *victim)
+log_free_(logfile_t *victim)
 {
   if (!victim)
     return;
   tor_free(victim->severities);
   tor_free(victim->filename);
+  tor_free(victim->android_tag);
   tor_free(victim);
 }
 
@@ -741,6 +831,7 @@ logs_free_all(void)
   logfiles = NULL;
   messages = pending_cb_messages;
   pending_cb_messages = NULL;
+  pending_cb_cb = NULL;
   messages2 = pending_startup_messages;
   pending_startup_messages = NULL;
   UNLOCK_LOGS();
@@ -807,7 +898,7 @@ close_log(logfile_t *victim)
       /* There are no other syslogs; close the logging facility. */
       closelog();
     }
-#endif
+#endif /* defined(HAVE_SYSLOG_H) */
   }
 }
 
@@ -904,6 +995,24 @@ add_temp_log(int min_severity)
 }
 
 /**
+ * Register "cb" as the callback to call when there are new pending log
+ * callbacks to be flushed with flush_pending_log_callbacks().
+ *
+ * Note that this callback, if present, can be invoked from any thread.
+ *
+ * This callback must not log.
+ *
+ * It is intentional that this function contains the name "callback" twice: it
+ * sets a "callback" to be called on the condition that there is a "pending
+ * callback".
+ **/
+void
+logs_set_pending_callback_callback(pending_callback_callback cb)
+{
+  pending_cb_cb = cb;
+}
+
+/**
  * Add a log handler to send messages in <b>severity</b>
  * to the function <b>cb</b>.
  */
@@ -1086,7 +1195,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename,
   int open_flags = O_WRONLY|O_CREAT;
   open_flags |= truncate_log ? O_TRUNC : O_APPEND;
 
-  fd = tor_open_cloexec(filename, open_flags, 0644);
+  fd = tor_open_cloexec(filename, open_flags, 0640);
   if (fd<0)
     return -1;
   if (tor_fd_seekend(fd)<0) {
@@ -1144,7 +1253,40 @@ add_syslog_log(const log_severity_list_t *severity,
   UNLOCK_LOGS();
   return 0;
 }
-#endif
+#endif /* defined(HAVE_SYSLOG_H) */
+
+#ifdef HAVE_ANDROID_LOG_H
+/**
+ * Add a log handler to send messages to the Android platform log facility.
+ */
+int
+add_android_log(const log_severity_list_t *severity,
+                const char *android_tag)
+{
+  logfile_t *lf = NULL;
+
+  lf = tor_malloc_zero(sizeof(logfile_t));
+  lf->fd = -1;
+  lf->severities = tor_memdup(severity, sizeof(log_severity_list_t));
+  lf->filename = tor_strdup("<android>");
+  lf->is_android = 1;
+
+  if (android_tag == NULL)
+    lf->android_tag = tor_strdup("Tor");
+  else {
+    char buf[256];
+    tor_snprintf(buf, sizeof(buf), "Tor-%s", android_tag);
+    lf->android_tag = tor_strdup(buf);
+  }
+
+  LOCK_LOGS();
+  lf->next = logfiles;
+  logfiles = lf;
+  log_global_min_severity_ = get_min_log_level();
+  UNLOCK_LOGS();
+  return 0;
+}
+#endif // HAVE_ANDROID_LOG_H.
 
 /** If <b>level</b> is a valid log severity, return the corresponding
  * numeric value.  Otherwise, return -1. */
@@ -1172,12 +1314,15 @@ log_level_to_string(int level)
 }
 
 /** NULL-terminated array of names for log domains such that domain_list[dom]
- * is a description of <b>dom</b>. */
+ * is a description of <b>dom</b>.
+ *
+ * Remember to update doc/tor.1.txt if you modify this list.
+ * */
 static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
 };
 
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
@@ -1313,16 +1458,15 @@ parse_log_severity_config(const char **cfg_ptr,
     if (!strcasecmpstart(cfg, "file") ||
         !strcasecmpstart(cfg, "stderr") ||
         !strcasecmpstart(cfg, "stdout") ||
-        !strcasecmpstart(cfg, "syslog")) {
+        !strcasecmpstart(cfg, "syslog") ||
+        !strcasecmpstart(cfg, "android")) {
       goto done;
     }
     if (got_an_unqualified_range > 1)
       return -1;
 
-    space = strchr(cfg, ' ');
+    space = find_whitespace(cfg);
     dash = strchr(cfg, '-');
-    if (!space)
-      space = strchr(cfg, '\0');
     if (dash && dash < space) {
       sev_lo = tor_strndup(cfg, dash-cfg);
       sev_hi = tor_strndup(dash+1, space-(dash+1));
diff --git a/src/common/memarea.c b/src/common/memarea.c
index 7d16b702e3..68c1625fe4 100644
--- a/src/common/memarea.c
+++ b/src/common/memarea.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2016, The Tor Project, Inc. */
+/* Copyright (c) 2008-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /** \file memarea.c
@@ -7,11 +7,15 @@
  */
 
 #include "orconfig.h"
+#include <stddef.h>
 #include <stdlib.h>
 #include "memarea.h"
 #include "util.h"
 #include "compat.h"
 #include "torlog.h"
+#include "container.h"
+
+#ifndef DISABLE_MEMORY_SENTINELS
 
 /** If true, we try to detect any attempts to write beyond the length of a
  * memarea. */
@@ -29,7 +33,7 @@
 #define MEMAREA_ALIGN_MASK ((uintptr_t)7)
 #else
 #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff."
-#endif
+#endif /* MEMAREA_ALIGN == 4 || ... */
 
 #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER)
 #define USE_ALIGNED_ATTRIBUTE
@@ -37,7 +41,7 @@
 #define U_MEM mem
 #else
 #define U_MEM u.mem
-#endif
+#endif /* defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) */
 
 #ifdef USE_SENTINELS
 /** Magic value that we stick at the end of a memarea so we can make sure
@@ -57,11 +61,11 @@
   uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]);     \
   tor_assert(sent_val == SENTINEL_VAL);                                 \
   STMT_END
-#else
+#else /* !(defined(USE_SENTINELS)) */
 #define SENTINEL_LEN 0
 #define SET_SENTINEL(chunk) STMT_NIL
 #define CHECK_SENTINEL(chunk) STMT_NIL
-#endif
+#endif /* defined(USE_SENTINELS) */
 
 /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */
 static inline void *
@@ -93,12 +97,12 @@ typedef struct memarea_chunk_t {
     void *void_for_alignment_; /**< Dummy; used to make sure mem is aligned. */
   } u; /**< Union used to enforce alignment when we don't have support for
         * doing it right. */
-#endif
+#endif /* defined(USE_ALIGNED_ATTRIBUTE) */
 } memarea_chunk_t;
 
 /** How many bytes are needed for overhead before we get to the memory part
  * of a chunk? */
-#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM)
+#define CHUNK_HEADER_SIZE offsetof(memarea_chunk_t, U_MEM)
 
 /** What's the smallest that we'll allocate a chunk? */
 #define CHUNK_SIZE 4096
@@ -149,7 +153,7 @@ memarea_new(void)
 /** Free <b>area</b>, invalidating all pointers returned from memarea_alloc()
  * and friends for this area */
 void
-memarea_drop_all(memarea_t *area)
+memarea_drop_all_(memarea_t *area)
 {
   memarea_chunk_t *chunk, *next;
   for (chunk = area->first; chunk; chunk = next) {
@@ -304,3 +308,91 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
+#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */
+
+struct memarea_t {
+  smartlist_t *pieces;
+};
+
+memarea_t *
+memarea_new(void)
+{
+  memarea_t *ma = tor_malloc_zero(sizeof(memarea_t));
+  ma->pieces = smartlist_new();
+  return ma;
+}
+void
+memarea_drop_all_(memarea_t *area)
+{
+  memarea_clear(area);
+  smartlist_free(area->pieces);
+  tor_free(area);
+}
+void
+memarea_clear(memarea_t *area)
+{
+  SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p));
+  smartlist_clear(area->pieces);
+}
+int
+memarea_owns_ptr(const memarea_t *area, const void *ptr)
+{
+  SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;);
+  return 0;
+}
+
+void *
+memarea_alloc(memarea_t *area, size_t sz)
+{
+  void *result = tor_malloc(sz);
+  smartlist_add(area->pieces, result);
+  return result;
+}
+
+void *
+memarea_alloc_zero(memarea_t *area, size_t sz)
+{
+  void *result = tor_malloc_zero(sz);
+  smartlist_add(area->pieces, result);
+  return result;
+}
+void *
+memarea_memdup(memarea_t *area, const void *s, size_t n)
+{
+  void *r = memarea_alloc(area, n);
+  memcpy(r, s, n);
+  return r;
+}
+char *
+memarea_strdup(memarea_t *area, const char *s)
+{
+  size_t n = strlen(s);
+  char *r = memarea_alloc(area, n+1);
+  memcpy(r, s, n);
+  r[n] = 0;
+  return r;
+}
+char *
+memarea_strndup(memarea_t *area, const char *s, size_t n)
+{
+  size_t ln = strnlen(s, n);
+  char *r = memarea_alloc(area, ln+1);
+  memcpy(r, s, ln);
+  r[ln] = 0;
+  return r;
+}
+void
+memarea_get_stats(memarea_t *area,
+                  size_t *allocated_out, size_t *used_out)
+{
+  (void)area;
+  *allocated_out = *used_out = 128;
+}
+void
+memarea_assert_ok(memarea_t *area)
+{
+  (void)area;
+}
+
+#endif /* !defined(DISABLE_MEMORY_SENTINELS) */
+
diff --git a/src/common/memarea.h b/src/common/memarea.h
index 85bca51ad3..5207e8a5bd 100644
--- a/src/common/memarea.h
+++ b/src/common/memarea.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2016, The Tor Project, Inc. */
+/* Copyright (c) 2008-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 /* Tor dependencies */
 
@@ -8,7 +8,12 @@
 typedef struct memarea_t memarea_t;
 
 memarea_t *memarea_new(void);
-void memarea_drop_all(memarea_t *area);
+void memarea_drop_all_(memarea_t *area);
+#define memarea_drop_all(area) \
+  do {                                          \
+    memarea_drop_all_(area);                    \
+    (area) = NULL;                              \
+  } while (0)
 void memarea_clear(memarea_t *area);
 int memarea_owns_ptr(const memarea_t *area, const void *ptr);
 void *memarea_alloc(memarea_t *area, size_t sz);
@@ -20,5 +25,5 @@ void memarea_get_stats(memarea_t *area,
                        size_t *allocated_out, size_t *used_out);
 void memarea_assert_ok(memarea_t *area);
 
-#endif
+#endif /* !defined(TOR_MEMAREA_H) */
 
diff --git a/src/common/procmon.c b/src/common/procmon.c
index c485c760c7..73c14cd584 100644
--- a/src/common/procmon.c
+++ b/src/common/procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,8 +10,6 @@
 
 #include "util.h"
 
-#include <event2/event.h>
-
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
 #endif
@@ -36,7 +34,7 @@ typedef int pid_t;
 #define PID_T_FORMAT I64_FORMAT
 #else
 #error Unknown: SIZEOF_PID_T
-#endif
+#endif /* (0 == SIZEOF_PID_T) && defined(_WIN32) || ... */
 
 /* Define to 1 if process-termination monitors on this OS and Libevent
    version must poll for process termination themselves. */
@@ -44,7 +42,7 @@ typedef int pid_t;
 /* Currently we need to poll in some way on all systems. */
 
 #ifdef PROCMON_POLLS
-static void tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
+static void tor_process_monitor_poll_cb(periodic_timer_t *ev,
                                         void *procmon_);
 #endif
 
@@ -71,7 +69,7 @@ parse_process_specifier(const char *process_spec,
 
   /* If we're lucky, long will turn out to be large enough to hold a
    * PID everywhere that Tor runs. */
-  pid_l = tor_parse_long(process_spec, 0, 1, LONG_MAX, &pid_ok, &pspec_next);
+  pid_l = tor_parse_long(process_spec, 10, 1, LONG_MAX, &pid_ok, &pspec_next);
 
   /* Reserve room in the ‘process specifier’ for additional
    * (platform-specific) identifying information beyond the PID, to
@@ -114,7 +112,7 @@ struct tor_process_monitor_t {
   HANDLE hproc;
   /* XXXX We should have Libevent watch hproc for us,
    * if/when some version of Libevent can be told to do so. */
-#endif
+#endif /* defined(_WIN32) */
 
   /* XXXX On Linux, we can and should receive the 22nd
    * (space-delimited) field (‘starttime’) of /proc/$PID/stat from the
@@ -136,7 +134,7 @@ struct tor_process_monitor_t {
 
   /** A Libevent event structure, to either poll for the process's
    * existence or receive a notification when the process ends. */
-  struct event *e;
+  periodic_timer_t *e;
 
   /** A callback to be called when the process ends. */
   tor_procmon_callback_t cb;
@@ -159,9 +157,6 @@ tor_validate_process_specifier(const char *process_spec,
   return parse_process_specifier(process_spec, &ppspec, msg);
 }
 
-/* XXXX we should use periodic_timer_new() for this stuff */
-#define PERIODIC_TIMER_FLAGS EV_PERSIST
-
 /* DOCDOC poll_interval_tv */
 static const struct timeval poll_interval_tv = {15, 0};
 
@@ -219,22 +214,18 @@ tor_process_monitor_new(struct event_base *base,
              "try again later.",
              procmon->pid);
   }
-#endif
+#endif /* defined(_WIN32) */
 
   procmon->cb = cb;
   procmon->cb_arg = cb_arg;
 
 #ifdef PROCMON_POLLS
-  procmon->e = tor_event_new(base, -1 /* no FD */, PERIODIC_TIMER_FLAGS,
-                             tor_process_monitor_poll_cb, procmon);
-  /* Note: If you port this file to plain Libevent 2, check that
-   * procmon->e is non-NULL.  We don't need to here because
-   * tor_evtimer_new never returns NULL. */
-
-  evtimer_add(procmon->e, &poll_interval_tv);
-#else
+  procmon->e = periodic_timer_new(base,
+                                  &poll_interval_tv,
+                                  tor_process_monitor_poll_cb, procmon);
+#else /* !(defined(PROCMON_POLLS)) */
 #error OOPS?
-#endif
+#endif /* defined(PROCMON_POLLS) */
 
   return procmon;
  err:
@@ -246,14 +237,12 @@ tor_process_monitor_new(struct event_base *base,
 /** Libevent callback to poll for the existence of the process
  * monitored by <b>procmon_</b>. */
 static void
-tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
-                            void *procmon_)
+tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
 {
+  (void)event;
   tor_process_monitor_t *procmon = (tor_process_monitor_t *)(procmon_);
   int its_dead_jim;
 
-  (void)unused1; (void)unused2;
-
   tor_assert(procmon != NULL);
 
 #ifdef _WIN32
@@ -306,11 +295,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
       tor_free(errmsg);
     }
   }
-#else
+#else /* !(defined(_WIN32)) */
   /* Unix makes this part easy, if a bit racy. */
   its_dead_jim = kill(procmon->pid, 0);
   its_dead_jim = its_dead_jim && (errno == ESRCH);
-#endif
+#endif /* defined(_WIN32) */
 
   tor_log(its_dead_jim ? LOG_NOTICE : LOG_INFO,
       procmon->log_domain, "Monitored process "PID_T_FORMAT" is %s.",
@@ -321,11 +310,11 @@ tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2,
     procmon->cb(procmon->cb_arg);
   }
 }
-#endif
+#endif /* defined(PROCMON_POLLS) */
 
 /** Free the process-termination monitor <b>procmon</b>. */
 void
-tor_process_monitor_free(tor_process_monitor_t *procmon)
+tor_process_monitor_free_(tor_process_monitor_t *procmon)
 {
   if (procmon == NULL)
     return;
@@ -336,7 +325,7 @@ tor_process_monitor_free(tor_process_monitor_t *procmon)
 #endif
 
   if (procmon->e != NULL)
-    tor_event_free(procmon->e);
+    periodic_timer_free(procmon->e);
 
   tor_free(procmon);
 }
diff --git a/src/common/procmon.h b/src/common/procmon.h
index 49ead24092..63777e4111 100644
--- a/src/common/procmon.h
+++ b/src/common/procmon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,7 +27,9 @@ tor_process_monitor_t *tor_process_monitor_new(struct event_base *base,
                                                tor_procmon_callback_t cb,
                                                void *cb_arg,
                                                const char **msg);
-void tor_process_monitor_free(tor_process_monitor_t *procmon);
+void tor_process_monitor_free_(tor_process_monitor_t *procmon);
+#define tor_process_monitor_free(procmon) \
+  FREE_AND_NULL(tor_process_monitor_t, tor_process_monitor_free_, (procmon))
 
-#endif
+#endif /* !defined(TOR_PROCMON_H) */
 
diff --git a/src/common/pubsub.c b/src/common/pubsub.c
index b3faf40e00..336e8a6e7f 100644
--- a/src/common/pubsub.c
+++ b/src/common/pubsub.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/pubsub.h b/src/common/pubsub.h
index bbb4f02a42..2bee3af085 100644
--- a/src/common/pubsub.h
+++ b/src/common/pubsub.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -175,5 +175,5 @@ int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn,
     pubsub_clear_(&name##_topic_);                                      \
   }
 
-#endif /* TOR_PUBSUB_H */
+#endif /* !defined(TOR_PUBSUB_H) */
 
diff --git a/src/common/sandbox.c b/src/common/sandbox.c
index 0a972d496b..ca7803ac9c 100644
--- a/src/common/sandbox.c
+++ b/src/common/sandbox.c
@@ -1,7 +1,7 @@
  /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,10 +17,16 @@
  * with the libevent fix.
  */
 #define _LARGEFILE64_SOURCE
-#endif
+#endif /* !defined(_LARGEFILE64_SOURCE) */
 
-/** Malloc mprotect limit in bytes. */
-#define MALLOC_MP_LIM 1048576
+/** Malloc mprotect limit in bytes.
+ *
+ * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced
+ * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but
+ * liblzma have a small overhead that we need to compensate for to avoid being
+ * killed by the sandbox.
+ */
+#define MALLOC_MP_LIM (20*1024*1024)
 
 #include <stdio.h>
 #include <string.h>
@@ -74,7 +80,7 @@
 #define USE_BACKTRACE
 #define EXPOSE_CLEAN_BACKTRACE
 #include "backtrace.h"
-#endif
+#endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */
 
 #ifdef USE_BACKTRACE
 #include <execinfo.h>
@@ -100,7 +106,12 @@
 
 #define M_SYSCALL arm_r7
 
-#endif
+#elif defined(__aarch64__) && defined(__LP64__)
+
+#define REG_SYSCALL 8
+#define M_SYSCALL regs[REG_SYSCALL]
+
+#endif /* defined(__i386__) || ... */
 
 /**Determines if at least one sandbox is active.*/
 static int sandbox_active = 0;
@@ -151,6 +162,7 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(fstat64),
 #endif
     SCMP_SYS(futex),
+    SCMP_SYS(getdents),
     SCMP_SYS(getdents64),
     SCMP_SYS(getegid),
 #ifdef __NR_getegid32
@@ -293,37 +305,6 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return rc;
 }
 
-#if 0
-/**
- * Function responsible for setting up the execve syscall for
- * the seccomp filter sandbox.
- */
-static int
-sb_execve(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
-{
-  int rc;
-  sandbox_cfg_t *elem = NULL;
-
-  // for each dynamic parameter filters
-  for (elem = filter; elem != NULL; elem = elem->next) {
-    smp_param_t *param = elem->param;
-
-    if (param != NULL && param->prot == 1 && param->syscall
-        == SCMP_SYS(execve)) {
-      rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(execve),
-               SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
-      if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add execve syscall, received "
-            "libseccomp error %d", rc);
-        return rc;
-      }
-    }
-  }
-
-  return 0;
-}
-#endif
-
 /**
  * Function responsible for setting up the time syscall for
  * the seccomp filter sandbox.
@@ -337,7 +318,7 @@ sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
        SCMP_CMP(0, SCMP_CMP_EQ, 0));
 #else
   return 0;
-#endif
+#endif /* defined(__NR_time) */
 }
 
 /**
@@ -356,7 +337,7 @@ sb_accept4(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   if (rc) {
     return rc;
   }
-#endif
+#endif /* defined(__i386__) */
 
   rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(accept4),
                    SCMP_CMP_MASKED(3, SOCK_CLOEXEC|SOCK_NONBLOCK, 0));
@@ -429,7 +410,7 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 
   return 0;
 }
-#endif
+#endif /* defined(__NR_mmap2) */
 
 #ifdef HAVE_GNU_LIBC_VERSION_H
 #ifdef HAVE_GNU_GET_LIBC_VERSION
@@ -457,9 +438,9 @@ libc_uses_openat_for_everything(void)
     return 1;
   else
     return 0;
-#else
+#else /* !(defined(CHECK_LIBC_VERSION)) */
   return 0;
-#endif
+#endif /* defined(CHECK_LIBC_VERSION) */
 }
 
 /** Allow a single file to be opened.  If <b>use_openat</b> is true,
@@ -540,7 +521,7 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chmod),
             SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
+        log_err(LD_BUG,"(Sandbox) failed to add chmod syscall, received "
             "libseccomp error %d", rc);
         return rc;
       }
@@ -565,7 +546,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown),
             SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
+        log_err(LD_BUG,"(Sandbox) failed to add chown syscall, received "
             "libseccomp error %d", rc);
         return rc;
       }
@@ -744,6 +725,25 @@ sb_socketpair(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return 0;
 }
 
+#ifdef HAVE_KIST_SUPPORT
+
+#include <linux/sockios.h>
+
+static int
+sb_ioctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
+{
+  int rc;
+  (void) filter;
+
+  rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ioctl),
+                          SCMP_CMP(1, SCMP_CMP_EQ, SIOCOUTQNSD));
+  if (rc)
+    return rc;
+  return 0;
+}
+
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
 /**
  * Function responsible for setting up the setsockopt syscall for
  * the seccomp filter sandbox.
@@ -784,7 +784,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUFFORCE));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 #ifdef IP_TRANSPARENT
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt),
@@ -792,7 +792,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, IP_TRANSPARENT));
   if (rc)
     return rc;
-#endif
+#endif /* defined(IP_TRANSPARENT) */
 
 #ifdef IPV6_V6ONLY
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt),
@@ -800,7 +800,7 @@ sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, IPV6_V6ONLY));
   if (rc)
     return rc;
-#endif
+#endif /* defined(IPV6_V6ONLY) */
 
   return 0;
 }
@@ -833,7 +833,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUF));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 #ifdef HAVE_LINUX_NETFILTER_IPV4_H
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
@@ -841,7 +841,7 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, SO_ORIGINAL_DST));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_LINUX_NETFILTER_IPV4_H) */
 
 #ifdef HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
@@ -849,7 +849,16 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       SCMP_CMP(2, SCMP_CMP_EQ, IP6T_SO_ORIGINAL_DST));
   if (rc)
     return rc;
-#endif
+#endif /* defined(HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H) */
+
+#ifdef HAVE_KIST_SUPPORT
+#include <netinet/tcp.h>
+  rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
+      SCMP_CMP(1, SCMP_CMP_EQ, SOL_TCP),
+      SCMP_CMP(2, SCMP_CMP_EQ, TCP_INFO));
+  if (rc)
+    return rc;
+#endif /* defined(HAVE_KIST_SUPPORT) */
 
   return 0;
 }
@@ -889,7 +898,7 @@ sb_fcntl64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 
   return 0;
 }
-#endif
+#endif /* defined(__NR_fcntl64) */
 
 /**
  * Function responsible for setting up the epoll_ctl syscall for
@@ -1086,8 +1095,8 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(stat64),
           SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
-            "libseccomp  error %d", rc);
+        log_err(LD_BUG,"(Sandbox) failed to add stat64 syscall, received "
+            "libseccomp error %d", rc);
         return rc;
       }
     }
@@ -1095,7 +1104,7 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 
   return 0;
 }
-#endif
+#endif /* defined(__NR_stat64) */
 
 static int
 sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
@@ -1107,7 +1116,7 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
        SCMP_CMP(1, SCMP_CMP_EQ, 0));
 #else
   return 0;
-#endif
+#endif /* defined(__NR_kill) */
 }
 
 /**
@@ -1117,9 +1126,6 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 static sandbox_filter_func_t filter_func[] = {
     sb_rt_sigaction,
     sb_rt_sigprocmask,
-#if 0
-    sb_execve,
-#endif
     sb_time,
     sb_accept4,
 #ifdef __NR_mmap2
@@ -1148,6 +1154,9 @@ static sandbox_filter_func_t filter_func[] = {
     sb_setsockopt,
     sb_getsockopt,
     sb_socketpair,
+#ifdef HAVE_KIST_SUPPORT
+    sb_ioctl,
+#endif
     sb_kill
 };
 
@@ -1374,10 +1383,6 @@ sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_stat, file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1391,10 +1396,6 @@ sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(open), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1408,10 +1409,6 @@ sandbox_cfg_allow_chmod_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(chmod), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1425,10 +1422,6 @@ sandbox_cfg_allow_chown_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(chown), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1443,11 +1436,6 @@ sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2)
 
   elem = new_element2(SCMP_SYS(rename), file1, file2);
 
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
-
   elem->next = *cfg;
   *cfg = elem;
 
@@ -1460,28 +1448,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   sandbox_cfg_t *elem = NULL;
 
   elem = new_element(SCMP_SYS(openat), file);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
-
-  elem->next = *cfg;
-  *cfg = elem;
-
-  return 0;
-}
-
-#if 0
-int
-sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
-{
-  sandbox_cfg_t *elem = NULL;
-
-  elem = new_element(SCMP_SYS(execve), com);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
 
   elem->next = *cfg;
   *cfg = elem;
@@ -1489,8 +1455,6 @@ sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
   return 0;
 }
 
-#endif
-
 /** Cache entry for getaddrinfo results; used when sandboxing is implemented
  * so that we can consult the cache when the sandbox prevents us from doing
  * getaddrinfo.
@@ -1521,8 +1485,12 @@ cached_getaddrinfo_items_eq(const cached_getaddrinfo_item_t *a,
   return (a->family == b->family) && 0 == strcmp(a->name, b->name);
 }
 
+#define cached_getaddrinfo_item_free(item)              \
+  FREE_AND_NULL(cached_getaddrinfo_item_t,              \
+                cached_getaddrinfo_item_free_, (item))
+
 static void
-cached_getaddrinfo_item_free(cached_getaddrinfo_item_t *item)
+cached_getaddrinfo_item_free_(cached_getaddrinfo_item_t *item)
 {
   if (item == NULL)
     return;
@@ -1614,7 +1582,7 @@ sandbox_getaddrinfo(const char *name, const char *servname,
     return err;
   }
 
-  /* Otherwise, the sanbox is on.  If we have an item, yield its cached
+  /* Otherwise, the sandbox is on.  If we have an item, yield its cached
      result. */
   if (item) {
     *res = item->res;
@@ -1747,7 +1715,9 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // loading the seccomp2 filter
   if ((rc = seccomp_load(ctx))) {
-    log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc,
+    log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)! "
+            "Are you sure that your kernel has seccomp2 support? The "
+            "sandbox won't work without it.", rc,
             strerror(-rc));
     goto end;
   }
@@ -1815,7 +1785,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
   /* Clean up the top stack frame so we get the real function
    * name for the most recently failing function. */
   clean_backtrace(syscall_cb_buf, depth, ctx);
-#endif
+#endif /* defined(USE_BACKTRACE) */
 
   syscall_name = get_syscall_name(syscall);
 
@@ -1831,7 +1801,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
 #endif
 
 #if defined(DEBUGGING_CLOSE)
-  _exit(1);
+  _exit(1); // exit ok: programming error has led to sandbox failure.
 #endif // DEBUGGING_CLOSE
 }
 
@@ -1889,7 +1859,7 @@ register_cfg(sandbox_cfg_t* cfg)
   return 0;
 }
 
-#endif // USE_LIBSECCOMP
+#endif /* defined(USE_LIBSECCOMP) */
 
 #ifdef USE_LIBSECCOMP
 /**
@@ -1919,7 +1889,7 @@ sandbox_is_active(void)
 {
   return sandbox_active != 0;
 }
-#endif // USE_LIBSECCOMP
+#endif /* defined(USE_LIBSECCOMP) */
 
 sandbox_cfg_t*
 sandbox_cfg_new(void)
@@ -1947,7 +1917,7 @@ sandbox_init(sandbox_cfg_t *cfg)
            "Currently, sandboxing is only implemented on Linux. The feature "
            "is disabled on your platform.");
   return 0;
-#endif
+#endif /* defined(USE_LIBSECCOMP) || ... */
 }
 
 #ifndef USE_LIBSECCOMP
@@ -1965,15 +1935,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
-#if 0
-int
-sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
-{
-  (void)cfg; (void)com;
-  return 0;
-}
-#endif
-
 int
 sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
 {
@@ -2012,5 +1973,5 @@ void
 sandbox_disable_getaddrinfo_cache(void)
 {
 }
-#endif
+#endif /* !defined(USE_LIBSECCOMP) */
 
diff --git a/src/common/sandbox.h b/src/common/sandbox.h
index c5963e3119..d0f85570f4 100644
--- a/src/common/sandbox.h
+++ b/src/common/sandbox.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,7 +23,7 @@
  */
 #define SYS_SECCOMP 1
 
-#endif
+#endif /* !defined(SYS_SECCOMP) */
 
 #if defined(HAVE_SECCOMP_H) && defined(__linux__)
 #define USE_LIBSECCOMP
@@ -101,7 +101,7 @@ typedef struct {
   sandbox_cfg_t *filter_dynamic;
 } sandbox_t;
 
-#endif // USE_LIBSECCOMP
+#endif /* defined(USE_LIBSECCOMP) */
 
 #ifdef USE_LIBSECCOMP
 /** Pre-calls getaddrinfo in order to pre-record result. */
@@ -114,7 +114,7 @@ int sandbox_getaddrinfo(const char *name, const char *servname,
                         struct addrinfo **res);
 void sandbox_freeaddrinfo(struct addrinfo *addrinfo);
 void sandbox_free_getaddrinfo_cache(void);
-#else
+#else /* !(defined(USE_LIBSECCOMP)) */
 #define sandbox_getaddrinfo(name, servname, hints, res)  \
   getaddrinfo((name),(servname), (hints),(res))
 #define sandbox_add_addrinfo(name) \
@@ -122,16 +122,16 @@ void sandbox_free_getaddrinfo_cache(void);
 #define sandbox_freeaddrinfo(addrinfo) \
   freeaddrinfo((addrinfo))
 #define sandbox_free_getaddrinfo_cache()
-#endif
+#endif /* defined(USE_LIBSECCOMP) */
 
 #ifdef USE_LIBSECCOMP
 /** Returns a registered protected string used with the sandbox, given that
  * it matches the parameter.
  */
 const char* sandbox_intern_string(const char *param);
-#else
+#else /* !(defined(USE_LIBSECCOMP)) */
 #define sandbox_intern_string(s) (s)
-#endif
+#endif /* defined(USE_LIBSECCOMP) */
 
 /** Creates an empty sandbox configuration file.*/
 sandbox_cfg_t * sandbox_cfg_new(void);
@@ -156,14 +156,6 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2);
  */
 int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file);
 
-#if 0
-/**
- * Function used to add a execve allowed filename to a supplied configuration.
- * The (char*) specifies the path to the allowed file; that pointer is stolen.
- */
-int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com);
-#endif
-
 /**
  * Function used to add a stat/stat64 allowed filename to a configuration.
  * The (char*) specifies the path to the allowed file; that pointer is stolen.
@@ -178,5 +170,5 @@ int sandbox_is_active(void);
 
 void sandbox_disable_getaddrinfo_cache(void);
 
-#endif /* SANDBOX_H_ */
+#endif /* !defined(SANDBOX_H_) */
 
diff --git a/src/common/storagedir.c b/src/common/storagedir.c
new file mode 100644
index 0000000000..e2c7b4bb87
--- /dev/null
+++ b/src/common/storagedir.c
@@ -0,0 +1,586 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "container.h"
+#include "compat.h"
+#include "confline.h"
+#include "memarea.h"
+#include "sandbox.h"
+#include "storagedir.h"
+#include "torlog.h"
+#include "util.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#define FNAME_MIN_NUM 1000
+
+/** A storage_dir_t represents a directory full of similar cached
+ * files. Filenames are decimal integers. Files can be cleaned as needed
+ * to limit total disk usage. */
+struct storage_dir_t {
+  /** Directory holding the files for this storagedir. */
+  char *directory;
+  /** Either NULL, or a directory listing of the directory (as a smartlist
+   * of strings */
+  smartlist_t *contents;
+  /** The largest number of non-temporary files we'll place in the
+   * directory. */
+  int max_files;
+  /** If true, then 'usage' has been computed. */
+  int usage_known;
+  /** The total number of bytes used in this directory */
+  uint64_t usage;
+};
+
+/** Create or open a new storage directory at <b>dirname</b>, with
+ * capacity for up to <b>max_files</b> files.
+ */
+storage_dir_t *
+storage_dir_new(const char *dirname, int max_files)
+{
+  if (check_private_dir(dirname, CPD_CREATE, NULL) < 0)
+    return NULL;
+
+  storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t));
+  d->directory = tor_strdup(dirname);
+  d->max_files = max_files;
+  return d;
+}
+
+/**
+ * Drop all in-RAM storage for <b>d</b>.  Does not delete any files.
+ */
+void
+storage_dir_free_(storage_dir_t *d)
+{
+  if (d == NULL)
+    return;
+  tor_free(d->directory);
+  if (d->contents) {
+    SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+    smartlist_free(d->contents);
+  }
+  tor_free(d);
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>d</b> will need.
+ *
+ * The presence of this function is why we need an upper limit on the
+ * number of files in a storage_dir_t: we need to approve file operations
+ * one by one.
+ */
+int
+storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg)
+{
+  int problems = 0;
+  int idx;
+  for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) {
+    char *path = NULL, *tmppath = NULL;
+    tor_asprintf(&path, "%s/%d", d->directory, idx);
+    tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx);
+
+    problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path));
+    problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath));
+    problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path));
+    problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath));
+    problems += sandbox_cfg_allow_rename(cfg,
+                                      tor_strdup(tmppath), tor_strdup(path));
+
+    tor_free(path);
+    tor_free(tmppath);
+  }
+
+  return problems ? -1 : 0;
+}
+
+/**
+ * Remove all files in <b>d</b> whose names end with ".tmp".
+ *
+ * Requires that the contents field of <b>d</b> is set.
+ */
+static void
+storage_dir_clean_tmpfiles(storage_dir_t *d)
+{
+  if (!d->contents)
+    return;
+  SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) {
+    if (strcmpend(fname, ".tmp"))
+      continue;
+    char *path = NULL;
+    tor_asprintf(&path, "%s/%s", d->directory, fname);
+    if (unlink(sandbox_intern_string(path))) {
+      log_warn(LD_FS, "Unable to unlink %s while cleaning "
+               "temporary files: %s", escaped(path), strerror(errno));
+      tor_free(path);
+      continue;
+    }
+    tor_free(path);
+    SMARTLIST_DEL_CURRENT(d->contents, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(fname);
+
+  d->usage_known = 0;
+}
+
+/**
+ * Re-scan the directory <b>d</b> to learn its contents.
+ */
+static int
+storage_dir_rescan(storage_dir_t *d)
+{
+  if (d->contents) {
+    SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+    smartlist_free(d->contents);
+  }
+  d->usage = 0;
+  d->usage_known = 0;
+  if (NULL == (d->contents = tor_listdir(d->directory))) {
+    return -1;
+  }
+  storage_dir_clean_tmpfiles(d);
+  return 0;
+}
+
+/**
+ * Return a smartlist containing the filenames within <b>d</b>.
+ */
+const smartlist_t *
+storage_dir_list(storage_dir_t *d)
+{
+  if (! d->contents)
+    storage_dir_rescan(d);
+  return d->contents;
+}
+
+/**
+ * Return the total number of bytes used for storage in <b>d</b>.
+ */
+uint64_t
+storage_dir_get_usage(storage_dir_t *d)
+{
+  if (d->usage_known)
+    return d->usage;
+
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) {
+    char *path = NULL;
+    struct stat st;
+    tor_asprintf(&path, "%s/%s", d->directory, cp);
+    if (stat(sandbox_intern_string(path), &st) == 0) {
+      total += st.st_size;
+    }
+    tor_free(path);
+  } SMARTLIST_FOREACH_END(cp);
+
+  d->usage = total;
+  d->usage_known = 1;
+  return d->usage;
+}
+
+/** Mmap a specified file within <b>d</b>.
+ *
+ * On failure, return NULL and set errno as for tor_mmap_file(). */
+tor_mmap_t *
+storage_dir_map(storage_dir_t *d, const char *fname)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  tor_mmap_t *result = tor_mmap_file(path);
+  int errval = errno;
+  tor_free(path);
+  if (result == NULL)
+    errno = errval;
+  return result;
+}
+
+/** Read a file within <b>d</b> into a newly allocated buffer.  Set
+ * *<b>sz_out</b> to its size. */
+uint8_t *
+storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
+{
+  const int flags = bin ? RFTS_BIN : 0;
+
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  struct stat st;
+  char *contents = read_file_to_str(path, flags, &st);
+  if (contents && sz_out) {
+    // it fits in RAM, so we know its size is less than SIZE_MAX
+#if UINT64_MAX > SIZE_MAX
+    tor_assert((uint64_t)st.st_size <= SIZE_MAX);
+#endif
+    *sz_out = (size_t) st.st_size;
+  }
+
+  tor_free(path);
+  return (uint8_t *) contents;
+}
+
+/** Helper: Find an unused filename within the directory */
+static char *
+find_unused_fname(storage_dir_t *d)
+{
+  if (!d->contents) {
+    if (storage_dir_rescan(d) < 0)
+      return NULL;
+  }
+
+  char buf[16];
+  int i;
+  /* Yuck; this is quadratic.  Fortunately, that shouldn't matter much,
+   * since disk writes are more expensive by a lot. */
+  for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) {
+    tor_snprintf(buf, sizeof(buf), "%d", i);
+    if (!smartlist_contains_string(d->contents, buf)) {
+      return tor_strdup(buf);
+    }
+  }
+  return NULL;
+}
+
+/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
+ * sized_chunk_t rather than a single byte array. */
+static int
+storage_dir_save_chunks_to_file(storage_dir_t *d,
+                                const smartlist_t *chunks,
+                                int binary,
+                                char **fname_out)
+{
+  uint64_t total_length = 0;
+  char *fname = find_unused_fname(d);
+  if (!fname)
+    return -1;
+
+  SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
+                    total_length += ch->len);
+
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+
+  int r = write_chunks_to_file(path, chunks, binary, 0);
+  if (r == 0) {
+    if (d->usage_known)
+      d->usage += total_length;
+    if (fname_out) {
+      *fname_out = tor_strdup(fname);
+    }
+    if (d->contents)
+      smartlist_add(d->contents, tor_strdup(fname));
+  }
+  tor_free(fname);
+  tor_free(path);
+  return r;
+}
+
+/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
+ * in <b>d</b>.  On success, return 0 and set *<b>fname_out</b> to a
+ * newly allocated string containing the filename.  On failure, return
+ * -1. */
+int
+storage_dir_save_bytes_to_file(storage_dir_t *d,
+                               const uint8_t *data,
+                               size_t length,
+                               int binary,
+                               char **fname_out)
+{
+  smartlist_t *chunks = smartlist_new();
+  sized_chunk_t chunk = { (const char *)data, length };
+  smartlist_add(chunks, &chunk);
+  int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
+  smartlist_free(chunks);
+  return r;
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string
+ * <b>str</b>.
+ */
+int
+storage_dir_save_string_to_file(storage_dir_t *d,
+                                const char *str,
+                                int binary,
+                                char **fname_out)
+{
+  return storage_dir_save_bytes_to_file(d,
+                (const uint8_t*)str, strlen(str), binary, fname_out);
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but associates the data with the
+ * key-value pairs in <b>labels</b>. Files stored in this format can be
+ * recovered with storage_dir_map_labeled() or storage_dir_read_labeled().
+ */
+int
+storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                  const config_line_t *labels,
+                                  const uint8_t *data,
+                                  size_t length,
+                                  char **fname_out)
+{
+  /*
+   * The storage format is to prefix the data with the key-value pairs in
+   * <b>labels</b>, and a single NUL separator.  But code outside this module
+   * MUST NOT rely on that format.
+   */
+
+  smartlist_t *chunks = smartlist_new();
+  memarea_t *area = memarea_new();
+  const config_line_t *line;
+  for (line = labels; line; line = line->next) {
+    sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
+    sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
+    const size_t allocated = sz->len + 1;
+    char *bytes = memarea_alloc(area, allocated);
+    tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
+    sz->bytes = bytes;
+    smartlist_add(chunks, sz);
+  }
+
+  sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
+  nul->len = 1;
+  nul->bytes = "\0";
+  smartlist_add(chunks, nul);
+
+  sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
+  datachunk->bytes = (const char *)data;
+  datachunk->len = length;
+  smartlist_add(chunks, datachunk);
+
+  int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
+  smartlist_free(chunks);
+  memarea_drop_all(area);
+  return r;
+}
+
+/**
+ * Map a file that was created with storage_dir_save_labeled_to_file().  On
+ * failure, return NULL.  On success, write a set of newly allocated labels
+ * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and
+ * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t
+ * object whose contents should not be used -- it needs to be kept around,
+ * though, for as long as <b>data_out</b> is going to be valid.
+ *
+ * On failure, set errno as for tor_mmap_file() if the file was missing or
+ * empty, and set errno to EINVAL if the file was not in the labeled
+ * format expected.
+ */
+tor_mmap_t *
+storage_dir_map_labeled(storage_dir_t *dir,
+                         const char *fname,
+                         config_line_t **labels_out,
+                         const uint8_t **data_out,
+                         size_t *sz_out)
+{
+  tor_mmap_t *m = storage_dir_map(dir, fname);
+  int errval;
+  if (! m) {
+    errval = errno;
+    goto err;
+  }
+  const char *nulp = memchr(m->data, '\0', m->size);
+  if (! nulp) {
+    errval = EINVAL;
+    goto err;
+  }
+  if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) {
+    errval = EINVAL;
+    goto err;
+  }
+  size_t offset = nulp - m->data + 1;
+  tor_assert(offset <= m->size);
+  *data_out = (const uint8_t *)(m->data + offset);
+  *sz_out = m->size - offset;
+
+  return m;
+ err:
+  tor_munmap_file(m);
+  errno = errval;
+  return NULL;
+}
+
+/** As storage_dir_map_labeled, but return a new byte array containing the
+ * data. */
+uint8_t *
+storage_dir_read_labeled(storage_dir_t *dir,
+                          const char *fname,
+                          config_line_t **labels_out,
+                          size_t *sz_out)
+{
+  const uint8_t *data = NULL;
+  tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
+                                           &data, sz_out);
+  if (m == NULL)
+    return NULL;
+  uint8_t *result = tor_memdup(data, *sz_out);
+  tor_munmap_file(m);
+  return result;
+}
+
+/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>.
+ * This function is a no-op if <b>d->usage_known</b> is 0. */
+static void
+storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size)
+{
+  if (d->usage_known) {
+    if (! BUG(d->usage < removed_file_size)) {
+      /* This bug can also be triggered if an external process resized a file
+       * between the call to storage_dir_get_usage() that last checked
+       * actual usage (rather than relaying on cached usage), and the call to
+       * this function. */
+      d->usage -= removed_file_size;
+    } else {
+      /* If we underflowed the cached directory size, re-check the sizes of all
+       * the files in the directory. This makes storage_dir_shrink() quadratic,
+       * but only if a process is continually changing file sizes in the
+       * storage directory (in which case, we have bigger issues).
+       *
+       * We can't just reset usage_known, because storage_dir_shrink() relies
+       * on knowing the usage. */
+      storage_dir_rescan(d);
+      (void)storage_dir_get_usage(d);
+    }
+  }
+}
+
+/**
+ * Remove the file called <b>fname</b> from <b>d</b>.
+ */
+void
+storage_dir_remove_file(storage_dir_t *d,
+                        const char *fname)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  const char *ipath = sandbox_intern_string(path);
+
+  uint64_t size = 0;
+  if (d->usage_known) {
+    struct stat st;
+    if (stat(ipath, &st) == 0) {
+      size = st.st_size;
+    }
+  }
+  if (unlink(ipath) == 0) {
+    storage_dir_reduce_usage(d, size);
+  } else {
+    log_warn(LD_FS, "Unable to unlink %s while removing file: %s",
+             escaped(path), strerror(errno));
+    tor_free(path);
+    return;
+  }
+  if (d->contents) {
+    smartlist_string_remove(d->contents, fname);
+  }
+
+  tor_free(path);
+}
+
+/** Helper type: used to sort the members of storage directory by mtime. */
+typedef struct shrinking_dir_entry_t {
+  time_t mtime;
+  uint64_t size;
+  char *path;
+} shrinking_dir_entry_t;
+
+/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */
+static int
+shrinking_dir_entry_compare(const void *a_, const void *b_)
+{
+  const shrinking_dir_entry_t *a = a_;
+  const shrinking_dir_entry_t *b = b_;
+
+  if (a->mtime < b->mtime)
+    return -1;
+  else if (a->mtime > b->mtime)
+    return 1;
+  else
+    return 0;
+}
+
+/**
+ * Try to free space by removing the oldest files in <b>d</b>. Delete
+ * until no more than <b>target_size</b> bytes are left, and at least
+ * <b>min_to_remove</b> files have been removed... or until there is
+ * nothing left to remove.
+ *
+ * Return 0 on success; -1 on failure.
+ */
+int
+storage_dir_shrink(storage_dir_t *d,
+                   uint64_t target_size,
+                   int min_to_remove)
+{
+  if (d->usage_known && d->usage <= target_size && !min_to_remove) {
+    /* Already small enough. */
+    return 0;
+  }
+
+  if (storage_dir_rescan(d) < 0)
+    return -1;
+
+  const uint64_t orig_usage = storage_dir_get_usage(d);
+  if (orig_usage <= target_size && !min_to_remove) {
+    /* Okay, small enough after rescan! */
+    return 0;
+  }
+
+  const int n = smartlist_len(d->contents);
+  shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t));
+  SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) {
+    shrinking_dir_entry_t *ent = &ents[fname_sl_idx];
+    struct stat st;
+    tor_asprintf(&ent->path, "%s/%s", d->directory, fname);
+    if (stat(sandbox_intern_string(ent->path), &st) == 0) {
+      ent->mtime = st.st_mtime;
+      ent->size = st.st_size;
+    }
+  } SMARTLIST_FOREACH_END(fname);
+
+  qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare);
+
+  int idx = 0;
+  while ((d->usage > target_size || min_to_remove > 0) && idx < n) {
+    if (unlink(sandbox_intern_string(ents[idx].path)) == 0) {
+      storage_dir_reduce_usage(d, ents[idx].size);
+      --min_to_remove;
+    }
+    ++idx;
+  }
+
+  for (idx = 0; idx < n; ++idx) {
+    tor_free(ents[idx].path);
+  }
+  tor_free(ents);
+
+  storage_dir_rescan(d);
+
+  return 0;
+}
+
+/** Remove all files in <b>d</b>. */
+int
+storage_dir_remove_all(storage_dir_t *d)
+{
+  return storage_dir_shrink(d, 0, d->max_files);
+}
+
+/**
+ * Return the largest number of non-temporary files we're willing to
+ * store in <b>d</b>.
+ */
+int
+storage_dir_get_max_files(storage_dir_t *d)
+{
+  return d->max_files;
+}
+
diff --git a/src/common/storagedir.h b/src/common/storagedir.h
new file mode 100644
index 0000000000..d99bd7ec52
--- /dev/null
+++ b/src/common/storagedir.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_STORAGEDIR_H
+#define TOR_STORAGEDIR_H
+
+typedef struct storage_dir_t storage_dir_t;
+struct config_line_t;
+struct sandbox_cfg_elem;
+
+storage_dir_t * storage_dir_new(const char *dirname, int n_files);
+void storage_dir_free_(storage_dir_t *d);
+#define storage_dir_free(d) \
+  FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d))
+
+int storage_dir_register_with_sandbox(storage_dir_t *d,
+                                      struct sandbox_cfg_elem **cfg);
+const smartlist_t *storage_dir_list(storage_dir_t *d);
+uint64_t storage_dir_get_usage(storage_dir_t *d);
+tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname);
+uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin,
+                          size_t *sz_out);
+int storage_dir_save_bytes_to_file(storage_dir_t *d,
+                                   const uint8_t *data,
+                                   size_t length,
+                                   int binary,
+                                   char **fname_out);
+int storage_dir_save_string_to_file(storage_dir_t *d,
+                                    const char *data,
+                                    int binary,
+                                    char **fname_out);
+int storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                      const struct config_line_t *labels,
+                                      const uint8_t *data,
+                                      size_t length,
+                                      char **fname_out);
+tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
+                                     const char *fname,
+                                     struct config_line_t **labels_out,
+                                     const uint8_t **data_out,
+                                     size_t *size_out);
+uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
+                                   struct config_line_t **labels_out,
+                                   size_t *sz_out);
+void storage_dir_remove_file(storage_dir_t *d,
+                             const char *fname);
+int storage_dir_shrink(storage_dir_t *d,
+                       uint64_t target_size,
+                       int min_to_remove);
+int storage_dir_remove_all(storage_dir_t *d);
+int storage_dir_get_max_files(storage_dir_t *d);
+
+#endif /* !defined(TOR_STORAGEDIR_H) */
+
diff --git a/src/common/testsupport.h b/src/common/testsupport.h
index 9ad2ba77e0..a3f2ff91ed 100644
--- a/src/common/testsupport.h
+++ b/src/common/testsupport.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TESTSUPPORT_H
@@ -10,7 +10,7 @@
 #else
 #define STATIC static
 #define EXTERN(type, name)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Quick and dirty macros to implement test mocking.
  *
@@ -76,15 +76,15 @@
   do {                                          \
     func = func ##__real;                       \
   } while (0)
-#else
+#else /* !(defined(TOR_UNIT_TESTS)) */
 #define MOCK_DECL(rv, funcname, arglist) \
   rv funcname arglist
 #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
   rv funcname arglist attr
 #define MOCK_IMPL(rv, funcname, arglist) \
   rv funcname arglist
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 /** @} */
 
-#endif
+#endif /* !defined(TOR_TESTSUPPORT_H) */
 
diff --git a/src/common/timers.c b/src/common/timers.c
index 41b2008ac4..6f6236ed3b 100644
--- a/src/common/timers.c
+++ b/src/common/timers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,14 +29,14 @@
 
 #include "orconfig.h"
 
+#define TOR_TIMERS_PRIVATE
+
 #include "compat.h"
 #include "compat_libevent.h"
 #include "timers.h"
 #include "torlog.h"
 #include "util.h"
 
-#include <event2/event.h>
-
 struct timeout_cb {
   timer_cb_fn_t cb;
   void *arg;
@@ -51,7 +51,7 @@ struct timeout_cb {
 #else
 /* We're not exposing any of the functions outside this file. */
 #define TIMEOUT_PUBLIC static
-#endif
+#endif /* defined(__GNUC__) */
 /* We're not using periodic events. */
 #define TIMEOUT_DISABLE_INTERVALS
 /* We always know the global_timeouts object, so we don't need each timeout
@@ -61,13 +61,18 @@ struct timeout_cb {
 #define TIMEOUT_CB_OVERRIDE
 /* We're going to support timers that are pretty far out in advance. Making
  * this big can be inefficient, but having a significant number of timers
- * above TIMEOUT_MAX can also be super-inefficent. Choosing 5 here sets
+ * above TIMEOUT_MAX can also be super-inefficient. Choosing 5 here sets
  * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */
 #define WHEEL_NUM 5
+#if SIZEOF_VOID_P == 4
+/* On 32-bit platforms, we want to override wheel_bit, so that timeout.c will
+ * use 32-bit math. */
+#define WHEEL_BIT 5
+#endif
 #include "src/ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
-static struct event *global_timer_event = NULL;
+static struct mainloop_event_t *global_timer_event = NULL;
 
 static monotime_t start_of_time;
 
@@ -145,20 +150,14 @@ libevent_timer_reschedule(void)
   if (delay > MIN_CHECK_TICKS)
     delay = MIN_CHECK_TICKS;
   timeout_to_tv(delay, &d);
-  event_add(global_timer_event, &d);
+  mainloop_event_schedule(global_timer_event, &d);
 }
 
-/**
- * Invoked when the libevent timer has expired: see which tor_timer_t events
- * have fired, activate their callbacks, and reschedule the libevent timer.
- */
-static void
-libevent_timer_callback(evutil_socket_t fd, short what, void *arg)
+/** Run the callback of every timer that has expired, based on the current
+ * output of monotime_get(). */
+STATIC void
+timers_run_pending(void)
 {
-  (void)fd;
-  (void)what;
-  (void)arg;
-
   monotime_t now;
   monotime_get(&now);
   timer_advance_to_cur_time(&now);
@@ -167,6 +166,19 @@ libevent_timer_callback(evutil_socket_t fd, short what, void *arg)
   while ((t = timeouts_get(global_timeouts))) {
     t->callback.cb(t, t->callback.arg, &now);
   }
+}
+
+/**
+ * Invoked when the libevent timer has expired: see which tor_timer_t events
+ * have fired, activate their callbacks, and reschedule the libevent timer.
+ */
+static void
+libevent_timer_callback(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+
+  timers_run_pending();
 
   libevent_timer_reschedule();
 }
@@ -181,7 +193,7 @@ timers_initialize(void)
   if (BUG(global_timeouts))
     return; // LCOV_EXCL_LINE
 
-  timeout_error_t err;
+  timeout_error_t err = 0;
   global_timeouts = timeouts_open(0, &err);
   if (!global_timeouts) {
     // LCOV_EXCL_START -- this can only fail on malloc failure.
@@ -193,9 +205,8 @@ timers_initialize(void)
   monotime_init();
   monotime_get(&start_of_time);
 
-  struct event *timer_event;
-  timer_event = tor_event_new(tor_libevent_get_base(),
-                              -1, 0, libevent_timer_callback, NULL);
+  mainloop_event_t *timer_event;
+  timer_event = mainloop_event_new(libevent_timer_callback, NULL);
   tor_assert(timer_event);
   global_timer_event = timer_event;
 
@@ -209,7 +220,7 @@ void
 timers_shutdown(void)
 {
   if (global_timer_event) {
-    tor_event_free(global_timer_event);
+    mainloop_event_free(global_timer_event);
     global_timer_event = NULL;
   }
   if (global_timeouts) {
@@ -235,7 +246,7 @@ timer_new(timer_cb_fn_t cb, void *arg)
  * scheduled.
  */
 void
-timer_free(tor_timer_t *t)
+timer_free_(tor_timer_t *t)
 {
   if (! t)
     return;
@@ -255,6 +266,20 @@ timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg)
 }
 
 /**
+ * Set *<b>cb_out</b> (if provided) to this timer's callback function,
+ * and *<b>arg_out</b> (if provided) to this timer's callback argument.
+ */
+void
+timer_get_cb(const tor_timer_t *t,
+             timer_cb_fn_t *cb_out, void **arg_out)
+{
+  if (cb_out)
+    *cb_out = t->callback.cb;
+  if (arg_out)
+    *arg_out = t->callback.arg;
+}
+
+/**
  * Schedule the timer t to fire at the current time plus a delay of
  * <b>delay</b> microseconds.  All times are relative to monotime_get().
  */
diff --git a/src/common/timers.h b/src/common/timers.h
index 5f918f8e15..6d27f3e01e 100644
--- a/src/common/timers.h
+++ b/src/common/timers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TIMERS_H
@@ -13,12 +13,19 @@ typedef void (*timer_cb_fn_t)(tor_timer_t *, void *,
                               const struct monotime_t *);
 tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg);
 void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg);
+void timer_get_cb(const tor_timer_t *t,
+                  timer_cb_fn_t *cb_out, void **arg_out);
 void timer_schedule(tor_timer_t *t, const struct timeval *delay);
 void timer_disable(tor_timer_t *t);
-void timer_free(tor_timer_t *t);
+void timer_free_(tor_timer_t *t);
+#define timer_free(t) FREE_AND_NULL(tor_timer_t, timer_free_, (t))
 
 void timers_initialize(void);
 void timers_shutdown(void);
 
+#ifdef TOR_TIMERS_PRIVATE
+STATIC void timers_run_pending(void);
 #endif
 
+#endif /* !defined(TOR_TIMERS_H) */
+
diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c
new file mode 100644
index 0000000000..f2396ec58a
--- /dev/null
+++ b/src/common/token_bucket.c
@@ -0,0 +1,255 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file token_bucket.c
+ * \brief Functions to use and manipulate token buckets, used for
+ *    rate-limiting on connections and globally.
+ *
+ * Tor uses these token buckets to keep track of bandwidth usage, and
+ * sometimes other things too.
+ *
+ * There are two layers of abstraction here: "raw" token buckets, in which all
+ * the pieces are decoupled, and "read-write" token buckets, which combine all
+ * the moving parts into one.
+ *
+ * Token buckets may become negative.
+ **/
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "token_bucket.h"
+#include "util_bug.h"
+
+/**
+ * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg.
+ *
+ * Note that the <b>rate</b> value is in arbitrary units, but those units will
+ * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and
+ * so on.
+ */
+void
+token_bucket_cfg_init(token_bucket_cfg_t *cfg,
+                      uint32_t rate,
+                      uint32_t burst)
+{
+  tor_assert_nonfatal(rate > 0);
+  tor_assert_nonfatal(burst > 0);
+  if (burst > TOKEN_BUCKET_MAX_BURST)
+    burst = TOKEN_BUCKET_MAX_BURST;
+
+  cfg->rate = rate;
+  cfg->burst = burst;
+}
+
+/**
+ * Initialize a raw token bucket and its associated timestamp to the "full"
+ * state, according to <b>cfg</b>.
+ */
+void
+token_bucket_raw_reset(token_bucket_raw_t *bucket,
+                       const token_bucket_cfg_t *cfg)
+{
+  bucket->bucket = cfg->burst;
+}
+
+/**
+ * Adust a preexisting token bucket to respect the new configuration
+ * <b>cfg</b>, by decreasing its current level if needed. */
+void
+token_bucket_raw_adjust(token_bucket_raw_t *bucket,
+                        const token_bucket_cfg_t *cfg)
+{
+  bucket->bucket = MIN(bucket->bucket, cfg->burst);
+}
+
+/**
+ * Given an amount of <b>elapsed</b> time units, and a bucket configuration
+ * <b>cfg</b>, refill the level of <b>bucket</b> accordingly.  Note that the
+ * units of time in <b>elapsed</b> must correspond to those used to set the
+ * rate in <b>cfg</b>, or the result will be illogical.
+ */
+int
+token_bucket_raw_refill_steps(token_bucket_raw_t *bucket,
+                              const token_bucket_cfg_t *cfg,
+                              const uint32_t elapsed)
+{
+  const int was_empty = (bucket->bucket <= 0);
+  /* The casts here prevent an underflow.
+   *
+   * Note that even if the bucket value is negative, subtracting it from
+   * "burst" will still produce a correct result.  If this result is
+   * ridiculously high, then the "elapsed > gap / rate" check below
+   * should catch it. */
+  const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket);
+
+  if (elapsed > gap / cfg->rate) {
+    bucket->bucket = cfg->burst;
+  } else {
+    bucket->bucket += cfg->rate * elapsed;
+  }
+
+  return was_empty && bucket->bucket > 0;
+}
+
+/**
+ * Decrement a provided bucket by <b>n</b> units.  Note that <b>n</b>
+ * must be nonnegative.
+ */
+int
+token_bucket_raw_dec(token_bucket_raw_t *bucket,
+                     ssize_t n)
+{
+  if (BUG(n < 0))
+    return 0;
+  const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket;
+  bucket->bucket -= n;
+  return becomes_empty;
+}
+
+/** Convert a rate in bytes per second to a rate in bytes per step */
+STATIC uint32_t
+rate_per_sec_to_rate_per_step(uint32_t rate)
+{
+  /*
+    The precise calculation we'd want to do is
+
+    (rate / 1000) * to_approximate_msec(TICKS_PER_STEP).  But to minimize
+    rounding error, we do it this way instead, and divide last.
+  */
+  uint64_t units = (uint64_t) rate * TICKS_PER_STEP;
+  uint32_t val = (uint32_t)
+    (monotime_coarse_stamp_units_to_approx_msec(units) / 1000);
+  return val ? val : 1;
+}
+
+/**
+ * Initialize a token bucket in *<b>bucket</b>, set up to allow <b>rate</b>
+ * bytes per second, with a maximum burst of <b>burst</b> bytes. The bucket
+ * is created such that <b>now_ts</b> is the current timestamp.  The bucket
+ * starts out full.
+ */
+void
+token_bucket_rw_init(token_bucket_rw_t *bucket,
+                     uint32_t rate,
+                     uint32_t burst,
+                     uint32_t now_ts)
+{
+  memset(bucket, 0, sizeof(token_bucket_rw_t));
+  token_bucket_rw_adjust(bucket, rate, burst);
+  token_bucket_rw_reset(bucket, now_ts);
+}
+
+/**
+ * Change the configured rate (in bytes per second) and burst (in bytes)
+ * for the token bucket in *<b>bucket</b>.
+ */
+void
+token_bucket_rw_adjust(token_bucket_rw_t *bucket,
+                       uint32_t rate,
+                       uint32_t burst)
+{
+  token_bucket_cfg_init(&bucket->cfg,
+                        rate_per_sec_to_rate_per_step(rate),
+                        burst);
+  token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg);
+  token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg);
+}
+
+/**
+ * Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>.
+ */
+void
+token_bucket_rw_reset(token_bucket_rw_t *bucket,
+                      uint32_t now_ts)
+{
+  token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg);
+  token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
+
+/**
+ * Refill <b>bucket</b> as appropriate, given that the current timestamp
+ * is <b>now_ts</b>.
+ *
+ * Return a bitmask containing TB_READ iff read bucket was empty and became
+ * nonempty, and TB_WRITE iff the write bucket was empty and became nonempty.
+ */
+int
+token_bucket_rw_refill(token_bucket_rw_t *bucket,
+                       uint32_t now_ts)
+{
+  const uint32_t elapsed_ticks =
+    (now_ts - bucket->last_refilled_at_timestamp);
+  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
+    /* Either about 48 days have passed since the last refill, or the
+     * monotonic clock has somehow moved backwards. (We're looking at you,
+     * Windows.).  We accept up to a 5 minute jump backwards as
+     * "unremarkable".
+     */
+    return 0;
+  }
+  const uint32_t elapsed_steps = elapsed_ticks / TICKS_PER_STEP;
+
+  if (!elapsed_steps) {
+    /* Note that if less than one whole step elapsed, we don't advance the
+     * time in last_refilled_at. That's intentional: we want to make sure
+     * that we add some bytes to it eventually. */
+    return 0;
+  }
+
+  int flags = 0;
+  if (token_bucket_raw_refill_steps(&bucket->read_bucket,
+                                    &bucket->cfg, elapsed_steps))
+    flags |= TB_READ;
+  if (token_bucket_raw_refill_steps(&bucket->write_bucket,
+                                    &bucket->cfg, elapsed_steps))
+    flags |= TB_WRITE;
+
+  bucket->last_refilled_at_timestamp = now_ts;
+  return flags;
+}
+
+/**
+ * Decrement the read token bucket in <b>bucket</b> by <b>n</b> bytes.
+ *
+ * Return true if the bucket was nonempty and became empty; return false
+ * otherwise.
+ */
+int
+token_bucket_rw_dec_read(token_bucket_rw_t *bucket,
+                         ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->read_bucket, n);
+}
+
+/**
+ * Decrement the write token bucket in <b>bucket</b> by <b>n</b> bytes.
+ *
+ * Return true if the bucket was nonempty and became empty; return false
+ * otherwise.
+ */
+int
+token_bucket_rw_dec_write(token_bucket_rw_t *bucket,
+                          ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->write_bucket, n);
+}
+
+/**
+ * As token_bucket_rw_dec_read and token_bucket_rw_dec_write, in a single
+ * operation.  Return a bitmask of TB_READ and TB_WRITE to indicate
+ * which buckets became empty.
+ */
+int
+token_bucket_rw_dec(token_bucket_rw_t *bucket,
+                    ssize_t n_read, ssize_t n_written)
+{
+  int flags = 0;
+  if (token_bucket_rw_dec_read(bucket, n_read))
+    flags |= TB_READ;
+  if (token_bucket_rw_dec_write(bucket, n_written))
+    flags |= TB_WRITE;
+  return flags;
+}
+
diff --git a/src/common/token_bucket.h b/src/common/token_bucket.h
new file mode 100644
index 0000000000..0e7832e838
--- /dev/null
+++ b/src/common/token_bucket.h
@@ -0,0 +1,118 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file token_bucket_rw.h
+ * \brief Headers for token_bucket_rw.c
+ **/
+
+#ifndef TOR_TOKEN_BUCKET_H
+#define TOR_TOKEN_BUCKET_H
+
+#include "torint.h"
+#include "testsupport.h"
+
+/** Largest allowable burst value for a token buffer. */
+#define TOKEN_BUCKET_MAX_BURST INT32_MAX
+
+/** A generic token buffer configuration: determines the number of tokens
+ * added to the bucket in each time unit (the "rate"), and the maximum number
+ * of tokens in the bucket (the "burst") */
+typedef struct token_bucket_cfg_t {
+  uint32_t rate;
+  int32_t burst;
+} token_bucket_cfg_t;
+
+/** A raw token bucket, decoupled from its configuration and timestamp. */
+typedef struct token_bucket_raw_t {
+  int32_t bucket;
+} token_bucket_raw_t;
+
+void token_bucket_cfg_init(token_bucket_cfg_t *cfg,
+                           uint32_t rate,
+                           uint32_t burst);
+
+void token_bucket_raw_adjust(token_bucket_raw_t *bucket,
+                             const token_bucket_cfg_t *cfg);
+
+void token_bucket_raw_reset(token_bucket_raw_t *bucket,
+                            const token_bucket_cfg_t *cfg);
+
+int token_bucket_raw_dec(token_bucket_raw_t *bucket,
+                         ssize_t n);
+
+int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket,
+                                  const token_bucket_cfg_t *cfg,
+                                  const uint32_t elapsed_steps);
+
+static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket);
+/** Return the current number of bytes set in a token bucket. */
+static inline size_t
+token_bucket_raw_get(const token_bucket_raw_t *bucket)
+{
+  return bucket->bucket >= 0 ? bucket->bucket : 0;
+}
+
+/** A convenience type containing all the pieces needed for a coupled
+ * read-bucket and write-bucket that have the same rate limit, and which use
+ * "timestamp units" (see compat_time.h) for their time. */
+typedef struct token_bucket_rw_t {
+  token_bucket_cfg_t cfg;
+  token_bucket_raw_t read_bucket;
+  token_bucket_raw_t write_bucket;
+  uint32_t last_refilled_at_timestamp;
+} token_bucket_rw_t;
+
+void token_bucket_rw_init(token_bucket_rw_t *bucket,
+                          uint32_t rate,
+                          uint32_t burst,
+                          uint32_t now_ts);
+
+void token_bucket_rw_adjust(token_bucket_rw_t *bucket,
+                            uint32_t rate, uint32_t burst);
+
+void token_bucket_rw_reset(token_bucket_rw_t *bucket,
+                           uint32_t now_ts);
+
+#define TB_READ 1
+#define TB_WRITE 2
+
+int token_bucket_rw_refill(token_bucket_rw_t *bucket,
+                           uint32_t now_ts);
+
+int token_bucket_rw_dec_read(token_bucket_rw_t *bucket,
+                             ssize_t n);
+int token_bucket_rw_dec_write(token_bucket_rw_t *bucket,
+                              ssize_t n);
+
+int token_bucket_rw_dec(token_bucket_rw_t *bucket,
+                        ssize_t n_read, ssize_t n_written);
+
+static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket);
+static inline size_t
+token_bucket_rw_get_read(const token_bucket_rw_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->read_bucket);
+}
+
+static inline size_t token_bucket_rw_get_write(
+                                            const token_bucket_rw_t *bucket);
+static inline size_t
+token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->write_bucket);
+}
+
+#ifdef TOKEN_BUCKET_PRIVATE
+
+/* To avoid making the rates too small, we consider units of "steps",
+ * where a "step" is defined as this many timestamp ticks.  Keep this
+ * a power of two if you can. */
+#define TICKS_PER_STEP 16
+
+STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
+
+#endif
+
+#endif /* TOR_TOKEN_BUCKET_H */
+
diff --git a/src/common/torgzip.c b/src/common/torgzip.c
deleted file mode 100644
index c44399aa74..0000000000
--- a/src/common/torgzip.c
+++ /dev/null
@@ -1,586 +0,0 @@
-/* Copyright (c) 2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torgzip.c
- * \brief A simple in-memory gzip implementation.
- **/
-
-#include "orconfig.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <string.h>
-#include "torint.h"
-
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-
-#include "util.h"
-#include "torlog.h"
-#include "torgzip.h"
-
-/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
-   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
-   that nobody will care if the compile outputs a no-such-identifier warning.
-
-   Sorry, but we like -Werror over here, so I guess we need to define these.
-   I hope that zlib 1.2.6 doesn't break these too.
-*/
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE 0
-#endif
-#ifndef _LFS64_LARGEFILE
-#define _LFS64_LARGEFILE 0
-#endif
-#ifndef _FILE_OFFSET_BITS
-#define _FILE_OFFSET_BITS 0
-#endif
-#ifndef off64_t
-#define off64_t int64_t
-#endif
-
-#include <zlib.h>
-
-#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
-#error "We require zlib version 1.2 or later."
-#endif
-
-static size_t tor_zlib_state_size_precalc(int inflate,
-                                          int windowbits, int memlevel);
-
-/** Total number of bytes allocated for zlib state */
-static size_t total_zlib_allocation = 0;
-
-/** Return a string representation of the version of the currently running
- * version of zlib. */
-const char *
-tor_zlib_get_version_str(void)
-{
-  return zlibVersion();
-}
-
-/** Return a string representation of the version of the version of zlib
-* used at compilation. */
-const char *
-tor_zlib_get_header_version_str(void)
-{
-  return ZLIB_VERSION;
-}
-
-/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
-static inline int
-method_bits(compress_method_t method, zlib_compression_level_t level)
-{
-  /* Bits+16 means "use gzip" in zlib >= 1.2 */
-  const int flag = method == GZIP_METHOD ? 16 : 0;
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return flag + 15;
-    case MEDIUM_COMPRESSION: return flag + 13;
-    case LOW_COMPRESSION: return flag + 11;
-  }
-}
-
-static inline int
-get_memlevel(zlib_compression_level_t level)
-{
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return 8;
-    case MEDIUM_COMPRESSION: return 7;
-    case LOW_COMPRESSION: return 6;
-  }
-}
-
-/** @{ */
-/* These macros define the maximum allowable compression factor.  Anything of
- * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to
- * have an uncompression factor (uncompressed size:compressed size ratio) of
- * any greater than MAX_UNCOMPRESSION_FACTOR.
- *
- * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to
- * be small to limit the attack multiplier, but we also want it to be large
- * enough so that no legitimate document --even ones we might invent in the
- * future -- ever compresses by a factor of greater than
- * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably
- * large range of possible values. IMO, anything over 8 is probably safe; IMO
- * anything under 50 is probably sufficient.
- */
-#define MAX_UNCOMPRESSION_FACTOR 25
-#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64)
-/** @} */
-
-/** Return true if uncompressing an input of size <b>in_size</b> to an input
- * of size at least <b>size_out</b> looks like a compression bomb. */
-static int
-is_compression_bomb(size_t size_in, size_t size_out)
-{
-  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
-    return 0;
-
-  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
-}
-
-/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
- * allocated buffer, using the method described in <b>method</b>.  Store the
- * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
- * Return 0 on success, -1 on failure.
- */
-int
-tor_gzip_compress(char **out, size_t *out_len,
-                  const char *in, size_t in_len,
-                  compress_method_t method)
-{
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                   method_bits(method, HIGH_COMPRESSION),
-                   get_memlevel(HIGH_COMPRESSION),
-                   Z_DEFAULT_STRATEGY) != Z_OK) {
-    //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
-    log_warn(LD_GENERAL, "Error from deflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    //LCOV_EXCL_STOP
-  }
-
-  /* Guess 50% compression. */
-  out_size = in_len / 2;
-  if (out_size < 1024) out_size = 1024;
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (deflate(stream, Z_FINISH))
-      {
-      case Z_STREAM_END:
-        goto done;
-      case Z_OK:
-        /* In case zlib doesn't work as I think .... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-        /* Falls through. */
-      case Z_BUF_ERROR:
-        offset = stream->next_out - ((unsigned char*)*out);
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in compression.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->total_out;
-#ifdef OPENBSD
-  /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
-   *    third-party API!"
-   * "Oh, that trick will just make people do unsafe casts to the unsigned
-   *    type in their cross-platform code!"
-   * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
-   *    the newly unsigned field isn't negative." */
-  tor_assert(stream->total_out >= 0);
-#endif
-  if (deflateEnd(stream)!=Z_OK) {
-    // LCOV_EXCL_START -- unreachable if we handled the zlib structure right
-    tor_assert_nonfatal_unreached();
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-  tor_free(stream);
-
-  if (is_compression_bomb(*out_len, in_len)) {
-    log_warn(LD_BUG, "We compressed something and got an insanely high "
-          "compression factor; other Tors would think this was a zlib bomb.");
-    goto err;
-  }
-
-  return 0;
- err:
-  if (stream) {
-    deflateEnd(stream);
-    tor_free(stream);
-  }
-  tor_free(*out);
-  return -1;
-}
-
-/** Given zero or more zlib-compressed or gzip-compressed strings of
- * total length
- * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated
- * buffer, using the method described in <b>method</b>.  Store the uncompressed
- * string in *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on
- * success, -1 on failure.
- *
- * If <b>complete_only</b> is true, we consider a truncated input as a
- * failure; otherwise we decompress as much as we can.  Warn about truncated
- * or corrupt inputs at <b>protocol_warn_level</b>.
- */
-int
-tor_gzip_uncompress(char **out, size_t *out_len,
-                    const char *in, size_t in_len,
-                    compress_method_t method,
-                    int complete_only,
-                    int protocol_warn_level)
-{
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-  int r;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (inflateInit2(stream,
-                   method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
-    // LCOV_EXCL_START -- can only hit this if we give bad inputs.
-    log_warn(LD_GENERAL, "Error from inflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-
-  out_size = in_len * 2;  /* guess 50% compression. */
-  if (out_size < 1024) out_size = 1024;
-  if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
-    goto err;
-
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
-      {
-      case Z_STREAM_END:
-        if (stream->avail_in == 0)
-          goto done;
-        /* There may be more compressed data here. */
-        if ((r = inflateEnd(stream)) != Z_OK) {
-          log_warn(LD_BUG, "Error freeing gzip structures");
-          goto err;
-        }
-        if (inflateInit2(stream,
-                         method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
-          log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
-                   stream->msg?stream->msg:"<no message>");
-          goto err;
-        }
-        break;
-      case Z_OK:
-        if (!complete_only && stream->avail_in == 0)
-          goto done;
-        /* In case zlib doesn't work as I think.... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-        /* Falls through. */
-      case Z_BUF_ERROR:
-        if (stream->avail_out > 0) {
-          log_fn(protocol_warn_level, LD_PROTOCOL,
-                 "possible truncated or corrupt zlib data");
-          goto err;
-        }
-        offset = stream->next_out - (unsigned char*)*out;
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in uncompression.");
-          goto err;
-        }
-        if (is_compression_bomb(in_len, out_size)) {
-          log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
-                   "not proceeding.");
-          goto err;
-        }
-        if (out_size >= SIZE_T_CEILING) {
-          log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->next_out - (unsigned char*)*out;
-  r = inflateEnd(stream);
-  tor_free(stream);
-  if (r != Z_OK) {
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-  }
-
-  /* NUL-terminate output. */
-  if (out_size == *out_len)
-    *out = tor_realloc(*out, out_size + 1);
-  (*out)[*out_len] = '\0';
-
-  return 0;
- err:
-  if (stream) {
-    inflateEnd(stream);
-    tor_free(stream);
-  }
-  if (*out) {
-    tor_free(*out);
-  }
-  return -1;
-}
-
-/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
- * to be compressed or not.  If it is, return the likeliest compression method.
- * Otherwise, return UNKNOWN_METHOD.
- */
-compress_method_t
-detect_compression_method(const char *in, size_t in_len)
-{
-  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
-    return GZIP_METHOD;
-  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
-             (ntohs(get_uint16(in)) % 31) == 0) {
-    return ZLIB_METHOD;
-  } else {
-    return UNKNOWN_METHOD;
-  }
-}
-
-/** Internal state for an incremental zlib compression/decompression.  The
- * body of this struct is not exposed. */
-struct tor_zlib_state_t {
-  struct z_stream_s stream; /**< The zlib stream */
-  int compress; /**< True if we are compressing; false if we are inflating */
-
-  /** Number of bytes read so far.  Used to detect zlib bombs. */
-  size_t input_so_far;
-  /** Number of bytes written so far.  Used to detect zlib bombs. */
-  size_t output_so_far;
-
-  /** Approximate number of bytes allocated for this object. */
-  size_t allocation;
-};
-
-/** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
- * <b>compress</b>, it's for compression; otherwise it's for
- * decompression. */
-tor_zlib_state_t *
-tor_zlib_new(int compress_, compress_method_t method,
-             zlib_compression_level_t compression_level)
-{
-  tor_zlib_state_t *out;
-  int bits, memlevel;
-
- if (! compress_) {
-   /* use this setting for decompression, since we might have the
-    * max number of window bits */
-   compression_level = HIGH_COMPRESSION;
- }
-
- out = tor_malloc_zero(sizeof(tor_zlib_state_t));
- out->stream.zalloc = Z_NULL;
- out->stream.zfree = Z_NULL;
- out->stream.opaque = NULL;
- out->compress = compress_;
- bits = method_bits(method, compression_level);
- memlevel = get_memlevel(compression_level);
- if (compress_) {
-   if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                    bits, memlevel,
-                    Z_DEFAULT_STRATEGY) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- } else {
-   if (inflateInit2(&out->stream, bits) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- }
- out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
-
- total_zlib_allocation += out->allocation;
-
- return out;
-
- err:
- tor_free(out);
- return NULL;
-}
-
-/** Compress/decompress some bytes using <b>state</b>.  Read up to
- * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
- * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
- * we've reached the end of the input.
- *
- * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression.
- * Return TOR_ZLIB_OK if we're processed everything from the input.
- * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>.
- * Return TOR_ZLIB_ERR if the stream is corrupt.
- */
-tor_zlib_output_t
-tor_zlib_process(tor_zlib_state_t *state,
-                 char **out, size_t *out_len,
-                 const char **in, size_t *in_len,
-                 int finish)
-{
-  int err;
-  tor_assert(*in_len <= UINT_MAX);
-  tor_assert(*out_len <= UINT_MAX);
-  state->stream.next_in = (unsigned char*) *in;
-  state->stream.avail_in = (unsigned int)*in_len;
-  state->stream.next_out = (unsigned char*) *out;
-  state->stream.avail_out = (unsigned int)*out_len;
-
-  if (state->compress) {
-    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
-  } else {
-    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
-  }
-
-  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
-  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
-
-  *out = (char*) state->stream.next_out;
-  *out_len = state->stream.avail_out;
-  *in = (const char *) state->stream.next_in;
-  *in_len = state->stream.avail_in;
-
-  if (! state->compress &&
-      is_compression_bomb(state->input_so_far, state->output_so_far)) {
-    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
-    return TOR_ZLIB_ERR;
-  }
-
-  switch (err)
-    {
-    case Z_STREAM_END:
-      return TOR_ZLIB_DONE;
-    case Z_BUF_ERROR:
-      if (state->stream.avail_in == 0 && !finish)
-        return TOR_ZLIB_OK;
-      return TOR_ZLIB_BUF_FULL;
-    case Z_OK:
-      if (state->stream.avail_out == 0 || finish)
-        return TOR_ZLIB_BUF_FULL;
-      return TOR_ZLIB_OK;
-    default:
-      log_warn(LD_GENERAL, "Gzip returned an error: %s",
-               state->stream.msg ? state->stream.msg : "<no message>");
-      return TOR_ZLIB_ERR;
-    }
-}
-
-/** Deallocate <b>state</b>. */
-void
-tor_zlib_free(tor_zlib_state_t *state)
-{
-  if (!state)
-    return;
-
- total_zlib_allocation -= state->allocation;
-
-  if (state->compress)
-    deflateEnd(&state->stream);
-  else
-    inflateEnd(&state->stream);
-
-  tor_free(state);
-}
-
-/** Return an approximate number of bytes used in RAM to hold a state with
- * window bits <b>windowBits</b> and compression level 'memlevel' */
-static size_t
-tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
-{
-  windowbits &= 15;
-
-#define A_FEW_KILOBYTES 2048
-
-  if (inflate_) {
-    /* From zconf.h:
-
-       "The memory requirements for inflate are (in bytes) 1 << windowBits
-       that is, 32K for windowBits=15 (default value) plus a few kilobytes
-       for small objects."
-    */
-    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
-      (1 << 15) + A_FEW_KILOBYTES;
-  } else {
-    /* Also from zconf.h:
-
-       "The memory requirements for deflate are (in bytes):
-            (1 << (windowBits+2)) +  (1 << (memLevel+9))
-        ... plus a few kilobytes for small objects."
-    */
-    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
-      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
-  }
-#undef A_FEW_KILOBYTES
-}
-
-/** Return the approximate number of bytes allocated for <b>state</b>. */
-size_t
-tor_zlib_state_size(const tor_zlib_state_t *state)
-{
-  return state->allocation;
-}
-
-/** Return the approximate number of bytes allocated for all zlib states. */
-size_t
-tor_zlib_get_total_allocation(void)
-{
-  return total_zlib_allocation;
-}
-
diff --git a/src/common/torgzip.h b/src/common/torgzip.h
deleted file mode 100644
index 00f62dcb45..0000000000
--- a/src/common/torgzip.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torgzip.h
- * \brief Headers for torgzip.h
- **/
-
-#ifndef TOR_TORGZIP_H
-#define TOR_TORGZIP_H
-
-/** Enumeration of what kind of compression to use.  Only ZLIB_METHOD is
- * guaranteed to be supported by the compress/uncompress functions here;
- * GZIP_METHOD may be supported if we built against zlib version 1.2 or later
- * and is_gzip_supported() returns true. */
-typedef enum {
-  NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3
-} compress_method_t;
-
-/**
- * Enumeration to define tradeoffs between memory usage and compression level.
- * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
- * memory.
- **/
-typedef enum {
-  HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
-} zlib_compression_level_t;
-
-int
-tor_gzip_compress(char **out, size_t *out_len,
-                  const char *in, size_t in_len,
-                  compress_method_t method);
-int
-tor_gzip_uncompress(char **out, size_t *out_len,
-                    const char *in, size_t in_len,
-                    compress_method_t method,
-                    int complete_only,
-                    int protocol_warn_level);
-
-int is_gzip_supported(void);
-
-const char *
-tor_zlib_get_version_str(void);
-
-const char *
-tor_zlib_get_header_version_str(void);
-
-compress_method_t detect_compression_method(const char *in, size_t in_len);
-
-/** Return values from tor_zlib_process; see that function's documentation for
- * details. */
-typedef enum {
-  TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR
-} tor_zlib_output_t;
-/** Internal state for an incremental zlib compression/decompression. */
-typedef struct tor_zlib_state_t tor_zlib_state_t;
-tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method,
-                               zlib_compression_level_t level);
-
-tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
-                                   char **out, size_t *out_len,
-                                   const char **in, size_t *in_len,
-                                   int finish);
-void tor_zlib_free(tor_zlib_state_t *state);
-
-size_t tor_zlib_state_size(const tor_zlib_state_t *state);
-size_t tor_zlib_get_total_allocation(void);
-
-#endif
-
diff --git a/src/common/torint.h b/src/common/torint.h
index 58c30f41a8..fc7818fe2c 100644
--- a/src/common/torint.h
+++ b/src/common/torint.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,12 +34,14 @@
       does the same thing (but doesn't defined __FreeBSD__).
      */
 #include <machine/limits.h>
-#endif
-#endif
+#endif /* !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__) */
+#endif /* defined(HAVE_MACHINE_LIMITS_H) */
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
 
+#include <stdbool.h>
+
 #if (SIZEOF_INT8_T != 0)
 #define HAVE_INT8_T
 #endif
@@ -80,7 +82,7 @@ typedef signed char int8_t;
 typedef unsigned char uint8_t;
 #define HAVE_UINT8_T
 #endif
-#endif
+#endif /* (SIZEOF_CHAR == 1) */
 
 #if (SIZEOF_SHORT == 2)
 #ifndef HAVE_INT16_T
@@ -91,7 +93,7 @@ typedef signed short int16_t;
 typedef unsigned short uint16_t;
 #define HAVE_UINT16_T
 #endif
-#endif
+#endif /* (SIZEOF_SHORT == 2) */
 
 #if (SIZEOF_INT == 2)
 #ifndef HAVE_INT16_T
@@ -129,7 +131,7 @@ typedef unsigned int uint32_t;
 #ifndef INT32_MIN
 #define INT32_MIN (-2147483647-1)
 #endif
-#endif
+#endif /* (SIZEOF_INT == 2) || ... */
 
 #if (SIZEOF_LONG == 4)
 #ifndef HAVE_INT32_T
@@ -142,7 +144,7 @@ typedef unsigned long uint32_t;
 #ifndef UINT32_MAX
 #define UINT32_MAX 0xfffffffful
 #endif
-#endif
+#endif /* !defined(HAVE_UINT32_T) */
 #elif (SIZEOF_LONG == 8)
 #ifndef HAVE_INT64_T
 typedef signed long int64_t;
@@ -155,7 +157,7 @@ typedef unsigned long uint64_t;
 #ifndef UINT64_MAX
 #define UINT64_MAX 0xfffffffffffffffful
 #endif
-#endif
+#endif /* (SIZEOF_LONG == 4) || ... */
 
 #if (SIZEOF_LONG_LONG == 8)
 #ifndef HAVE_INT64_T
@@ -172,7 +174,7 @@ typedef unsigned long long uint64_t;
 #ifndef INT64_MAX
 #define INT64_MAX 0x7fffffffffffffffll
 #endif
-#endif
+#endif /* (SIZEOF_LONG_LONG == 8) */
 
 #if (SIZEOF___INT64 == 8)
 #ifndef HAVE_INT64_T
@@ -189,7 +191,7 @@ typedef unsigned __int64 uint64_t;
 #ifndef INT64_MAX
 #define INT64_MAX 0x7fffffffffffffffi64
 #endif
-#endif
+#endif /* (SIZEOF___INT64 == 8) */
 
 #ifndef INT64_MIN
 #define INT64_MIN ((- INT64_MAX) - 1)
@@ -202,8 +204,8 @@ typedef unsigned __int64 uint64_t;
 #define SIZE_MAX UINT32_MAX
 #else
 #error "Can't define SIZE_MAX"
-#endif
-#endif
+#endif /* SIZEOF_SIZE_T == 8 || ... */
+#endif /* !defined(SIZE_MAX) */
 
 #ifndef HAVE_SSIZE_T
 #if SIZEOF_SIZE_T == 8
@@ -212,8 +214,8 @@ typedef int64_t ssize_t;
 typedef int32_t ssize_t;
 #else
 #error "Can't define ssize_t."
-#endif
-#endif
+#endif /* SIZEOF_SIZE_T == 8 || ... */
+#endif /* !defined(HAVE_SSIZE_T) */
 
 #if (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8)
 #ifndef HAVE_INTPTR_T
@@ -235,7 +237,7 @@ typedef uint32_t uintptr_t;
 #endif
 #else
 #error "void * is either >8 bytes or <= 2.  In either case, I am confused."
-#endif
+#endif /* (SIZEOF_VOID_P > 4 && SIZEOF_VOID_P <= 8) || ... */
 
 #ifndef HAVE_INT8_T
 #error "Missing type int8_t"
@@ -275,8 +277,8 @@ typedef uint32_t uintptr_t;
 #define LONG_MAX 0x7fffffffffffffffL
 #else
 #error "Can't define LONG_MAX"
-#endif
-#endif
+#endif /* (SIZEOF_LONG == 4) || ... */
+#endif /* !defined(LONG_MAX) */
 
 #ifndef INT_MAX
 #if (SIZEOF_INT == 4)
@@ -285,8 +287,8 @@ typedef uint32_t uintptr_t;
 #define INT_MAX 0x7fffffffffffffffL
 #else
 #error "Can't define INT_MAX"
-#endif
-#endif
+#endif /* (SIZEOF_INT == 4) || ... */
+#endif /* !defined(INT_MAX) */
 
 #ifndef UINT_MAX
 #if (SIZEOF_INT == 2)
@@ -297,8 +299,8 @@ typedef uint32_t uintptr_t;
 #define UINT_MAX 0xffffffffffffffffu
 #else
 #error "Can't define UINT_MAX"
-#endif
-#endif
+#endif /* (SIZEOF_INT == 2) || ... */
+#endif /* !defined(UINT_MAX) */
 
 #ifndef SHORT_MAX
 #if (SIZEOF_SHORT == 2)
@@ -307,8 +309,8 @@ typedef uint32_t uintptr_t;
 #define SHORT_MAX 0x7fffffff
 #else
 #error "Can't define SHORT_MAX"
-#endif
-#endif
+#endif /* (SIZEOF_SHORT == 2) || ... */
+#endif /* !defined(SHORT_MAX) */
 
 #ifndef TIME_MAX
 
@@ -320,9 +322,9 @@ typedef uint32_t uintptr_t;
 #define TIME_MAX ((time_t)INT64_MAX)
 #else
 #error "Can't define TIME_MAX"
-#endif
+#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */
 
-#endif /* ifndef(TIME_MAX) */
+#endif /* !defined(TIME_MAX) */
 
 #ifndef TIME_MIN
 
@@ -334,9 +336,9 @@ typedef uint32_t uintptr_t;
 #define TIME_MIN ((time_t)INT64_MIN)
 #else
 #error "Can't define TIME_MIN"
-#endif
+#endif /* (SIZEOF_TIME_T == SIZEOF_INT) || ... */
 
-#endif /* ifndef(TIME_MIN) */
+#endif /* !defined(TIME_MIN) */
 
 #ifndef SIZE_MAX
 #if (SIZEOF_SIZE_T == 4)
@@ -345,7 +347,17 @@ typedef uint32_t uintptr_t;
 #define SIZE_MAX UINT64_MAX
 #else
 #error "Can't define SIZE_MAX"
-#endif
+#endif /* (SIZEOF_SIZE_T == 4) || ... */
+#endif /* !defined(SIZE_MAX) */
+
+#ifdef _WIN32
+#  ifdef _WIN64
+#    define TOR_PRIuSZ PRIu64
+#  else
+#    define TOR_PRIuSZ PRIu32
+#  endif
+#else
+#  define TOR_PRIuSZ "zu"
 #endif
 
 #ifndef SSIZE_MAX
@@ -355,13 +367,13 @@ typedef uint32_t uintptr_t;
 #define SSIZE_MAX INT64_MAX
 #else
 #error "Can't define SSIZE_MAX"
-#endif
-#endif
+#endif /* (SIZEOF_SIZE_T == 4) || ... */
+#endif /* !defined(SSIZE_MAX) */
 
 /** Any ssize_t larger than this amount is likely to be an underflow. */
 #define SSIZE_T_CEILING ((ssize_t)(SSIZE_MAX-16))
 /** Any size_t larger than this amount is likely to be an underflow. */
 #define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
 
-#endif /* __TORINT_H */
+#endif /* !defined(TOR_TORINT_H) */
 
diff --git a/src/common/torlog.h b/src/common/torlog.h
index 20b7d938f0..de389883c0 100644
--- a/src/common/torlog.h
+++ b/src/common/torlog.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,7 @@
 #error "Your syslog.h thinks high numbers are more important.  " \
        "We aren't prepared to deal with that."
 #endif
-#else
+#else /* !(defined(HAVE_SYSLOG_H)) */
 /* Note: Syslog's logging code refers to priorities, with 0 being the most
  * important.  Thus, all our comparisons needed to be reversed when we added
  * syslog support.
@@ -48,7 +48,7 @@
 /** Error-level severity: for messages that only appear when something has gone
  * very wrong, and the Tor process can no longer proceed. */
 #define LOG_ERR     3
-#endif
+#endif /* defined(HAVE_SYSLOG_H) */
 
 /* Logging domains */
 
@@ -99,10 +99,14 @@
 #define LD_CHANNEL   (1u<<21)
 /** Scheduler */
 #define LD_SCHED     (1u<<22)
+/** Guard nodes */
+#define LD_GUARD     (1u<<23)
+/** Generation and application of consensus diffs. */
+#define LD_CONSDIFF  (1u<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<23)
+#define LD_DOS       (1u<<25)
 /** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 24
+#define N_LOGGING_DOMAINS 26
 
 /** This log message is not safe to send to a callback-based logger
  * immediately.  Used as a flag, not a log domain. */
@@ -144,8 +148,14 @@ int add_file_log(const log_severity_list_t *severity, const char *filename,
 #ifdef HAVE_SYSLOG_H
 int add_syslog_log(const log_severity_list_t *severity,
                    const char* syslog_identity_tag);
-#endif
+#endif // HAVE_SYSLOG_H.
+#ifdef HAVE_ANDROID_LOG_H
+int add_android_log(const log_severity_list_t *severity,
+                    const char *android_identity_tag);
+#endif // HAVE_ANDROID_LOG_H.
 int add_callback_log(const log_severity_list_t *severity, log_callback cb);
+typedef void (*pending_callback_callback)(void);
+void logs_set_pending_callback_callback(pending_callback_callback cb);
 void logs_set_domain_logging(int enabled);
 int get_min_log_level(void);
 void switch_logs_debug(void);
@@ -183,6 +193,10 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
                      const char *format, ...)
   CHECK_PRINTF(5,6);
 
+int log_message_is_interesting(int severity, log_domain_mask_t domain);
+void tor_log_string(int severity, log_domain_mask_t domain,
+                    const char *function, const char *string);
+
 #if defined(__GNUC__) && __GNUC__ <= 3
 
 /* These are the GCC varidaic macros, so that older versions of GCC don't
@@ -211,7 +225,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
 #define log_err(domain, args...)                            \
   log_fn_(LOG_ERR, domain, __FUNCTION__, args)
 
-#else /* ! defined(__GNUC__) */
+#else /* !(defined(__GNUC__) && __GNUC__ <= 3) */
 
 /* Here are the c99 variadic macros, to work with non-GCC compilers */
 
@@ -238,7 +252,17 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity,
 #define log_fn_ratelim(ratelim, severity, domain, args,...)      \
   log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, \
                   args, ##__VA_ARGS__)
-#endif
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
+
+/** This defines log levels that are linked in the Rust log module, rather
+ * than re-defining these in both Rust and C.
+ *
+ * C_RUST_COUPLED src/rust/tor_log LogSeverity, LogDomain
+ */
+extern const int LOG_WARN_;
+extern const int LOG_NOTICE_;
+extern const log_domain_mask_t LD_NET_;
+extern const log_domain_mask_t LD_GENERAL_;
 
 #ifdef LOG_PRIVATE
 MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
@@ -247,5 +271,5 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
 #endif
 
 # define TOR_TORLOG_H
-#endif
+#endif /* !defined(TOR_TORLOG_H) */
 
diff --git a/src/common/tortls.c b/src/common/tortls.c
index f79969d0da..6ff513b14f 100644
--- a/src/common/tortls.c
+++ b/src/common/tortls.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "orconfig.h"
 
 #define TORTLS_PRIVATE
+#define TORTLS_OPENSSL_PRIVATE
 
 #include <assert.h>
 #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/
@@ -24,6 +25,9 @@
   #include <ws2tcpip.h>
 #endif
 
+#include "crypto.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "compat.h"
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
@@ -31,7 +35,6 @@
 DISABLE_GCC_WARNING(redundant-decls)
 
 #include <openssl/opensslv.h>
-#include "crypto.h"
 
 #ifdef OPENSSL_NO_EC
 #error "We require OpenSSL with ECC support"
@@ -55,10 +58,25 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include "container.h"
 #include <string.h>
 
+#ifdef OPENSSL_1_1_API
+#define X509_get_notBefore_const(cert) \
+    X509_get0_notBefore(cert)
+#define X509_get_notAfter_const(cert) \
+    X509_get0_notAfter(cert)
+#ifndef X509_get_notBefore
+#define X509_get_notBefore(cert) \
+    X509_getm_notBefore(cert)
+#endif
+#ifndef X509_get_notAfter
+#define X509_get_notAfter(cert) \
+    X509_getm_notAfter(cert)
+#endif
+#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
 #define X509_get_notBefore_const(cert) \
   ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
 #define X509_get_notAfter_const(cert) \
   ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
+#endif
 
 /* Copied from or.h */
 #define LEGAL_NICKNAME_CHARACTERS \
@@ -75,7 +93,7 @@ ENABLE_GCC_WARNING(redundant-decls)
  * SSL3 safely at the same time.
  */
 #define DISABLE_SSL3_HANDSHAKE
-#endif
+#endif /* OPENSSL_VERSION_NUMBER <  OPENSSL_V(1,0,0,'f') */
 
 /* We redefine these so that we can run correctly even if the vendor gives us
  * a version of OpenSSL that does not match its header files.  (Apple: I am
@@ -88,6 +106,9 @@ ENABLE_GCC_WARNING(redundant-decls)
 #define SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION 0x0010
 #endif
 
+/** Set to true iff openssl bug 7712 has been detected. */
+static int openssl_bug_7712_is_present = 0;
+
 /** Return values for tor_tls_classify_client_ciphers.
  *
  * @{
@@ -136,6 +157,7 @@ static void tor_tls_context_decref(tor_tls_context_t *ctx);
 static void tor_tls_context_incref(tor_tls_context_t *ctx);
 
 static int check_cert_lifetime_internal(int severity, const X509 *cert,
+                                   time_t now,
                                    int past_tolerance, int future_tolerance);
 
 /** Global TLS contexts. We keep them here because nobody else needs
@@ -353,8 +375,12 @@ tor_tls_init(void)
   check_no_tls_errors();
 
   if (!tls_library_is_initialized) {
+#ifdef OPENSSL_1_1_API
+    OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL);
+#else
     SSL_library_init();
     SSL_load_error_strings();
+#endif
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
@@ -388,7 +414,7 @@ tor_tls_init(void)
                    "when configuring it) would make ECDH much faster.");
     }
     /* LCOV_EXCL_STOP */
-#endif
+#endif /* (SIZEOF_VOID_P >= 8 &&                              ... */
 
     tor_tls_allocate_tor_tls_object_ex_data_index();
 
@@ -442,8 +468,9 @@ tor_x509_name_new(const char *cname)
     goto error;
   /* LCOV_EXCL_BR_STOP */
   return name;
- error:
+
   /* LCOV_EXCL_START : these lines will only execute on out of memory errors*/
+ error:
   X509_NAME_free(name);
   return NULL;
   /* LCOV_EXCL_STOP */
@@ -458,11 +485,11 @@ tor_x509_name_new(const char *cname)
  * Return a certificate on success, NULL on failure.
  */
 MOCK_IMPL(STATIC X509 *,
-          tor_tls_create_certificate,(crypto_pk_t *rsa,
-                                      crypto_pk_t *rsa_sign,
-                                      const char *cname,
-                                      const char *cname_sign,
-                                      unsigned int cert_lifetime))
+tor_tls_create_certificate,(crypto_pk_t *rsa,
+                            crypto_pk_t *rsa_sign,
+                            const char *cname,
+                            const char *cname_sign,
+                            unsigned int cert_lifetime))
 {
   /* OpenSSL generates self-signed certificates with random 64-bit serial
    * numbers, so let's do that too. */
@@ -482,8 +509,25 @@ MOCK_IMPL(STATIC X509 *,
    * then we might pick a time where we're about to expire. Lastly, be
    * sure to start on a day boundary. */
   time_t now = time(NULL);
-  start_time = crypto_rand_time_range(now - cert_lifetime, now) + 2*24*3600;
-  start_time -= start_time % (24*3600);
+  /* Our certificate lifetime will be cert_lifetime no matter what, but if we
+   * start cert_lifetime in the past, we'll have 0 real lifetime.  instead we
+   * start up to (cert_lifetime - min_real_lifetime - start_granularity) in
+   * the past. */
+  const time_t min_real_lifetime = 24*3600;
+  const time_t start_granularity = 24*3600;
+  time_t earliest_start_time;
+  /* Don't actually start in the future! */
+  if (cert_lifetime <= min_real_lifetime + start_granularity) {
+    earliest_start_time = now - 1;
+  } else {
+    earliest_start_time = now + min_real_lifetime + start_granularity
+      - cert_lifetime;
+  }
+  start_time = crypto_rand_time_range(earliest_start_time, now);
+  /* Round the start time back to the start of a day. */
+  start_time -= start_time % start_granularity;
+
+  end_time = start_time + cert_lifetime;
 
   tor_assert(rsa);
   tor_assert(cname);
@@ -517,12 +561,12 @@ MOCK_IMPL(STATIC X509 *,
 
   if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time))
     goto error;
-  end_time = start_time + cert_lifetime;
   if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time))
     goto error;
   if (!X509_set_pubkey(x509, pkey))
     goto error;
-  if (!X509_sign(x509, sign_pkey, EVP_sha1()))
+
+  if (!X509_sign(x509, sign_pkey, EVP_sha256()))
     goto error;
 
   goto done;
@@ -605,6 +649,12 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256
        TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 ":"
 #endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_CCM
+       TLS1_TXT_DHE_RSA_WITH_AES_256_CCM ":"
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_CCM
+       TLS1_TXT_DHE_RSA_WITH_AES_128_CCM ":"
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256
        TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 ":"
 #endif
@@ -614,8 +664,14 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
        /* Required */
        TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
        /* Required */
-       TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
-       ;
+       TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":"
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
+       TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 ":"
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305
+       TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305
+#endif
+  ;
 
 /* Note: to set up your own private testing network with link crypto
  * disabled, set your Tors' cipher list to
@@ -637,7 +693,7 @@ static const char CLIENT_CIPHER_LIST[] =
 
 /** Free all storage held in <b>cert</b> */
 void
-tor_x509_cert_free(tor_x509_cert_t *cert)
+tor_x509_cert_free_(tor_x509_cert_t *cert)
 {
   if (! cert)
     return;
@@ -656,7 +712,7 @@ tor_x509_cert_free(tor_x509_cert_t *cert)
  * Steals a reference to x509_cert.
  */
 MOCK_IMPL(STATIC tor_x509_cert_t *,
-          tor_x509_cert_new,(X509 *x509_cert))
+tor_x509_cert_new,(X509 *x509_cert))
 {
   tor_x509_cert_t *cert;
   EVP_PKEY *pkey;
@@ -670,12 +726,7 @@ MOCK_IMPL(STATIC tor_x509_cert_t *,
   length = i2d_X509(x509_cert, &buf);
   cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
   if (length <= 0 || buf == NULL) {
-    /* LCOV_EXCL_START for the same reason as the exclusion above */
-    tor_free(cert);
-    log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate");
-    X509_free(x509_cert);
-    return NULL;
-    /* LCOV_EXCL_STOP */
+    goto err;
   }
   cert->encoded_len = (size_t) length;
   cert->encoded = tor_malloc(length);
@@ -690,13 +741,25 @@ MOCK_IMPL(STATIC tor_x509_cert_t *,
   if ((pkey = X509_get_pubkey(x509_cert)) &&
       (rsa = EVP_PKEY_get1_RSA(pkey))) {
     crypto_pk_t *pk = crypto_new_pk_from_rsa_(rsa);
-    crypto_pk_get_common_digests(pk, &cert->pkey_digests);
+    if (crypto_pk_get_common_digests(pk, &cert->pkey_digests) < 0) {
+      crypto_pk_free(pk);
+      EVP_PKEY_free(pkey);
+      goto err;
+    }
+
     cert->pkey_digests_set = 1;
     crypto_pk_free(pk);
     EVP_PKEY_free(pkey);
   }
 
   return cert;
+ err:
+  /* LCOV_EXCL_START for the same reason as the exclusion above */
+  tor_free(cert);
+  log_err(LD_CRYPTO, "Couldn't wrap encoded X509 certificate.");
+  X509_free(x509_cert);
+  return NULL;
+  /* LCOV_EXCL_STOP */
 }
 
 /** Return a new copy of <b>cert</b>. */
@@ -800,8 +863,8 @@ tor_tls_context_decref(tor_tls_context_t *ctx)
 /** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate
  * and ID certificate that we're currently using for our V3 in-protocol
  * handshake's certificate chain.  If <b>server</b> is true, provide the certs
- * that we use in server mode; otherwise, provide the certs that we use in
- * client mode. */
+ * that we use in server mode (auth, ID); otherwise, provide the certs that we
+ * use in client mode. (link, ID) */
 int
 tor_tls_get_my_certs(int server,
                      const tor_x509_cert_t **link_cert_out,
@@ -831,7 +894,7 @@ tor_tls_get_my_client_auth_key(void)
 
 /**
  * Return a newly allocated copy of the public key that a certificate
- * certifies.  Return NULL if the cert's key is not RSA.
+ * certifies. Watch out! This returns NULL if the cert's key is not RSA.
  */
 crypto_pk_t *
 tor_tls_cert_get_key(tor_x509_cert_t *cert)
@@ -888,6 +951,7 @@ int
 tor_tls_cert_is_valid(int severity,
                       const tor_x509_cert_t *cert,
                       const tor_x509_cert_t *signing_cert,
+                      time_t now,
                       int check_rsa_1024)
 {
   check_no_tls_errors();
@@ -907,7 +971,7 @@ tor_tls_cert_is_valid(int severity,
 
   /* okay, the signature checked out right.  Now let's check the check the
    * lifetime. */
-  if (check_cert_lifetime_internal(severity, cert->cert,
+  if (check_cert_lifetime_internal(severity, cert->cert, now,
                                    48*60*60, 30*24*60*60) < 0)
     goto bad;
 
@@ -1052,6 +1116,8 @@ tor_tls_context_init_one(tor_tls_context_t **ppcontext,
 /** The group we should use for ecdhe when none was selected. */
 #define  NID_tor_default_ecdhe_group NID_X9_62_prime256v1
 
+#define RSA_LINK_KEY_BITS 2048
+
 /** Create a new TLS context for use with Tor TLS handshakes.
  * <b>identity</b> should be set to the identity key used to sign the
  * certificate.
@@ -1077,7 +1143,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
   /* Generate short-term RSA key for use with TLS. */
   if (!(rsa = crypto_pk_new()))
     goto error;
-  if (crypto_pk_generate_key(rsa)<0)
+  if (crypto_pk_generate_key_with_bits(rsa, RSA_LINK_KEY_BITS)<0)
     goto error;
   if (!is_client) {
     /* Generate short-term RSA key for use in the in-protocol ("v3")
@@ -1120,7 +1186,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
    * with existing Tors. */
   if (!(result->ctx = SSL_CTX_new(TLSv1_method())))
     goto error;
-#endif
+#endif /* 0 */
 
   /* Tell OpenSSL to use TLS 1.0 or later but not SSL2 or SSL3. */
 #ifdef HAVE_TLS_METHOD
@@ -1129,7 +1195,8 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
 #else
   if (!(result->ctx = SSL_CTX_new(SSLv23_method())))
     goto error;
-#endif
+#endif /* defined(HAVE_TLS_METHOD) */
+
 #ifdef HAVE_SSL_CTX_SET_SECURITY_LEVEL
   /* Level 1 re-enables RSA1024 and DH1024 for compatibility with old tors */
   SSL_CTX_set_security_level(result->ctx, 1);
@@ -1172,17 +1239,20 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     SSL_CTX_set_options(result->ctx,
                         SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
   }
+
+  /* Don't actually allow compression; it uses RAM and time, it makes TLS
+   * vulnerable to CRIME-style attacks, and most of the data we transmit over
+   * TLS is encrypted (and therefore uncompressible) anyway. */
 #ifdef SSL_OP_NO_COMPRESSION
   SSL_CTX_set_options(result->ctx, SSL_OP_NO_COMPRESSION);
 #endif
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0)
 #ifndef OPENSSL_NO_COMP
-  /* Don't actually allow compression; it uses ram and time, but the data
-   * we transmit is all encrypted anyway. */
   if (result->ctx->comp_methods)
     result->ctx->comp_methods = NULL;
 #endif
-#endif
+#endif /* OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0) */
+
 #ifdef SSL_MODE_RELEASE_BUFFERS
   SSL_CTX_set_mode(result->ctx, SSL_MODE_RELEASE_BUFFERS);
 #endif
@@ -1359,7 +1429,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher);
     return c != NULL;
   }
-#else
+#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */
 
 # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR)
   if (m && m->get_cipher_by_char) {
@@ -1373,7 +1443,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((c->id & 0xffff) == cipher);
     return c != NULL;
   }
-# endif
+#endif /* defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) */
 # ifndef OPENSSL_1_1_API
   if (m && m->get_cipher && m->num_ciphers) {
     /* It would seem that some of the "let's-clean-up-openssl" forks have
@@ -1389,12 +1459,12 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
     }
     return 0;
   }
-# endif
+#endif /* !defined(OPENSSL_1_1_API) */
   (void) ssl;
   (void) m;
   (void) cipher;
   return 1; /* No way to search */
-#endif
+#endif /* defined(HAVE_SSL_CIPHER_FIND) */
 }
 
 /** Remove from v2_cipher_list every cipher that we don't support, so that
@@ -1522,7 +1592,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl)
     return CIPHERS_ERR;
   }
   ciphers = session->ciphers;
-#endif
+#endif /* defined(HAVE_SSL_GET_CLIENT_CIPHERS) */
 
   return tor_tls_classify_client_ciphers(ssl, ciphers) >= CIPHERS_V2;
 }
@@ -1653,6 +1723,13 @@ tor_tls_new(int sock, int isServer)
     SSL_set_tlsext_host_name(result->ssl, fake_hostname);
     tor_free(fake_hostname);
   }
+#endif /* defined(SSL_set_tlsext_host_name) */
+
+#ifdef SSL_CTRL_SET_MAX_PROTO_VERSION
+  if (openssl_bug_7712_is_present) {
+    /* We can't actually use TLS 1.3 until this bug is fixed. */
+    SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION);
+  }
 #endif
 
   if (!SSL_set_cipher_list(result->ssl,
@@ -1780,7 +1857,7 @@ tor_tls_assert_renegotiation_unblocked(tor_tls_t *tls)
   tor_assert(0 != (options & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION));
 #else
   (void) tls;
-#endif
+#endif /* defined(SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION) && ... */
 }
 
 /** Return whether this tls initiated the connect (client) or
@@ -1796,7 +1873,7 @@ tor_tls_is_server(tor_tls_t *tls)
  * underlying file descriptor.
  */
 void
-tor_tls_free(tor_tls_t *tls)
+tor_tls_free_(tor_tls_t *tls)
 {
   if (!tls)
     return;
@@ -1943,7 +2020,7 @@ tor_tls_handshake(tor_tls_t *tls)
   return r;
 }
 
-/** Perform the final part of the intial TLS handshake on <b>tls</b>.  This
+/** Perform the final part of the initial TLS handshake on <b>tls</b>.  This
  * should be called for the first handshake only: it determines whether the v1
  * or the v2 handshake was used, and adjusts things for the renegotiation
  * handshake as appropriate.
@@ -2097,13 +2174,13 @@ tor_tls_get_own_cert,(tor_tls_t *tls))
 
 /** Warn that a certificate lifetime extends through a certain range. */
 static void
-log_cert_lifetime(int severity, const X509 *cert, const char *problem)
+log_cert_lifetime(int severity, const X509 *cert, const char *problem,
+                  time_t now)
 {
   BIO *bio = NULL;
   BUF_MEM *buf;
   char *s1=NULL, *s2=NULL;
   char mytime[33];
-  time_t now = time(NULL);
   struct tm tm;
   size_t n;
 
@@ -2251,6 +2328,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity_key)
  */
 int
 tor_tls_check_lifetime(int severity, tor_tls_t *tls,
+                       time_t now,
                        int past_tolerance, int future_tolerance)
 {
   X509 *cert;
@@ -2259,7 +2337,7 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls,
   if (!(cert = SSL_get_peer_certificate(tls->ssl)))
     goto done;
 
-  if (check_cert_lifetime_internal(severity, cert,
+  if (check_cert_lifetime_internal(severity, cert, now,
                                    past_tolerance, future_tolerance) < 0)
     goto done;
 
@@ -2275,30 +2353,48 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls,
 
 /** Helper: check whether <b>cert</b> is expired give or take
  * <b>past_tolerance</b> seconds, or not-yet-valid give or take
- * <b>future_tolerance</b> seconds.  If it is live, return 0.  If it is not
- * live, log a message and return -1. */
+ * <b>future_tolerance</b> seconds.  (Relative to the current time
+ * <b>now</b>.)  If it is live, return 0.  If it is not live, log a message
+ * and return -1. */
 static int
 check_cert_lifetime_internal(int severity, const X509 *cert,
+                             time_t now,
                              int past_tolerance, int future_tolerance)
 {
-  time_t now, t;
-
-  now = time(NULL);
+  time_t t;
 
   t = now + future_tolerance;
   if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) {
-    log_cert_lifetime(severity, cert, "not yet valid");
+    log_cert_lifetime(severity, cert, "not yet valid", now);
     return -1;
   }
   t = now - past_tolerance;
   if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) {
-    log_cert_lifetime(severity, cert, "already expired");
+    log_cert_lifetime(severity, cert, "already expired", now);
     return -1;
   }
 
   return 0;
 }
 
+#ifdef TOR_UNIT_TESTS
+/* Testing only: return a new x509 cert with the same contents as <b>inp</b>,
+   but with the expiration time <b>new_expiration_time</b>, signed with
+   <b>signing_key</b>. */
+STATIC tor_x509_cert_t *
+tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
+                                 time_t new_expiration_time,
+                                 crypto_pk_t *signing_key)
+{
+  X509 *newc = X509_dup(inp->cert);
+  X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time);
+  EVP_PKEY *pk = crypto_pk_get_evp_pkey_(signing_key, 1);
+  tor_assert(X509_sign(newc, pk, EVP_sha256()));
+  EVP_PKEY_free(pk);
+  return tor_x509_cert_new(newc);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Return the number of bytes available for reading from <b>tls</b>.
  */
 int
@@ -2341,10 +2437,10 @@ tor_tls_get_n_raw_bytes(tor_tls_t *tls, size_t *n_read, size_t *n_written)
   if (BIO_method_type(wbio) == BIO_TYPE_BUFFER &&
         (tmpbio = BIO_next(wbio)) != NULL)
     wbio = tmpbio;
-#else
+#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5)) */
   if (wbio->method == BIO_f_buffer() && (tmpbio = BIO_next(wbio)) != NULL)
     wbio = tmpbio;
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_VER(1,1,0,0,5) */
   w = (unsigned long) BIO_number_written(wbio);
 
   /* We are ok with letting these unsigned ints go "negative" here:
@@ -2423,7 +2519,7 @@ SSL_get_client_random(SSL *s, uint8_t *out, size_t len)
   memcpy(out, s->s3->client_random, len);
   return len;
 }
-#endif
+#endif /* !defined(HAVE_SSL_GET_CLIENT_RANDOM) */
 
 #ifndef HAVE_SSL_GET_SERVER_RANDOM
 static size_t
@@ -2436,7 +2532,7 @@ SSL_get_server_random(SSL *s, uint8_t *out, size_t len)
   memcpy(out, s->s3->server_random, len);
   return len;
 }
-#endif
+#endif /* !defined(HAVE_SSL_GET_SERVER_RANDOM) */
 
 #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
 STATIC size_t
@@ -2450,7 +2546,7 @@ SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, size_t len)
   memcpy(out, s->master_key, len);
   return len;
 }
-#endif
+#endif /* !defined(HAVE_SSL_SESSION_GET_MASTER_KEY) */
 
 /** Set the DIGEST256_LEN buffer at <b>secrets_out</b> to the value used in
  * the v3 handshake to prove that the client knows the TLS secrets for the
@@ -2517,6 +2613,62 @@ tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out))
   return 0;
 }
 
+/** Using the RFC5705 key material exporting construction, and the
+ * provided <b>context</b> (<b>context_len</b> bytes long) and
+ * <b>label</b> (a NUL-terminated string), compute a 32-byte secret in
+ * <b>secrets_out</b> that only the parties to this TLS session can
+ * compute.  Return 0 on success; -1 on failure; and -2 on failure
+ * caused by OpenSSL bug 7712.
+ */
+MOCK_IMPL(int,
+tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
+                             const uint8_t *context,
+                             size_t context_len,
+                             const char *label))
+{
+  tor_assert(tls);
+  tor_assert(tls->ssl);
+
+  int r = SSL_export_keying_material(tls->ssl,
+                                     secrets_out, DIGEST256_LEN,
+                                     label, strlen(label),
+                                     context, context_len, 1);
+
+  if (r != 1) {
+    int severity = openssl_bug_7712_is_present ? LOG_WARN : LOG_DEBUG;
+    tls_log_errors(tls, severity, LD_NET, "exporting keying material");
+  }
+
+#ifdef TLS1_3_VERSION
+  if (r != 1 &&
+      strlen(label) > 12 &&
+      SSL_version(tls->ssl) >= TLS1_3_VERSION) {
+
+    if (! openssl_bug_7712_is_present) {
+      /* We might have run into OpenSSL issue 7712, which caused OpenSSL
+       * 1.1.1a to not handle long labels.  Let's test to see if we have.
+       */
+      r = SSL_export_keying_material(tls->ssl, secrets_out, DIGEST256_LEN,
+                                     "short", 5, context, context_len, 1);
+      if (r == 1) {
+        /* A short label succeeds, but a long label fails. This was openssl
+         * issue 7712. */
+        openssl_bug_7712_is_present = 1;
+        log_warn(LD_GENERAL, "Detected OpenSSL bug 7712: disabling TLS 1.3 on "
+                 "future connections. A fix is expected to appear in OpenSSL "
+                 "1.1.1b.");
+      }
+    }
+    if (openssl_bug_7712_is_present)
+      return -2;
+    else
+      return -1;
+  }
+#endif
+
+  return (r == 1) ? 0 : -1;
+}
+
 /** Examine the amount of memory used and available for buffers in <b>tls</b>.
  * Set *<b>rbuf_capacity</b> to the amount of storage allocated for the read
  * buffer and *<b>rbuf_bytes</b> to the amount actually used.
@@ -2537,7 +2689,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls,
   (void)wbuf_bytes;
 
   return -1;
-#else
+#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)) */
   if (tls->ssl->s3->rbuf.buf)
     *rbuf_capacity = tls->ssl->s3->rbuf.len;
   else
@@ -2549,7 +2701,7 @@ tor_tls_get_buffer_sizes(tor_tls_t *tls,
   *rbuf_bytes = tls->ssl->s3->rbuf.left;
   *wbuf_bytes = tls->ssl->s3->wbuf.left;
   return 0;
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
 }
 
 /** Check whether the ECC group requested is supported by the current OpenSSL
diff --git a/src/common/tortls.h b/src/common/tortls.h
index f018c45c82..7c867bfff2 100644
--- a/src/common/tortls.h
+++ b/src/common/tortls.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TORTLS_H
@@ -11,7 +11,7 @@
  * \brief Headers for tortls.c
  **/
 
-#include "crypto.h"
+#include "crypto_rsa.h"
 #include "compat_openssl.h"
 #include "compat.h"
 #include "testsupport.h"
@@ -63,12 +63,17 @@ typedef enum {
 } tor_tls_state_t;
 #define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t)
 
+struct x509_st;
+struct ssl_st;
+struct ssl_ctx_st;
+struct ssl_session_st;
+
 /** Holds a SSL_CTX object and related state used to configure TLS
  * connections.
  */
 typedef struct tor_tls_context_t {
   int refcnt;
-  SSL_CTX *ctx;
+  struct ssl_ctx_st *ctx;
   tor_x509_cert_t *my_link_cert;
   tor_x509_cert_t *my_id_cert;
   tor_x509_cert_t *my_auth_cert;
@@ -78,7 +83,7 @@ typedef struct tor_tls_context_t {
 
 /** Structure that we use for a single certificate. */
 struct tor_x509_cert_t {
-  X509 *cert;
+  struct x509_st *cert;
   uint8_t *encoded;
   size_t encoded_len;
   unsigned pkey_digests_set : 1;
@@ -92,7 +97,7 @@ struct tor_x509_cert_t {
 struct tor_tls_t {
   uint32_t magic;
   tor_tls_context_t *context; /** A link to the context object for this tls. */
-  SSL *ssl; /**< An OpenSSL SSL object. */
+  struct ssl_st *ssl; /**< An OpenSSL SSL object. */
   int socket; /**< The underlying file descriptor for this TLS connection. */
   char *address; /**< An address to log when describing this connection. */
   tor_tls_state_bitfield_t state : 3; /**< The current SSL state,
@@ -128,35 +133,45 @@ struct tor_tls_t {
 STATIC int tor_errno_to_tls_error(int e);
 STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra,
                   const char *doing, int severity, int domain);
-STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl);
+STATIC tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl);
 STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void);
+#ifdef TORTLS_OPENSSL_PRIVATE
 STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx);
-STATIC int tor_tls_classify_client_ciphers(const SSL *ssl,
+STATIC int tor_tls_classify_client_ciphers(const struct ssl_st *ssl,
                                            STACK_OF(SSL_CIPHER) *peer_ciphers);
-STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl);
+#endif
+STATIC int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl);
 MOCK_DECL(STATIC void, try_to_extract_certs_from_tls,
-          (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out));
+          (int severity, tor_tls_t *tls, struct x509_st **cert_out,
+           struct x509_st **id_cert_out));
 #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
-STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out,
+STATIC size_t SSL_SESSION_get_master_key(struct ssl_session_st *s,
+                                         uint8_t *out,
                                          size_t len);
 #endif
-STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val);
-STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val);
-STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret,
+STATIC void tor_tls_debug_state_callback(const struct ssl_st *ssl,
+                                         int type, int val);
+STATIC void tor_tls_server_info_callback(const struct ssl_st *ssl,
+                                         int type, int val);
+#ifdef TORTLS_OPENSSL_PRIVATE
+STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
                             int *secret_len,
                             STACK_OF(SSL_CIPHER) *peer_ciphers,
                             CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher,
                             void *arg);
 STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
                              uint16_t cipher);
-MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa,
+#endif /* defined(TORTLS_OPENSSL_PRIVATE) */
+MOCK_DECL(STATIC struct x509_st *, tor_tls_create_certificate,
+                                                   (crypto_pk_t *rsa,
                                                     crypto_pk_t *rsa_sign,
                                                     const char *cname,
                                                     const char *cname_sign,
                                                   unsigned int cert_lifetime));
 STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity,
                    unsigned int key_lifetime, unsigned flags, int is_client);
-MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert));
+MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,
+          (struct x509_st *x509_cert));
 STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext,
                                     crypto_pk_t *identity,
                                     unsigned int key_lifetime,
@@ -172,10 +187,16 @@ extern tor_tls_context_t *client_tls_context;
 extern uint16_t v2_cipher_list[];
 extern uint64_t total_bytes_written_over_tls;
 extern uint64_t total_bytes_written_by_tls;
-#endif
 
-#endif /* endif TORTLS_PRIVATE */
+STATIC tor_x509_cert_t *tor_x509_cert_replace_expiration(
+                                               const tor_x509_cert_t *inp,
+                                               time_t new_expiration_time,
+                                               crypto_pk_t *signing_key);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(TORTLS_PRIVATE) */
 
+tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 const char *tor_tls_err_to_string(int err);
 void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz);
 
@@ -195,14 +216,15 @@ void tor_tls_set_renegotiate_callback(tor_tls_t *tls,
                                       void (*cb)(tor_tls_t *, void *arg),
                                       void *arg);
 int tor_tls_is_server(tor_tls_t *tls);
-void tor_tls_free(tor_tls_t *tls);
+void tor_tls_free_(tor_tls_t *tls);
+#define tor_tls_free(tls) FREE_AND_NULL(tor_tls_t, tor_tls_free_, (tls))
 int tor_tls_peer_has_cert(tor_tls_t *tls);
-tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 MOCK_DECL(tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls));
 MOCK_DECL(tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls));
 int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity);
 int tor_tls_check_lifetime(int severity,
-                           tor_tls_t *tls, int past_tolerance,
+                           tor_tls_t *tls, time_t now,
+                           int past_tolerance,
                            int future_tolerance);
 MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len));
 int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n);
@@ -228,6 +250,11 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls);
 int tor_tls_get_num_server_handshakes(tor_tls_t *tls);
 int tor_tls_server_got_renegotiate(tor_tls_t *tls);
 MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out));
+MOCK_DECL(int,tor_tls_export_key_material,(
+                     tor_tls_t *tls, uint8_t *secrets_out,
+                     const uint8_t *context,
+                     size_t context_len,
+                     const char *label));
 
 /* Log and abort if there are unhandled TLS errors in OpenSSL's error stack.
  */
@@ -237,7 +264,9 @@ void check_no_tls_errors_(const char *fname, int line);
 void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
                            int severity, int domain, const char *doing);
 
-void tor_x509_cert_free(tor_x509_cert_t *cert);
+void tor_x509_cert_free_(tor_x509_cert_t *cert);
+#define tor_x509_cert_free(c) \
+  FREE_AND_NULL(tor_x509_cert_t, tor_x509_cert_free_, (c))
 tor_x509_cert_t *tor_x509_cert_decode(const uint8_t *certificate,
                             size_t certificate_len);
 void tor_x509_cert_get_der(const tor_x509_cert_t *cert,
@@ -256,10 +285,11 @@ MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls,
 int tor_tls_cert_is_valid(int severity,
                           const tor_x509_cert_t *cert,
                           const tor_x509_cert_t *signing_cert,
+                          time_t now,
                           int check_rsa_1024);
 const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls);
 
 int evaluate_ecgroup_for_tls(const char *ecgroup);
 
-#endif
+#endif /* !defined(TOR_TORTLS_H) */
 
diff --git a/src/common/util.c b/src/common/util.c
index d2cbacde31..dece5877f1 100644
--- a/src/common/util.c
+++ b/src/common/util.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #define UTIL_PRIVATE
 #include "util.h"
 #include "torlog.h"
-#include "crypto.h"
+#include "crypto_digest.h"
 #include "torint.h"
 #include "container.h"
 #include "address.h"
@@ -31,11 +31,11 @@
 #include <process.h>
 #include <tchar.h>
 #include <winbase.h>
-#else
+#else /* !(defined(_WIN32)) */
 #include <dirent.h>
 #include <pwd.h>
 #include <grp.h>
-#endif
+#endif /* defined(_WIN32) */
 
 /* math.h needs this on Linux */
 #ifndef _USE_ISOC99_
@@ -79,13 +79,13 @@
 #include <malloc/malloc.h>
 #endif
 #ifdef HAVE_MALLOC_H
-#if !defined(OPENBSD) && !defined(__FreeBSD__)
+#if !defined(OpenBSD) && !defined(__FreeBSD__)
 /* OpenBSD has a malloc.h, but for our purposes, it only exists in order to
  * scold us for being so stupid as to autodetect its presence.  To be fair,
  * they've done this since 1996, when autoconf was only 5 years old. */
 #include <malloc.h>
-#endif
-#endif
+#endif /* !defined(OpenBSD) && !defined(__FreeBSD__) */
+#endif /* defined(HAVE_MALLOC_H) */
 #ifdef HAVE_MALLOC_NP_H
 #include <malloc_np.h>
 #endif
@@ -116,12 +116,12 @@
          dmalloc_strndup(file, line, (string), -1, xalloc_b)
  #else
  #error "No dmalloc_strdup or equivalent"
- #endif
+#endif /* defined(HAVE_DMALLOC_STRDUP) || ... */
 
-#else /* not using dmalloc */
+#else /* !(defined(USE_DMALLOC)) */
 
  #define DMALLOC_FN_ARGS
-#endif
+#endif /* defined(USE_DMALLOC) */
 
 /** Allocate a chunk of <b>size</b> bytes of memory, and return a pointer to
  * result.  On error, log and terminate the process.  (Same as malloc(size),
@@ -142,7 +142,7 @@ tor_malloc_(size_t size DMALLOC_PARAMS)
   if (size==0) {
     size=1;
   }
-#endif
+#endif /* !defined(MALLOC_ZERO_WORKS) */
 
 #ifdef USE_DMALLOC
   result = dmalloc_malloc(file, line, size, DMALLOC_FUNC_MALLOC, 0, 0);
@@ -156,7 +156,7 @@ tor_malloc_(size_t size DMALLOC_PARAMS)
     /* If these functions die within a worker process, they won't call
      * spawn_exit, but that's ok, since the parent will run out of memory soon
      * anyway. */
-    exit(1);
+    exit(1); // exit ok: alloc failed.
     /* LCOV_EXCL_STOP */
   }
   return result;
@@ -187,8 +187,9 @@ tor_malloc_zero_(size_t size DMALLOC_PARAMS)
  * 0xfffe0001. */
 #define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4))
 
-/** Return non-zero if and only if the product of the arguments is exact. */
-static inline int
+/** Return non-zero if and only if the product of the arguments is exact,
+ * and cannot overflow. */
+int
 size_mul_check(const size_t x, const size_t y)
 {
   /* This first check is equivalent to
@@ -202,15 +203,6 @@ size_mul_check(const size_t x, const size_t y)
           x <= SIZE_MAX / y);
 }
 
-#ifdef TOR_UNIT_TESTS
-/** Exposed for unit tests only */
-int
-size_mul_check__(const size_t x, const size_t y)
-{
-  return size_mul_check(x,y);
-}
-#endif
-
 /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill
  * the memory with zero bytes, and return a pointer to the result.
  * Log and terminate the process on error.  (Same as
@@ -241,7 +233,7 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS)
   if (size==0) {
     size=1;
   }
-#endif
+#endif /* !defined(MALLOC_ZERO_WORKS) */
 
 #ifdef USE_DMALLOC
   result = dmalloc_realloc(file, line, ptr, size, DMALLOC_FUNC_REALLOC, 0);
@@ -252,7 +244,7 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS)
   if (PREDICT_UNLIKELY(result == NULL)) {
     /* LCOV_EXCL_START */
     log_err(LD_MM,"Out of memory on realloc(). Dying.");
-    exit(1);
+    exit(1); // exit ok: alloc failed.
     /* LCOV_EXCL_STOP */
   }
   return result;
@@ -290,7 +282,7 @@ tor_strdup_(const char *s DMALLOC_PARAMS)
   if (PREDICT_UNLIKELY(duplicate == NULL)) {
     /* LCOV_EXCL_START */
     log_err(LD_MM,"Out of memory on strdup(). Dying.");
-    exit(1);
+    exit(1); // exit ok: alloc failed.
     /* LCOV_EXCL_STOP */
   }
   return duplicate;
@@ -370,16 +362,16 @@ tor_log_mallinfo(int severity)
       mi.arena, mi.ordblks, mi.smblks, mi.hblks,
       mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks,
       mi.keepcost);
-#else
+#else /* !(defined(HAVE_MALLINFO)) */
   (void)severity;
-#endif
+#endif /* defined(HAVE_MALLINFO) */
 #ifdef USE_DMALLOC
   dmalloc_log_changed(0, /* Since the program started. */
                       1, /* Log info about non-freed pointers. */
                       0, /* Do not log info about freed pointers. */
                       0  /* Do not log individual pointers. */
                       );
-#endif
+#endif /* defined(USE_DMALLOC) */
 }
 ENABLE_GCC_WARNING(aggregate-return)
 
@@ -409,7 +401,7 @@ tor_lround(double d)
   return (long)rint(d);
 #else
   return (long)(d > 0 ? d + 0.5 : ceil(d - 0.5));
-#endif
+#endif /* defined(HAVE_LROUND) || ... */
 }
 
 /** Return the 64-bit integer closest to d.  We define this wrapper here so
@@ -424,7 +416,7 @@ tor_llround(double d)
   return (int64_t)rint(d);
 #else
   return (int64_t)(d > 0 ? d + 0.5 : ceil(d - 0.5));
-#endif
+#endif /* defined(HAVE_LLROUND) || ... */
 }
 
 /** Returns floor(log2(u64)).  If u64 is 0, (incorrectly) returns 0. */
@@ -453,7 +445,7 @@ tor_log2(uint64_t u64)
     r += 2;
   }
   if (u64 >= (U64_LITERAL(1)<<1)) {
-    u64 >>= 1;
+    // u64 >>= 1; // not using this any more.
     r += 1;
   }
   return r;
@@ -485,7 +477,7 @@ round_to_power_of_2(uint64_t u64)
 
 /** Return the lowest x such that x is at least <b>number</b>, and x modulo
  * <b>divisor</b> == 0.  If no such x can be expressed as an unsigned, return
- * UINT_MAX */
+ * UINT_MAX. Asserts if divisor is zero. */
 unsigned
 round_to_next_multiple_of(unsigned number, unsigned divisor)
 {
@@ -499,7 +491,7 @@ round_to_next_multiple_of(unsigned number, unsigned divisor)
 
 /** Return the lowest x such that x is at least <b>number</b>, and x modulo
  * <b>divisor</b> == 0. If no such x can be expressed as a uint32_t, return
- * UINT32_MAX */
+ * UINT32_MAX. Asserts if divisor is zero. */
 uint32_t
 round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
 {
@@ -514,7 +506,7 @@ round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
 
 /** Return the lowest x such that x is at least <b>number</b>, and x modulo
  * <b>divisor</b> == 0. If no such x can be expressed as a uint64_t, return
- * UINT64_MAX */
+ * UINT64_MAX. Asserts if divisor is zero. */
 uint64_t
 round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
 {
@@ -580,6 +572,19 @@ add_laplace_noise(int64_t signal_, double random_, double delta_f,
     return signal_ + noise;
 }
 
+/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather
+ * than overflow */
+uint32_t
+tor_add_u32_nowrap(uint32_t a, uint32_t b)
+{
+  /* a+b > UINT32_MAX check, without overflow */
+  if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) {
+    return UINT32_MAX;
+  } else {
+    return a+b;
+  }
+}
+
 /* Helper: return greatest common divisor of a,b */
 static uint64_t
 gcd64(uint64_t a, uint64_t b)
@@ -712,6 +717,19 @@ tor_strisnonupper(const char *s)
   return 1;
 }
 
+/** Return true iff every character in <b>s</b> is whitespace space; else
+ * return false. */
+int
+tor_strisspace(const char *s)
+{
+  while (*s) {
+    if (!TOR_ISSPACE(*s))
+      return 0;
+    s++;
+  }
+  return 1;
+}
+
 /** As strcmp, except that either string may be NULL.  The NULL string is
  * considered to be before any non-NULL string. */
 int
@@ -1066,6 +1084,36 @@ string_is_valid_ipv6_address(const char *string)
   return (tor_inet_pton(AF_INET6,string,&addr) == 1);
 }
 
+/** Return true iff <b>string</b> is a valid destination address,
+ * i.e. either a DNS hostname or IPv4/IPv6 address string.
+ */
+int
+string_is_valid_dest(const char *string)
+{
+  char *tmp = NULL;
+  int retval;
+  size_t len;
+
+  if (string == NULL)
+    return 0;
+
+  len = strlen(string);
+
+  if (len == 0)
+    return 0;
+
+  if (string[0] == '[' && string[len - 1] == ']')
+    string = tmp = tor_strndup(string + 1, len - 2);
+
+  retval = string_is_valid_ipv4_address(string) ||
+    string_is_valid_ipv6_address(string) ||
+    string_is_valid_nonrfc_hostname(string);
+
+  tor_free(tmp);
+
+  return retval;
+}
+
 /** Return true iff <b>string</b> matches a pattern of DNS names
  * that we allow Tor clients to connect to.
  *
@@ -1073,37 +1121,51 @@ string_is_valid_ipv6_address(const char *string)
  * with misconfigured zones that have been encountered in the wild.
  */
 int
-string_is_valid_hostname(const char *string)
+string_is_valid_nonrfc_hostname(const char *string)
 {
   int result = 1;
+  int has_trailing_dot;
+  char *last_label;
   smartlist_t *components;
 
+  if (!string || strlen(string) == 0)
+    return 0;
+
+  if (string_is_valid_ipv4_address(string))
+    return 0;
+
   components = smartlist_new();
 
   smartlist_split_string(components,string,".",0,0);
 
+  if (BUG(smartlist_len(components) == 0))
+    return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks.
+
+  /* Allow a single terminating '.' used rarely to indicate domains
+   * are FQDNs rather than relative. */
+  last_label = (char *)smartlist_get(components,
+                                     smartlist_len(components) - 1);
+  has_trailing_dot = (last_label[0] == '\0');
+  if (has_trailing_dot) {
+    smartlist_pop_last(components);
+    tor_free(last_label);
+    last_label = NULL;
+  }
+
   SMARTLIST_FOREACH_BEGIN(components, char *, c) {
     if ((c[0] == '-') || (*c == '_')) {
       result = 0;
       break;
     }
 
-    /* Allow a single terminating '.' used rarely to indicate domains
-     * are FQDNs rather than relative. */
-    if ((c_sl_idx > 0) && (c_sl_idx + 1 == c_sl_len) && !*c) {
-      continue;
-    }
-
     do {
-      if ((*c >= 'a' && *c <= 'z') ||
-          (*c >= 'A' && *c <= 'Z') ||
-          (*c >= '0' && *c <= '9') ||
-          (*c == '-') || (*c == '_'))
-        c++;
-      else
-        result = 0;
+      result = (TOR_ISALNUM(*c) || (*c == '-') || (*c == '_'));
+      c++;
     } while (result && *c);
 
+    if (result == 0) {
+      break;
+    }
   } SMARTLIST_FOREACH_END(c);
 
   SMARTLIST_FOREACH_BEGIN(components, char *, c) {
@@ -1167,7 +1229,7 @@ tor_parse_long(const char *s, int base, long min, long max,
   char *endptr;
   long r;
 
-  if (base < 0) {
+  if (BUG(base < 0)) {
     if (ok)
       *ok = 0;
     return 0;
@@ -1186,7 +1248,7 @@ tor_parse_ulong(const char *s, int base, unsigned long min,
   char *endptr;
   unsigned long r;
 
-  if (base < 0) {
+  if (BUG(base < 0)) {
     if (ok)
       *ok = 0;
     return 0;
@@ -1218,7 +1280,7 @@ tor_parse_uint64(const char *s, int base, uint64_t min,
   char *endptr;
   uint64_t r;
 
-  if (base < 0) {
+  if (BUG(base < 0)) {
     if (ok)
       *ok = 0;
     return 0;
@@ -1228,20 +1290,12 @@ tor_parse_uint64(const char *s, int base, uint64_t min,
 #ifdef HAVE_STRTOULL
   r = (uint64_t)strtoull(s, &endptr, base);
 #elif defined(_WIN32)
-#if defined(_MSC_VER) && _MSC_VER < 1300
-  tor_assert(base <= 10);
-  r = (uint64_t)_atoi64(s);
-  endptr = (char*)s;
-  while (TOR_ISSPACE(*endptr)) endptr++;
-  while (TOR_ISDIGIT(*endptr)) endptr++;
-#else
   r = (uint64_t)_strtoui64(s, &endptr, base);
-#endif
 #elif SIZEOF_LONG == 8
   r = (uint64_t)strtoul(s, &endptr, base);
 #else
 #error "I don't know how to parse 64-bit numbers."
-#endif
+#endif /* defined(HAVE_STRTOULL) || ... */
 
   CHECK_STRTOX_RESULT();
 }
@@ -1639,7 +1693,7 @@ tor_timegm(const struct tm *tm, time_t *time_out)
     log_warn(LD_BUG, "Result does not fit in tor_timegm");
     return -1;
   }
-#endif
+#endif /* SIZEOF_TIME_T < 8 */
   *time_out = (time_t)seconds;
   return 0;
 }
@@ -1780,6 +1834,15 @@ format_iso_time(char *buf, time_t t)
   strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm));
 }
 
+/** As format_local_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
+ * embedding an internal space. */
+void
+format_local_iso_time_nospace(char *buf, time_t t)
+{
+  format_local_iso_time(buf, t);
+  buf[10] = 'T';
+}
+
 /** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid
  * embedding an internal space. */
 void
@@ -1803,17 +1866,26 @@ format_iso_time_nospace_usec(char *buf, const struct timeval *tv)
 /** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>,
  * parse it and store its value in *<b>t</b>.  Return 0 on success, -1 on
  * failure.  Ignore extraneous stuff in <b>cp</b> after the end of the time
- * string, unless <b>strict</b> is set. */
+ * string, unless <b>strict</b> is set. If <b>nospace</b> is set,
+ * expect the YYYY-MM-DDTHH:MM:SS format. */
 int
-parse_iso_time_(const char *cp, time_t *t, int strict)
+parse_iso_time_(const char *cp, time_t *t, int strict, int nospace)
 {
   struct tm st_tm;
   unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0;
   int n_fields;
-  char extra_char;
-  n_fields = tor_sscanf(cp, "%u-%2u-%2u %2u:%2u:%2u%c", &year, &month,
-                        &day, &hour, &minute, &second, &extra_char);
-  if (strict ? (n_fields != 6) : (n_fields < 6)) {
+  char extra_char, separator_char;
+  n_fields = tor_sscanf(cp, "%u-%2u-%2u%c%2u:%2u:%2u%c",
+                        &year, &month, &day,
+                        &separator_char,
+                        &hour, &minute, &second, &extra_char);
+  if (strict ? (n_fields != 7) : (n_fields < 7)) {
+    char *esc = esc_for_log(cp);
+    log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
+    tor_free(esc);
+    return -1;
+  }
+  if (separator_char != (nospace ? 'T' : ' ')) {
     char *esc = esc_for_log(cp);
     log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
     tor_free(esc);
@@ -1855,7 +1927,16 @@ parse_iso_time_(const char *cp, time_t *t, int strict)
 int
 parse_iso_time(const char *cp, time_t *t)
 {
-  return parse_iso_time_(cp, t, 1);
+  return parse_iso_time_(cp, t, 1, 0);
+}
+
+/**
+ * As parse_iso_time, but parses a time encoded by format_iso_time_nospace().
+ */
+int
+parse_iso_time_nospace(const char *cp, time_t *t)
+{
+  return parse_iso_time_(cp, t, 1, 1);
 }
 
 /** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh),
@@ -1930,7 +2011,7 @@ parse_http_time(const char *date, struct tm *tm)
 
 /** Given an <b>interval</b> in seconds, try to write it to the
  * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form.
- * Return 0 on success, -1 on failure.
+ * Returns a non-negative integer on success, -1 on failure.
  */
 int
 format_time_interval(char *out, size_t out_len, long interval)
@@ -1999,7 +2080,7 @@ update_approx_time(time_t now)
 {
   cached_approx_time = now;
 }
-#endif
+#endif /* !defined(TIME_IS_FAST) */
 
 /* =====
  * Rate limiting
@@ -2095,7 +2176,7 @@ read_all(tor_socket_t fd, char *buf, size_t count, int isSocket)
     return -1;
   }
 
-  while (numread != count) {
+  while (numread < count) {
     if (isSocket)
       result = tor_socket_recv(fd, buf+numread, count-numread, 0);
     else
@@ -2128,9 +2209,9 @@ clean_name_for_stat(char *name)
       return;
     name[len-1]='\0';
   }
-#else
+#else /* !(defined(_WIN32)) */
   (void)name;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Wrapper for unlink() to make it mockable for the test suite; returns 0
@@ -2270,10 +2351,14 @@ check_private_dir,(const char *dirname, cpd_check_t check,
        * permissions on the directory will be checked again below.*/
       fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
 
-      if (fd == -1)
+      if (fd == -1) {
+        log_warn(LD_FS, "Could not reopen recently created directory %s: %s",
+                 dirname,
+                 strerror(errno));
         return -1;
-      else
+      } else {
         close(fd);
+      }
 
     } else if (!(check & CPD_CHECK)) {
       log_warn(LD_FS, "Directory %s does not exist.", dirname);
@@ -2323,21 +2408,27 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     running_gid = getgid();
   }
   if (st.st_uid != running_uid) {
-    const struct passwd *pw_uid = NULL;
-    char *process_ownername = NULL;
+    char *process_ownername = NULL, *file_ownername = NULL;
 
-    pw_uid = tor_getpwuid(running_uid);
-    process_ownername = pw_uid ? tor_strdup(pw_uid->pw_name) :
-      tor_strdup("<unknown>");
+    {
+      const struct passwd *pw_running = tor_getpwuid(running_uid);
+      process_ownername = pw_running ? tor_strdup(pw_running->pw_name) :
+        tor_strdup("<unknown>");
+    }
 
-    pw_uid = tor_getpwuid(st.st_uid);
+    {
+      const struct passwd *pw_stat = tor_getpwuid(st.st_uid);
+      file_ownername = pw_stat ? tor_strdup(pw_stat->pw_name) :
+        tor_strdup("<unknown>");
+    }
 
     log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by "
         "%s (%d). Perhaps you are running Tor as the wrong user?",
-                         dirname, process_ownername, (int)running_uid,
-                         pw ? pw->pw_name : "<unknown>", (int)st.st_uid);
+             dirname, process_ownername, (int)running_uid,
+             file_ownername, (int)st.st_uid);
 
     tor_free(process_ownername);
+    tor_free(file_ownername);
     close(fd);
     return -1;
   }
@@ -2394,7 +2485,7 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     }
   }
   close(fd);
-#else
+#else /* !(!defined(_WIN32)) */
   /* Win32 case: we can't open() a directory. */
   (void)effective_user;
 
@@ -2428,7 +2519,7 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     return -1;
   }
 
-#endif
+#endif /* !defined(_WIN32) */
   return 0;
 }
 
@@ -2448,7 +2539,7 @@ write_str_to_file,(const char *fname, const char *str, int bin))
              "We're writing a text string that already contains a CR to %s",
              escaped(fname));
   }
-#endif
+#endif /* defined(_WIN32) */
   return write_bytes_to_file(fname, str, strlen(str), bin);
 }
 
@@ -2601,6 +2692,14 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
 
   if (file_data->rename_on_close) {
     tor_assert(file_data->tempname && file_data->filename);
+    if (!abort_write) {
+      tor_assert(strcmp(file_data->filename, file_data->tempname));
+      if (replace_file(file_data->tempname, file_data->filename)) {
+        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
+                 strerror(errno));
+        abort_write = r = -1;
+      }
+    }
     if (abort_write) {
       int res = unlink(file_data->tempname);
       if (res != 0) {
@@ -2609,13 +2708,6 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
                  file_data->tempname, strerror(errno));
         r = -1;
       }
-    } else {
-      tor_assert(strcmp(file_data->filename, file_data->tempname));
-      if (replace_file(file_data->tempname, file_data->filename)) {
-        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
-                 strerror(errno));
-        r = -1;
-      }
     }
   }
 
@@ -2847,7 +2939,7 @@ read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
       errno = save_errno;
     return string;
   }
-#endif
+#endif /* !defined(_WIN32) */
 
   if ((uint64_t)(statbuf.st_size)+1 >= SIZE_T_CEILING) {
     close(fd);
@@ -2880,7 +2972,7 @@ read_file_to_str, (const char *filename, int flags, struct stat *stat_out))
   if (!bin) {
     statbuf.st_size = (size_t) r;
   } else
-#endif
+#endif /* defined(_WIN32) || defined(__CYGWIN__) */
     if (r != statbuf.st_size) {
       /* Unless we're using text mode on win32, we'd better have an exact
        * match for size. */
@@ -2954,8 +3046,9 @@ unescape_string(const char *s, char **result, size_t *size_out)
         *out = '\0';
         if (size_out) *size_out = out - *result;
         return cp+1;
-      case '\0':
+
         /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */
+      case '\0':
         tor_fragile_assert();
         tor_free(*result);
         return NULL;
@@ -3003,8 +3096,9 @@ unescape_string(const char *s, char **result, size_t *size_out)
             *out++ = cp[1];
             cp += 2;
             break;
-          default:
+
             /* LCOV_EXCL_START */
+          default:
             /* we caught this above in the initial loop. */
             tor_assert_nonfatal_unreached();
             tor_free(*result); return NULL;
@@ -3017,135 +3111,39 @@ unescape_string(const char *s, char **result, size_t *size_out)
   }
 }
 
-/** Given a string containing part of a configuration file or similar format,
- * advance past comments and whitespace and try to parse a single line.  If we
- * parse a line successfully, set *<b>key_out</b> to a new string holding the
- * key portion and *<b>value_out</b> to a new string holding the value portion
- * of the line, and return a pointer to the start of the next line.  If we run
- * out of data, return a pointer to the end of the string.  If we encounter an
- * error, return NULL and set *<b>err_out</b> (if provided) to an error
- * message.
- */
-const char *
-parse_config_line_from_str_verbose(const char *line, char **key_out,
-                                   char **value_out,
-                                   const char **err_out)
+/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
+ * enclosing quotes. Backslashes are not unescaped. Return the unquoted
+ * <b>path</b> on success or 0 if <b>path</b> is not quoted correctly. */
+char *
+get_unquoted_path(const char *path)
 {
-  /*
-    See torrc_format.txt for a description of the (silly) format this parses.
-   */
-  const char *key, *val, *cp;
-  int continuation = 0;
-
-  tor_assert(key_out);
-  tor_assert(value_out);
+  size_t len = strlen(path);
 
-  *key_out = *value_out = NULL;
-  key = val = NULL;
-  /* Skip until the first keyword. */
-  while (1) {
-    while (TOR_ISSPACE(*line))
-      ++line;
-    if (*line == '#') {
-      while (*line && *line != '\n')
-        ++line;
-    } else {
-      break;
-    }
+  if (len == 0) {
+    return tor_strdup("");
   }
 
-  if (!*line) { /* End of string? */
-    *key_out = *value_out = NULL;
-    return line;
+  int has_start_quote = (path[0] == '\"');
+  int has_end_quote = (len > 0 && path[len-1] == '\"');
+  if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) {
+    return NULL;
   }
 
-  /* Skip until the next space or \ followed by newline. */
-  key = line;
-  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
-         ! (line[0] == '\\' && line[1] == '\n'))
-    ++line;
-  *key_out = tor_strndup(key, line-key);
-
-  /* Skip until the value. */
-  while (*line == ' ' || *line == '\t')
-    ++line;
-
-  val = line;
-
-  /* Find the end of the line. */
-  if (*line == '\"') { // XXX No continuation handling is done here
-    if (!(line = unescape_string(line, value_out, NULL))) {
-      if (err_out)
-        *err_out = "Invalid escape sequence in quoted string";
-      return NULL;
-    }
-    while (*line == ' ' || *line == '\t')
-      ++line;
-    if (*line == '\r' && *(++line) == '\n')
-      ++line;
-    if (*line && *line != '#' && *line != '\n') {
-      if (err_out)
-        *err_out = "Excess data after quoted string";
+  char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1);
+  char *s = unquoted_path;
+  size_t i;
+  for (i = has_start_quote; i < len - has_end_quote; i++) {
+    if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) {
+      *(s-1) = path[i];
+    } else if (path[i] != '\"') {
+      *s++ = path[i];
+    } else {  /* unescaped quote */
+      tor_free(unquoted_path);
       return NULL;
     }
-  } else {
-    /* Look for the end of the line. */
-    while (*line && *line != '\n' && (*line != '#' || continuation)) {
-      if (*line == '\\' && line[1] == '\n') {
-        continuation = 1;
-        line += 2;
-      } else if (*line == '#') {
-        do {
-          ++line;
-        } while (*line && *line != '\n');
-        if (*line == '\n')
-          ++line;
-      } else {
-        ++line;
-      }
-    }
-
-    if (*line == '\n') {
-      cp = line++;
-    } else {
-      cp = line;
-    }
-    /* Now back cp up to be the last nonspace character */
-    while (cp>val && TOR_ISSPACE(*(cp-1)))
-      --cp;
-
-    tor_assert(cp >= val);
-
-    /* Now copy out and decode the value. */
-    *value_out = tor_strndup(val, cp-val);
-    if (continuation) {
-      char *v_out, *v_in;
-      v_out = v_in = *value_out;
-      while (*v_in) {
-        if (*v_in == '#') {
-          do {
-            ++v_in;
-          } while (*v_in && *v_in != '\n');
-          if (*v_in == '\n')
-            ++v_in;
-        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
-          v_in += 2;
-        } else {
-          *v_out++ = *v_in++;
-        }
-      }
-      *v_out = '\0';
-    }
-  }
-
-  if (*line == '#') {
-    do {
-      ++line;
-    } while (*line && *line != '\n');
   }
-  while (TOR_ISSPACE(*line)) ++line;
-
-  return line;
+  *s = '\0';
+  return unquoted_path;
 }
 
 /** Expand any homedir prefix on <b>filename</b>; return a newly allocated
@@ -3160,7 +3158,7 @@ expand_filename(const char *filename)
    *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
    */
   return tor_strdup(filename);
-#else
+#else /* !(defined(_WIN32)) */
   if (*filename == '~') {
     char *home, *result=NULL;
     const char *rest;
@@ -3190,10 +3188,10 @@ expand_filename(const char *filename)
       }
       tor_free(username);
       rest = slash ? (slash+1) : "";
-#else
+#else /* !(defined(HAVE_PWD_H)) */
       log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
       return tor_strdup(filename);
-#endif
+#endif /* defined(HAVE_PWD_H) */
     }
     tor_assert(home);
     /* Remove trailing slash. */
@@ -3206,7 +3204,7 @@ expand_filename(const char *filename)
   } else {
     return tor_strdup(filename);
   }
-#endif
+#endif /* defined(_WIN32) */
 }
 
 #define MAX_SCANF_WIDTH 9999
@@ -3534,6 +3532,17 @@ smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
   smartlist_add(sl, str);
 }
 
+/** Append a copy of string to sl */
+void
+smartlist_add_strdup(struct smartlist_t *sl, const char *string)
+{
+  char *copy;
+
+  copy = tor_strdup(string);
+
+  smartlist_add(sl, copy);
+}
+
 /** Return a new list containing the filenames in the directory <b>dirname</b>.
  * Return NULL on error or if <b>dirname</b> is not a directory.
  */
@@ -3564,10 +3573,10 @@ tor_listdir, (const char *dirname))
     name[sizeof(name)-1] = '\0';
 #else
     strlcpy(name,findData.cFileName,sizeof(name));
-#endif
+#endif /* defined(UNICODE) */
     if (strcmp(name, ".") &&
         strcmp(name, "..")) {
-      smartlist_add(result, tor_strdup(name));
+      smartlist_add_strdup(result, name);
     }
     if (!FindNextFile(handle, &findData)) {
       DWORD err;
@@ -3581,7 +3590,7 @@ tor_listdir, (const char *dirname))
   }
   FindClose(handle);
   tor_free(pattern);
-#else
+#else /* !(defined(_WIN32)) */
   const char *prot_dname = sandbox_intern_string(dirname);
   DIR *d;
   struct dirent *de;
@@ -3593,10 +3602,10 @@ tor_listdir, (const char *dirname))
     if (!strcmp(de->d_name, ".") ||
         !strcmp(de->d_name, ".."))
       continue;
-    smartlist_add(result, tor_strdup(de->d_name));
+    smartlist_add_strdup(result, de->d_name);
   }
   closedir(d);
-#endif
+#endif /* defined(_WIN32) */
   return result;
 }
 
@@ -3612,7 +3621,7 @@ path_is_relative(const char *filename)
   else if (filename && strlen(filename)>3 && TOR_ISALPHA(filename[0]) &&
            filename[1] == ':' && filename[2] == '\\')
     return 0;
-#endif
+#endif /* defined(_WIN32) */
   else
     return 1;
 }
@@ -3647,14 +3656,14 @@ start_daemon(void)
   if (pipe(daemon_filedes)) {
     /* LCOV_EXCL_START */
     log_err(LD_GENERAL,"pipe failed; exiting. Error was %s", strerror(errno));
-    exit(1);
+    exit(1); // exit ok: during daemonize, pipe failed.
     /* LCOV_EXCL_STOP */
   }
   pid = fork();
   if (pid < 0) {
     /* LCOV_EXCL_START */
     log_err(LD_GENERAL,"fork failed. Exiting.");
-    exit(1);
+    exit(1); // exit ok: during daemonize, fork failed
     /* LCOV_EXCL_STOP */
   }
   if (pid) {  /* Parent */
@@ -3669,13 +3678,13 @@ start_daemon(void)
     }
     fflush(stdout);
     if (ok == 1)
-      exit(0);
+      exit(0); // exit ok: during daemonize, daemonizing.
     else
-      exit(1); /* child reported error */
+      exit(1); /* child reported error. exit ok: daemonize failed. */
   } else { /* Child */
     close(daemon_filedes[0]); /* we only write */
 
-    pid = setsid(); /* Detach from controlling terminal */
+    (void) setsid(); /* Detach from controlling terminal */
     /*
      * Fork one more time, so the parent (the session group leader) can exit.
      * This means that we, as a non-session group leader, can never regain a
@@ -3683,7 +3692,7 @@ start_daemon(void)
      * _Advanced Programming in the Unix Environment_.
      */
     if (fork() != 0) {
-      exit(0);
+      exit(0); // exit ok: during daemonize, fork failed (2)
     }
     set_main_thread(); /* We are now the main thread. */
 
@@ -3712,14 +3721,14 @@ finish_daemon(const char *desired_cwd)
    /* Don't hold the wrong FS mounted */
   if (chdir(desired_cwd) < 0) {
     log_err(LD_GENERAL,"chdir to \"%s\" failed. Exiting.",desired_cwd);
-    exit(1);
+    exit(1); // exit ok: during daemonize, chdir failed.
   }
 
   nullfd = tor_open_cloexec("/dev/null", O_RDWR, 0);
   if (nullfd < 0) {
     /* LCOV_EXCL_START */
     log_err(LD_GENERAL,"/dev/null can't be opened. Exiting.");
-    exit(1);
+    exit(1); // exit ok: during daemonize, couldn't open /dev/null
     /* LCOV_EXCL_STOP */
   }
   /* close fds linking to invoking terminal, but
@@ -3731,7 +3740,7 @@ finish_daemon(const char *desired_cwd)
       dup2(nullfd,2) < 0) {
     /* LCOV_EXCL_START */
     log_err(LD_GENERAL,"dup2 failed. Exiting.");
-    exit(1);
+    exit(1); // exit ok: during daemonize, dup2 failed.
     /* LCOV_EXCL_STOP */
   }
   if (nullfd > 2)
@@ -3742,7 +3751,7 @@ finish_daemon(const char *desired_cwd)
   }
   close(daemon_filedes[1]);
 }
-#else
+#else /* !(!defined(_WIN32)) */
 /* defined(_WIN32) */
 void
 start_daemon(void)
@@ -3753,11 +3762,12 @@ finish_daemon(const char *cp)
 {
   (void)cp;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /** Write the current process ID, followed by NL, into <b>filename</b>.
+ * Return 0 on success, -1 on failure.
  */
-void
+int
 write_pidfile(const char *filename)
 {
   FILE *pidfile;
@@ -3765,13 +3775,19 @@ write_pidfile(const char *filename)
   if ((pidfile = fopen(filename, "w")) == NULL) {
     log_warn(LD_FS, "Unable to open \"%s\" for writing: %s", filename,
              strerror(errno));
+    return -1;
   } else {
 #ifdef _WIN32
-    fprintf(pidfile, "%d\n", (int)_getpid());
+    int pid = (int)_getpid();
 #else
-    fprintf(pidfile, "%d\n", (int)getpid());
+    int pid = (int)getpid();
 #endif
-    fclose(pidfile);
+    int rv = 0;
+    if (fprintf(pidfile, "%d\n", pid) < 0)
+      rv = -1;
+    if (fclose(pidfile) < 0)
+      rv = -1;
+    return rv;
   }
 }
 
@@ -3788,7 +3804,7 @@ load_windows_system_library(const TCHAR *library_name)
   _tcscat(path, library_name);
   return LoadLibrary(path);
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Format a single argument for being put on a Windows command line.
  * Returns a newly allocated string */
@@ -3948,7 +3964,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
  * call it with a signed int and an unsigned char, and since the C standard
  * does not guarantee that an int is wider than a char (an int must be at
  * least 16 bits but it is permitted for a char to be that wide as well), we
- * can't assume a signed int is sufficient to accomodate an unsigned char.
+ * can't assume a signed int is sufficient to accommodate an unsigned char.
  * Thus, format_helper_exit_status() will still need to emit any require '-'
  * on its own.
  *
@@ -3978,7 +3994,7 @@ format_dec_number_sigsafe(unsigned long x, char *buf, int buf_len)
  *
  * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
  * with spaces. CHILD_STATE indicates where
- * in the processs of starting the child process did the failure occur (see
+ * in the process of starting the child process did the failure occur (see
  * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
  * errno when the failure occurred.
  *
@@ -4083,7 +4099,7 @@ format_helper_exit_status(unsigned char child_state, int saved_errno,
  done:
   return res;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /* Maximum number of file descriptors, if we cannot get it via sysconf() */
 #define DEFAULT_MAX_FD 256
@@ -4107,12 +4123,12 @@ tor_terminate_process(process_handle_t *process_handle)
     else
       return 0;
   }
-#else /* Unix */
+#else /* !(defined(_WIN32)) */
   if (process_handle->waitpid_cb) {
     /* We haven't got a waitpid yet, so we can just kill off the process. */
     return kill(process_handle->pid, SIGTERM);
   }
-#endif
+#endif /* defined(_WIN32) */
 
   return 0; /* We didn't need to kill the process, so report success */
 }
@@ -4134,14 +4150,14 @@ tor_process_get_stdout_pipe(process_handle_t *process_handle)
 {
   return process_handle->stdout_pipe;
 }
-#else
+#else /* !(defined(_WIN32)) */
 /* DOCDOC tor_process_get_stdout_pipe */
-FILE *
+int
 tor_process_get_stdout_pipe(process_handle_t *process_handle)
 {
-  return process_handle->stdout_handle;
+  return process_handle->stdout_pipe;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /* DOCDOC process_handle_new */
 static process_handle_t *
@@ -4157,7 +4173,7 @@ process_handle_new(void)
   out->stdin_pipe = -1;
   out->stdout_pipe = -1;
   out->stderr_pipe = -1;
-#endif
+#endif /* defined(_WIN32) */
 
   return out;
 }
@@ -4177,7 +4193,7 @@ process_handle_waitpid_cb(int status, void *arg)
     process_handle->status = PROCESS_STATUS_NOTRUNNING;
   process_handle->waitpid_cb = 0;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /**
  * @name child-process states
@@ -4199,6 +4215,20 @@ process_handle_waitpid_cb(int status, void *arg)
 #define CHILD_STATE_EXEC 8
 #define CHILD_STATE_FAILEXEC 9
 /** @} */
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
 /** Start a program in the background. If <b>filename</b> contains a '/', then
  * it will be treated as an absolute or relative path.  Otherwise, on
  * non-Windows systems, the system path will be searched for <b>filename</b>.
@@ -4223,6 +4253,12 @@ tor_spawn_background(const char *const filename, const char **argv,
                      process_environment_t *env,
                      process_handle_t **process_handle_out)
 {
+  if (BUG(may_spawn_background_process == 0)) {
+    /* We should never reach this point if we're forbidden to spawn
+     * processes. Instead we should have caught the attempt earlier. */
+    return PROCESS_STATUS_ERROR;
+  }
+
 #ifdef _WIN32
   HANDLE stdout_pipe_read = NULL;
   HANDLE stdout_pipe_write = NULL;
@@ -4340,13 +4376,12 @@ tor_spawn_background(const char *const filename, const char **argv,
   /* TODO: Close pipes on exit */
   *process_handle_out = process_handle;
   return status;
-#else // _WIN32
+#else /* !(defined(_WIN32)) */
   pid_t pid;
   int stdout_pipe[2];
   int stderr_pipe[2];
   int stdin_pipe[2];
   int fd, retval;
-  ssize_t nbytes;
   process_handle_t *process_handle;
   int status;
 
@@ -4367,7 +4402,7 @@ tor_spawn_background(const char *const filename, const char **argv,
      and we are not allowed to use unsafe functions between fork and exec */
   error_message_length = strlen(error_message);
 
-  child_state = CHILD_STATE_PIPE;
+  // child_state = CHILD_STATE_PIPE;
 
   /* Set up pipe for redirecting stdout, stderr, and stdin of child */
   retval = pipe(stdout_pipe);
@@ -4404,7 +4439,7 @@ tor_spawn_background(const char *const filename, const char **argv,
     return status;
   }
 
-  child_state = CHILD_STATE_MAXFD;
+  // child_state = CHILD_STATE_MAXFD;
 
 #ifdef _SC_OPEN_MAX
   if (-1 == max_fd) {
@@ -4415,11 +4450,11 @@ tor_spawn_background(const char *const filename, const char **argv,
                "Cannot find maximum file descriptor, assuming %d", max_fd);
     }
   }
-#else
+#else /* !(defined(_SC_OPEN_MAX)) */
   max_fd = DEFAULT_MAX_FD;
-#endif
+#endif /* defined(_SC_OPEN_MAX) */
 
-  child_state = CHILD_STATE_FORK;
+  // child_state = CHILD_STATE_FORK;
 
   pid = fork();
   if (0 == pid) {
@@ -4432,7 +4467,7 @@ tor_spawn_background(const char *const filename, const char **argv,
      * than nothing.
      */
     prctl(PR_SET_PDEATHSIG, SIGTERM);
-#endif
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
 
     child_state = CHILD_STATE_DUPOUT;
 
@@ -4455,7 +4490,7 @@ tor_spawn_background(const char *const filename, const char **argv,
     if (-1 == retval)
       goto error;
 
-    child_state = CHILD_STATE_CLOSEFD;
+    // child_state = CHILD_STATE_CLOSEFD;
 
     close(stderr_pipe[0]);
     close(stderr_pipe[1]);
@@ -4471,7 +4506,7 @@ tor_spawn_background(const char *const filename, const char **argv,
       close(fd);
     }
 
-    child_state = CHILD_STATE_EXEC;
+    // child_state = CHILD_STATE_EXEC;
 
     /* Call the requested program. We need the cast because
        execvp doesn't define argv as const, even though it
@@ -4490,7 +4525,8 @@ tor_spawn_background(const char *const filename, const char **argv,
   error:
     {
       /* XXX: are we leaking fds from the pipe? */
-      int n;
+      int n, err=0;
+      ssize_t nbytes;
 
       n = format_helper_exit_status(child_state, errno, hex_errno);
 
@@ -4499,13 +4535,14 @@ tor_spawn_background(const char *const filename, const char **argv,
            value, but there is nothing we can do if it fails */
         /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
         nbytes = write(STDOUT_FILENO, error_message, error_message_length);
+        err = (nbytes < 0);
         nbytes = write(STDOUT_FILENO, hex_errno, n);
+        err += (nbytes < 0);
       }
-    }
 
-    (void) nbytes;
+      _exit(err?254:255); // exit ok: in child.
+    }
 
-    _exit(255);
     /* Never reached, but avoids compiler warning */
     return status; // LCOV_EXCL_LINE
   }
@@ -4570,14 +4607,10 @@ tor_spawn_background(const char *const filename, const char **argv,
     log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
              "nonblocking in parent process: %s", strerror(errno));
   }
-  /* Open the buffered IO streams */
-  process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r");
-  process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r");
-  process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r");
 
   *process_handle_out = process_handle;
-  return process_handle->status;
-#endif // _WIN32
+  return status;
+#endif /* defined(_WIN32) */
 }
 
 /** Destroy all resources allocated by the process handle in
@@ -4619,18 +4652,13 @@ tor_process_handle_destroy,(process_handle_t *process_handle,
 
   if (process_handle->stdin_pipe)
     CloseHandle(process_handle->stdin_pipe);
-#else
-  if (process_handle->stdout_handle)
-    fclose(process_handle->stdout_handle);
-
-  if (process_handle->stderr_handle)
-    fclose(process_handle->stderr_handle);
-
-  if (process_handle->stdin_handle)
-    fclose(process_handle->stdin_handle);
+#else /* !(defined(_WIN32)) */
+  close(process_handle->stdout_pipe);
+  close(process_handle->stderr_pipe);
+  close(process_handle->stdin_pipe);
 
   clear_waitpid_callback(process_handle->waitpid_cb);
-#endif
+#endif /* defined(_WIN32) */
 
   memset(process_handle, 0x0f, sizeof(process_handle_t));
   tor_free(process_handle);
@@ -4683,7 +4711,7 @@ tor_get_exit_code(process_handle_t *process_handle,
       return PROCESS_EXIT_ERROR;
     }
   }
-#else
+#else /* !(defined(_WIN32)) */
   int stat_loc;
   int retval;
 
@@ -4718,7 +4746,7 @@ tor_get_exit_code(process_handle_t *process_handle,
 
   if (exit_code != NULL)
     *exit_code = WEXITSTATUS(stat_loc);
-#endif // _WIN32
+#endif /* defined(_WIN32) */
 
   return PROCESS_EXIT_EXITED;
 }
@@ -4751,7 +4779,7 @@ environment_variable_names_equal(const char *s1, const char *s2)
 /** Free <b>env</b> (assuming it was produced by
  * process_environment_make). */
 void
-process_environment_free(process_environment_t *env)
+process_environment_free_(process_environment_t *env)
 {
   if (env == NULL) return;
 
@@ -4773,8 +4801,8 @@ process_environment_t *
 process_environment_make(struct smartlist_t *env_vars)
 {
   process_environment_t *env = tor_malloc_zero(sizeof(process_environment_t));
-  size_t n_env_vars = smartlist_len(env_vars);
-  size_t i;
+  int n_env_vars = smartlist_len(env_vars);
+  int i;
   size_t total_env_length;
   smartlist_t *env_vars_sorted;
 
@@ -4864,7 +4892,7 @@ get_current_process_environment_variables(void)
 
   char **environ_tmp; /* Not const char ** ? Really? */
   for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) {
-    smartlist_add(sl, tor_strdup(*environ_tmp));
+    smartlist_add_strdup(sl, *environ_tmp);
   }
 
   return sl;
@@ -4913,7 +4941,7 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count,
   if (count > SIZE_T_CEILING || count > SSIZE_MAX)
     return -1;
 
-  while (numread != count) {
+  while (numread < count) {
     /* Check if there is anything to read */
     retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
     if (!retval) {
@@ -4958,20 +4986,20 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count,
   }
   return (ssize_t)numread;
 }
-#else
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
+#else /* !(defined(_WIN32)) */
+/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
  * <b>process</b> is NULL, the function will return immediately if there is
  * nothing more to read. Otherwise data will be read until end of file, or
  * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
  * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
  * file has been reached. */
 ssize_t
-tor_read_all_handle(FILE *h, char *buf, size_t count,
+tor_read_all_handle(int fd, char *buf, size_t count,
                     const process_handle_t *process,
                     int *eof)
 {
   size_t numread = 0;
-  char *retval;
+  ssize_t result;
 
   if (eof)
     *eof = 0;
@@ -4979,37 +5007,31 @@ tor_read_all_handle(FILE *h, char *buf, size_t count,
   if (count > SIZE_T_CEILING || count > SSIZE_MAX)
     return -1;
 
-  while (numread != count) {
-    /* Use fgets because that is what we use in log_from_pipe() */
-    retval = fgets(buf+numread, (int)(count-numread), h);
-    if (NULL == retval) {
-      if (feof(h)) {
-        log_debug(LD_GENERAL, "fgets() reached end of file");
-        if (eof)
-          *eof = 1;
+  while (numread < count) {
+    result = read(fd, buf+numread, count-numread);
+
+    if (result == 0) {
+      log_debug(LD_GENERAL, "read() reached end of file");
+      if (eof)
+        *eof = 1;
+      break;
+    } else if (result < 0 && errno == EAGAIN) {
+      if (process)
+        continue;
+      else
         break;
-      } else {
-        if (EAGAIN == errno) {
-          if (process)
-            continue;
-          else
-            break;
-        } else {
-          log_warn(LD_GENERAL, "fgets() from handle failed: %s",
-                   strerror(errno));
-          return -1;
-        }
-      }
+    } else if (result < 0) {
+      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
+      return -1;
     }
-    tor_assert(retval != NULL);
-    tor_assert(strlen(retval) + numread <= count);
-    numread += strlen(retval);
+
+    numread += result;
   }
 
-  log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread);
+  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
   return (ssize_t)numread;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Read from stdout of a process until the process exits. */
 ssize_t
@@ -5020,9 +5042,9 @@ tor_read_all_from_process_stdout(const process_handle_t *process_handle,
   return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
                              process_handle);
 #else
-  return tor_read_all_handle(process_handle->stdout_handle, buf, count,
+  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
                              process_handle, NULL);
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Read from stdout of a process until the process exits. */
@@ -5034,9 +5056,9 @@ tor_read_all_from_process_stderr(const process_handle_t *process_handle,
   return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
                              process_handle);
 #else
-  return tor_read_all_handle(process_handle->stderr_handle, buf, count,
+  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
                              process_handle, NULL);
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
@@ -5111,30 +5133,6 @@ stream_status_to_string(enum stream_status stream_status)
   }
 }
 
-/* DOCDOC */
-static void
-log_portfw_spawn_error_message(const char *buf,
-                               const char *executable, int *child_status)
-{
-  /* Parse error message */
-  int retval, child_state, saved_errno;
-  retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x",
-                      &child_state, &saved_errno);
-  if (retval == 2) {
-    log_warn(LD_GENERAL,
-             "Failed to start child process \"%s\" in state %d: %s",
-             executable, child_state, strerror(saved_errno));
-    if (child_status)
-      *child_status = 1;
-  } else {
-    /* Failed to parse message from child process, log it as a
-       warning */
-    log_warn(LD_GENERAL,
-             "Unexpected message from port forwarding helper \"%s\": %s",
-             executable, buf);
-  }
-}
-
 #ifdef _WIN32
 
 /** Return a smartlist containing lines outputted from
@@ -5180,59 +5178,13 @@ tor_get_lines_from_handle, (HANDLE *handle,
   return lines;
 }
 
-/** Read from stream, and send lines to log at the specified log level.
- * Returns -1 if there is a error reading, and 0 otherwise.
- * If the generated stream is flushed more often than on new lines, or
- * a read exceeds 256 bytes, lines will be truncated. This should be fixed,
- * along with the corresponding problem on *nix (see bug #2045).
- */
-static int
-log_from_handle(HANDLE *pipe, int severity)
-{
-  char buf[256];
-  int pos;
-  smartlist_t *lines;
-
-  pos = tor_read_all_handle(pipe, buf, sizeof(buf) - 1, NULL);
-  if (pos < 0) {
-    /* Error */
-    log_warn(LD_GENERAL, "Failed to read data from subprocess");
-    return -1;
-  }
-
-  if (0 == pos) {
-    /* There's nothing to read (process is busy or has exited) */
-    log_debug(LD_GENERAL, "Subprocess had nothing to say");
-    return 0;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  buf[pos] = '\0';
-  log_debug(LD_GENERAL, "Subprocess had %d bytes to say", pos);
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, buf, pos);
-
-  /* Log each line */
-  SMARTLIST_FOREACH(lines, char *, line,
-  {
-    log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", line);
-  });
-  smartlist_free(lines);
-
-  return 0;
-}
-
-#else
+#else /* !(defined(_WIN32)) */
 
 /** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
+ *  <b>fd</b>. Return NULL on error, and set
  *  <b>stream_status_out</b> appropriately. */
 MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (FILE *handle,
-                            enum stream_status *stream_status_out))
+tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
 {
   enum stream_status stream_status;
   char stdout_buf[400];
@@ -5241,13 +5193,13 @@ tor_get_lines_from_handle, (FILE *handle,
   while (1) {
     memset(stdout_buf, 0, sizeof(stdout_buf));
 
-    stream_status = get_string_from_pipe(handle,
+    stream_status = get_string_from_pipe(fd,
                                          stdout_buf, sizeof(stdout_buf) - 1);
     if (stream_status != IO_STREAM_OKAY)
       goto done;
 
     if (!lines) lines = smartlist_new();
-    smartlist_add(lines, tor_strdup(stdout_buf));
+    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
   }
 
  done:
@@ -5255,45 +5207,9 @@ tor_get_lines_from_handle, (FILE *handle,
   return lines;
 }
 
-/** Read from stream, and send lines to log at the specified log level.
- * Returns 1 if stream is closed normally, -1 if there is a error reading, and
- * 0 otherwise. Handles lines from tor-fw-helper and
- * tor_spawn_background() specially.
- */
-static int
-log_from_pipe(FILE *stream, int severity, const char *executable,
-              int *child_status)
-{
-  char buf[256];
-  enum stream_status r;
-
-  for (;;) {
-    r = get_string_from_pipe(stream, buf, sizeof(buf) - 1);
-
-    if (r == IO_STREAM_CLOSED) {
-      return 1;
-    } else if (r == IO_STREAM_EAGAIN) {
-      return 0;
-    } else if (r == IO_STREAM_TERM) {
-      return -1;
-    }
-
-    tor_assert(r == IO_STREAM_OKAY);
-
-    /* Check if buf starts with SPAWN_ERROR_MESSAGE */
-    if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) {
-      log_portfw_spawn_error_message(buf, executable, child_status);
-    } else {
-      log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf);
-    }
-  }
-
-  /* We should never get here */
-  return -1;
-}
-#endif
+#endif /* defined(_WIN32) */
 
-/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making
+/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
  *  sure it's below <b>count</b> bytes.
  *  If the string has a trailing newline, we strip it off.
  *
@@ -5309,340 +5225,28 @@ log_from_pipe(FILE *stream, int severity, const char *executable,
  * IO_STREAM_OKAY: If everything went okay and we got a string
  *  in <b>buf_out</b>. */
 enum stream_status
-get_string_from_pipe(FILE *stream, char *buf_out, size_t count)
+get_string_from_pipe(int fd, char *buf_out, size_t count)
 {
-  char *retval;
-  size_t len;
+  ssize_t ret;
 
   tor_assert(count <= INT_MAX);
 
-  retval = fgets(buf_out, (int)count, stream);
+  ret = read(fd, buf_out, count);
 
-  if (!retval) {
-    if (feof(stream)) {
-      /* Program has closed stream (probably it exited) */
-      /* TODO: check error */
-      return IO_STREAM_CLOSED;
-    } else {
-      if (EAGAIN == errno) {
-        /* Nothing more to read, try again next time */
-        return IO_STREAM_EAGAIN;
-      } else {
-        /* There was a problem, abandon this child process */
-        return IO_STREAM_TERM;
-      }
-    }
-  } else {
-    len = strlen(buf_out);
-    if (len == 0) {
-      /* this probably means we got a NUL at the start of the string. */
-      return IO_STREAM_EAGAIN;
-    }
+  if (ret == 0)
+    return IO_STREAM_CLOSED;
+  else if (ret < 0 && errno == EAGAIN)
+    return IO_STREAM_EAGAIN;
+  else if (ret < 0)
+    return IO_STREAM_TERM;
 
-    if (buf_out[len - 1] == '\n') {
-      /* Remove the trailing newline */
-      buf_out[len - 1] = '\0';
-    } else {
-      /* No newline; check whether we overflowed the buffer */
-      if (!feof(stream))
-        log_info(LD_GENERAL,
-                 "Line from stream was truncated: %s", buf_out);
-      /* TODO: What to do with this error? */
-    }
-
-    return IO_STREAM_OKAY;
-  }
-
-  /* We should never get here */
-  return IO_STREAM_TERM;
-}
-
-/** Parse a <b>line</b> from tor-fw-helper and issue an appropriate
- *  log message to our user. */
-static void
-handle_fw_helper_line(const char *executable, const char *line)
-{
-  smartlist_t *tokens = smartlist_new();
-  char *message = NULL;
-  char *message_for_log = NULL;
-  const char *external_port = NULL;
-  const char *internal_port = NULL;
-  const char *result = NULL;
-  int port = 0;
-  int success = 0;
-
-  if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) {
-    /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's
-     * possible that it got sent after we tried to read it in log_from_pipe.
-     *
-     * XXX Ideally, we should be using one of stdout/stderr for the real
-     * output, and one for the output of the startup code.  We used to do that
-     * before cd05f35d2c.
-     */
-    int child_status;
-    log_portfw_spawn_error_message(line, executable, &child_status);
-    goto done;
-  }
-
-  smartlist_split_string(tokens, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-
-  if (smartlist_len(tokens) < 5)
-    goto err;
-
-  if (strcmp(smartlist_get(tokens, 0), "tor-fw-helper") ||
-      strcmp(smartlist_get(tokens, 1), "tcp-forward"))
-    goto err;
-
-  external_port = smartlist_get(tokens, 2);
-  internal_port = smartlist_get(tokens, 3);
-  result = smartlist_get(tokens, 4);
-
-  if (smartlist_len(tokens) > 5) {
-    /* If there are more than 5 tokens, they are part of [<message>].
-       Let's use a second smartlist to form the whole message;
-       strncat loops suck. */
-    int i;
-    int message_words_n = smartlist_len(tokens) - 5;
-    smartlist_t *message_sl = smartlist_new();
-    for (i = 0; i < message_words_n; i++)
-      smartlist_add(message_sl, smartlist_get(tokens, 5+i));
-
-    tor_assert(smartlist_len(message_sl) > 0);
-    message = smartlist_join_strings(message_sl, " ", 0, NULL);
-
-    /* wrap the message in log-friendly wrapping */
-    tor_asprintf(&message_for_log, " ('%s')", message);
-
-    smartlist_free(message_sl);
-  }
-
-  port = atoi(external_port);
-  if (port < 1 || port > 65535)
-    goto err;
-
-  port = atoi(internal_port);
-  if (port < 1 || port > 65535)
-    goto err;
-
-  if (!strcmp(result, "SUCCESS"))
-    success = 1;
-  else if (!strcmp(result, "FAIL"))
-    success = 0;
-  else
-    goto err;
-
-  if (!success) {
-    log_warn(LD_GENERAL, "Tor was unable to forward TCP port '%s' to '%s'%s. "
-             "Please make sure that your router supports port "
-             "forwarding protocols (like NAT-PMP). Note that if '%s' is "
-             "your ORPort, your relay will be unable to receive inbound "
-             "traffic.", external_port, internal_port,
-             message_for_log ? message_for_log : "",
-             internal_port);
-  } else {
-    log_info(LD_GENERAL,
-             "Tor successfully forwarded TCP port '%s' to '%s'%s.",
-             external_port, internal_port,
-             message_for_log ? message_for_log : "");
-  }
-
-  goto done;
-
- err:
-  log_warn(LD_GENERAL, "tor-fw-helper sent us a string we could not "
-           "parse (%s).", line);
-
- done:
-  SMARTLIST_FOREACH(tokens, char *, cp, tor_free(cp));
-  smartlist_free(tokens);
-  tor_free(message);
-  tor_free(message_for_log);
-}
-
-/** Read what tor-fw-helper has to say in its stdout and handle it
- *  appropriately */
-static int
-handle_fw_helper_output(const char *executable,
-                        process_handle_t *process_handle)
-{
-  smartlist_t *fw_helper_output = NULL;
-  enum stream_status stream_status = 0;
-
-  fw_helper_output =
-    tor_get_lines_from_handle(tor_process_get_stdout_pipe(process_handle),
-                              &stream_status);
-  if (!fw_helper_output) { /* didn't get any output from tor-fw-helper */
-    /* if EAGAIN we should retry in the future */
-    return (stream_status == IO_STREAM_EAGAIN) ? 0 : -1;
-  }
-
-  /* Handle the lines we got: */
-  SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) {
-    handle_fw_helper_line(executable, line);
-    tor_free(line);
-  } SMARTLIST_FOREACH_END(line);
-
-  smartlist_free(fw_helper_output);
-
-  return 0;
-}
-
-/** Spawn tor-fw-helper and ask it to forward the ports in
- *  <b>ports_to_forward</b>. <b>ports_to_forward</b> contains strings
- *  of the form "<external port>:<internal port>", which is the format
- *  that tor-fw-helper expects. */
-void
-tor_check_port_forwarding(const char *filename,
-                          smartlist_t *ports_to_forward,
-                          time_t now)
-{
-/* When fw-helper succeeds, how long do we wait until running it again */
-#define TIME_TO_EXEC_FWHELPER_SUCCESS 300
-/* When fw-helper failed to start, how long do we wait until running it again
- */
-#define TIME_TO_EXEC_FWHELPER_FAIL 60
-
-  /* Static variables are initialized to zero, so child_handle.status=0
-   * which corresponds to it not running on startup */
-  static process_handle_t *child_handle=NULL;
-
-  static time_t time_to_run_helper = 0;
-  int stderr_status, retval;
-  int stdout_status = 0;
-
-  tor_assert(filename);
-
-  /* Start the child, if it is not already running */
-  if ((!child_handle || child_handle->status != PROCESS_STATUS_RUNNING) &&
-      time_to_run_helper < now) {
-    /*tor-fw-helper cli looks like this: tor_fw_helper -p :5555 -p 4555:1111 */
-    const char **argv; /* cli arguments */
-    int args_n, status;
-    int argv_index = 0; /* index inside 'argv' */
-
-    tor_assert(smartlist_len(ports_to_forward) > 0);
-
-    /* check for overflow during 'argv' allocation:
-       (len(ports_to_forward)*2 + 2)*sizeof(char*) > SIZE_MAX ==
-       len(ports_to_forward) > (((SIZE_MAX/sizeof(char*)) - 2)/2) */
-    if ((size_t) smartlist_len(ports_to_forward) >
-        (((SIZE_MAX/sizeof(char*)) - 2)/2)) {
-      log_warn(LD_GENERAL,
-               "Overflow during argv allocation. This shouldn't happen.");
-      return;
-    }
-    /* check for overflow during 'argv_index' increase:
-       ((len(ports_to_forward)*2 + 2) > INT_MAX) ==
-       len(ports_to_forward) > (INT_MAX - 2)/2 */
-    if (smartlist_len(ports_to_forward) > (INT_MAX - 2)/2) {
-      log_warn(LD_GENERAL,
-               "Overflow during argv_index increase. This shouldn't happen.");
-      return;
-    }
-
-    /* Calculate number of cli arguments: one for the filename, two
-       for each smartlist element (one for "-p" and one for the
-       ports), and one for the final NULL. */
-    args_n = 1 + 2*smartlist_len(ports_to_forward) + 1;
-    argv = tor_calloc(args_n, sizeof(char *));
-
-    argv[argv_index++] = filename;
-    SMARTLIST_FOREACH_BEGIN(ports_to_forward, const char *, port) {
-      argv[argv_index++] = "-p";
-      argv[argv_index++] = port;
-    } SMARTLIST_FOREACH_END(port);
-    argv[argv_index] = NULL;
-
-    /* Assume tor-fw-helper will succeed, start it later*/
-    time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS;
-
-    if (child_handle) {
-      tor_process_handle_destroy(child_handle, 1);
-      child_handle = NULL;
-    }
-
-#ifdef _WIN32
-    /* Passing NULL as lpApplicationName makes Windows search for the .exe */
-    status = tor_spawn_background(NULL, argv, NULL, &child_handle);
-#else
-    status = tor_spawn_background(filename, argv, NULL, &child_handle);
-#endif
-
-    tor_free_((void*)argv);
-    argv=NULL;
-
-    if (PROCESS_STATUS_ERROR == status) {
-      log_warn(LD_GENERAL, "Failed to start port forwarding helper %s",
-              filename);
-      time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL;
-      return;
-    }
-
-    log_info(LD_GENERAL,
-             "Started port forwarding helper (%s) with pid '%d'",
-             filename, tor_process_get_pid(child_handle));
-  }
-
-  /* If child is running, read from its stdout and stderr) */
-  if (child_handle && PROCESS_STATUS_RUNNING == child_handle->status) {
-    /* Read from stdout/stderr and log result */
-    retval = 0;
-#ifdef _WIN32
-    stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO);
-#else
-    stderr_status = log_from_pipe(child_handle->stderr_handle,
-                                  LOG_INFO, filename, &retval);
-#endif
-    if (handle_fw_helper_output(filename, child_handle) < 0) {
-      log_warn(LD_GENERAL, "Failed to handle fw helper output.");
-      stdout_status = -1;
-      retval = -1;
-    }
-
-    if (retval) {
-      /* There was a problem in the child process */
-      time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL;
-    }
-
-    /* Combine the two statuses in order of severity */
-    if (-1 == stdout_status || -1 == stderr_status)
-      /* There was a failure */
-      retval = -1;
-#ifdef _WIN32
-    else if (!child_handle || tor_get_exit_code(child_handle, 0, NULL) !=
-             PROCESS_EXIT_RUNNING) {
-      /* process has exited or there was an error */
-      /* TODO: Do something with the process return value */
-      /* TODO: What if the process output something since
-       * between log_from_handle and tor_get_exit_code? */
-      retval = 1;
-    }
-#else
-    else if (1 == stdout_status || 1 == stderr_status)
-      /* stdout or stderr was closed, the process probably
-       * exited. It will be reaped by waitpid() in main.c */
-      /* TODO: Do something with the process return value */
-      retval = 1;
-#endif
-    else
-      /* Both are fine */
-      retval = 0;
-
-    /* If either pipe indicates a failure, act on it */
-    if (0 != retval) {
-      if (1 == retval) {
-        log_info(LD_GENERAL, "Port forwarding helper terminated");
-        child_handle->status = PROCESS_STATUS_NOTRUNNING;
-      } else {
-        log_warn(LD_GENERAL, "Failed to read from port forwarding helper");
-        child_handle->status = PROCESS_STATUS_ERROR;
-      }
+  if (buf_out[ret - 1] == '\n') {
+    /* Remove the trailing newline */
+    buf_out[ret - 1] = '\0';
+  } else
+    buf_out[ret] = '\0';
 
-      /* TODO: The child might not actually be finished (maybe it failed or
-         closed stdout/stderr), so maybe we shouldn't start another? */
-    }
-  }
+  return IO_STREAM_OKAY;
 }
 
 /** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */
@@ -5695,7 +5299,7 @@ clamp_double_to_int64(double number)
 {
   int exponent;
 
-#if (defined(__MINGW32__) || defined(__MINGW64__)) && GCC_VERSION >= 409
+#if defined(MINGW_ANY) && GCC_VERSION >= 409
 /*
   Mingw's math.h uses gcc's __builtin_choose_expr() facility to declare
   isnan, isfinite, and signbit.  But as implemented in at least some
@@ -5704,7 +5308,7 @@ clamp_double_to_int64(double number)
 */
 #define PROBLEMATIC_FLOAT_CONVERSION_WARNING
 DISABLE_GCC_WARNING(float-conversion)
-#endif
+#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */
 
 /*
   With clang 4.0 we apparently run into "double promotion" warnings here,
@@ -5715,7 +5319,7 @@ DISABLE_GCC_WARNING(float-conversion)
 #define PROBLEMATIC_DOUBLE_PROMOTION_WARNING
 DISABLE_GCC_WARNING(double-promotion)
 #endif
-#endif
+#endif /* defined(__clang__) */
 
   /* NaN is a special case that can't be used with the logic below. */
   if (isnan(number)) {
@@ -5762,7 +5366,7 @@ tor_htonll(uint64_t a)
   /* Little endian. The worst... */
   return htonl((uint32_t)(a>>32)) |
     (((uint64_t)htonl((uint32_t)a))<<32);
-#endif /* WORDS_BIGENDIAN */
+#endif /* defined(WORDS_BIGENDIAN) */
 }
 
 /** Return a uint64_t value from <b>a</b> in host byte order. */
diff --git a/src/common/util.h b/src/common/util.h
index 479fc8d610..7172b7da08 100644
--- a/src/common/util.h
+++ b/src/common/util.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -45,7 +45,7 @@
 #else
 #define DMALLOC_PARAMS
 #define DMALLOC_ARGS
-#endif
+#endif /* defined(USE_DMALLOC) */
 
 /* Memory management */
 void *tor_malloc_(size_t size DMALLOC_PARAMS) ATTR_MALLOC;
@@ -72,21 +72,32 @@ extern int dmalloc_free(const char *file, const int line, void *pnt,
       (p)=NULL;                                     \
     }                                               \
   STMT_END
-#else
-/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc.
- * Unlike the free() function, tor_free() will still work on NULL pointers,
- * and it sets the pointer value to NULL after freeing it.
+#else /* !(defined(USE_DMALLOC)) */
+/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup,
+ * etc.  Unlike the free() function, the tor_free() macro sets the
+ * pointer value to NULL after freeing it.
  *
  * This is a macro.  If you need a function pointer to release memory from
  * tor_malloc(), use tor_free_().
+ *
+ * Note that this macro takes the address of the pointer it is going to
+ * free and clear.  If that pointer is stored with a nonstandard
+ * alignment (eg because of a "packed" pragma) it is not correct to use
+ * tor_free().
  */
+#ifdef __GNUC__
+#define tor_free(p) STMT_BEGIN                                 \
+    typeof(&(p)) tor_free__tmpvar = &(p);                      \
+    raw_free(*tor_free__tmpvar);                               \
+    *tor_free__tmpvar=NULL;                                    \
+  STMT_END
+#else
 #define tor_free(p) STMT_BEGIN                                 \
-    if (PREDICT_LIKELY((p)!=NULL)) {                           \
-      raw_free(p);                                             \
-      (p)=NULL;                                                \
-    }                                                          \
+  raw_free(p);                                                 \
+  (p)=NULL;                                                    \
   STMT_END
 #endif
+#endif /* defined(USE_DMALLOC) */
 
 #define tor_malloc(size)       tor_malloc_(size DMALLOC_ARGS)
 #define tor_malloc_zero(size)  tor_malloc_zero_(size DMALLOC_ARGS)
@@ -109,19 +120,22 @@ extern int dmalloc_free(const char *file, const int line, void *pnt,
 
 void tor_log_mallinfo(int severity);
 
-/** Return the offset of <b>member</b> within the type <b>tp</b>, in bytes */
-#if defined(__GNUC__) && __GNUC__ > 3
-#define STRUCT_OFFSET(tp, member) __builtin_offsetof(tp, member)
-#else
- #define STRUCT_OFFSET(tp, member) \
-   ((off_t) (((char*)&((tp*)0)->member)-(char*)0))
-#endif
+/* Helper macro: free a variable of type 'typename' using freefn, and
+ * set the variable to NULL.
+ */
+#define FREE_AND_NULL(typename, freefn, var)                            \
+  do {                                                                  \
+    /* only evaluate (var) once. */                                     \
+    typename **tmp__free__ptr ## freefn = &(var);                       \
+    freefn(*tmp__free__ptr ## freefn);                                  \
+    (*tmp__free__ptr ## freefn) = NULL;                                 \
+  } while (0)
 
 /** Macro: yield a pointer to the field at position <b>off</b> within the
  * structure <b>st</b>.  Example:
  * <pre>
  *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = STRUCT_OFFSET(struct a, bar);
+ *   off_t bar_offset = offsetof(struct a, bar);
  *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
  *   *bar_p = 3;
  * </pre>
@@ -138,7 +152,7 @@ void tor_log_mallinfo(int severity);
  * </pre>
  */
 #define SUBTYPE_P(p, subtype, basemember) \
-  ((void*) ( ((char*)(p)) - STRUCT_OFFSET(subtype, basemember) ))
+  ((void*) ( ((char*)(p)) - offsetof(subtype, basemember) ))
 
 /* Logic */
 /** Macro: true if two values have the same boolean value. */
@@ -162,10 +176,12 @@ int n_bits_set_u8(uint8_t v);
 int64_t clamp_double_to_int64(double number);
 void simplify_fraction64(uint64_t *numer, uint64_t *denom);
 
+uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
+
 /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
- * and positive <b>b</b>.  Works on integer types only. Not defined if a+b can
- * overflow. */
-#define CEIL_DIV(a,b) (((a)+(b)-1)/(b))
+ * and positive <b>b</b>.  Works on integer types only. Not defined if a+(b-1)
+ * can overflow. */
+#define CEIL_DIV(a,b) (((a)+((b)-1))/(b))
 
 /* Return <b>v</b> if it's between <b>min</b> and <b>max</b>.  Otherwise
  * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if
@@ -186,6 +202,7 @@ void tor_strlower(char *s) ATTR_NONNULL((1));
 void tor_strupper(char *s) ATTR_NONNULL((1));
 int tor_strisprint(const char *s) ATTR_NONNULL((1));
 int tor_strisnonupper(const char *s) ATTR_NONNULL((1));
+int tor_strisspace(const char *s);
 int strcmp_opt(const char *s1, const char *s2);
 int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2));
 int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2));
@@ -214,7 +231,8 @@ const char *find_str_at_start_of_line(const char *haystack,
                                       const char *needle);
 int string_is_C_identifier(const char *string);
 int string_is_key_value(int severity, const char *string);
-int string_is_valid_hostname(const char *string);
+int string_is_valid_dest(const char *string);
+int string_is_valid_nonrfc_hostname(const char *string);
 int string_is_valid_ipv4_address(const char *string);
 int string_is_valid_ipv6_address(const char *string);
 
@@ -239,6 +257,7 @@ void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
 void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
                              va_list args)
   CHECK_PRINTF(2, 0);
+void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
 
 /* Time helpers */
 long tv_udiff(const struct timeval *start, const struct timeval *end);
@@ -252,10 +271,12 @@ int parse_rfc1123_time(const char *buf, time_t *t);
 #define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7)
 void format_local_iso_time(char *buf, time_t t);
 void format_iso_time(char *buf, time_t t);
+void format_local_iso_time_nospace(char *buf, time_t t);
 void format_iso_time_nospace(char *buf, time_t t);
 void format_iso_time_nospace_usec(char *buf, const struct timeval *tv);
-int parse_iso_time_(const char *cp, time_t *t, int strict);
+int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace);
 int parse_iso_time(const char *buf, time_t *t);
+int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
@@ -266,7 +287,7 @@ int format_time_interval(char *out, size_t out_len, long interval);
 #else
 time_t approx_time(void);
 void update_approx_time(time_t now);
-#endif
+#endif /* defined(TIME_IS_FAST) */
 
 /* Rate-limiter */
 
@@ -319,7 +340,7 @@ enum stream_status {
 
 const char *stream_status_to_string(enum stream_status stream_status);
 
-enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count);
+enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
 
 MOCK_DECL(int,tor_unlink,(const char *pathname));
 
@@ -386,9 +407,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
                                  size_t *sz_out)
   ATTR_MALLOC;
 const char *unescape_string(const char *s, char **result, size_t *size_out);
-const char *parse_config_line_from_str_verbose(const char *line,
-                                       char **key_out, char **value_out,
-                                       const char **err_out);
+char *get_unquoted_path(const char *path);
 char *expand_filename(const char *filename);
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 int path_is_relative(const char *filename);
@@ -396,12 +415,9 @@ int path_is_relative(const char *filename);
 /* Process helpers */
 void start_daemon(void);
 void finish_daemon(const char *desired_cwd);
-void write_pidfile(const char *filename);
+int write_pidfile(const char *filename);
 
-/* Port forwarding */
-void tor_check_port_forwarding(const char *filename,
-                               struct smartlist_t *ports_to_forward,
-                               time_t now);
+void tor_disable_spawning_background_processes(void);
 
 typedef struct process_handle_t process_handle_t;
 typedef struct process_environment_t process_environment_t;
@@ -428,7 +444,9 @@ struct process_environment_t {
 };
 
 process_environment_t *process_environment_make(struct smartlist_t *env_vars);
-void process_environment_free(process_environment_t *env);
+void process_environment_free_(process_environment_t *env);
+#define process_environment_free(env) \
+  FREE_AND_NULL(process_environment_t, process_environment_free_, (env))
 
 struct smartlist_t *get_current_process_environment_variables(void);
 
@@ -437,9 +455,7 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
                                            void (*free_old)(void*),
                                            int free_p);
 
-/* Values of process_handle_t.status. PROCESS_STATUS_NOTRUNNING must be
- * 0 because tor_check_port_forwarding depends on this being the initial
- * statue of the static instance of process_handle_t */
+/* Values of process_handle_t.status. */
 #define PROCESS_STATUS_NOTRUNNING 0
 #define PROCESS_STATUS_RUNNING 1
 #define PROCESS_STATUS_ERROR -1
@@ -456,13 +472,10 @@ struct process_handle_t {
   HANDLE stdout_pipe;
   HANDLE stderr_pipe;
   PROCESS_INFORMATION pid;
-#else
+#else /* !(defined(_WIN32)) */
   int stdin_pipe;
   int stdout_pipe;
   int stderr_pipe;
-  FILE *stdin_handle;
-  FILE *stdout_handle;
-  FILE *stderr_handle;
   pid_t pid;
   /** If the process has not given us a SIGCHLD yet, this has the
    * waitpid_callback_t that gets invoked once it has. Otherwise this
@@ -470,9 +483,9 @@ struct process_handle_t {
   struct waitpid_callback_t *waitpid_cb;
   /** The exit status reported by waitpid. */
   int waitpid_exit_status;
-#endif // _WIN32
+#endif /* defined(_WIN32) */
 };
-#endif
+#endif /* defined(UTIL_PRIVATE) */
 
 /* Return values of tor_get_exit_code() */
 #define PROCESS_EXIT_RUNNING 1
@@ -485,10 +498,10 @@ int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
 ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
                             const process_handle_t *process);
 #else
-ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count,
+ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
                             const process_handle_t *process,
                             int *eof);
-#endif
+#endif /* defined(_WIN32) */
 ssize_t tor_read_all_from_process_stdout(
     const process_handle_t *process_handle, char *buf, size_t count);
 ssize_t tor_read_all_from_process_stderr(
@@ -499,7 +512,7 @@ int tor_process_get_pid(process_handle_t *process_handle);
 #ifdef _WIN32
 HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
 #else
-FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle);
+int tor_process_get_stdout_pipe(process_handle_t *process_handle);
 #endif
 
 #ifdef _WIN32
@@ -508,9 +521,9 @@ tor_get_lines_from_handle,(HANDLE *handle,
                            enum stream_status *stream_status));
 #else
 MOCK_DECL(struct smartlist_t *,
-tor_get_lines_from_handle,(FILE *handle,
+tor_get_lines_from_handle,(int fd,
                            enum stream_status *stream_status));
-#endif
+#endif /* defined(_WIN32) */
 
 int
 tor_terminate_process(process_handle_t *process_handle);
@@ -547,15 +560,13 @@ STATIC int format_helper_exit_status(unsigned char child_state,
    leading minus) and newline (no null) */
 #define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
                         1 + sizeof(int) * 2 + 1)
-#endif
+#endif /* !defined(_WIN32) */
 
-#endif
+#endif /* defined(UTIL_PRIVATE) */
 
-#ifdef TOR_UNIT_TESTS
-int size_mul_check__(const size_t x, const size_t y);
-#endif
+int size_mul_check(const size_t x, const size_t y);
 
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
-#endif
+#endif /* !defined(TOR_UTIL_H) */
 
diff --git a/src/common/util_bug.c b/src/common/util_bug.c
index 08aba47974..126e843866 100644
--- a/src/common/util_bug.c
+++ b/src/common/util_bug.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,10 @@
 #include "backtrace.h"
 #include "container.h"
 
+#ifdef __COVERITY__
+int bug_macro_deadcode_dummy__ = 0;
+#endif
+
 #ifdef TOR_UNIT_TESTS
 static void (*failed_assertion_cb)(void) = NULL;
 static int n_bugs_to_capture = 0;
@@ -44,7 +48,7 @@ static void
 add_captured_bug(const char *s)
 {
   --n_bugs_to_capture;
-  smartlist_add(bug_messages, tor_strdup(s));
+  smartlist_add_strdup(bug_messages, s);
 }
 /** Set a callback to be invoked when we get any tor_bug_occurred_
  * invocation. We use this in the unit tests so that a nonfatal
@@ -55,10 +59,10 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 {
   failed_assertion_cb = fn;
 }
-#else
+#else /* !(defined(TOR_UNIT_TESTS)) */
 #define capturing_bugs() (0)
 #define add_captured_bug(s) do { } while (0)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Helper for tor_assert: report the assertion failure. */
 void
diff --git a/src/common/util_bug.h b/src/common/util_bug.h
index 0db6bb6ab0..50becd0c33 100644
--- a/src/common/util_bug.h
+++ b/src/common/util_bug.h
@@ -1,10 +1,36 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file util_bug.h
+ *
+ * \brief Macros to manage assertions, fatal and non-fatal.
+ *
+ * Guidelines: All the different kinds of assertion in this file are for
+ * bug-checking only. Don't write code that can assert based on bad inputs.
+ *
+ * We provide two kinds of assertion here: "fatal" and "nonfatal". Use
+ * nonfatal assertions for any bug you can reasonably recover from -- and
+ * please, try to recover!  Many severe bugs in Tor have been caused by using
+ * a regular assertion when a nonfatal assertion would have been better.
+ *
+ * If you need to check a condition with a nonfatal assertion, AND recover
+ * from that same condition, consider using the BUG() macro inside a
+ * conditional.  For example:
+ *
+ * <code>
+ *  // wrong -- use tor_assert_nonfatal() if you just want an assertion.
+ *  BUG(ptr == NULL);
+ *
+ *  // okay, but needlessly verbose
+ *  tor_assert_nonfatal(ptr != NULL);
+ *  if (ptr == NULL) { ... }
+ *
+ *  // this is how we do it:
+ *  if (BUG(ptr == NULL)) { ... }
+ * </code>
  **/
 
 #ifndef TOR_UTIL_BUG_H
@@ -27,7 +53,7 @@
  * security-critical properties.
  */
 #error "Sorry; we don't support building with NDEBUG."
-#endif
+#endif /* defined(NDEBUG) */
 
 #if defined(TOR_UNIT_TESTS) && defined(__GNUC__)
 /* We define this GCC macro as a replacement for PREDICT_UNLIKELY() in this
@@ -74,9 +100,14 @@
     tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr);     \
     abort();                                                            \
   } STMT_END
-#endif
+#endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */
 
-#define tor_assert_unreached() tor_assert(0)
+#define tor_assert_unreached()                                  \
+  STMT_BEGIN {                                                  \
+    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__,     \
+                          "line should be unreached");          \
+    abort();                                                    \
+  } STMT_END
 
 /* Non-fatal bug assertions. The "unreached" variants mean "this line should
  * never be reached." The "once" variants mean "Don't log a warning more than
@@ -89,11 +120,14 @@
  */
 
 #ifdef __COVERITY__
+extern int bug_macro_deadcode_dummy__;
 #undef BUG
 // Coverity defines this in global headers; let's override it.  This is a
 // magic coverity-only preprocessor thing.
-#nodef BUG(x) ((x)?(__coverity_panic__(),1):0)
-#endif
+// We use this "deadcode_dummy__" trick to prevent coverity from
+// complaining about unreachable bug cases.
+#nodef BUG(x) ((x)?(__coverity_panic__(),1):(0+bug_macro_deadcode_dummy__))
+#endif /* defined(__COVERITY__) */
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
 // We're running with a static analysis tool: let's treat even nonfatal
@@ -146,7 +180,7 @@
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
    (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \
    : 0)
-#endif
+#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
 
 #ifdef __GNUC__
 #define IF_BUG_ONCE__(cond,var)                                         \
@@ -159,7 +193,7 @@
                           "!("#cond")", 1);                             \
       }                                                                 \
       bool_result; } ))
-#else
+#else /* !(defined(__GNUC__)) */
 #define IF_BUG_ONCE__(cond,var)                                         \
   static int var = 0;                                                   \
   if ((cond) ?                                                          \
@@ -169,7 +203,7 @@
                            "!("#cond")", 1),                            \
         1))                                                             \
       : 0)
-#endif
+#endif /* defined(__GNUC__) */
 #define IF_BUG_ONCE_VARNAME_(a)               \
   warning_logged_on_ ## a ## __
 #define IF_BUG_ONCE_VARNAME__(a)              \
@@ -199,6 +233,6 @@ void tor_capture_bugs_(int n);
 void tor_end_capture_bugs_(void);
 const struct smartlist_t *tor_get_captured_bug_log_(void);
 void tor_set_failed_assertion_callback(void (*fn)(void));
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_UTIL_BUG_H) */
diff --git a/src/common/util_format.c b/src/common/util_format.c
index aef9db85c8..e51757a4e8 100644
--- a/src/common/util_format.c
+++ b/src/common/util_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,13 +22,16 @@
 #include <stdlib.h>
 
 /* Return the base32 encoded size in bytes using the source length srclen.
- * The NUL terminated byte is added as well since every base32 encoding
- * requires enough space for it. */
+ *
+ * (WATCH OUT: This API counts the terminating NUL byte, but
+ * base64_encode_size does not.)
+ */
 size_t
 base32_encoded_size(size_t srclen)
 {
   size_t enclen;
-  enclen = CEIL_DIV(srclen*8, 5) + 1;
+  tor_assert(srclen < SIZE_T_CEILING / 8);
+  enclen = BASE32_NOPAD_BUFSIZE(srclen);
   tor_assert(enclen < INT_MAX && enclen > srclen);
   return enclen;
 }
@@ -41,7 +44,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   size_t nbits = srclen * 8;
   size_t bit;
 
-  tor_assert(srclen < SIZE_T_CEILING/8);
   /* We need enough space for the encoded data and the extra NUL byte. */
   tor_assert(base32_encoded_size(srclen) <= destlen);
   tor_assert(destlen < SIZE_T_CEILING);
@@ -51,9 +53,10 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   for (i=0,bit=0; bit < nbits; ++i, bit+=5) {
     /* set v to the 16-bit value starting at src[bits/8], 0-padded. */
-    v = ((uint8_t)src[bit/8]) << 8;
-    if (bit+5<nbits)
-      v += (uint8_t)src[(bit/8)+1];
+    size_t idx = bit / 8;
+    v = ((uint8_t)src[idx]) << 8;
+    if (idx+1 < srclen)
+      v += (uint8_t)src[idx+1];
     /* set u to the 5-bit value at the bit'th bit of buf. */
     u = (v >> (11-(bit%8))) & 0x1F;
     dest[i] = BASE32_CHARS[u];
@@ -133,6 +136,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 /** Return the Base64 encoded size of <b>srclen</b> bytes of data in
  * bytes.
  *
+ * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte,
+ * but base32_encoded_size does.)
+ *
  * If <b>flags</b>&amp;BASE64_ENCODE_MULTILINE is true, return the size
  * of the encoded output as multiline output (64 character, `\n' terminated
  * lines).
@@ -141,19 +147,16 @@ size_t
 base64_encode_size(size_t srclen, int flags)
 {
   size_t enclen;
+
+  /* Use INT_MAX for overflow checking because base64_encode() returns int. */
   tor_assert(srclen < INT_MAX);
+  tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4);
 
-  if (srclen == 0)
-    return 0;
+  enclen = BASE64_LEN(srclen);
+  if (flags & BASE64_ENCODE_MULTILINE)
+    enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN);
 
-  enclen = ((srclen - 1) / 3) * 4 + 4;
-  if (flags & BASE64_ENCODE_MULTILINE) {
-    size_t remainder = enclen % BASE64_OPENSSL_LINELEN;
-    enclen += enclen / BASE64_OPENSSL_LINELEN;
-    if (remainder)
-      enclen++;
-  }
-  tor_assert(enclen < INT_MAX && enclen > srclen);
+  tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen));
   return enclen;
 }
 
@@ -263,10 +266,13 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
     ENCODE_N(3);
     ENCODE_PAD();
     break;
+  // LCOV_EXCL_START -- we can't reach this point, because we enforce
+  // 0 <= ncov_idx < 3 in the loop above.
   default:
     /* Something went catastrophically wrong. */
-    tor_fragile_assert(); // LCOV_EXCL_LINE
+    tor_fragile_assert();
     return -1;
+  // LCOV_EXCL_STOP
   }
 
 #undef ENCODE_N
@@ -310,39 +316,6 @@ base64_encode_nopad(char *dest, size_t destlen,
   return (int)(out - dest);
 }
 
-/** As base64_decode, but do not require any padding on the input */
-int
-base64_decode_nopad(uint8_t *dest, size_t destlen,
-                    const char *src, size_t srclen)
-{
-  if (srclen > SIZE_T_CEILING - 4)
-    return -1;
-  char *buf = tor_malloc(srclen + 4);
-  memcpy(buf, src, srclen+1);
-  size_t buflen;
-  switch (srclen % 4)
-    {
-    case 0:
-    default:
-      buflen = srclen;
-      break;
-    case 1:
-      tor_free(buf);
-      return -1;
-    case 2:
-      memcpy(buf+srclen, "==", 3);
-      buflen = srclen + 2;
-      break;
-    case 3:
-      memcpy(buf+srclen, "=", 2);
-      buflen = srclen + 1;
-      break;
-  }
-  int n = base64_decode((char*)dest, destlen, buf, buflen);
-  tor_free(buf);
-  return n;
-}
-
 #undef BASE64_OPENSSL_LINELEN
 
 /** @{ */
@@ -392,15 +365,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   const char *eos = src+srclen;
   uint32_t n=0;
   int n_idx=0;
-  char *dest_orig = dest;
+  size_t di = 0;
 
-  /* Max number of bits == srclen*6.
-   * Number of bytes required to hold all bits == (srclen*6)/8.
-   * Yes, we want to round down: anything that hangs over the end of a
-   * byte is padding. */
-  if (destlen < (srclen*3)/4)
-    return -1;
-  if (destlen > SIZE_T_CEILING)
+  if (destlen > INT_MAX)
     return -1;
 
   /* Make sure we leave no uninitialized data in the destination buffer. */
@@ -428,9 +395,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
         n = (n<<6) | v;
         if ((++n_idx) == 4) {
           /* We've accumulated 24 bits in n. Flush them. */
-          *dest++ = (n>>16);
-          *dest++ = (n>>8) & 0xff;
-          *dest++ = (n) & 0xff;
+          if (destlen < 3 || di > destlen - 3)
+            return -1;
+          dest[di++] = (n>>16);
+          dest[di++] = (n>>8) & 0xff;
+          dest[di++] = (n) & 0xff;
           n_idx = 0;
           n = 0;
         }
@@ -448,18 +417,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
       return -1;
     case 2:
       /* 12 leftover bits: The last 4 are padding and the first 8 are data. */
-      *dest++ = n >> 4;
+      if (destlen < 1 || di > destlen - 1)
+        return -1;
+      dest[di++] = n >> 4;
       break;
     case 3:
       /* 18 leftover bits: The last 2 are padding and the first 16 are data. */
-      *dest++ = n >> 10;
-      *dest++ = n >> 2;
+      if (destlen < 2 || di > destlen - 2)
+        return -1;
+      dest[di++] = n >> 10;
+      dest[di++] = n >> 2;
   }
 
-  tor_assert((dest-dest_orig) <= (ssize_t)destlen);
-  tor_assert((dest-dest_orig) <= INT_MAX);
+  tor_assert(di <= destlen);
 
-  return (int)(dest-dest_orig);
+  return (int)di;
 }
 #undef X
 #undef SP
@@ -475,7 +447,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   const char *end;
   char *cp;
 
-  tor_assert(destlen >= srclen*2+1);
+  tor_assert(srclen < SIZE_T_CEILING / 2 - 1);
+  tor_assert(destlen >= BASE16_BUFSIZE(srclen));
   tor_assert(destlen < SIZE_T_CEILING);
 
   /* Make sure we leave no uninitialized data in the destination buffer. */
diff --git a/src/common/util_format.h b/src/common/util_format.h
index 20ac711d10..0aefe3a44e 100644
--- a/src/common/util_format.h
+++ b/src/common/util_format.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_UTIL_FORMAT_H
@@ -10,6 +10,26 @@
 #include "testsupport.h"
 #include "torint.h"
 
+/** @{ */
+/** These macros don't check for overflow.  Use them only for constant inputs
+ * (like array declarations).  The *_LEN macros are the raw encoding lengths
+ * (without terminating NUL), while the *_BUFSIZE macros count the terminating
+ * NUL. */
+#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4)
+#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8)
+#define BASE16_LEN(n) ((n) * 2)
+
+#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1)
+#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1)
+#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1)
+
+#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3))
+#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5))
+
+#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1)
+#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1)
+/** @} */
+
 #define BASE64_ENCODE_MULTILINE 1
 size_t base64_encode_size(size_t srclen, int flags);
 int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
@@ -17,8 +37,6 @@ int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
 int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base64_encode_nopad(char *dest, size_t destlen,
                         const uint8_t *src, size_t srclen);
-int base64_decode_nopad(uint8_t *dest, size_t destlen,
-                        const char *src, size_t srclen);
 
 /** Characters that can appear (case-insensitively) in a base32 encoding. */
 #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567"
@@ -30,5 +48,5 @@ int hex_decode_digit(char c);
 void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
-#endif
+#endif /* !defined(TOR_UTIL_FORMAT_H) */
 
diff --git a/src/common/util_process.c b/src/common/util_process.c
index abda63720c..c2826152e9 100644
--- a/src/common/util_process.c
+++ b/src/common/util_process.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -154,5 +154,5 @@ notify_pending_waitpid_callbacks(void)
   }
 }
 
-#endif
+#endif /* !defined(_WIN32) */
 
diff --git a/src/common/util_process.h b/src/common/util_process.h
index d38301a354..c9aa771b77 100644
--- a/src/common/util_process.h
+++ b/src/common/util_process.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,7 @@ waitpid_callback_t *set_waitpid_callback(pid_t pid,
                                          void (*fn)(int, void *), void *arg);
 void clear_waitpid_callback(waitpid_callback_t *ent);
 void notify_pending_waitpid_callbacks(void);
-#endif
+#endif /* !defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_UTIL_PROCESS_H) */
 
diff --git a/src/common/workqueue.c b/src/common/workqueue.c
index e1fb663a2a..563a98af96 100644
--- a/src/common/workqueue.c
+++ b/src/common/workqueue.c
@@ -1,3 +1,4 @@
+
 /* copyright (c) 2013-2015, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
@@ -24,12 +25,23 @@
 
 #include "orconfig.h"
 #include "compat.h"
+#include "compat_libevent.h"
 #include "compat_threads.h"
+#include "crypto_rand.h"
 #include "util.h"
 #include "workqueue.h"
 #include "tor_queue.h"
 #include "torlog.h"
 
+#include <event2/event.h>
+
+#define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH
+#define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW
+#define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1)
+
+TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s);
+typedef struct work_tailq_t work_tailq_t;
+
 struct threadpool_s {
   /** An array of pointers to workerthread_t: one for each running worker
    * thread. */
@@ -38,8 +50,12 @@ struct threadpool_s {
   /** Condition variable that we wait on when we have no work, and which
    * gets signaled when our queue becomes nonempty. */
   tor_cond_t condition;
-  /** Queue of pending work that we have to do. */
-  TOR_TAILQ_HEAD(, workqueue_entry_s) work;
+  /** Queues of pending work that we have to do. The queue with priority
+   * <b>p</b> is work[p]. */
+  work_tailq_t work[WORKQUEUE_N_PRIORITIES];
+
+  /** Weak RNG, used to decide when to ignore priority. */
+  tor_weak_rng_t weak_rng;
 
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
@@ -51,6 +67,9 @@ struct threadpool_s {
   void (*free_update_arg_fn)(void *);
   /** Array of n_threads update arguments. */
   void **update_args;
+  /** Event to notice when another thread has sent a reply. */
+  struct event *reply_event;
+  void (*reply_cb)(threadpool_t *);
 
   /** Number of elements in threads. */
   int n_threads;
@@ -66,6 +85,11 @@ struct threadpool_s {
   void *new_thread_state_arg;
 };
 
+/** Used to put a workqueue_priority_t value into a bitfield. */
+#define workqueue_priority_bitfield_t ENUM_BF(workqueue_priority_t)
+/** Number of bits needed to hold all legal values of workqueue_priority_t */
+#define WORKQUEUE_PRIORITY_BITS 2
+
 struct workqueue_entry_s {
   /** The next workqueue_entry_t that's pending on the same thread or
    * reply queue. */
@@ -76,6 +100,8 @@ struct workqueue_entry_s {
   struct threadpool_s *on_pool;
   /** True iff this entry is waiting for a worker to start processing it. */
   uint8_t pending;
+  /** Priority of this entry. */
+  workqueue_priority_bitfield_t priority : WORKQUEUE_PRIORITY_BITS;
   /** Function to run in the worker thread. */
   workqueue_reply_t (*fn)(void *state, void *arg);
   /** Function to run while processing the reply queue. */
@@ -94,9 +120,7 @@ struct replyqueue_s {
   alert_sockets_t alert;
 };
 
-/** A worker thread represents a single thread in a thread pool.  To avoid
- * contention, each gets its own queue. This breaks the guarantee that that
- * queued work will get executed strictly in order. */
+/** A worker thread represents a single thread in a thread pool. */
 typedef struct workerthread_s {
   /** Which thread it this?  In range 0..in_pool->n_threads-1 */
   int index;
@@ -109,6 +133,8 @@ typedef struct workerthread_s {
   replyqueue_t *reply_queue;
   /** The current update generation of this thread */
   unsigned generation;
+  /** One over the probability of taking work from a lower-priority queue. */
+  int32_t lower_priority_chance;
 } workerthread_t;
 
 static void queue_reply(replyqueue_t *queue, workqueue_entry_t *work);
@@ -125,15 +151,19 @@ workqueue_entry_new(workqueue_reply_t (*fn)(void*, void*),
   ent->fn = fn;
   ent->reply_fn = reply_fn;
   ent->arg = arg;
+  ent->priority = WQ_PRI_HIGH;
   return ent;
 }
 
+#define workqueue_entry_free(ent) \
+  FREE_AND_NULL(workqueue_entry_t, workqueue_entry_free_, (ent))
+
 /**
  * Release all storage held in <b>ent</b>. Call only when <b>ent</b> is not on
  * any queue.
  */
 static void
-workqueue_entry_free(workqueue_entry_t *ent)
+workqueue_entry_free_(workqueue_entry_t *ent)
 {
   if (!ent)
     return;
@@ -161,8 +191,9 @@ workqueue_entry_cancel(workqueue_entry_t *ent)
   int cancelled = 0;
   void *result = NULL;
   tor_mutex_acquire(&ent->on_pool->lock);
+  workqueue_priority_t prio = ent->priority;
   if (ent->pending) {
-    TOR_TAILQ_REMOVE(&ent->on_pool->work, ent, next_work);
+    TOR_TAILQ_REMOVE(&ent->on_pool->work[prio], ent, next_work);
     cancelled = 1;
     result = ent->arg;
   }
@@ -180,8 +211,46 @@ workqueue_entry_cancel(workqueue_entry_t *ent)
 static int
 worker_thread_has_work(workerthread_t *thread)
 {
-  return !TOR_TAILQ_EMPTY(&thread->in_pool->work) ||
-    thread->generation != thread->in_pool->generation;
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    if (!TOR_TAILQ_EMPTY(&thread->in_pool->work[i]))
+        return 1;
+  }
+  return thread->generation != thread->in_pool->generation;
+}
+
+/** Extract the next workqueue_entry_t from the the thread's pool, removing
+ * it from the relevant queues and marking it as non-pending.
+ *
+ * The caller must hold the lock. */
+static workqueue_entry_t *
+worker_thread_extract_next_work(workerthread_t *thread)
+{
+  threadpool_t *pool = thread->in_pool;
+  work_tailq_t *queue = NULL, *this_queue;
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    this_queue = &pool->work[i];
+    if (!TOR_TAILQ_EMPTY(this_queue)) {
+      queue = this_queue;
+      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+                                     thread->lower_priority_chance)) {
+        /* Usually we'll just break now, so that we can get out of the loop
+         * and use the queue where we found work. But with a small
+         * probability, we'll keep looking for lower priority work, so that
+         * we don't ignore our low-priority queues entirely. */
+        break;
+      }
+    }
+  }
+
+  if (queue == NULL)
+    return NULL;
+
+  workqueue_entry_t *work = TOR_TAILQ_FIRST(queue);
+  TOR_TAILQ_REMOVE(queue, work, next_work);
+  work->pending = 0;
+  return work;
 }
 
 /**
@@ -217,9 +286,9 @@ worker_thread_main(void *thread_)
         tor_mutex_acquire(&pool->lock);
         continue;
       }
-      work = TOR_TAILQ_FIRST(&pool->work);
-      TOR_TAILQ_REMOVE(&pool->work, work, next_work);
-      work->pending = 0;
+      work = worker_thread_extract_next_work(thread);
+      if (BUG(work == NULL))
+        break;
       tor_mutex_release(&pool->lock);
 
       /* We run the work function without holding the thread lock. This
@@ -268,12 +337,14 @@ queue_reply(replyqueue_t *queue, workqueue_entry_t *work)
 /** Allocate and start a new worker thread to use state object <b>state</b>,
  * and send responses to <b>replyqueue</b>. */
 static workerthread_t *
-workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue)
+workerthread_new(int32_t lower_priority_chance,
+                 void *state, threadpool_t *pool, replyqueue_t *replyqueue)
 {
   workerthread_t *thr = tor_malloc_zero(sizeof(workerthread_t));
   thr->state = state;
   thr->reply_queue = replyqueue;
   thr->in_pool = pool;
+  thr->lower_priority_chance = lower_priority_chance;
 
   if (spawn_func(worker_thread_main, thr) < 0) {
     //LCOV_EXCL_START
@@ -299,24 +370,34 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue)
  * function's responsibility to free the work object.
  *
  * On success, return a workqueue_entry_t object that can be passed to
- * workqueue_entry_cancel(). On failure, return NULL.
+ * workqueue_entry_cancel(). On failure, return NULL.  (Failure is not
+ * currently possible, but callers should check anyway.)
+ *
+ * Items are executed in a loose priority order -- each thread will usually
+ * take from the queued work with the highest prioirity, but will occasionally
+ * visit lower-priority queues to keep them from starving completely.
  *
- * Note that because each thread has its own work queue, work items may not
+ * Note that because of priorities and thread behavior, work items may not
  * be executed strictly in order.
  */
 workqueue_entry_t *
-threadpool_queue_work(threadpool_t *pool,
-                      workqueue_reply_t (*fn)(void *, void *),
-                      void (*reply_fn)(void *),
-                      void *arg)
+threadpool_queue_work_priority(threadpool_t *pool,
+                               workqueue_priority_t prio,
+                               workqueue_reply_t (*fn)(void *, void *),
+                               void (*reply_fn)(void *),
+                               void *arg)
 {
+  tor_assert(((int)prio) >= WORKQUEUE_PRIORITY_FIRST &&
+             ((int)prio) <= WORKQUEUE_PRIORITY_LAST);
+
   workqueue_entry_t *ent = workqueue_entry_new(fn, reply_fn, arg);
   ent->on_pool = pool;
   ent->pending = 1;
+  ent->priority = prio;
 
   tor_mutex_acquire(&pool->lock);
 
-  TOR_TAILQ_INSERT_TAIL(&pool->work, ent, next_work);
+  TOR_TAILQ_INSERT_TAIL(&pool->work[prio], ent, next_work);
 
   tor_cond_signal_one(&pool->condition);
 
@@ -325,6 +406,16 @@ threadpool_queue_work(threadpool_t *pool,
   return ent;
 }
 
+/** As threadpool_queue_work_priority(), but assumes WQ_PRI_HIGH */
+workqueue_entry_t *
+threadpool_queue_work(threadpool_t *pool,
+                      workqueue_reply_t (*fn)(void *, void *),
+                      void (*reply_fn)(void *),
+                      void *arg)
+{
+  return threadpool_queue_work_priority(pool, WQ_PRI_HIGH, fn, reply_fn, arg);
+}
+
 /**
  * Queue a copy of a work item for every thread in a pool.  This can be used,
  * for example, to tell the threads to update some parameter in their states.
@@ -388,6 +479,14 @@ threadpool_queue_update(threadpool_t *pool,
 /** Don't have more than this many threads per pool. */
 #define MAX_THREADS 1024
 
+/** For half of our threads, choose lower priority queues with probability
+ * 1/N for each of these values. Both are chosen somewhat arbitrarily.  If
+ * CHANCE_PERMISSIVE is too low, then we have a risk of low-priority tasks
+ * stalling forever.  If it's too high, we have a risk of low-priority tasks
+ * grabbing half of the threads. */
+#define CHANCE_PERMISSIVE 37
+#define CHANCE_STRICT INT32_MAX
+
 /** Launch threads until we have <b>n</b>. */
 static int
 threadpool_start_threads(threadpool_t *pool, int n)
@@ -404,8 +503,14 @@ threadpool_start_threads(threadpool_t *pool, int n)
                                      sizeof(workerthread_t*), n);
 
   while (pool->n_threads < n) {
+    /* For half of our threads, we'll choose lower priorities permissively;
+     * for the other half, we'll stick more strictly to higher priorities.
+     * This keeps slow low-priority tasks from taking over completely. */
+    int32_t chance = (pool->n_threads & 1) ? CHANCE_STRICT : CHANCE_PERMISSIVE;
+
     void *state = pool->new_thread_state_fn(pool->new_thread_state_arg);
-    workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue);
+    workerthread_t *thr = workerthread_new(chance,
+                                           state, pool, pool->reply_queue);
 
     if (!thr) {
       //LCOV_EXCL_START
@@ -441,7 +546,15 @@ threadpool_new(int n_threads,
   pool = tor_malloc_zero(sizeof(threadpool_t));
   tor_mutex_init_nonrecursive(&pool->lock);
   tor_cond_init(&pool->condition);
-  TOR_TAILQ_INIT(&pool->work);
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    TOR_TAILQ_INIT(&pool->work[i]);
+  }
+  {
+    unsigned seed;
+    crypto_rand((void*)&seed, sizeof(seed));
+    tor_init_weak_random(&pool->weak_rng, seed);
+  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -491,15 +604,41 @@ replyqueue_new(uint32_t alertsocks_flags)
   return rq;
 }
 
-/**
- * Return the "read socket" for a given reply queue.  The main thread should
- * listen for read events on this socket, and call replyqueue_process() every
- * time it triggers.
+/** Internal: Run from the libevent mainloop when there is work to handle in
+ * the reply queue handler. */
+static void
+reply_event_cb(evutil_socket_t sock, short events, void *arg)
+{
+  threadpool_t *tp = arg;
+  (void) sock;
+  (void) events;
+  replyqueue_process(tp->reply_queue);
+  if (tp->reply_cb)
+    tp->reply_cb(tp);
+}
+
+/** Register the threadpool <b>tp</b>'s reply queue with the libevent
+ * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
+ * each time there is work to process from the reply queue. Return 0 on
+ * success, -1 on failure.
  */
-tor_socket_t
-replyqueue_get_socket(replyqueue_t *rq)
+int
+threadpool_register_reply_event(threadpool_t *tp,
+                                void (*cb)(threadpool_t *tp))
 {
-  return rq->alert.read_fd;
+  struct event_base *base = tor_libevent_get_base();
+
+  if (tp->reply_event) {
+    tor_event_free(tp->reply_event);
+  }
+  tp->reply_event = tor_event_new(base,
+                                  tp->reply_queue->alert.read_fd,
+                                  EV_READ|EV_PERSIST,
+                                  reply_event_cb,
+                                  tp);
+  tor_assert(tp->reply_event);
+  tp->reply_cb = cb;
+  return event_add(tp->reply_event, NULL);
 }
 
 /**
@@ -510,12 +649,13 @@ replyqueue_get_socket(replyqueue_t *rq)
 void
 replyqueue_process(replyqueue_t *queue)
 {
-  if (queue->alert.drain_fn(queue->alert.read_fd) < 0) {
+  int r = queue->alert.drain_fn(queue->alert.read_fd);
+  if (r < 0) {
     //LCOV_EXCL_START
     static ratelim_t warn_limit = RATELIM_INIT(7200);
     log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL,
                  "Failure from drain_fd: %s",
-                 tor_socket_strerror(tor_socket_errno(queue->alert.read_fd)));
+                   tor_socket_strerror(-r));
     //LCOV_EXCL_STOP
   }
 
diff --git a/src/common/workqueue.h b/src/common/workqueue.h
index 54276767b0..e1fe612e2b 100644
--- a/src/common/workqueue.h
+++ b/src/common/workqueue.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_WORKQUEUE_H
@@ -16,12 +16,26 @@ typedef struct threadpool_s threadpool_t;
 typedef struct workqueue_entry_s workqueue_entry_t;
 
 /** Possible return value from a work function: */
-typedef enum {
+typedef enum workqueue_reply_t {
   WQ_RPL_REPLY = 0, /** indicates success */
   WQ_RPL_ERROR = 1, /** indicates fatal error */
   WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */
 } workqueue_reply_t;
 
+/** Possible priorities for work.  Lower numeric values are more important. */
+typedef enum workqueue_priority_t {
+  WQ_PRI_HIGH = 0,
+  WQ_PRI_MED  = 1,
+  WQ_PRI_LOW  = 2,
+} workqueue_priority_t;
+
+workqueue_entry_t *threadpool_queue_work_priority(threadpool_t *pool,
+                                    workqueue_priority_t prio,
+                                    workqueue_reply_t (*fn)(void *,
+                                                            void *),
+                                    void (*reply_fn)(void *),
+                                    void *arg);
+
 workqueue_entry_t *threadpool_queue_work(threadpool_t *pool,
                                          workqueue_reply_t (*fn)(void *,
                                                                  void *),
@@ -42,8 +56,11 @@ threadpool_t *threadpool_new(int n_threads,
 replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
-tor_socket_t replyqueue_get_socket(replyqueue_t *rq);
 void replyqueue_process(replyqueue_t *queue);
 
-#endif
+struct event_base;
+int threadpool_register_reply_event(threadpool_t *tp,
+                                    void (*cb)(threadpool_t *tp));
+
+#endif /* !defined(TOR_WORKQUEUE_H) */
 
diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging
index c537c51f9b..86429f1176 100644
--- a/src/config/torrc.minimal.in-staging
+++ b/src/config/torrc.minimal.in-staging
@@ -1,5 +1,5 @@
 ## Configuration file for a typical Tor user
-## Last updated 22 September 2015 for Tor 0.2.7.3-alpha.
+## Last updated 22 December 2017 for Tor 0.3.2.8-rc.
 ## (may or may not work for much older or much newer versions of Tor.)
 ##
 ## Lines that begin with "## " try to explain what's going on. Lines
@@ -100,7 +100,7 @@
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
 ## contain only the alphanumeric characters (a-z, A-Z, 0-9). No unicode,
-## no emoji.
+## no emoji. If not set, "Unnamed" will be used.
 #Nickname ididnteditheconfig
 
 ## Define these to limit how much relayed traffic you will allow. Your
@@ -132,6 +132,9 @@
 ## spammers might also collect them. You may want to obscure the fact that
 ## it's an email address and/or generate a new address for this purpose.
 ## Notice that "<" and ">" are recommended.
+##
+## If you are running multiple relays, you MUST set this option.
+##
 #ContactInfo Random Person <nobody AT example dot com>
 ## You might also include your PGP or GPG fingerprint if you have one.
 ## Use the full fingerprint, not just a (short) KeyID: KeyIDs are easy
@@ -161,8 +164,15 @@
 ## See https://www.torproject.org/docs/faq#MultipleRelays
 ## However, you should never include a bridge's fingerprint here, as it would
 ## break its concealability and potentially reveal its IP/TCP address.
+##
+## If you are running multiple relays, you MUST set this option.
+##
 #MyFamily $keyid,$keyid,...
 
+## Uncomment this if you want your relay to allow IPv6 exit traffic.
+## (Relays only allow IPv4 exit traffic by default.)
+#IPv6Exit 1
+
 ## A comma-separated list of exit policies. They're considered first
 ## to last, and the first match wins.
 ##
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index 5328206da9..72cca0be31 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -1,5 +1,5 @@
 ## Configuration file for a typical Tor user
-## Last updated 22 September 2015 for Tor 0.2.7.3-alpha.
+## Last updated 22 December 2017 for Tor 0.3.2.8-rc.
 ## (may or may not work for much older or much newer versions of Tor.)
 ##
 ## Lines that begin with "## " try to explain what's going on. Lines
@@ -95,11 +95,18 @@
 
 ## If you have multiple network interfaces, you can specify one for
 ## outgoing traffic to use.
-# OutboundBindAddress 10.0.0.5
+## OutboundBindAddressExit will be used for all exit traffic, while
+## OutboundBindAddressOR will be used for all OR and Dir connections
+## (DNS connections ignore OutboundBindAddress).
+## If you do not wish to differentiate, use OutboundBindAddress to
+## specify the same address for both in a single line.
+#OutboundBindAddressExit 10.0.0.4
+#OutboundBindAddressOR 10.0.0.5
 
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
 ## contain only the characters [a-zA-Z0-9].
+## If not set, "Unnamed" will be used.
 #Nickname ididnteditheconfig
 
 ## Define these to limit how much relayed traffic you will allow. Your
@@ -130,6 +137,9 @@
 ## descriptors containing these lines and that Google indexes them, so
 ## spammers might also collect them. You may want to obscure the fact that
 ## it's an email address and/or generate a new address for this purpose.
+##
+## If you are running multiple relays, you MUST set this option.
+##
 #ContactInfo Random Person <nobody AT example dot com>
 ## You might also include your PGP or GPG fingerprint if you have one:
 #ContactInfo 0xFFFFFFFF Random Person <nobody AT example dot com>
@@ -156,8 +166,19 @@
 ## https://www.torproject.org/docs/faq#MultipleRelays
 ## However, you should never include a bridge's fingerprint here, as it would
 ## break its concealability and potentially reveal its IP/TCP address.
+##
+## If you are running multiple relays, you MUST set this option.
+##
 #MyFamily $keyid,$keyid,...
 
+## Uncomment this if you do *not* want your relay to allow any exit traffic.
+## (Relays allow exit traffic by default.)
+#ExitRelay 0
+
+## Uncomment this if you want your relay to allow IPv6 exit traffic.
+## (Relays only allow IPv4 exit traffic by default.)
+#IPv6Exit 1
+
 ## A comma-separated list of exit policies. They're considered first
 ## to last, and the first match wins.
 ##
@@ -204,3 +225,12 @@
 ## address manually to your friends, uncomment this line:
 #PublishServerDescriptor 0
 
+## Configuration options can be imported from files or folders using the %include
+## option with the value being a path. If the path is a file, the options from the
+## file will be parsed as if they were written where the %include option is. If
+## the path is a folder, all files on that folder will be parsed following lexical
+## order. Files starting with a dot are ignored. Files on subfolders are ignored.
+## The %include option can be used recursively.
+#%include /etc/torrc.d/
+#%include /etc/torrc.custom
+
diff --git a/src/ext/byteorder.h b/src/ext/byteorder.h
new file mode 100644
index 0000000000..c8ba52184b
--- /dev/null
+++ b/src/ext/byteorder.h
@@ -0,0 +1,67 @@
+/* <MIT License>
+ Copyright (c) 2013-2014  Marek Majkowski <marek@popcount.org>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ </MIT License>
+
+ Original location:
+    https://github.com/majek/csiphash/
+
+ Solution inspired by code from:
+    Samuel Neves (supercop/crypto_auth/siphash24/little)
+    djb (supercop/crypto_auth/siphash24/little2)
+    Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c)
+*/
+
+/* This code is extracted from csiphash.h */
+
+#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) &&      \
+	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#  define _le64toh(x) ((uint64_t)(x))
+#elif defined(_WIN32)
+/* Windows is always little endian, unless you're on xbox360
+   http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
+#  define _le64toh(x) ((uint64_t)(x))
+#elif defined(__APPLE__)
+#  include <libkern/OSByteOrder.h>
+#  define _le64toh(x) OSSwapLittleToHostInt64(x)
+#elif defined(sun) || defined(__sun)
+#  include <sys/byteorder.h>
+#  define _le64toh(x) LE_64(x)
+
+#else
+
+/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
+#  if defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD)
+#    include <sys/endian.h>
+#  else
+#    include <endian.h>
+#  endif
+#  if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
+	__BYTE_ORDER == __LITTLE_ENDIAN
+#    define _le64toh(x) ((uint64_t)(x))
+#  else
+#    if defined(OpenBSD)
+#      define _le64toh(x) letoh64(x)
+#    else
+#      define _le64toh(x) le64toh(x)
+#    endif
+#  endif
+
+#endif
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index 49a6dc4778..0427c87950 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -35,41 +35,7 @@
 #include "util.h"
 /* for memcpy */
 #include <string.h>
-
-#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
-	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#  define _le64toh(x) ((uint64_t)(x))
-#elif defined(_WIN32)
-/* Windows is always little endian, unless you're on xbox360
-   http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
-#  define _le64toh(x) ((uint64_t)(x))
-#elif defined(__APPLE__)
-#  include <libkern/OSByteOrder.h>
-#  define _le64toh(x) OSSwapLittleToHostInt64(x)
-#elif defined(sun) || defined(__sun)
-#  include <sys/byteorder.h>
-#  define _le64toh(x) LE_64(x)
-
-#else
-
-/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
-#  if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
-#    include <sys/endian.h>
-#  else
-#    include <endian.h>
-#  endif
-#  if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
-	__BYTE_ORDER == __LITTLE_ENDIAN
-#    define _le64toh(x) ((uint64_t)(x))
-#  else
-#    if defined(__OpenBSD__)
-#      define _le64toh(x) letoh64(x)
-#    else
-#      define _le64toh(x) le64toh(x)
-#    endif
-#  endif
-
-#endif
+#include "byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -157,3 +123,9 @@ void siphash_set_global_key(const struct sipkey *key)
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;
 }
+
+void siphash_unset_global_key(void)
+{
+	the_siphash_key_is_set = 0;
+	memset(&the_siphash_key, 0, sizeof(the_siphash_key));
+}
diff --git a/src/ext/ed25519/donna/ed25519-hash-custom.h b/src/ext/ed25519/donna/ed25519-hash-custom.h
index 7dc249129d..cdeab3e45b 100644
--- a/src/ext/ed25519/donna/ed25519-hash-custom.h
+++ b/src/ext/ed25519/donna/ed25519-hash-custom.h
@@ -9,3 +9,34 @@
 	void ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen);
 */
 
+#include "crypto_digest.h"
+
+typedef struct ed25519_hash_context {
+  crypto_digest_t *ctx;
+} ed25519_hash_context;
+
+
+static void
+ed25519_hash_init(ed25519_hash_context *ctx)
+{
+  ctx->ctx = crypto_digest512_new(DIGEST_SHA512);
+}
+static void
+ed25519_hash_update(ed25519_hash_context *ctx, const uint8_t *in, size_t inlen)
+{
+  crypto_digest_add_bytes(ctx->ctx, (const char *)in, inlen);
+}
+static void
+ed25519_hash_final(ed25519_hash_context *ctx, uint8_t *hash)
+{
+  crypto_digest_get_digest(ctx->ctx, (char *)hash, DIGEST512_LEN);
+  crypto_digest_free(ctx->ctx);
+  ctx->ctx = NULL;
+}
+static void
+ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen)
+{
+  crypto_digest512((char *)hash, (const char *)in, inlen,
+		   DIGEST_SHA512);
+}
+
diff --git a/src/ext/ed25519/donna/ed25519-randombytes-custom.h b/src/ext/ed25519/donna/ed25519-randombytes-custom.h
index 3fb0959fc4..27eade4f95 100644
--- a/src/ext/ed25519/donna/ed25519-randombytes-custom.h
+++ b/src/ext/ed25519/donna/ed25519-randombytes-custom.h
@@ -8,7 +8,7 @@
 */
 
 /* Tor: Instead of calling OpenSSL's CSPRNG directly, call the wrapper. */
-#include "crypto.h"
+#include "crypto_rand.h"
 
 static void
 ED25519_FN(ed25519_randombytes_unsafe) (void *p, size_t len)
diff --git a/src/ext/ed25519/donna/ed25519_donna_tor.h b/src/ext/ed25519/donna/ed25519_donna_tor.h
index d225407b1c..7d7b8c0625 100644
--- a/src/ext/ed25519/donna/ed25519_donna_tor.h
+++ b/src/ext/ed25519/donna/ed25519_donna_tor.h
@@ -30,4 +30,9 @@ int ed25519_donna_blind_public_key(unsigned char *out, const unsigned char *inp,
 int ed25519_donna_pubkey_from_curve25519_pubkey(unsigned char *out,
   const unsigned char *inp, int signbit);
 
+
+int
+ed25519_donna_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey);
+
 #endif
diff --git a/src/ext/ed25519/donna/ed25519_tor.c b/src/ext/ed25519/donna/ed25519_tor.c
index 9537ae66a1..43de9faaea 100644
--- a/src/ext/ed25519/donna/ed25519_tor.c
+++ b/src/ext/ed25519/donna/ed25519_tor.c
@@ -40,6 +40,8 @@
 #include "ed25519-randombytes.h"
 #include "ed25519-hash.h"
 
+#include "crypto_util.h"
+
 typedef unsigned char ed25519_signature[64];
 typedef unsigned char ed25519_public_key[32];
 typedef unsigned char ed25519_secret_key[32];
@@ -132,7 +134,7 @@ ED25519_FN(curved25519_scalarmult_basepoint) (curved25519_key pk, const curved25
 }
 
 /*
-   Tor has a specific idea of how an Ed25519 implementaion should behave.
+   Tor has a specific idea of how an Ed25519 implementation should behave.
    Implement such a beast using the ed25519-donna primitives/internals.
 
     * Private key generation using Tor's CSPRNG.
@@ -245,13 +247,7 @@ ed25519_donna_sign(unsigned char *sig, const unsigned char *m, size_t mlen,
 static void
 ed25519_donna_gettweak(unsigned char *out, const unsigned char *param)
 {
-  static const char str[] = "Derive temporary signing key";
-  ed25519_hash_context ctx;
-
-  ed25519_hash_init(&ctx);
-  ed25519_hash_update(&ctx, (const unsigned char*)str, strlen(str));
-  ed25519_hash_update(&ctx, param, 32);
-  ed25519_hash_final(&ctx, out);
+  memcpy(out, param, 32);
 
   out[0] &= 248;  /* Is this necessary ? */
   out[31] &= 63;
@@ -304,7 +300,9 @@ ed25519_donna_blind_public_key(unsigned char *out, const unsigned char *inp,
   /* No "ge25519_unpack", negate the public key. */
   memcpy(pkcopy, inp, 32);
   pkcopy[31] ^= (1<<7);
-  ge25519_unpack_negative_vartime(&A, pkcopy);
+  if (!ge25519_unpack_negative_vartime(&A, pkcopy)) {
+    return -1;
+  }
 
   /* A' = [tweak] * A + [0] * basepoint. */
   ge25519_double_scalarmult_vartime(&Aprime, &A, t, zero);
@@ -340,5 +338,32 @@ ed25519_donna_pubkey_from_curve25519_pubkey(unsigned char *out,
   return 0;
 }
 
+/* Do the scalar multiplication of <b>pubkey</b> with the group order
+ * <b>modm_m</b>.  Place the result in <b>out</b> which must be at least 32
+ * bytes long. */
+int
+ed25519_donna_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey)
+{
+  static const bignum256modm ALIGN(16) zero = { 0 };
+  unsigned char pkcopy[32];
+  ge25519 ALIGN(16) Point, Result;
+
+  /* No "ge25519_unpack", negate the public key and unpack it back.
+   * See ed25519_donna_blind_public_key() */
+  memcpy(pkcopy, pubkey, 32);
+  pkcopy[31] ^= (1<<7);
+  if (!ge25519_unpack_negative_vartime(&Point, pkcopy)) {
+    return -1; /* error: bail out */
+  }
+
+  /* There is no regular scalarmult function so we have to do:
+   * Result = l*P + 0*B */
+  ge25519_double_scalarmult_vartime(&Result, &Point, modm_m, zero);
+  ge25519_pack(out, &Result);
+
+  return 0;
+}
+
 #include "test-internals.c"
 
diff --git a/src/ext/ed25519/ref10/blinding.c b/src/ext/ed25519/ref10/blinding.c
index ee3e8666fa..88e84cac20 100644
--- a/src/ext/ed25519/ref10/blinding.c
+++ b/src/ext/ed25519/ref10/blinding.c
@@ -7,13 +7,13 @@
 #include "ed25519_ref10.h"
 
 #include <string.h>
-#include "crypto.h"
+#include "crypto_util.h"
 
 static void
 ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param)
 {
-  const char str[] = "Derive temporary signing key";
-  crypto_hash_sha512_2(out, (const unsigned char*)str, strlen(str), param, 32);
+  memcpy(out, param, 32);
+
   out[0] &= 248;  /* Is this necessary necessary ? */
   out[31] &= 63;
   out[31] |= 64;
@@ -49,6 +49,7 @@ int ed25519_ref10_blind_public_key(unsigned char *out,
   unsigned char pkcopy[32];
   ge_p3 A;
   ge_p2 Aprime;
+  int retval = -1;
 
   ed25519_ref10_gettweak(tweak, param);
 
@@ -62,15 +63,57 @@ int ed25519_ref10_blind_public_key(unsigned char *out,
    * "ge_frombytes", we'd use that, but there isn't. */
   memcpy(pkcopy, inp, 32);
   pkcopy[31] ^= (1<<7);
-  ge_frombytes_negate_vartime(&A, pkcopy);
+  if (ge_frombytes_negate_vartime(&A, pkcopy) != 0) {
+    goto done;
+  }
   /* There isn't a regular ge_scalarmult -- we have to do tweak*A + zero*B. */
   ge_double_scalarmult_vartime(&Aprime, tweak, &A, zero);
   ge_tobytes(out, &Aprime);
 
+  retval = 0;
+
+ done:
   memwipe(tweak, 0, sizeof(tweak));
   memwipe(&A, 0, sizeof(A));
   memwipe(&Aprime, 0, sizeof(Aprime));
   memwipe(pkcopy, 0, sizeof(pkcopy));
 
+  return retval;
+}
+
+/* This is the group order encoded in a format that
+ * ge_double_scalarmult_vartime() understands. The group order m is:
+ * m = 	 2^252 +  27742317777372353535851937790883648493 =
+ *       0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed
+ */
+static const uint8_t modm_m[32] = {0xed,0xd3,0xf5,0x5c,0x1a,0x63,0x12,0x58,
+                                   0xd6,0x9c,0xf7,0xa2,0xde,0xf9,0xde,0x14,
+                                   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+                                   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10};
+
+/* Do the scalar multiplication of <b>pubkey</b> with the group order
+ * <b>modm_m</b>.  Place the result in <b>out</b> which must be at least 32
+ * bytes long. */
+int
+ed25519_ref10_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey)
+{
+  unsigned char pkcopy[32];
+  unsigned char zero[32] = {0};
+  ge_p3 Point;
+  ge_p2 Result;
+
+  /* All this is done to fit 'pubkey' in 'Point' so that it can be used by
+   * ed25519 ref code. Same thing as in blinding function */
+  memcpy(pkcopy, pubkey, 32);
+  pkcopy[31] ^= (1<<7);
+  if (ge_frombytes_negate_vartime(&Point, pkcopy) != 0) {
+    return -1; /* error: bail out */
+  }
+
+  /* There isn't a regular scalarmult -- we have to do r = l*P + 0*B */
+  ge_double_scalarmult_vartime(&Result, modm_m, &Point, zero);
+  ge_tobytes(out, &Result);
+
   return 0;
 }
diff --git a/src/ext/ed25519/ref10/crypto_hash_sha512.h b/src/ext/ed25519/ref10/crypto_hash_sha512.h
index 0278571522..7faddb1597 100644
--- a/src/ext/ed25519/ref10/crypto_hash_sha512.h
+++ b/src/ext/ed25519/ref10/crypto_hash_sha512.h
@@ -1,30 +1,32 @@
 /* Added for Tor. */
-#include <openssl/sha.h>
+#include "crypto_digest.h"
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len'-byte string in 'inp' */
 #define crypto_hash_sha512(out, inp, len) \
-  SHA512((inp), (len), (out))
+  crypto_digest512((char *)(out), (const char *)(inp), (len), DIGEST_SHA512)
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len1'-byte string in 'inp1',
  * concatenated with the 'len2'-byte string in 'inp2'. */
 #define crypto_hash_sha512_2(out, inp1, len1, inp2, len2)               \
   do {                                                                  \
-      SHA512_CTX sha_ctx_;                                              \
-      SHA512_Init(&sha_ctx_);                                           \
-      SHA512_Update(&sha_ctx_, (inp1), (len1));                         \
-      SHA512_Update(&sha_ctx_, (inp2), (len2));                         \
-      SHA512_Final((out), &sha_ctx_);                                   \
- } while(0)
+    crypto_digest_t *sha_ctx_;						\
+    sha_ctx_ = crypto_digest512_new(DIGEST_SHA512);			\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp1), (len1));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp2), (len2));	\
+    crypto_digest_get_digest(sha_ctx_, (char *)out, DIGEST512_LEN);	\
+    crypto_digest_free(sha_ctx_);					\
+  } while (0)
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len1'-byte string in 'inp1',
  * concatenated with the 'len2'-byte string in 'inp2', concatenated with
  * the 'len3'-byte string in 'len3'. */
 #define crypto_hash_sha512_3(out, inp1, len1, inp2, len2, inp3, len3)   \
   do {                                                                  \
-      SHA512_CTX sha_ctx_;                                              \
-      SHA512_Init(&sha_ctx_);                                           \
-      SHA512_Update(&sha_ctx_, (inp1), (len1));                         \
-      SHA512_Update(&sha_ctx_, (inp2), (len2));                         \
-      SHA512_Update(&sha_ctx_, (inp3), (len3));                         \
-      SHA512_Final((out), &sha_ctx_);                                   \
+    crypto_digest_t *sha_ctx_;						\
+    sha_ctx_ = crypto_digest512_new(DIGEST_SHA512);			\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp1), (len1));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp2), (len2));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp3), (len3));	\
+    crypto_digest_get_digest(sha_ctx_, (char *)out, DIGEST512_LEN);	\
+    crypto_digest_free(sha_ctx_);					\
  } while(0)
diff --git a/src/ext/ed25519/ref10/ed25519_ref10.h b/src/ext/ed25519/ref10/ed25519_ref10.h
index af7e21a2ad..5965694977 100644
--- a/src/ext/ed25519/ref10/ed25519_ref10.h
+++ b/src/ext/ed25519/ref10/ed25519_ref10.h
@@ -27,4 +27,8 @@ int ed25519_ref10_blind_public_key(unsigned char *out,
                               const unsigned char *inp,
                               const unsigned char *param);
 
+int
+ed25519_ref10_scalarmult_with_group_order(unsigned char *out,
+                                          const unsigned char *pubkey);
+
 #endif
diff --git a/src/ext/ed25519/ref10/keypair.c b/src/ext/ed25519/ref10/keypair.c
index 68a88f9adc..c437f0a4f2 100644
--- a/src/ext/ed25519/ref10/keypair.c
+++ b/src/ext/ed25519/ref10/keypair.c
@@ -6,6 +6,9 @@
 #include "crypto_hash_sha512.h"
 #include "ge.h"
 
+#include "crypto_rand.h"
+#include "crypto_util.h"
+
 int
 crypto_sign_seckey(unsigned char *sk)
 {
diff --git a/src/ext/ed25519/ref10/randombytes.h b/src/ext/ed25519/ref10/randombytes.h
index 8bf31631f0..a21dde8540 100644
--- a/src/ext/ed25519/ref10/randombytes.h
+++ b/src/ext/ed25519/ref10/randombytes.h
@@ -1,4 +1,4 @@
 /* Added for Tor. */
-#include "crypto.h"
+#include "crypto_rand.h"
 #define randombytes(b, n) \
   (crypto_strongest_rand((b), (n)), 0)
diff --git a/src/ext/ht.h b/src/ext/ht.h
index a441d0b685..99da773faf 100644
--- a/src/ext/ht.h
+++ b/src/ext/ht.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2002, Christopher Clark.
  * Copyright (c) 2005-2006, Nick Mathewson.
- * Copyright (c) 2007-2015, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See license at end. */
 
 /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */
@@ -150,6 +150,8 @@
 #define HT_CLEAR(name, head)         name##_HT_CLEAR(head)
 #define HT_INIT(name, head)          name##_HT_INIT(head)
 #define HT_REP_IS_BAD_(name, head)    name##_HT_REP_IS_BAD_(head)
+#define HT_FOREACH_FN(name, head, fn, data) \
+   name##_HT_FOREACH_FN((head), (fn), (data))
 /* Helper: */
 static inline unsigned
 ht_improve_hash(unsigned h)
diff --git a/src/ext/include.am b/src/ext/include.am
index f00f3e031e..7ec2e43312 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -5,6 +5,7 @@ EXTRA_DIST += src/ext/README
 
 EXTHEADERS = \
   src/ext/ht.h		\
+  src/ext/byteorder.h   \
   src/ext/tinytest.h	\
   src/ext/tor_readpassphrase.h \
   src/ext/strlcat.c	\
@@ -100,6 +101,7 @@ noinst_LIBRARIES += $(LIBED25519_REF10)
 src_ext_ed25519_donna_libed25519_donna_a_CFLAGS=\
   @CFLAGS_CONSTTIME@ \
   -DED25519_CUSTOMRANDOM \
+  -DED25519_CUSTOMHASH \
   -DED25519_SUFFIX=_donna
 
 src_ext_ed25519_donna_libed25519_donna_a_SOURCES= \
diff --git a/src/ext/keccak-tiny/keccak-tiny-unrolled.c b/src/ext/keccak-tiny/keccak-tiny-unrolled.c
index d1342c3601..07e8c95bcf 100644
--- a/src/ext/keccak-tiny/keccak-tiny-unrolled.c
+++ b/src/ext/keccak-tiny/keccak-tiny-unrolled.c
@@ -9,29 +9,22 @@
 #include "keccak-tiny.h"
 
 #include <string.h>
-#include "crypto.h"
+#include "crypto_util.h"
+#include "byteorder.h"
 
 /******** Endianness conversion helpers ********/
 
 static inline uint64_t
 loadu64le(const unsigned char *x) {
-  uint64_t r = 0;
-  size_t i;
-
-  for (i = 0; i < 8; ++i) {
-    r |= (uint64_t)x[i] << 8 * i;
-  }
-  return r;
+  uint64_t r;
+  memcpy(&r, x, sizeof(r));
+  return _le64toh(r);
 }
 
 static inline void
 storeu64le(uint8_t *x, uint64_t u) {
-  size_t i;
-
-  for(i=0; i<8; ++i) {
-    x[i] = u;
-    u >>= 8;
-  }
+  uint64_t val = _le64toh(u);
+  memcpy(x, &val, sizeof(u));
 }
 
 /******** The Keccak-f[1600] permutation ********/
diff --git a/src/ext/rust b/src/ext/rust
new file mode 160000
+Subproject aa37fb84fb829902e83ca11a7244bbc6b86b809
diff --git a/src/ext/siphash.h b/src/ext/siphash.h
index d9b34b8980..730e49937d 100644
--- a/src/ext/siphash.h
+++ b/src/ext/siphash.h
@@ -9,5 +9,6 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 
 void siphash_set_global_key(const struct sipkey *key);
 uint64_t siphash24g(const void *src, unsigned long src_sz);
+void siphash_unset_global_key(void);
 
 #endif
diff --git a/src/ext/timeouts/timeout-bitops.c b/src/ext/timeouts/timeout-bitops.c
index 45466f6cb3..68db817933 100644
--- a/src/ext/timeouts/timeout-bitops.c
+++ b/src/ext/timeouts/timeout-bitops.c
@@ -231,7 +231,8 @@ main(int c, char **v)
 	int result = 0;
 
 	for (i = 0; i <= 63; ++i) {
-		uint64_t x = 1 << i;
+		uint64_t x = 1;
+		x <<= i;
 		if (!check(x))
 			result = 1;
 		--x;
diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c
index 713ec219ce..d4b514d2c5 100644
--- a/src/ext/timeouts/timeout.c
+++ b/src/ext/timeouts/timeout.c
@@ -150,7 +150,7 @@
 #else
 #define ctz(n) ctz32(n)
 #define clz(n) clz32(n)
-#define fls(n) ((int)(32 - clz32(n)))
+#define fls(n) ((int)(32 - clz32((uint32_t)n)))
 #endif
 
 #if WHEEL_BIT == 6
@@ -432,7 +432,7 @@ TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) {
 			 * or can be replaced with a simpler operation.
 			 */
 			oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT));
-			pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot);
+			pending = rotl(((WHEEL_C(1) << _elapsed) - 1), oslot);
 
 			nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT));
 			pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed);
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 3ffde6e09b..b233cf7631 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,11 +1,11 @@
-/* trunnel-impl.h -- copied from Trunnel v1.4.6
+/* trunnel-impl.h -- copied from Trunnel v1.5.2
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel-impl.h -- Implementation helpers for trunnel, included by
  * generated trunnel files
  *
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2017, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  */
 
@@ -17,6 +17,7 @@
 #include "trunnel.h"
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 
 #if defined(_MSC_VER) && (_MSC_VER < 1600)
 #define uint8_t unsigned char
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3994422643..b749d8136f 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,10 +1,10 @@
-/* trunnel.c -- copied from Trunnel v1.4.6
+/* trunnel.c -- copied from Trunnel v1.5.2
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel.c -- Helper functions to implement trunnel.
  *
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2017, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  *
  * See trunnel-impl.h for documentation of these functions.
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 41068b8fb3..32c80bac23 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,11 +1,11 @@
-/* trunnel.h -- copied from Trunnel v1.4.6
+/* trunnel.h -- copied from Trunnel v1.5.2
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel.h -- Public declarations for trunnel, to be included
  * in trunnel header files.
 
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2017, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  */
 
diff --git a/src/include.am b/src/include.am
index c468af3649..90ecf90d45 100644
--- a/src/include.am
+++ b/src/include.am
@@ -2,8 +2,10 @@ include src/ext/include.am
 include src/trunnel/include.am
 include src/common/include.am
 include src/or/include.am
+include src/rust/include.am
 include src/test/include.am
 include src/tools/include.am
 include src/win32/include.am
 include src/config/include.am
-
+include src/test/fuzz/include.am
+include src/trace/include.am
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake
index 2ac98cd372..429ae67858 100644
--- a/src/or/Makefile.nmake
+++ b/src/or/Makefile.nmake
@@ -14,6 +14,7 @@ LIBTOR_OBJECTS = \
   addressmap.obj \
   buffers.obj \
   channel.obj \
+  channelpadding.obj \
   channeltls.obj \
   circpathbias.obj \
   circuitbuild.obj \
diff --git a/src/or/addressmap.c b/src/or/addressmap.c
index 33fd7e0f4a..7f861e4d24 100644
--- a/src/or/addressmap.c
+++ b/src/or/addressmap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,9 +21,10 @@
 #include "config.h"
 #include "connection_edge.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "dns.h"
-#include "routerset.h"
 #include "nodelist.h"
+#include "routerset.h"
 
 /** A client-side struct to remember requests to rewrite addresses
  * to new addresses. These structs are stored in the hash table
@@ -90,34 +91,47 @@ addressmap_init(void)
   virtaddress_reversemap = strmap_new();
 }
 
+#define addressmap_ent_free(ent)                                        \
+  FREE_AND_NULL(addressmap_entry_t, addressmap_ent_free_, (ent))
+
 /** Free the memory associated with the addressmap entry <b>_ent</b>. */
 static void
-addressmap_ent_free(void *_ent)
+addressmap_ent_free_(addressmap_entry_t *ent)
 {
-  addressmap_entry_t *ent;
-  if (!_ent)
+  if (!ent)
     return;
 
-  ent = _ent;
   tor_free(ent->new_address);
   tor_free(ent);
 }
 
+static void
+addressmap_ent_free_void(void *ent)
+{
+  addressmap_ent_free_(ent);
+}
+
+#define addressmap_virtaddress_ent_free(ent)                            \
+  FREE_AND_NULL(virtaddress_entry_t, addressmap_virtaddress_ent_free_, (ent))
+
 /** Free storage held by a virtaddress_entry_t* entry in <b>_ent</b>. */
 static void
-addressmap_virtaddress_ent_free(void *_ent)
+addressmap_virtaddress_ent_free_(virtaddress_entry_t *ent)
 {
-  virtaddress_entry_t *ent;
-  if (!_ent)
+  if (!ent)
     return;
-
-  ent = _ent;
   tor_free(ent->ipv4_address);
   tor_free(ent->ipv6_address);
   tor_free(ent->hostname_address);
   tor_free(ent);
 }
 
+static void
+addressmap_virtaddress_ent_free_void(void *ent)
+{
+  addressmap_virtaddress_ent_free_(ent);
+}
+
 /** Remove <b>address</b> (which must map to <b>ent</b>) from the
  * virtual address map. */
 static void
@@ -213,8 +227,8 @@ addressmap_clear_excluded_trackexithosts(const or_options_t *options)
     while (dot > target && *dot != '.')
       dot--;
     if (*dot == '.') dot++;
-    nodename = tor_strndup(dot, len-5-(dot-target));;
-    node = node_get_by_nickname(nodename, 0);
+    nodename = tor_strndup(dot, len-5-(dot-target));
+    node = node_get_by_nickname(nodename, NNF_NO_WARN_UNNAMED);
     tor_free(nodename);
     if (!node ||
         (allow_nodes && !routerset_contains_node(allow_nodes, node)) ||
@@ -311,10 +325,10 @@ addressmap_clean(time_t now)
 void
 addressmap_free_all(void)
 {
-  strmap_free(addressmap, addressmap_ent_free);
+  strmap_free(addressmap, addressmap_ent_free_void);
   addressmap = NULL;
 
-  strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free);
+  strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free_void);
   virtaddress_reversemap = NULL;
 }
 
@@ -376,29 +390,38 @@ addressmap_rewrite(char *address, size_t maxlen,
   char *addr_orig = tor_strdup(address);
   char *log_addr_orig = NULL;
 
+  /* We use a loop here to limit the total number of rewrites we do,
+   * so that we can't hit an infinite loop. */
   for (rewrites = 0; rewrites < 16; rewrites++) {
     int exact_match = 0;
     log_addr_orig = tor_strdup(escaped_safe_str_client(address));
 
+    /* First check to see if there's an exact match for this address */
     ent = strmap_get(addressmap, address);
 
     if (!ent || !ent->new_address) {
+      /* And if we don't have an exact match, try to check whether
+       * we have a pattern-based match.
+       */
       ent = addressmap_match_superdomains(address);
     } else {
       if (ent->src_wildcard && !ent->dst_wildcard &&
           !strcasecmp(address, ent->new_address)) {
-        /* This is a rule like *.example.com example.com, and we just got
-         * "example.com" */
+        /* This is a rule like "rewrite *.example.com to example.com", and we
+         * just got "example.com". Instead of calling it an infinite loop,
+         * call it complete. */
         goto done;
       }
-
       exact_match = 1;
     }
 
     if (!ent || !ent->new_address) {
+      /* We still have no match at all.  We're done! */
       goto done;
     }
 
+    /* Check wither the flags we were passed tell us not to use this
+     * mapping. */
     switch (ent->source) {
       case ADDRMAPSRC_DNS:
         {
@@ -431,6 +454,8 @@ addressmap_rewrite(char *address, size_t maxlen,
         goto done;
     }
 
+    /* Now fill in the address with the new address. That might be via
+     * appending some new stuff to the end, or via just replacing it. */
     if (ent->dst_wildcard && !exact_match) {
       strlcat(address, ".", maxlen);
       strlcat(address, ent->new_address, maxlen);
@@ -438,6 +463,7 @@ addressmap_rewrite(char *address, size_t maxlen,
       strlcpy(address, ent->new_address, maxlen);
     }
 
+    /* Is this now a .exit address?  If so, remember where we got it.*/
     if (!strcmpend(address, ".exit") &&
         strcmpend(addr_orig, ".exit") &&
         exit_source == ADDRMAPSRC_NONE) {
@@ -529,7 +555,7 @@ addressmap_have_mapping(const char *address, int update_expiry)
  * (virtual address mapping) from the controller.)
  *
  * <b>new_address</b> should be a newly dup'ed string, which we'll use or
- * free as appropriate. We will leave address alone.
+ * free as appropriate. We will leave <b>address</b> alone.
  *
  * If <b>wildcard_addr</b> is true, then the mapping will match any address
  * equal to <b>address</b>, or any address ending with a period followed by
@@ -542,7 +568,6 @@ addressmap_have_mapping(const char *address, int update_expiry)
  * <b>wildcard_new_addr</b>, remove any mappings that exist from
  * <b>address</b>.
  *
- *
  * It is an error to set <b>wildcard_new_addr</b> if <b>wildcard_addr</b> is
  * not set. */
 void
@@ -802,7 +827,7 @@ parse_virtual_addr_network(const char *val, sa_family_t family,
                    ipv6?"IPv6":"");
     return -1;
   }
-#endif
+#endif /* 0 */
 
   if (bits > max_prefix_bits) {
     if (msg)
@@ -935,9 +960,11 @@ addressmap_get_virtual_address(int type)
         char tmp[TOR_ADDR_BUF_LEN];
         tor_addr_to_str(tmp, &addr, sizeof(tmp), 0);
         if (strmap_get(addressmap, tmp)) {
+          // LCOV_EXCL_START
           log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.",
                    buf, tmp);
           continue;
+          // LCOV_EXCL_STOP
         }
 
         return tor_strdup(buf);
@@ -946,8 +973,10 @@ addressmap_get_virtual_address(int type)
     log_warn(LD_CONFIG, "Ran out of virtual addresses!");
     return NULL;
   } else {
+    // LCOV_EXCL_START
     log_warn(LD_BUG, "Called with unsupported address type (%d)", type);
     return NULL;
+    // LCOV_EXCL_STOP
   }
 }
 
@@ -1032,7 +1061,7 @@ addressmap_register_virtual_address(int type, char *new_address)
              safe_str_client(*addrp),
              safe_str_client(new_address));
   }
-#endif
+#endif /* 0 */
 
   return *addrp;
 }
diff --git a/src/or/addressmap.h b/src/or/addressmap.h
index 67648d0518..1544b76e10 100644
--- a/src/or/addressmap.h
+++ b/src/or/addressmap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ADDRESSMAP_H
@@ -59,7 +59,7 @@ typedef struct virtual_addr_conf_t {
 
 STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf,
                                     tor_addr_t *addr_out);
-#endif
+#endif /* defined(ADDRESSMAP_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_ADDRESSMAP_H) */
 
diff --git a/src/or/bridges.c b/src/or/bridges.c
new file mode 100644
index 0000000000..699e030e6c
--- /dev/null
+++ b/src/or/bridges.c
@@ -0,0 +1,932 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.c
+ * \brief Code to manage bridges and bridge selection.
+ *
+ * Bridges are fixed entry nodes, used for censorship circumvention.
+ **/
+
+#define TOR_BRIDGES_PRIVATE
+
+#include "or.h"
+#include "bridges.h"
+#include "circuitbuild.h"
+#include "config.h"
+#include "connection.h"
+#include "directory.h"
+#include "entrynodes.h"
+#include "nodelist.h"
+#include "policies.h"
+#include "router.h"
+#include "routerlist.h"
+#include "routerset.h"
+#include "transports.h"
+
+/** Information about a configured bridge. Currently this just matches the
+ * ones in the torrc file, but one day we may be able to learn about new
+ * bridges on our own, and remember them in the state file. */
+struct bridge_info_t {
+  /** Address and port of the bridge, as configured by the user.*/
+  tor_addr_port_t addrport_configured;
+  /** Address of the bridge. */
+  tor_addr_t addr;
+  /** TLS port for the bridge. */
+  uint16_t port;
+  /** Boolean: We are re-parsing our bridge list, and we are going to remove
+   * this one if we don't find it in the list of configured bridges. */
+  unsigned marked_for_removal : 1;
+  /** Expected identity digest, or all zero bytes if we don't know what the
+   * digest should be. */
+  char identity[DIGEST_LEN];
+
+  /** Name of pluggable transport protocol taken from its config line. */
+  char *transport_name;
+
+  /** When should we next try to fetch a descriptor for this bridge? */
+  download_status_t fetch_status;
+
+  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
+      transports are used for this bridge. */
+  smartlist_t *socks_args;
+};
+
+#define bridge_free(bridge) \
+  FREE_AND_NULL(bridge_info_t, bridge_free_, (bridge))
+
+static void bridge_free_(bridge_info_t *bridge);
+static void rewrite_node_address_for_bridge(const bridge_info_t *bridge,
+                                            node_t *node);
+
+/** A list of configured bridges. Whenever we actually get a descriptor
+ * for one, we add it as an entry guard.  Note that the order of bridges
+ * in this list does not necessarily correspond to the order of bridges
+ * in the torrc. */
+static smartlist_t *bridge_list = NULL;
+
+/** Mark every entry of the bridge list to be removed on our next call to
+ * sweep_bridge_list unless it has first been un-marked. */
+void
+mark_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
+                    b->marked_for_removal = 1);
+}
+
+/** Remove every entry of the bridge list that was marked with
+ * mark_bridge_list if it has not subsequently been un-marked. */
+void
+sweep_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+    if (b->marked_for_removal) {
+      SMARTLIST_DEL_CURRENT(bridge_list, b);
+      bridge_free(b);
+    }
+  } SMARTLIST_FOREACH_END(b);
+}
+
+/** Initialize the bridge list to empty, creating it if needed. */
+STATIC void
+clear_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
+  smartlist_clear(bridge_list);
+}
+
+/** Free the bridge <b>bridge</b>. */
+static void
+bridge_free_(bridge_info_t *bridge)
+{
+  if (!bridge)
+    return;
+
+  tor_free(bridge->transport_name);
+  if (bridge->socks_args) {
+    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
+    smartlist_free(bridge->socks_args);
+  }
+
+  tor_free(bridge);
+}
+
+/** Return a list of all the configured bridges, as bridge_info_t pointers. */
+const smartlist_t *
+bridge_list_get(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  return bridge_list;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its RSA identity digest, or
+ * NULL if we don't know one for it.
+ */
+const uint8_t *
+bridge_get_rsa_id_digest(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  if (tor_digest_is_zero(bridge->identity))
+    return NULL;
+  else
+    return (const uint8_t *) bridge->identity;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its configured addr:port
+ * combination.
+ */
+const tor_addr_port_t *
+bridge_get_addr_port(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  return &bridge->addrport_configured;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches any of the
+ * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
+ * NULL. */
+STATIC bridge_info_t *
+get_configured_bridge_by_orports_digest(const char *digest,
+                                        const smartlist_t *orports)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if (tor_digest_is_zero(bridge->identity)) {
+        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
+          {
+            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
+                bridge->port == ap->port)
+              return bridge;
+          }
+        SMARTLIST_FOREACH_END(ap);
+      }
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+bridge_info_t *
+get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
+          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+          bridge->port == port)
+        return bridge;
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/**
+ * As get_configured_bridge_by_addr_port, but require that the
+ * address match <b>addr</b>:<b>port</b>, and that the ID digest match
+ * <b>digest</b>.  (The other function will ignore the address if the
+ * digest matches.)
+ */
+bridge_info_t *
+get_configured_bridge_by_exact_addr_port_digest(const tor_addr_t *addr,
+                                                uint16_t port,
+                                                const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (!tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+        bridge->port == port) {
+
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+      else if (!digest || tor_digest_is_zero(bridge->identity))
+        return bridge;
+    }
+
+  } SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return 1.  Else return 0. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+int
+addr_is_a_configured_bridge(const tor_addr_t *addr,
+                                uint16_t port,
+                                const char *digest)
+{
+  tor_assert(addr);
+  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
+}
+
+/** If we have a bridge configured whose digest matches
+ * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
+ * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
+ * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
+int
+extend_info_is_a_configured_bridge(const extend_info_t *ei)
+{
+  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
+  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
+}
+
+/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
+ * it up via router descriptor <b>ri</b>. */
+static bridge_info_t *
+get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
+{
+  bridge_info_t *bi = NULL;
+  smartlist_t *orports = router_get_all_orports(ri);
+  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
+                                               orports);
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return bi;
+}
+
+/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
+int
+routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
+{
+  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
+}
+
+/** Return 1 if <b>node</b> is one of our configured bridges, else 0. */
+int
+node_is_a_configured_bridge(const node_t *node)
+{
+  int retval = 0;
+  smartlist_t *orports = node_get_all_orports(node);
+  retval = get_configured_bridge_by_orports_digest(node->identity,
+                                                   orports) != NULL;
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return retval;
+}
+
+/** We made a connection to a router at <b>addr</b>:<b>port</b>
+ * without knowing its digest. Its digest turned out to be <b>digest</b>.
+ * If it was a bridge, and we still don't know its digest, record it.
+ */
+void
+learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                        const char *digest,
+                        const ed25519_public_key_t *ed_id)
+{
+  // XXXX prop220 use ed_id here, once there is some way to specify
+  (void)ed_id;
+  int learned = 0;
+  bridge_info_t *bridge =
+    get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  if (bridge && tor_digest_is_zero(bridge->identity)) {
+    memcpy(bridge->identity, digest, DIGEST_LEN);
+    learned = 1;
+  }
+  /* XXXX prop220 remember bridge ed25519 identities -- add a field */
+#if 0
+  if (bridge && ed_id &&
+      ed25519_public_key_is_zero(&bridge->ed25519_identity) &&
+      !ed25519_public_key_is_zero(ed_id)) {
+    memcpy(&bridge->ed25519_identity, ed_id, sizeof(*ed_id));
+    learned = 1;
+  }
+#endif /* 0 */
+  if (learned) {
+    char *transport_info = NULL;
+    const char *transport_name =
+      find_transport_name_by_bridge_addrport(addr, port);
+    if (transport_name)
+      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
+
+    // XXXX prop220 log both fingerprints.
+    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
+               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
+               transport_info ? transport_info : "");
+    tor_free(transport_info);
+    entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                                        (const uint8_t *)digest);
+  }
+}
+
+/** Return true if <b>bridge</b> has the same identity digest as
+ *  <b>digest</b>. If <b>digest</b> is NULL, it matches
+ *  bridges with unspecified identity digests. */
+static int
+bridge_has_digest(const bridge_info_t *bridge, const char *digest)
+{
+  if (digest)
+    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
+  else
+    return tor_digest_is_zero(bridge->identity);
+}
+
+/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
+ * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
+ * existing bridge with the same address and port, and warn the user as
+ * appropriate.
+ */
+STATIC void
+bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
+                         const char *digest, const char *transport_name)
+{
+  /* Iterate the already-registered bridge list:
+
+     If you find a bridge with the same address and port, mark it for
+     removal. It doesn't make sense to have two active bridges with
+     the same IP:PORT. If the bridge in question has a different
+     digest or transport than <b>digest</b>/<b>transport_name</b>,
+     it's probably a misconfiguration and we should warn the user.
+  */
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (bridge->marked_for_removal)
+      continue;
+
+    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
+
+      bridge->marked_for_removal = 1;
+
+      if (!bridge_has_digest(bridge, digest) ||
+          strcmp_opt(bridge->transport_name, transport_name)) {
+        /* warn the user */
+        char *bridge_description_new, *bridge_description_old;
+        tor_asprintf(&bridge_description_new, "%s:%s:%s",
+                     fmt_addrport(addr, port),
+                     digest ? hex_str(digest, DIGEST_LEN) : "",
+                     transport_name ? transport_name : "");
+        tor_asprintf(&bridge_description_old, "%s:%s:%s",
+                     fmt_addrport(&bridge->addr, bridge->port),
+                     tor_digest_is_zero(bridge->identity) ?
+                     "" : hex_str(bridge->identity,DIGEST_LEN),
+                     bridge->transport_name ? bridge->transport_name : "");
+
+        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
+                 " with the already registered bridge '%s'. We will discard"
+                 " the old bridge and keep '%s'. If this is not what you"
+                 " wanted, please change your configuration file accordingly.",
+                 bridge_description_new, bridge_description_old,
+                 bridge_description_new);
+
+        tor_free(bridge_description_new);
+        tor_free(bridge_description_old);
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+}
+
+/** Return True if we have a bridge that uses a transport with name
+ *  <b>transport_name</b>. */
+MOCK_IMPL(int,
+transport_is_needed, (const char *transport_name))
+{
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (bridge->transport_name &&
+        !strcmp(bridge->transport_name, transport_name))
+      return 1;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return 0;
+}
+
+/** Register the bridge information in <b>bridge_line</b> to the
+ *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
+void
+bridge_add_from_config(bridge_line_t *bridge_line)
+{
+  bridge_info_t *b;
+
+  // XXXX prop220 add a way to specify ed25519 ID to bridge_line_t.
+
+  { /* Log the bridge we are about to register: */
+    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
+              fmt_addrport(&bridge_line->addr, bridge_line->port),
+              bridge_line->transport_name ?
+              bridge_line->transport_name : "no transport",
+              tor_digest_is_zero(bridge_line->digest) ?
+              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
+
+    if (bridge_line->socks_args) { /* print socks arguments */
+      int i = 0;
+
+      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
+
+      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
+                smartlist_len(bridge_line->socks_args));
+      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
+                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
+    }
+  }
+
+  bridge_resolve_conflicts(&bridge_line->addr,
+                           bridge_line->port,
+                           bridge_line->digest,
+                           bridge_line->transport_name);
+
+  b = tor_malloc_zero(sizeof(bridge_info_t));
+  tor_addr_copy(&b->addrport_configured.addr, &bridge_line->addr);
+  b->addrport_configured.port = bridge_line->port;
+  tor_addr_copy(&b->addr, &bridge_line->addr);
+  b->port = bridge_line->port;
+  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
+  if (bridge_line->transport_name)
+    b->transport_name = bridge_line->transport_name;
+  b->fetch_status.schedule = DL_SCHED_BRIDGE;
+  b->fetch_status.increment_on = DL_SCHED_INCREMENT_ATTEMPT;
+  /* We can't reset the bridge's download status here, because UseBridges
+   * might be 0 now, and it might be changed to 1 much later. */
+  b->socks_args = bridge_line->socks_args;
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+
+  tor_free(bridge_line); /* Deallocate bridge_line now. */
+
+  smartlist_add(bridge_list, b);
+}
+
+/** If <b>digest</b> is one of our known bridges, return it. */
+STATIC bridge_info_t *
+find_bridge_by_digest(const char *digest)
+{
+  if (! bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
+    {
+      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    });
+  return NULL;
+}
+
+/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
+ *  supports a pluggable transport, return its name. Otherwise, return
+ *  NULL. */
+const char *
+find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  if (!bridge_list)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port))
+      return bridge->transport_name;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return NULL;
+}
+
+/** If <b>addr</b> and <b>port</b> match the address and port of a
+ * bridge of ours that uses pluggable transports, place its transport
+ * in <b>transport</b>.
+ *
+ * Return 0 on success (found a transport, or found a bridge with no
+ * transport, or found no bridge); return -1 if we should be using a
+ * transport, but the transport could not be found.
+ */
+int
+get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                  const transport_t **transport)
+{
+  *transport = NULL;
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port)) { /* bridge matched */
+      if (bridge->transport_name) { /* it also uses pluggable transports */
+        *transport = transport_get_by_name(bridge->transport_name);
+        if (*transport == NULL) { /* it uses pluggable transports, but
+                                     the transport could not be found! */
+          return -1;
+        }
+        return 0;
+      } else { /* bridge matched, but it doesn't use transports. */
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+
+  *transport = NULL;
+  return 0;
+}
+
+/** Return a smartlist containing all the SOCKS arguments that we
+ *  should pass to the SOCKS proxy. */
+const smartlist_t *
+get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
+                                                                    port,
+                                                                    NULL);
+  return bridge ? bridge->socks_args : NULL;
+}
+
+/** We need to ask <b>bridge</b> for its server descriptor. */
+static void
+launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
+{
+  const or_options_t *options = get_options();
+  circuit_guard_state_t *guard_state = NULL;
+
+  if (connection_get_by_type_addr_port_purpose(
+      CONN_TYPE_DIR, &bridge->addr, bridge->port,
+      DIR_PURPOSE_FETCH_SERVERDESC))
+    return; /* it's already on the way */
+
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+    download_status_mark_impossible(&bridge->fetch_status);
+    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+    return;
+  }
+
+  /* Until we get a descriptor for the bridge, we only know one address for
+   * it. */
+  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                            FIREWALL_OR_CONNECTION, 0, 0)) {
+    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
+               "bridge, but that bridge is not reachable through our "
+               "firewall.");
+    return;
+  }
+
+  /* If we already have a node_t for this bridge, rewrite its address now. */
+  node_t *node = node_get_mutable_by_id(bridge->identity);
+  if (node) {
+    rewrite_node_address_for_bridge(bridge, node);
+  }
+
+  tor_addr_port_t bridge_addrport;
+  memcpy(&bridge_addrport.addr, &bridge->addr, sizeof(tor_addr_t));
+  bridge_addrport.port = bridge->port;
+
+  guard_state = get_guard_state_for_bridge_desc_fetch(bridge->identity);
+
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+  directory_request_set_or_addr_port(req, &bridge_addrport);
+  directory_request_set_directory_id_digest(req, bridge->identity);
+  directory_request_set_router_purpose(req, ROUTER_PURPOSE_BRIDGE);
+  directory_request_set_resource(req, "authority.z");
+  if (guard_state) {
+    directory_request_set_guard_state(req, guard_state);
+  }
+  directory_initiate_request(req);
+  directory_request_free(req);
+}
+
+/** Fetching the bridge descriptor from the bridge authority returned a
+ * "not found". Fall back to trying a direct fetch. */
+void
+retry_bridge_descriptor_fetch_directly(const char *digest)
+{
+  bridge_info_t *bridge = find_bridge_by_digest(digest);
+  if (!bridge)
+    return; /* not found? oh well. */
+
+  launch_direct_bridge_descriptor_fetch(bridge);
+}
+
+/** For each bridge in our list for which we don't currently have a
+ * descriptor, fetch a new copy of its descriptor -- either directly
+ * from the bridge or via a bridge authority. */
+void
+fetch_bridge_descriptors(const or_options_t *options, time_t now)
+{
+  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
+  int ask_bridge_directly;
+  int can_use_bridge_authority;
+
+  if (!bridge_list)
+    return;
+
+  /* If we still have unconfigured managed proxies, don't go and
+     connect to a bridge. */
+  if (pt_proxies_configuration_pending())
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      /* This resets the download status on first use */
+      if (!download_status_is_ready(&bridge->fetch_status, now))
+        continue; /* don't bother, no need to retry yet */
+      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+        download_status_mark_impossible(&bridge->fetch_status);
+        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+        continue;
+      }
+
+      /* schedule the next attempt
+       * we can't increment after a failure, because sometimes we use the
+       * bridge authority, and sometimes we use the bridge direct */
+      download_status_increment_attempt(
+                        &bridge->fetch_status,
+                        safe_str_client(fmt_and_decorate_addr(&bridge->addr)),
+                        now);
+
+      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
+                                 num_bridge_auths;
+      ask_bridge_directly = !can_use_bridge_authority ||
+                            !options->UpdateBridgesFromAuthority;
+      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
+                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
+                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
+
+      if (ask_bridge_directly &&
+          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                                FIREWALL_OR_CONNECTION, 0,
+                                                0)) {
+        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
+                   "firewall policy. %s.",
+                   fmt_addrport(&bridge->addr, bridge->port),
+                   can_use_bridge_authority ?
+                     "Asking bridge authority instead" : "Skipping");
+        if (can_use_bridge_authority)
+          ask_bridge_directly = 0;
+        else
+          continue;
+      }
+
+      if (ask_bridge_directly) {
+        /* we need to ask the bridge itself for its descriptor. */
+        launch_direct_bridge_descriptor_fetch(bridge);
+      } else {
+        /* We have a digest and we want to ask an authority. We could
+         * combine all the requests into one, but that may give more
+         * hints to the bridge authority than we want to give. */
+        char resource[10 + HEX_DIGEST_LEN];
+        memcpy(resource, "fp/", 3);
+        base16_encode(resource+3, HEX_DIGEST_LEN+1,
+                      bridge->identity, DIGEST_LEN);
+        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
+        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
+                 resource);
+        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
+      }
+    }
+  SMARTLIST_FOREACH_END(bridge);
+}
+
+/** If our <b>bridge</b> is configured to be a different address than
+ * the bridge gives in <b>node</b>, rewrite the routerinfo
+ * we received to use the address we meant to use. Now we handle
+ * multihomed bridges better.
+ */
+static void
+rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
+{
+  /* XXXX move this function. */
+  /* XXXX overridden addresses should really live in the node_t, so that the
+   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
+   *   do that safely if we know that no function that connects to an OR
+   *   does so through an address from any source other than node_get_addr().
+   */
+  tor_addr_t addr;
+  const or_options_t *options = get_options();
+
+  if (node->ri) {
+    routerinfo_t *ri = node->ri;
+    tor_addr_from_ipv4h(&addr, ri->addr);
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+         bridge->port == ri->or_port) ||
+        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
+         bridge->port == ri->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
+        ri->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s:%d.",
+                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
+        ri->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s.",
+                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+       the same relay, warn the user */
+    if (!tor_addr_is_null(&ri->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 ri->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+  if (node->rs) {
+    routerstatus_t *rs = node->rs;
+    tor_addr_from_ipv4h(&addr, rs->addr);
+
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+        bridge->port == rs->or_port) ||
+       (!tor_addr_compare(&bridge->addr, &rs->ipv6_addr, CMP_EXACT) &&
+        bridge->port == rs->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        rs->addr = tor_addr_to_ipv4h(&bridge->addr);
+        rs->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerstatus for '%s' to match "
+                 "configured address %s.",
+                 rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
+      /* set IPv6 preferences even if there is no ri */
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&rs->ipv6_addr, &bridge->addr);
+        rs->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerstatus for '%s' to match configured"
+                 " address %s.",
+                 rs->nickname, fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->rs->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->rs->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+    the same relay, warn the user */
+    if (!tor_addr_is_null(&rs->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 rs->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+}
+
+/** We just learned a descriptor for a bridge. See if that
+ * digest is in our entry guard list, and add it if not. */
+void
+learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
+{
+  tor_assert(ri);
+  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
+  if (get_options()->UseBridges) {
+    /* Retry directory downloads whenever we get a bridge descriptor:
+     * - when bootstrapping, and
+     * - when we aren't sure if any of our bridges are reachable.
+     * Keep on retrying until we have at least one reachable bridge. */
+    int first = num_bridges_usable(0) < 1;
+    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
+    time_t now = time(NULL);
+    router_set_status(ri->cache_info.identity_digest, 1);
+
+    if (bridge) { /* if we actually want to use this one */
+      node_t *node;
+      /* it's here; schedule its re-fetch for a long time from now. */
+      if (!from_cache) {
+        /* This schedules the re-fetch at a constant interval, which produces
+         * a pattern of bridge traffic. But it's better than trying all
+         * configured briges several times in the first few minutes. */
+        download_status_reset(&bridge->fetch_status);
+      }
+
+      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      tor_assert(node);
+      rewrite_node_address_for_bridge(bridge, node);
+      if (tor_digest_is_zero(bridge->identity)) {
+        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
+        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
+                   hex_str(bridge->identity, DIGEST_LEN),
+                   fmt_and_decorate_addr(&bridge->addr),
+                   (int) bridge->port);
+      }
+      entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                              (const uint8_t*)ri->cache_info.identity_digest);
+
+      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
+                 from_cache ? "cached" : "fresh", router_describe(ri));
+      /* If we didn't have a reachable bridge before this one, try directory
+       * documents again. */
+      if (first) {
+        routerlist_retry_directory_downloads(now);
+      }
+    }
+  }
+}
+
+/** Return a smartlist containing all bridge identity digests */
+MOCK_IMPL(smartlist_t *,
+list_bridge_identities, (void))
+{
+  smartlist_t *result = NULL;
+  char *digest_tmp;
+
+  if (get_options()->UseBridges && bridge_list) {
+    result = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      digest_tmp = tor_malloc(DIGEST_LEN);
+      memcpy(digest_tmp, b->identity, DIGEST_LEN);
+      smartlist_add(result, digest_tmp);
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return result;
+}
+
+/** Get the download status for a bridge descriptor given its identity */
+MOCK_IMPL(download_status_t *,
+get_bridge_dl_status_by_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+
+  if (digest && get_options()->UseBridges && bridge_list) {
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
+        dl = &(b->fetch_status);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return dl;
+}
+
+/** Release all storage held in bridges.c */
+void
+bridges_free_all(void)
+{
+  clear_bridge_list();
+  smartlist_free(bridge_list);
+  bridge_list = NULL;
+}
+
diff --git a/src/or/bridges.h b/src/or/bridges.h
new file mode 100644
index 0000000000..3108eb555d
--- /dev/null
+++ b/src/or/bridges.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_BRIDGES_H
+#define TOR_BRIDGES_H
+
+struct bridge_line_t;
+
+/* Opaque handle to a configured bridge */
+typedef struct bridge_info_t bridge_info_t;
+
+void mark_bridge_list(void);
+void sweep_bridge_list(void);
+const smartlist_t *bridge_list_get(void);
+const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge);
+const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge);
+bridge_info_t *get_configured_bridge_by_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+bridge_info_t *get_configured_bridge_by_exact_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+
+int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
+                                const char *digest);
+int extend_info_is_a_configured_bridge(const extend_info_t *ei);
+int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
+int node_is_a_configured_bridge(const node_t *node);
+void learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                             const char *digest,
+                             const ed25519_public_key_t *ed_id);
+
+void bridge_add_from_config(struct bridge_line_t *bridge_line);
+void retry_bridge_descriptor_fetch_directly(const char *digest);
+void fetch_bridge_descriptors(const or_options_t *options, time_t now);
+void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
+const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
+                                                     uint16_t port);
+
+int any_bridges_dont_support_microdescriptors(void);
+
+const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
+                                                   uint16_t port);
+struct transport_t;
+int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                      const struct transport_t **transport);
+
+MOCK_DECL(int, transport_is_needed, (const char *transport_name));
+int validate_pluggable_transports_config(void);
+
+MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
+MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
+          (const char *digest));
+
+void bridges_free_all(void);
+
+#ifdef TOR_BRIDGES_PRIVATE
+STATIC void clear_bridge_list(void);
+STATIC bridge_info_t *find_bridge_by_digest(const char *digest);
+STATIC bridge_info_t *get_configured_bridge_by_orports_digest(
+                                                  const char *digest,
+                                                  const smartlist_t *orports);
+STATIC void bridge_resolve_conflicts(const tor_addr_t *addr,
+                                     uint16_t port,
+                                     const char *digest,
+                                     const char *transport_name);
+#endif /* defined(TOR_BRIDGES_PRIVATE) */
+
+#endif /* !defined(TOR_BRIDGES_H) */
+
diff --git a/src/or/buffers.c b/src/or/buffers.c
deleted file mode 100644
index 89382d1d8e..0000000000
--- a/src/or/buffers.c
+++ /dev/null
@@ -1,2063 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file buffers.c
- * \brief Implements a generic buffer interface.
- *
- * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
- * from memory, sockets, file descriptors, TLS connections, or another buf_t.
- * Buffers are implemented as linked lists of memory chunks.
- *
- * All socket-backed and TLS-based connection_t objects have a pair of
- * buffers: one for incoming data, and one for outcoming data.  These are fed
- * and drained from functions in connection.c, trigged by events that are
- * monitored in main.c.
- *
- * This module has basic support for reading and writing on buf_t objects. It
- * also contains specialized functions for handling particular protocols
- * on a buf_t backend, including SOCKS (used in connection_edge.c), Tor cells
- * (used in connection_or.c and channeltls.c), HTTP (used in directory.c), and
- * line-oriented communication (used in control.c).
- **/
-#define BUFFERS_PRIVATE
-#include "or.h"
-#include "addressmap.h"
-#include "buffers.h"
-#include "config.h"
-#include "connection_edge.h"
-#include "connection_or.h"
-#include "control.h"
-#include "reasons.h"
-#include "ext_orport.h"
-#include "util.h"
-#include "torlog.h"
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-//#define PARANOIA
-
-#ifdef PARANOIA
-/** Helper: If PARANOIA is defined, assert that the buffer in local variable
- * <b>buf</b> is well-formed. */
-#define check() STMT_BEGIN assert_buf_ok(buf); STMT_END
-#else
-#define check() STMT_NIL
-#endif
-
-/* Implementation notes:
- *
- * After flirting with memmove, and dallying with ring-buffers, we're finally
- * getting up to speed with the 1970s and implementing buffers as a linked
- * list of small chunks.  Each buffer has such a list; data is removed from
- * the head of the list, and added at the tail.  The list is singly linked,
- * and the buffer keeps a pointer to the head and the tail.
- *
- * Every chunk, except the tail, contains at least one byte of data.  Data in
- * each chunk is contiguous.
- *
- * When you need to treat the first N characters on a buffer as a contiguous
- * string, use the buf_pullup function to make them so.  Don't do this more
- * than necessary.
- *
- * The major free Unix kernels have handled buffers like this since, like,
- * forever.
- */
-
-static void socks_request_set_socks5_error(socks_request_t *req,
-                              socks5_reply_status_t reason);
-
-static int parse_socks(const char *data, size_t datalen, socks_request_t *req,
-                       int log_sockstype, int safe_socks, ssize_t *drain_out,
-                       size_t *want_length_out);
-static int parse_socks_client(const uint8_t *data, size_t datalen,
-                              int state, char **reason,
-                              ssize_t *drain_out);
-
-/* Chunk manipulation functions */
-
-#define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0])
-
-/* We leave this many NUL bytes at the end of the buffer. */
-#define SENTINEL_LEN 4
-
-/* Header size plus NUL bytes at the end */
-#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
-
-/** Return the number of bytes needed to allocate a chunk to hold
- * <b>memlen</b> bytes. */
-#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
-/** Return the number of usable bytes in a chunk allocated with
- * malloc(<b>memlen</b>). */
-#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
-
-#define DEBUG_SENTINEL
-
-#ifdef DEBUG_SENTINEL
-#define DBG_S(s) s
-#else
-#define DBG_S(s) (void)0
-#endif
-
-#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
-    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
-    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
-    DBG_S(tor_assert(a == b));                                          \
-    memset(a,0,SENTINEL_LEN);                                           \
-  } while (0)
-
-/** Return the next character in <b>chunk</b> onto which data can be appended.
- * If the chunk is full, this might be off the end of chunk->mem. */
-static inline char *
-CHUNK_WRITE_PTR(chunk_t *chunk)
-{
-  return chunk->data + chunk->datalen;
-}
-
-/** Return the number of bytes that can be written onto <b>chunk</b> without
- * running out of space. */
-static inline size_t
-CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
-{
-  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
-}
-
-/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
- * to free up space at the end. */
-static inline void
-chunk_repack(chunk_t *chunk)
-{
-  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
-    memmove(chunk->mem, chunk->data, chunk->datalen);
-  }
-  chunk->data = &chunk->mem[0];
-}
-
-/** Keep track of total size of allocated chunks for consistency asserts */
-static size_t total_bytes_allocated_in_chunks = 0;
-static void
-buf_chunk_free_unchecked(chunk_t *chunk)
-{
-  if (!chunk)
-    return;
-#ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
-#endif
-  tor_assert(total_bytes_allocated_in_chunks >=
-             CHUNK_ALLOC_SIZE(chunk->memlen));
-  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
-  tor_free(chunk);
-}
-static inline chunk_t *
-chunk_new_with_alloc_size(size_t alloc)
-{
-  chunk_t *ch;
-  ch = tor_malloc(alloc);
-  ch->next = NULL;
-  ch->datalen = 0;
-#ifdef DEBUG_CHUNK_ALLOC
-  ch->DBG_alloc = alloc;
-#endif
-  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
-  total_bytes_allocated_in_chunks += alloc;
-  ch->data = &ch->mem[0];
-  CHUNK_SET_SENTINEL(ch, alloc);
-  return ch;
-}
-
-/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
- * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
-static inline chunk_t *
-chunk_grow(chunk_t *chunk, size_t sz)
-{
-  off_t offset;
-  const size_t memlen_orig = chunk->memlen;
-  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
-  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
-  tor_assert(sz > chunk->memlen);
-  offset = chunk->data - chunk->mem;
-  chunk = tor_realloc(chunk, new_alloc);
-  chunk->memlen = sz;
-  chunk->data = chunk->mem + offset;
-#ifdef DEBUG_CHUNK_ALLOC
-  tor_assert(chunk->DBG_alloc == orig_alloc);
-  chunk->DBG_alloc = new_alloc;
-#endif
-  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
-  CHUNK_SET_SENTINEL(chunk, new_alloc);
-  return chunk;
-}
-
-/** If a read onto the end of a chunk would be smaller than this number, then
- * just start a new chunk. */
-#define MIN_READ_LEN 8
-/** Every chunk should take up at least this many bytes. */
-#define MIN_CHUNK_ALLOC 256
-/** No chunk should take up more than this many bytes. */
-#define MAX_CHUNK_ALLOC 65536
-
-/** Return the allocation size we'd like to use to hold <b>target</b>
- * bytes. */
-STATIC size_t
-preferred_chunk_size(size_t target)
-{
-  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
-  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
-    return CHUNK_ALLOC_SIZE(target);
-  size_t sz = MIN_CHUNK_ALLOC;
-  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
-    sz <<= 1;
-  }
-  return sz;
-}
-
-/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
- * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
- * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
- */
-STATIC void
-buf_pullup(buf_t *buf, size_t bytes)
-{
-  chunk_t *dest, *src;
-  size_t capacity;
-  if (!buf->head)
-    return;
-
-  check();
-  if (buf->datalen < bytes)
-    bytes = buf->datalen;
-
-  capacity = bytes;
-  if (buf->head->datalen >= bytes)
-    return;
-
-  if (buf->head->memlen >= capacity) {
-    /* We don't need to grow the first chunk, but we might need to repack it.*/
-    size_t needed = capacity - buf->head->datalen;
-    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
-      chunk_repack(buf->head);
-    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
-  } else {
-    chunk_t *newhead;
-    size_t newsize;
-    /* We need to grow the chunk. */
-    chunk_repack(buf->head);
-    newsize = CHUNK_SIZE_WITH_ALLOC(preferred_chunk_size(capacity));
-    newhead = chunk_grow(buf->head, newsize);
-    tor_assert(newhead->memlen >= capacity);
-    if (newhead != buf->head) {
-      if (buf->tail == buf->head)
-        buf->tail = newhead;
-      buf->head = newhead;
-    }
-  }
-
-  dest = buf->head;
-  while (dest->datalen < bytes) {
-    size_t n = bytes - dest->datalen;
-    src = dest->next;
-    tor_assert(src);
-    if (n >= src->datalen) {
-      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
-      dest->datalen += src->datalen;
-      dest->next = src->next;
-      if (buf->tail == src)
-        buf->tail = dest;
-      buf_chunk_free_unchecked(src);
-    } else {
-      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
-      dest->datalen += n;
-      src->data += n;
-      src->datalen -= n;
-      tor_assert(dest->datalen == bytes);
-    }
-  }
-
-  check();
-}
-
-#ifdef TOR_UNIT_TESTS
-void
-buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz)
-{
-  if (!buf || !buf->head) {
-    *cp = NULL;
-    *sz = 0;
-  } else {
-    *cp = buf->head->data;
-    *sz = buf->head->datalen;
-  }
-}
-#endif
-
-/** Remove the first <b>n</b> bytes from buf. */
-static inline void
-buf_remove_from_front(buf_t *buf, size_t n)
-{
-  tor_assert(buf->datalen >= n);
-  while (n) {
-    tor_assert(buf->head);
-    if (buf->head->datalen > n) {
-      buf->head->datalen -= n;
-      buf->head->data += n;
-      buf->datalen -= n;
-      return;
-    } else {
-      chunk_t *victim = buf->head;
-      n -= victim->datalen;
-      buf->datalen -= victim->datalen;
-      buf->head = victim->next;
-      if (buf->tail == victim)
-        buf->tail = NULL;
-      buf_chunk_free_unchecked(victim);
-    }
-  }
-  check();
-}
-
-/** Create and return a new buf with default chunk capacity <b>size</b>.
- */
-buf_t *
-buf_new_with_capacity(size_t size)
-{
-  buf_t *b = buf_new();
-  b->default_chunk_size = preferred_chunk_size(size);
-  return b;
-}
-
-/** Allocate and return a new buffer with default capacity. */
-buf_t *
-buf_new(void)
-{
-  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
-  buf->magic = BUFFER_MAGIC;
-  buf->default_chunk_size = 4096;
-  return buf;
-}
-
-size_t
-buf_get_default_chunk_size(const buf_t *buf)
-{
-  return buf->default_chunk_size;
-}
-
-/** Remove all data from <b>buf</b>. */
-void
-buf_clear(buf_t *buf)
-{
-  chunk_t *chunk, *next;
-  buf->datalen = 0;
-  for (chunk = buf->head; chunk; chunk = next) {
-    next = chunk->next;
-    buf_chunk_free_unchecked(chunk);
-  }
-  buf->head = buf->tail = NULL;
-}
-
-/** Return the number of bytes stored in <b>buf</b> */
-MOCK_IMPL(size_t,
-buf_datalen, (const buf_t *buf))
-{
-  return buf->datalen;
-}
-
-/** Return the total length of all chunks used in <b>buf</b>. */
-size_t
-buf_allocation(const buf_t *buf)
-{
-  size_t total = 0;
-  const chunk_t *chunk;
-  for (chunk = buf->head; chunk; chunk = chunk->next) {
-    total += CHUNK_ALLOC_SIZE(chunk->memlen);
-  }
-  return total;
-}
-
-/** Return the number of bytes that can be added to <b>buf</b> without
- * performing any additional allocation. */
-size_t
-buf_slack(const buf_t *buf)
-{
-  if (!buf->tail)
-    return 0;
-  else
-    return CHUNK_REMAINING_CAPACITY(buf->tail);
-}
-
-/** Release storage held by <b>buf</b>. */
-void
-buf_free(buf_t *buf)
-{
-  if (!buf)
-    return;
-
-  buf_clear(buf);
-  buf->magic = 0xdeadbeef;
-  tor_free(buf);
-}
-
-/** Return a new copy of <b>in_chunk</b> */
-static chunk_t *
-chunk_copy(const chunk_t *in_chunk)
-{
-  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
-  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
-#ifdef DEBUG_CHUNK_ALLOC
-  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
-#endif
-  newch->next = NULL;
-  if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
-    newch->data = newch->mem + offset;
-  }
-  return newch;
-}
-
-/** Return a new copy of <b>buf</b> */
-buf_t *
-buf_copy(const buf_t *buf)
-{
-  chunk_t *ch;
-  buf_t *out = buf_new();
-  out->default_chunk_size = buf->default_chunk_size;
-  for (ch = buf->head; ch; ch = ch->next) {
-    chunk_t *newch = chunk_copy(ch);
-    if (out->tail) {
-      out->tail->next = newch;
-      out->tail = newch;
-    } else {
-      out->head = out->tail = newch;
-    }
-  }
-  out->datalen = buf->datalen;
-  return out;
-}
-
-/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
- * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
- * than MAX_CHUNK_ALLOC. */
-static chunk_t *
-buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
-{
-  chunk_t *chunk;
-
-  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
-    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
-  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
-    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
-  } else {
-    chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity));
-  }
-
-  chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec();
-
-  if (buf->tail) {
-    tor_assert(buf->head);
-    buf->tail->next = chunk;
-    buf->tail = chunk;
-  } else {
-    tor_assert(!buf->head);
-    buf->head = buf->tail = chunk;
-  }
-  check();
-  return chunk;
-}
-
-/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
- * milliseconds.  Requires the current monotonic time, in truncated msec,
- * as its input <b>now</b>.
- */
-uint32_t
-buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
-{
-  if (buf->head) {
-    return now - buf->head->inserted_time;
-  } else {
-    return 0;
-  }
-}
-
-size_t
-buf_get_total_allocation(void)
-{
-  return total_bytes_allocated_in_chunks;
-}
-
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
- * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
-static inline int
-read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
-{
-  ssize_t read_result;
-  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
-    at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
-
-  if (read_result < 0) {
-    int e = tor_socket_errno(fd);
-    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
-#ifdef _WIN32
-      if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
-#endif
-      *socket_error = e;
-      return -1;
-    }
-    return 0; /* would block. */
-  } else if (read_result == 0) {
-    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
-    *reached_eof = 1;
-    return 0;
-  } else { /* actually got bytes. */
-    buf->datalen += read_result;
-    chunk->datalen += read_result;
-    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
-              (int)buf->datalen);
-    tor_assert(read_result < INT_MAX);
-    return (int)read_result;
-  }
-}
-
-/** As read_to_chunk(), but return (negative) error code on error, blocking,
- * or TLS, and the number of bytes read otherwise. */
-static inline int
-read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
-                  size_t at_most)
-{
-  int read_result;
-
-  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
-  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
-  if (read_result < 0)
-    return read_result;
-  buf->datalen += read_result;
-  chunk->datalen += read_result;
-  return read_result;
-}
-
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
- */
-/* XXXX indicate "read blocked" somehow? */
-int
-read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
-            int *socket_error)
-{
-  /* XXXX It's stupid to overload the return values for these functions:
-   * "error status" and "number of bytes read" are not mutually exclusive.
-   */
-  int r = 0;
-  size_t total_read = 0;
-
-  check();
-  tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
-    check();
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
-      break;
-    }
-  }
-  return (int)total_read;
-}
-
-/** As read_to_buf, but reads from a TLS connection, and returns a TLS
- * status value rather than the number of bytes read.
- *
- * Using TLS on OR connections complicates matters in two ways.
- *
- * First, a TLS stream has its own read buffer independent of the
- * connection's read buffer.  (TLS needs to read an entire frame from
- * the network before it can decrypt any data.  Thus, trying to read 1
- * byte from TLS can require that several KB be read from the network
- * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
- * Because the data hasn't been read by Tor (it's still inside the TLS),
- * this means that sometimes a connection "has stuff to read" even when
- * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
- * used in connection.c to detect TLS objects with non-empty internal
- * buffers and read from them again.
- *
- * Second, the TLS stream's events do not correspond directly to network
- * events: sometimes, before a TLS stream can read, the network must be
- * ready to write -- or vice versa.
- */
-int
-read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf)
-{
-  int r = 0;
-  size_t total_read = 0;
-
-  check_no_tls_errors();
-
-  check();
-
-  while (at_most > total_read) {
-    size_t readlen = at_most - total_read;
-    chunk_t *chunk;
-    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
-      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
-      if (readlen > chunk->memlen)
-        readlen = chunk->memlen;
-    } else {
-      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
-      chunk = buf->tail;
-      if (cap < readlen)
-        readlen = cap;
-    }
-
-    r = read_to_chunk_tls(buf, chunk, tls, readlen);
-    check();
-    if (r < 0)
-      return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
-    total_read += r;
-    if ((size_t)r < readlen) /* eof, block, or no more to read. */
-      break;
-  }
-  return (int)total_read;
-}
-
-/** Helper for flush_buf(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
- */
-static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
-{
-  ssize_t write_result;
-
-  if (sz > chunk->datalen)
-    sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
-
-  if (write_result < 0) {
-    int e = tor_socket_errno(s);
-    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
-#ifdef _WIN32
-      if (e == WSAENOBUFS)
-        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
-#endif
-      return -1;
-    }
-    log_debug(LD_NET,"write() would block, returning.");
-    return 0;
-  } else {
-    *buf_flushlen -= write_result;
-    buf_remove_from_front(buf, write_result);
-    tor_assert(write_result < INT_MAX);
-    return (int)write_result;
-  }
-}
-
-/** Helper for flush_buf_tls(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
- * more if there is a forced pending write size.)  On success, deduct the
- * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, and a TOR_TLS error code on failure or blocking.
- */
-static inline int
-flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
-                size_t sz, size_t *buf_flushlen)
-{
-  int r;
-  size_t forced;
-  char *data;
-
-  forced = tor_tls_get_forced_write_size(tls);
-  if (forced > sz)
-    sz = forced;
-  if (chunk) {
-    data = chunk->data;
-    tor_assert(sz <= chunk->datalen);
-  } else {
-    data = NULL;
-    tor_assert(sz == 0);
-  }
-  r = tor_tls_write(tls, data, sz);
-  if (r < 0)
-    return r;
-  if (*buf_flushlen > (size_t)r)
-    *buf_flushlen -= r;
-  else
-    *buf_flushlen = 0;
-  buf_remove_from_front(buf, r);
-  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
-            r,(int)*buf_flushlen,(int)buf->datalen);
-  return r;
-}
-
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
- * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
- * the number of bytes actually written, and remove the written bytes
- * from the buffer.  Return the number of bytes written on success,
- * -1 on failure.  Return 0 if write() would block.
- */
-int
-flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen)
-{
-  /* XXXX It's stupid to overload the return values for these functions:
-   * "error status" and "number of bytes flushed" are not mutually exclusive.
-   */
-  int r;
-  size_t flushed = 0;
-  tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
-  tor_assert(*buf_flushlen <= buf->datalen);
-  tor_assert(sz <= *buf_flushlen);
-
-  check();
-  while (sz) {
-    size_t flushlen0;
-    tor_assert(buf->head);
-    if (buf->head->datalen >= sz)
-      flushlen0 = sz;
-    else
-      flushlen0 = buf->head->datalen;
-
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
-    check();
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
-      break;
-  }
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-
-/** As flush_buf(), but writes data to a TLS connection.  Can write more than
- * <b>flushlen</b> bytes.
- */
-int
-flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t flushlen,
-              size_t *buf_flushlen)
-{
-  int r;
-  size_t flushed = 0;
-  ssize_t sz;
-  tor_assert(buf_flushlen);
-  tor_assert(*buf_flushlen <= buf->datalen);
-  tor_assert(flushlen <= *buf_flushlen);
-  sz = (ssize_t) flushlen;
-
-  /* we want to let tls write even if flushlen is zero, because it might
-   * have a partial record pending */
-  check_no_tls_errors();
-
-  check();
-  do {
-    size_t flushlen0;
-    if (buf->head) {
-      if ((ssize_t)buf->head->datalen >= sz)
-        flushlen0 = sz;
-      else
-        flushlen0 = buf->head->datalen;
-    } else {
-      flushlen0 = 0;
-    }
-
-    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
-    check();
-    if (r < 0)
-      return r;
-    flushed += r;
-    sz -= r;
-    if (r == 0) /* Can't flush any more now. */
-      break;
-  } while (sz > 0);
-  tor_assert(flushed < INT_MAX);
-  return (int)flushed;
-}
-
-/** Append <b>string_len</b> bytes from <b>string</b> to the end of
- * <b>buf</b>.
- *
- * Return the new length of the buffer on success, -1 on failure.
- */
-int
-write_to_buf(const char *string, size_t string_len, buf_t *buf)
-{
-  if (!string_len)
-    return (int)buf->datalen;
-  check();
-
-  while (string_len) {
-    size_t copy;
-    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
-      buf_add_chunk_with_capacity(buf, string_len, 1);
-
-    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
-    if (copy > string_len)
-      copy = string_len;
-    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
-    string_len -= copy;
-    string += copy;
-    buf->datalen += copy;
-    buf->tail->datalen += copy;
-  }
-
-  check();
-  tor_assert(buf->datalen < INT_MAX);
-  return (int)buf->datalen;
-}
-
-/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
- * onto <b>string</b>.
- */
-static inline void
-peek_from_buf(char *string, size_t string_len, const buf_t *buf)
-{
-  chunk_t *chunk;
-
-  tor_assert(string);
-  /* make sure we don't ask for too much */
-  tor_assert(string_len <= buf->datalen);
-  /* assert_buf_ok(buf); */
-
-  chunk = buf->head;
-  while (string_len) {
-    size_t copy = string_len;
-    tor_assert(chunk);
-    if (chunk->datalen < copy)
-      copy = chunk->datalen;
-    memcpy(string, chunk->data, copy);
-    string_len -= copy;
-    string += copy;
-    chunk = chunk->next;
-  }
-}
-
-/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
- * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
- * must be \<= the number of bytes on the buffer.
- */
-int
-fetch_from_buf(char *string, size_t string_len, buf_t *buf)
-{
-  /* There must be string_len bytes in buf; write them onto string,
-   * then memmove buf back (that is, remove them from buf).
-   *
-   * Return the number of bytes still on the buffer. */
-
-  check();
-  peek_from_buf(string, string_len, buf);
-  buf_remove_from_front(buf, string_len);
-  check();
-  tor_assert(buf->datalen < INT_MAX);
-  return (int)buf->datalen;
-}
-
-/** True iff the cell command <b>command</b> is one that implies a
- * variable-length cell in Tor link protocol <b>linkproto</b>. */
-static inline int
-cell_command_is_var_length(uint8_t command, int linkproto)
-{
-  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
-   * work as implemented here. If it's 1, there are no variable-length cells.
-   * Tor does not support other versions right now, and so can't negotiate
-   * them.
-   */
-  switch (linkproto) {
-  case 1:
-    /* Link protocol version 1 has no variable-length cells. */
-    return 0;
-  case 2:
-    /* In link protocol version 2, VERSIONS is the only variable-length cell */
-    return command == CELL_VERSIONS;
-  case 0:
-  case 3:
-  default:
-    /* In link protocol version 3 and later, and in version "unknown",
-     * commands 128 and higher indicate variable-length. VERSIONS is
-     * grandfathered in. */
-    return command == CELL_VERSIONS || command >= 128;
-  }
-}
-
-/** Check <b>buf</b> for a variable-length cell according to the rules of link
- * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
- * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
- * Return 0 if whatever is on the start of buf_t is not a variable-length
- * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
- * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
- * yet. */
-int
-fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
-{
-  char hdr[VAR_CELL_MAX_HEADER_SIZE];
-  var_cell_t *result;
-  uint8_t command;
-  uint16_t length;
-  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
-  const int circ_id_len = get_circ_id_size(wide_circ_ids);
-  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
-  check();
-  *out = NULL;
-  if (buf->datalen < header_len)
-    return 0;
-  peek_from_buf(hdr, header_len, buf);
-
-  command = get_uint8(hdr + circ_id_len);
-  if (!(cell_command_is_var_length(command, linkproto)))
-    return 0;
-
-  length = ntohs(get_uint16(hdr + circ_id_len + 1));
-  if (buf->datalen < (size_t)(header_len+length))
-    return 1;
-  result = var_cell_new(length);
-  result->command = command;
-  if (wide_circ_ids)
-    result->circ_id = ntohl(get_uint32(hdr));
-  else
-    result->circ_id = ntohs(get_uint16(hdr));
-
-  buf_remove_from_front(buf, header_len);
-  peek_from_buf((char*) result->payload, length, buf);
-  buf_remove_from_front(buf, length);
-  check();
-
-  *out = result;
-  return 1;
-}
-
-/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
- * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
- * Return the number of bytes actually copied.
- */
-int
-move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
-{
-  /* We can do way better here, but this doesn't turn up in any profiles. */
-  char b[4096];
-  size_t cp, len;
-  len = *buf_flushlen;
-  if (len > buf_in->datalen)
-    len = buf_in->datalen;
-
-  cp = len; /* Remember the number of bytes we intend to copy. */
-  tor_assert(cp < INT_MAX);
-  while (len) {
-    /* This isn't the most efficient implementation one could imagine, since
-     * it does two copies instead of 1, but I kinda doubt that this will be
-     * critical path. */
-    size_t n = len > sizeof(b) ? sizeof(b) : len;
-    fetch_from_buf(b, n, buf_in);
-    write_to_buf(b, n, buf_out);
-    len -= n;
-  }
-  *buf_flushlen -= cp;
-  return (int)cp;
-}
-
-/** Internal structure: represents a position in a buffer. */
-typedef struct buf_pos_t {
-  const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
-  size_t chunk_pos; /**< Total length of all previous chunks. */
-} buf_pos_t;
-
-/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
-static void
-buf_pos_init(const buf_t *buf, buf_pos_t *out)
-{
-  out->chunk = buf->head;
-  out->pos = 0;
-  out->chunk_pos = 0;
-}
-
-/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
- * position of <b>out</b>, or later.  Return -1 if no instances are found;
- * otherwise returns the absolute position of the character. */
-static off_t
-buf_find_pos_of_char(char ch, buf_pos_t *out)
-{
-  const chunk_t *chunk;
-  int pos;
-  tor_assert(out);
-  if (out->chunk) {
-    if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
-    } else {
-      tor_assert(out->pos == 0);
-    }
-  }
-  pos = out->pos;
-  for (chunk = out->chunk; chunk; chunk = chunk->next) {
-    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
-    if (cp) {
-      out->chunk = chunk;
-      tor_assert(cp - chunk->data < INT_MAX);
-      out->pos = (int)(cp - chunk->data);
-      return out->chunk_pos + out->pos;
-    } else {
-      out->chunk_pos += chunk->datalen;
-      pos = 0;
-    }
-  }
-  return -1;
-}
-
-/** Advance <b>pos</b> by a single character, if there are any more characters
- * in the buffer.  Returns 0 on success, -1 on failure. */
-static inline int
-buf_pos_inc(buf_pos_t *pos)
-{
-  ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
-    if (!pos->chunk->next)
-      return -1;
-    pos->chunk_pos += pos->chunk->datalen;
-    pos->chunk = pos->chunk->next;
-    pos->pos = 0;
-  }
-  return 0;
-}
-
-/** Return true iff the <b>n</b>-character string in <b>s</b> appears
- * (verbatim) at <b>pos</b>. */
-static int
-buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
-{
-  buf_pos_t p;
-  if (!n)
-    return 1;
-
-  memcpy(&p, pos, sizeof(p));
-
-  while (1) {
-    char ch = p.chunk->data[p.pos];
-    if (ch != *s)
-      return 0;
-    ++s;
-    /* If we're out of characters that don't match, we match.  Check this
-     * _before_ we test incrementing pos, in case we're at the end of the
-     * string. */
-    if (--n == 0)
-      return 1;
-    if (buf_pos_inc(&p)<0)
-      return 0;
-  }
-}
-
-/** Return the first position in <b>buf</b> at which the <b>n</b>-character
- * string <b>s</b> occurs, or -1 if it does not occur. */
-STATIC int
-buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
-{
-  buf_pos_t pos;
-  buf_pos_init(buf, &pos);
-  while (buf_find_pos_of_char(*s, &pos) >= 0) {
-    if (buf_matches_at_pos(&pos, s, n)) {
-      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
-      return (int)(pos.chunk_pos + pos.pos);
-    } else {
-      if (buf_pos_inc(&pos)<0)
-        return -1;
-    }
-  }
-  return -1;
-}
-
-/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
- * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
- * If a) the headers include a Content-Length field and all bytes in
- * the body are present, or b) there's no Content-Length field and
- * all headers are present, then:
- *
- *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
- *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
- *  - Then remove them from <b>buf</b>, and return 1.
- *
- *  - If headers or body is NULL, discard that part of the buf.
- *  - If a headers or body doesn't fit in the arg, return -1.
- *  (We ensure that the headers or body don't exceed max len,
- *   _even if_ we're planning to discard them.)
- *  - If force_complete is true, then succeed even if not all of the
- *    content has arrived.
- *
- * Else, change nothing and return 0.
- */
-int
-fetch_from_buf_http(buf_t *buf,
-                    char **headers_out, size_t max_headerlen,
-                    char **body_out, size_t *body_used, size_t max_bodylen,
-                    int force_complete)
-{
-  char *headers, *p;
-  size_t headerlen, bodylen, contentlen;
-  int crlf_offset;
-
-  check();
-  if (!buf->head)
-    return 0;
-
-  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
-  if (crlf_offset > (int)max_headerlen ||
-      (crlf_offset < 0 && buf->datalen > max_headerlen)) {
-    log_debug(LD_HTTP,"headers too long.");
-    return -1;
-  } else if (crlf_offset < 0) {
-    log_debug(LD_HTTP,"headers not all here yet.");
-    return 0;
-  }
-  /* Okay, we have a full header.  Make sure it all appears in the first
-   * chunk. */
-  if ((int)buf->head->datalen < crlf_offset + 4)
-    buf_pullup(buf, crlf_offset+4);
-  headerlen = crlf_offset + 4;
-
-  headers = buf->head->data;
-  bodylen = buf->datalen - headerlen;
-  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
-
-  if (max_headerlen <= headerlen) {
-    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
-             (int)headerlen, (int)max_headerlen-1);
-    return -1;
-  }
-  if (max_bodylen <= bodylen) {
-    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
-             (int)bodylen, (int)max_bodylen-1);
-    return -1;
-  }
-
-#define CONTENT_LENGTH "\r\nContent-Length: "
-  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
-  if (p) {
-    int i;
-    i = atoi(p+strlen(CONTENT_LENGTH));
-    if (i < 0) {
-      log_warn(LD_PROTOCOL, "Content-Length is less than zero; it looks like "
-               "someone is trying to crash us.");
-      return -1;
-    }
-    contentlen = i;
-    /* if content-length is malformed, then our body length is 0. fine. */
-    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
-    if (bodylen < contentlen) {
-      if (!force_complete) {
-        log_debug(LD_HTTP,"body not all here yet.");
-        return 0; /* not all there yet */
-      }
-    }
-    if (bodylen > contentlen) {
-      bodylen = contentlen;
-      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
-    }
-  }
-  /* all happy. copy into the appropriate places, and return 1 */
-  if (headers_out) {
-    *headers_out = tor_malloc(headerlen+1);
-    fetch_from_buf(*headers_out, headerlen, buf);
-    (*headers_out)[headerlen] = 0; /* NUL terminate it */
-  }
-  if (body_out) {
-    tor_assert(body_used);
-    *body_used = bodylen;
-    *body_out = tor_malloc(bodylen+1);
-    fetch_from_buf(*body_out, bodylen, buf);
-    (*body_out)[bodylen] = 0; /* NUL terminate it */
-  }
-  check();
-  return 1;
-}
-
-/**
- * Wait this many seconds before warning the user about using SOCKS unsafely
- * again (requires that WarnUnsafeSocks is turned on). */
-#define SOCKS_WARN_INTERVAL 5
-
-/** Warn that the user application has made an unsafe socks request using
- * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
- * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
-static void
-log_unsafe_socks_warning(int socks_protocol, const char *address,
-                         uint16_t port, int safe_socks)
-{
-  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
-
-  const or_options_t *options = get_options();
-  if (! options->WarnUnsafeSocks)
-    return;
-  if (safe_socks) {
-    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
-             "Your application (using socks%d to port %d) is giving "
-             "Tor only an IP address. Applications that do DNS resolves "
-             "themselves may leak information. Consider using Socks4A "
-             "(e.g. via privoxy or socat) instead. For more information, "
-             "please see https://wiki.torproject.org/TheOnionRouter/"
-             "TorFAQ#SOCKSAndDNS.%s",
-             socks_protocol,
-             (int)port,
-             safe_socks ? " Rejecting." : "");
-  }
-  control_event_client_status(LOG_WARN,
-                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
-                              socks_protocol, address, (int)port);
-}
-
-/** Do not attempt to parse socks messages longer than this.  This value is
- * actually significantly higher than the longest possible socks message. */
-#define MAX_SOCKS_MESSAGE_LEN 512
-
-/** Return a new socks_request_t. */
-socks_request_t *
-socks_request_new(void)
-{
-  return tor_malloc_zero(sizeof(socks_request_t));
-}
-
-/** Free all storage held in the socks_request_t <b>req</b>. */
-void
-socks_request_free(socks_request_t *req)
-{
-  if (!req)
-    return;
-  if (req->username) {
-    memwipe(req->username, 0x10, req->usernamelen);
-    tor_free(req->username);
-  }
-  if (req->password) {
-    memwipe(req->password, 0x04, req->passwordlen);
-    tor_free(req->password);
-  }
-  memwipe(req, 0xCC, sizeof(socks_request_t));
-  tor_free(req);
-}
-
-/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
- * of the forms
- *  - socks4: "socksheader username\\0"
- *  - socks4a: "socksheader username\\0 destaddr\\0"
- *  - socks5 phase one: "version #methods methods"
- *  - socks5 phase two: "version command 0 addresstype..."
- * If it's a complete and valid handshake, and destaddr fits in
- *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
- *   assign to <b>req</b>, and return 1.
- *
- * If it's invalid or too big, return -1.
- *
- * Else it's not all there yet, leave buf alone and return 0.
- *
- * If you want to specify the socks reply, write it into <b>req->reply</b>
- *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
- *
- * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
- * the connection is possibly leaking DNS requests locally or not.
- *
- * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
- *
- * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
- * undefined.
- */
-int
-fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                     int log_sockstype, int safe_socks)
-{
-  int res;
-  ssize_t n_drain;
-  size_t want_length = 128;
-
-  if (buf->datalen < 2) /* version and another byte */
-    return 0;
-
-  do {
-    n_drain = 0;
-    buf_pullup(buf, want_length);
-    tor_assert(buf->head && buf->head->datalen >= 2);
-    want_length = 0;
-
-    res = parse_socks(buf->head->data, buf->head->datalen, req, log_sockstype,
-                      safe_socks, &n_drain, &want_length);
-
-    if (n_drain < 0)
-      buf_clear(buf);
-    else if (n_drain > 0)
-      buf_remove_from_front(buf, n_drain);
-
-  } while (res == 0 && buf->head && want_length < buf->datalen &&
-           buf->datalen >= 2);
-
-  return res;
-}
-
-/** The size of the header of an Extended ORPort message: 2 bytes for
- *  COMMAND, 2 bytes for BODYLEN */
-#define EXT_OR_CMD_HEADER_SIZE 4
-
-/** Read <b>buf</b>, which should contain an Extended ORPort message
- *  from a transport proxy. If well-formed, create and populate
- *  <b>out</b> with the Extended ORport message. Return 0 if the
- *  buffer was incomplete, 1 if it was well-formed and -1 if we
- *  encountered an error while parsing it.  */
-int
-fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
-{
-  char hdr[EXT_OR_CMD_HEADER_SIZE];
-  uint16_t len;
-
-  check();
-  if (buf->datalen < EXT_OR_CMD_HEADER_SIZE)
-    return 0;
-  peek_from_buf(hdr, sizeof(hdr), buf);
-  len = ntohs(get_uint16(hdr+2));
-  if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
-    return 0;
-  *out = ext_or_cmd_new(len);
-  (*out)->cmd = ntohs(get_uint16(hdr));
-  (*out)->len = len;
-  buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE);
-  fetch_from_buf((*out)->body, len, buf);
-  return 1;
-}
-
-/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
- * have Tor send it as error response to <b>req</b>.
- */
-static void
-socks_request_set_socks5_error(socks_request_t *req,
-                  socks5_reply_status_t reason)
-{
-   req->replylen = 10;
-   memset(req->reply,0,10);
-
-   req->reply[0] = 0x05;   // VER field.
-   req->reply[1] = reason; // REP field.
-   req->reply[3] = 0x01;   // ATYP field.
-}
-
-/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
- * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
- * <b>data</b>. Instead of removing data from the buffer, we set
- * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
- * buffer should be cleared).  Instead of pulling more data into the first
- * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
- * we'd like to see in the input buffer, if they're available. */
-static int
-parse_socks(const char *data, size_t datalen, socks_request_t *req,
-            int log_sockstype, int safe_socks, ssize_t *drain_out,
-            size_t *want_length_out)
-{
-  unsigned int len;
-  char tmpbuf[TOR_ADDR_BUF_LEN+1];
-  tor_addr_t destaddr;
-  uint32_t destip;
-  uint8_t socksver;
-  char *next, *startaddr;
-  unsigned char usernamelen, passlen;
-  struct in_addr in;
-
-  if (datalen < 2) {
-    /* We always need at least 2 bytes. */
-    *want_length_out = 2;
-    return 0;
-  }
-
-  if (req->socks_version == 5 && !req->got_auth) {
-    /* See if we have received authentication.  Strictly speaking, we should
-       also check whether we actually negotiated username/password
-       authentication.  But some broken clients will send us authentication
-       even if we negotiated SOCKS_NO_AUTH. */
-    if (*data == 1) { /* username/pass version 1 */
-      /* Format is: authversion [1 byte] == 1
-                    usernamelen [1 byte]
-                    username    [usernamelen bytes]
-                    passlen     [1 byte]
-                    password    [passlen bytes] */
-      usernamelen = (unsigned char)*(data + 1);
-      if (datalen < 2u + usernamelen + 1u) {
-        *want_length_out = 2u + usernamelen + 1u;
-        return 0;
-      }
-      passlen = (unsigned char)*(data + 2u + usernamelen);
-      if (datalen < 2u + usernamelen + 1u + passlen) {
-        *want_length_out = 2u + usernamelen + 1u + passlen;
-        return 0;
-      }
-      req->replylen = 2; /* 2 bytes of response */
-      req->reply[0] = 1; /* authversion == 1 */
-      req->reply[1] = 0; /* authentication successful */
-      log_debug(LD_APP,
-               "socks5: Accepted username/password without checking.");
-      if (usernamelen) {
-        req->username = tor_memdup(data+2u, usernamelen);
-        req->usernamelen = usernamelen;
-      }
-      if (passlen) {
-        req->password = tor_memdup(data+3u+usernamelen, passlen);
-        req->passwordlen = passlen;
-      }
-      *drain_out = 2u + usernamelen + 1u + passlen;
-      req->got_auth = 1;
-      *want_length_out = 7; /* Minimal socks5 command. */
-      return 0;
-    } else if (req->auth_type == SOCKS_USER_PASS) {
-      /* unknown version byte */
-      log_warn(LD_APP, "Socks5 username/password version %d not recognized; "
-               "rejecting.", (int)*data);
-      return -1;
-    }
-  }
-
-  socksver = *data;
-
-  switch (socksver) { /* which version of socks? */
-    case 5: /* socks5 */
-
-      if (req->socks_version != 5) { /* we need to negotiate a method */
-        unsigned char nummethods = (unsigned char)*(data+1);
-        int have_user_pass, have_no_auth;
-        int r=0;
-        tor_assert(!req->socks_version);
-        if (datalen < 2u+nummethods) {
-          *want_length_out = 2u+nummethods;
-          return 0;
-        }
-        if (!nummethods)
-          return -1;
-        req->replylen = 2; /* 2 bytes of response */
-        req->reply[0] = 5; /* socks5 reply */
-        have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL);
-        have_no_auth   = (memchr(data+2, SOCKS_NO_AUTH,   nummethods) !=NULL);
-        if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
-          req->auth_type = SOCKS_USER_PASS;
-          req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass"
-                                              auth method */
-          req->socks_version = 5; /* remember we've already negotiated auth */
-          log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
-          r=0;
-        } else if (have_no_auth) {
-          req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth
-                                            method */
-          req->socks_version = 5; /* remember we've already negotiated auth */
-          log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
-          r=0;
-        } else {
-          log_warn(LD_APP,
-                    "socks5: offered methods don't include 'no auth' or "
-                    "username/password. Rejecting.");
-          req->reply[1] = '\xFF'; /* reject all methods */
-          r=-1;
-        }
-        /* Remove packet from buf. Some SOCKS clients will have sent extra
-         * junk at this point; let's hope it's an authentication message. */
-        *drain_out = 2u + nummethods;
-
-        return r;
-      }
-      if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) {
-        log_warn(LD_APP,
-                 "socks5: negotiated authentication, but none provided");
-        return -1;
-      }
-      /* we know the method; read in the request */
-      log_debug(LD_APP,"socks5: checking request");
-      if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */
-        *want_length_out = 7;
-        return 0; /* not yet */
-      }
-      req->command = (unsigned char) *(data+1);
-      if (req->command != SOCKS_COMMAND_CONNECT &&
-          req->command != SOCKS_COMMAND_RESOLVE &&
-          req->command != SOCKS_COMMAND_RESOLVE_PTR) {
-        /* not a connect or resolve or a resolve_ptr? we don't support it. */
-        socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
-
-        log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.",
-                 req->command);
-        return -1;
-      }
-      switch (*(data+3)) { /* address type */
-        case 1: /* IPv4 address */
-        case 4: /* IPv6 address */ {
-          const int is_v6 = *(data+3) == 4;
-          const unsigned addrlen = is_v6 ? 16 : 4;
-          log_debug(LD_APP,"socks5: ipv4 address type");
-          if (datalen < 6+addrlen) {/* ip/port there? */
-            *want_length_out = 6+addrlen;
-            return 0; /* not yet */
-          }
-
-          if (is_v6)
-            tor_addr_from_ipv6_bytes(&destaddr, data+4);
-          else
-            tor_addr_from_ipv4n(&destaddr, get_uint32(data+4));
-
-          tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1);
-
-          if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-            log_warn(LD_APP,
-                     "socks5 IP takes %d bytes, which doesn't fit in %d. "
-                     "Rejecting.",
-                     (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN);
-            return -1;
-          }
-          strlcpy(req->address,tmpbuf,sizeof(req->address));
-          req->port = ntohs(get_uint16(data+4+addrlen));
-          *drain_out = 6+addrlen;
-          if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
-              !addressmap_have_mapping(req->address,0)) {
-            log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
-            if (safe_socks) {
-              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
-              return -1;
-            }
-          }
-          return 1;
-        }
-        case 3: /* fqdn */
-          log_debug(LD_APP,"socks5: fqdn address type");
-          if (req->command == SOCKS_COMMAND_RESOLVE_PTR) {
-            socks_request_set_socks5_error(req,
-                                           SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
-            log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
-                     "hostname type. Rejecting.");
-            return -1;
-          }
-          len = (unsigned char)*(data+4);
-          if (datalen < 7+len) { /* addr/port there? */
-            *want_length_out = 7+len;
-            return 0; /* not yet */
-          }
-          if (len+1 > MAX_SOCKS_ADDR_LEN) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-            log_warn(LD_APP,
-                     "socks5 hostname is %d bytes, which doesn't fit in "
-                     "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN);
-            return -1;
-          }
-          memcpy(req->address,data+5,len);
-          req->address[len] = 0;
-          req->port = ntohs(get_uint16(data+5+len));
-          *drain_out = 5+len+2;
-
-          if (string_is_valid_ipv4_address(req->address) ||
-              string_is_valid_ipv6_address(req->address)) {
-            log_unsafe_socks_warning(5,req->address,req->port,safe_socks);
-
-            if (safe_socks) {
-              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
-              return -1;
-            }
-          } else if (!string_is_valid_hostname(req->address)) {
-            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
-
-            log_warn(LD_PROTOCOL,
-                     "Your application (using socks5 to port %d) gave Tor "
-                     "a malformed hostname: %s. Rejecting the connection.",
-                     req->port, escaped_safe_str_client(req->address));
-            return -1;
-          }
-          if (log_sockstype)
-            log_notice(LD_APP,
-                  "Your application (using socks5 to port %d) instructed "
-                  "Tor to take care of the DNS resolution itself if "
-                  "necessary. This is good.", req->port);
-          return 1;
-        default: /* unsupported */
-          socks_request_set_socks5_error(req,
-                                         SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
-          log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.",
-                   (int) *(data+3));
-          return -1;
-      }
-      tor_assert(0);
-      break;
-    case 4: { /* socks4 */
-      enum {socks4, socks4a} socks4_prot = socks4a;
-      const char *authstart, *authend;
-      /* http://ss5.sourceforge.net/socks4.protocol.txt */
-      /* http://ss5.sourceforge.net/socks4A.protocol.txt */
-
-      req->socks_version = 4;
-      if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */
-        *want_length_out = SOCKS4_NETWORK_LEN;
-        return 0; /* not yet */
-      }
-      // buf_pullup(buf, 1280);
-      req->command = (unsigned char) *(data+1);
-      if (req->command != SOCKS_COMMAND_CONNECT &&
-          req->command != SOCKS_COMMAND_RESOLVE) {
-        /* not a connect or resolve? we don't support it. (No resolve_ptr with
-         * socks4.) */
-        log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.",
-                 req->command);
-        return -1;
-      }
-
-      req->port = ntohs(get_uint16(data+2));
-      destip = ntohl(get_uint32(data+4));
-      if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) {
-        log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting.");
-        return -1;
-      }
-      if (destip >> 8) {
-        log_debug(LD_APP,"socks4: destip not in form 0.0.0.x.");
-        in.s_addr = htonl(destip);
-        tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf));
-        if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) {
-          log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.",
-                    (int)strlen(tmpbuf));
-          return -1;
-        }
-        log_debug(LD_APP,
-                  "socks4: successfully read destip (%s)",
-                  safe_str_client(tmpbuf));
-        socks4_prot = socks4;
-      }
-
-      authstart = data + SOCKS4_NETWORK_LEN;
-      next = memchr(authstart, 0,
-                    datalen-SOCKS4_NETWORK_LEN);
-      if (!next) {
-        if (datalen >= 1024) {
-          log_debug(LD_APP, "Socks4 user name too long; rejecting.");
-          return -1;
-        }
-        log_debug(LD_APP,"socks4: Username not here yet.");
-        *want_length_out = datalen+1024; /* More than we need, but safe */
-        return 0;
-      }
-      authend = next;
-      tor_assert(next < data+datalen);
-
-      startaddr = NULL;
-      if (socks4_prot != socks4a &&
-          !addressmap_have_mapping(tmpbuf,0)) {
-        log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks);
-
-        if (safe_socks)
-          return -1;
-      }
-      if (socks4_prot == socks4a) {
-        if (next+1 == data+datalen) {
-          log_debug(LD_APP,"socks4: No part of destaddr here yet.");
-          *want_length_out = datalen + 1024; /* More than we need, but safe */
-          return 0;
-        }
-        startaddr = next+1;
-        next = memchr(startaddr, 0, data + datalen - startaddr);
-        if (!next) {
-          if (datalen >= 1024) {
-            log_debug(LD_APP,"socks4: Destaddr too long.");
-            return -1;
-          }
-          log_debug(LD_APP,"socks4: Destaddr not all here yet.");
-          *want_length_out = datalen + 1024; /* More than we need, but safe */
-          return 0;
-        }
-        if (MAX_SOCKS_ADDR_LEN <= next-startaddr) {
-          log_warn(LD_APP,"socks4: Destaddr too long. Rejecting.");
-          return -1;
-        }
-        // tor_assert(next < buf->cur+buf->datalen);
-
-        if (log_sockstype)
-          log_notice(LD_APP,
-                     "Your application (using socks4a to port %d) instructed "
-                     "Tor to take care of the DNS resolution itself if "
-                     "necessary. This is good.", req->port);
-      }
-      log_debug(LD_APP,"socks4: Everything is here. Success.");
-      strlcpy(req->address, startaddr ? startaddr : tmpbuf,
-              sizeof(req->address));
-      if (!tor_strisprint(req->address) || strchr(req->address,'\"')) {
-        log_warn(LD_PROTOCOL,
-                 "Your application (using socks4 to port %d) gave Tor "
-                 "a malformed hostname: %s. Rejecting the connection.",
-                 req->port, escaped_safe_str_client(req->address));
-        return -1;
-      }
-      if (authend != authstart) {
-        req->got_auth = 1;
-        req->usernamelen = authend - authstart;
-        req->username = tor_memdup(authstart, authend - authstart);
-      }
-      /* next points to the final \0 on inbuf */
-      *drain_out = next - data + 1;
-      return 1;
-    }
-    case 'G': /* get */
-    case 'H': /* head */
-    case 'P': /* put/post */
-    case 'C': /* connect */
-      strlcpy((char*)req->reply,
-"HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
-"Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
-"<html>\n"
-"<head>\n"
-"<title>Tor is not an HTTP Proxy</title>\n"
-"</head>\n"
-"<body>\n"
-"<h1>Tor is not an HTTP Proxy</h1>\n"
-"<p>\n"
-"It appears you have configured your web browser to use Tor as an HTTP proxy."
-"\n"
-"This is not correct: Tor is a SOCKS proxy, not an HTTP proxy.\n"
-"Please configure your client accordingly.\n"
-"</p>\n"
-"<p>\n"
-"See <a href=\"https://www.torproject.org/documentation.html\">"
-           "https://www.torproject.org/documentation.html</a> for more "
-           "information.\n"
-"<!-- Plus this comment, to make the body response more than 512 bytes, so "
-"     IE will be willing to display it. Comment comment comment comment "
-"     comment comment comment comment comment comment comment comment.-->\n"
-"</p>\n"
-"</body>\n"
-"</html>\n"
-             , MAX_SOCKS_REPLY_LEN);
-      req->replylen = strlen((char*)req->reply)+1;
-      /* fall through */
-    default: /* version is not socks4 or socks5 */
-      log_warn(LD_APP,
-               "Socks version %d not recognized. (Tor is not an http proxy.)",
-               *(data));
-      {
-        /* Tell the controller the first 8 bytes. */
-        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
-        control_event_client_status(LOG_WARN,
-                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
-                                    escaped(tmp));
-        tor_free(tmp);
-      }
-      return -1;
-  }
-}
-
-/** Inspect a reply from SOCKS server stored in <b>buf</b> according
- * to <b>state</b>, removing the protocol data upon success. Return 0 on
- * incomplete response, 1 on success and -1 on error, in which case
- * <b>reason</b> is set to a descriptive message (free() when finished
- * with it).
- *
- * As a special case, 2 is returned when user/pass is required
- * during SOCKS5 handshake and user/pass is configured.
- */
-int
-fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
-{
-  ssize_t drain = 0;
-  int r;
-  if (buf->datalen < 2)
-    return 0;
-
-  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN);
-  tor_assert(buf->head && buf->head->datalen >= 2);
-
-  r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen,
-                         state, reason, &drain);
-  if (drain > 0)
-    buf_remove_from_front(buf, drain);
-  else if (drain < 0)
-    buf_clear(buf);
-
-  return r;
-}
-
-/** Implementation logic for fetch_from_*_socks_client. */
-static int
-parse_socks_client(const uint8_t *data, size_t datalen,
-                   int state, char **reason,
-                   ssize_t *drain_out)
-{
-  unsigned int addrlen;
-  *drain_out = 0;
-  if (datalen < 2)
-    return 0;
-
-  switch (state) {
-    case PROXY_SOCKS4_WANT_CONNECT_OK:
-      /* Wait for the complete response */
-      if (datalen < 8)
-        return 0;
-
-      if (data[1] != 0x5a) {
-        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
-        return -1;
-      }
-
-      /* Success */
-      *drain_out = 8;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
-      /* we don't have any credentials */
-      if (data[1] != 0x00) {
-        *reason = tor_strdup("server doesn't support any of our "
-                             "available authentication methods");
-        return -1;
-      }
-
-      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
-      *drain_out = -1;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
-      /* we have a username and password. return 1 if we can proceed without
-       * providing authentication, or 2 otherwise. */
-      switch (data[1]) {
-        case 0x00:
-          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
-                            "doesn't require authentication.");
-          *drain_out = -1;
-          return 1;
-        case 0x02:
-          log_info(LD_NET, "SOCKS 5 client: need authentication.");
-          *drain_out = -1;
-          return 2;
-        /* fall through */
-      }
-
-      *reason = tor_strdup("server doesn't support any of our available "
-                           "authentication methods");
-      return -1;
-
-    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
-      /* handle server reply to rfc1929 authentication */
-      if (data[1] != 0x00) {
-        *reason = tor_strdup("authentication failed");
-        return -1;
-      }
-
-      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
-      *drain_out = -1;
-      return 1;
-
-    case PROXY_SOCKS5_WANT_CONNECT_OK:
-      /* response is variable length. BND.ADDR, etc, isn't needed
-       * (don't bother with buf_pullup()), but make sure to eat all
-       * the data used */
-
-      /* wait for address type field to arrive */
-      if (datalen < 4)
-        return 0;
-
-      switch (data[3]) {
-        case 0x01: /* ip4 */
-          addrlen = 4;
-          break;
-        case 0x04: /* ip6 */
-          addrlen = 16;
-          break;
-        case 0x03: /* fqdn (can this happen here?) */
-          if (datalen < 5)
-            return 0;
-          addrlen = 1 + data[4];
-          break;
-        default:
-          *reason = tor_strdup("invalid response to connect request");
-          return -1;
-      }
-
-      /* wait for address and port */
-      if (datalen < 6 + addrlen)
-        return 0;
-
-      if (data[1] != 0x00) {
-        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
-        return -1;
-      }
-
-      *drain_out = 6 + addrlen;
-      return 1;
-  }
-
-  /* shouldn't get here... */
-  tor_assert(0);
-
-  return -1;
-}
-
-/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
- * command on it than any valid v1 controller command. */
-int
-peek_buf_has_control0_command(buf_t *buf)
-{
-  if (buf->datalen >= 4) {
-    char header[4];
-    uint16_t cmd;
-    peek_from_buf(header, sizeof(header), buf);
-    cmd = ntohs(get_uint16(header+2));
-    if (cmd <= 0x14)
-      return 1; /* This is definitely not a v1 control command. */
-  }
-  return 0;
-}
-
-/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
- * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
-buf_find_offset_of_char(buf_t *buf, char ch)
-{
-  chunk_t *chunk;
-  off_t offset = 0;
-  for (chunk = buf->head; chunk; chunk = chunk->next) {
-    char *cp = memchr(chunk->data, ch, chunk->datalen);
-    if (cp)
-      return offset + (cp - chunk->data);
-    else
-      offset += chunk->datalen;
-  }
-  return -1;
-}
-
-/** Try to read a single LF-terminated line from <b>buf</b>, and write it
- * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
- * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
- * line, not counting the terminating NUL.  Return 1 if we read a whole line,
- * return 0 if we don't have a whole line yet, and return -1 if the line
- * length exceeds *<b>data_len</b>.
- */
-int
-fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len)
-{
-  size_t sz;
-  off_t offset;
-
-  if (!buf->head)
-    return 0;
-
-  offset = buf_find_offset_of_char(buf, '\n');
-  if (offset < 0)
-    return 0;
-  sz = (size_t) offset;
-  if (sz+2 > *data_len) {
-    *data_len = sz + 2;
-    return -1;
-  }
-  fetch_from_buf(data_out, sz+1, buf);
-  data_out[sz+1] = '\0';
-  *data_len = sz+1;
-  return 1;
-}
-
-/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the
- * zlib state <b>state</b>, appending the result to <b>buf</b>.  If
- * <b>done</b> is true, flush the data in the state and finish the
- * compression/uncompression.  Return -1 on failure, 0 on success. */
-int
-write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
-                  const char *data, size_t data_len,
-                  int done)
-{
-  char *next;
-  size_t old_avail, avail;
-  int over = 0;
-
-  do {
-    int need_new_chunk = 0;
-    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
-      size_t cap = data_len / 4;
-      buf_add_chunk_with_capacity(buf, cap, 1);
-    }
-    next = CHUNK_WRITE_PTR(buf->tail);
-    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
-    switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) {
-      case TOR_ZLIB_DONE:
-        over = 1;
-        break;
-      case TOR_ZLIB_ERR:
-        return -1;
-      case TOR_ZLIB_OK:
-        if (data_len == 0)
-          over = 1;
-        break;
-      case TOR_ZLIB_BUF_FULL:
-        if (avail) {
-          /* Zlib says we need more room (ZLIB_BUF_FULL).  Start a new chunk
-           * automatically, whether were going to or not. */
-          need_new_chunk = 1;
-        }
-        break;
-    }
-    buf->datalen += old_avail - avail;
-    buf->tail->datalen += old_avail - avail;
-    if (need_new_chunk) {
-      buf_add_chunk_with_capacity(buf, data_len/4, 1);
-    }
-
-  } while (!over);
-  check();
-  return 0;
-}
-
-/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
-int
-buf_set_to_copy(buf_t **output,
-                const buf_t *input)
-{
-  if (*output)
-    buf_free(*output);
-  *output = buf_copy(input);
-  return 0;
-}
-
-/** Log an error and exit if <b>buf</b> is corrupted.
- */
-void
-assert_buf_ok(buf_t *buf)
-{
-  tor_assert(buf);
-  tor_assert(buf->magic == BUFFER_MAGIC);
-
-  if (! buf->head) {
-    tor_assert(!buf->tail);
-    tor_assert(buf->datalen == 0);
-  } else {
-    chunk_t *ch;
-    size_t total = 0;
-    tor_assert(buf->tail);
-    for (ch = buf->head; ch; ch = ch->next) {
-      total += ch->datalen;
-      tor_assert(ch->datalen <= ch->memlen);
-      tor_assert(ch->data >= &ch->mem[0]);
-      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
-      if (ch->data == &ch->mem[0]+ch->memlen) {
-        static int warned = 0;
-        if (! warned) {
-          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
-          warned = 1;
-        }
-      }
-      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
-      if (!ch->next)
-        tor_assert(ch == buf->tail);
-    }
-    tor_assert(buf->datalen == total);
-  }
-}
-
diff --git a/src/or/buffers.h b/src/or/buffers.h
deleted file mode 100644
index 52b21d5885..0000000000
--- a/src/or/buffers.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file buffers.h
- * \brief Header file for buffers.c.
- **/
-
-#ifndef TOR_BUFFERS_H
-#define TOR_BUFFERS_H
-
-#include "testsupport.h"
-
-buf_t *buf_new(void);
-buf_t *buf_new_with_capacity(size_t size);
-size_t buf_get_default_chunk_size(const buf_t *buf);
-void buf_free(buf_t *buf);
-void buf_clear(buf_t *buf);
-buf_t *buf_copy(const buf_t *buf);
-
-MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
-size_t buf_allocation(const buf_t *buf);
-size_t buf_slack(const buf_t *buf);
-
-uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
-size_t buf_get_total_allocation(void);
-
-int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
-                int *socket_error);
-int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf);
-
-int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen);
-int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen);
-
-int write_to_buf(const char *string, size_t string_len, buf_t *buf);
-int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
-                      const char *data, size_t data_len, int done);
-int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
-int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
-int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto);
-int fetch_from_buf_http(buf_t *buf,
-                        char **headers_out, size_t max_headerlen,
-                        char **body_out, size_t *body_used, size_t max_bodylen,
-                        int force_complete);
-socks_request_t *socks_request_new(void);
-void socks_request_free(socks_request_t *req);
-int fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
-                         int log_sockstype, int safe_socks);
-int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
-int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len);
-
-int peek_buf_has_control0_command(buf_t *buf);
-
-int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out);
-
-int buf_set_to_copy(buf_t **output,
-                    const buf_t *input);
-
-void assert_buf_ok(buf_t *buf);
-
-#ifdef BUFFERS_PRIVATE
-STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
-STATIC void buf_pullup(buf_t *buf, size_t bytes);
-void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz);
-STATIC size_t preferred_chunk_size(size_t target);
-
-#define DEBUG_CHUNK_ALLOC
-/** A single chunk on a buffer. */
-typedef struct chunk_t {
-  struct chunk_t *next; /**< The next chunk on the buffer. */
-  size_t datalen; /**< The number of bytes stored in this chunk */
-  size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */
-#ifdef DEBUG_CHUNK_ALLOC
-  size_t DBG_alloc;
-#endif
-  char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */
-  uint32_t inserted_time; /**< Timestamp in truncated ms since epoch
-                           * when this chunk was inserted. */
-  char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in
-                * this chunk. */
-} chunk_t;
-
-/** Magic value for buf_t.magic, to catch pointer errors. */
-#define BUFFER_MAGIC 0xB0FFF312u
-/** A resizeable buffer, optimized for reading and writing. */
-struct buf_t {
-  uint32_t magic; /**< Magic cookie for debugging: Must be set to
-                   *   BUFFER_MAGIC. */
-  size_t datalen; /**< How many bytes is this buffer holding right now? */
-  size_t default_chunk_size; /**< Don't allocate any chunks smaller than
-                              * this for this buffer. */
-  chunk_t *head; /**< First chunk in the list, or NULL for none. */
-  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
-};
-#endif
-
-#endif
-
diff --git a/src/or/channel.c b/src/or/channel.c
index 54e10666d2..c30e508018 100644
--- a/src/or/channel.c
+++ b/src/or/channel.c
@@ -1,13 +1,13 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file channel.c
  *
  * \brief OR/OP-to-OR channel abstraction layer. A channel's job is to
- * transfer cells from Tor instance to Tor instance.
- * Currently, there is only one implementation of the channel abstraction: in
- * channeltls.c.
+ * transfer cells from Tor instance to Tor instance. Currently, there is only
+ * one implementation of the channel abstraction: in channeltls.c.
  *
  * Channels are a higher-level abstraction than or_connection_t: In general,
  * any means that two Tor relays use to exchange cells, or any means that a
@@ -24,23 +24,34 @@
  * connection.
  *
  * Every channel implementation is responsible for being able to transmit
- * cells that are added to it with channel_write_cell() and related functions,
- * and to receive incoming cells with the channel_queue_cell() and related
- * functions.  See the channel_t documentation for more information.
- *
- * When new cells arrive on a channel, they are passed to cell handler
- * functions, which can be set by channel_set_cell_handlers()
- * functions. (Tor's cell handlers are in command.c.)
- *
- * Tor flushes cells to channels from relay.c in
- * channel_flush_from_first_active_circuit().
+ * cells that are passed to it
+ *
+ * For *inbound* cells, the entry point is: channel_process_cell(). It takes a
+ * cell and will pass it to the cell handler set by
+ * channel_set_cell_handlers(). Currently, this is passed back to the command
+ * subsystem which is command_process_cell().
+ *
+ * NOTE: For now, the separation between channels and specialized channels
+ * (like channeltls) is not that well defined. So the channeltls layer calls
+ * channel_process_cell() which originally comes from the connection subsytem.
+ * This should be hopefully be fixed with #23993.
+ *
+ * For *outbound* cells, the entry point is: channel_write_packed_cell().
+ * Only packed cells are dequeued from the circuit queue by the scheduler
+ * which uses channel_flush_from_first_active_circuit() to decide which cells
+ * to flush from which circuit on the channel. They are then passed down to
+ * the channel subsystem. This calls the low layer with the function pointer
+ * .write_packed_cell().
+ *
+ * Each specialized channel (currently only channeltls_t) MUST implement a
+ * series of function found in channel_t. See channel.h for more
+ * documentation.
  **/
 
 /*
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-
 #define TOR_CHANNEL_INTERNAL_
 
 /* This one's for stuff only channel.c and the test suite should see */
@@ -49,6 +60,7 @@
 #include "or.h"
 #include "channel.h"
 #include "channeltls.h"
+#include "channelpadding.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuitstats.h"
@@ -57,12 +69,16 @@
 #include "circuitmux.h"
 #include "entrynodes.h"
 #include "geoip.h"
+#include "main.h"
 #include "nodelist.h"
 #include "relay.h"
 #include "rephist.h"
 #include "router.h"
 #include "routerlist.h"
 #include "scheduler.h"
+#include "compat_time.h"
+#include "networkstatus.h"
+#include "rendservice.h"
 
 /* Global lists of channels */
 
@@ -84,61 +100,30 @@ static smartlist_t *active_listeners = NULL;
 /* All channel_listener_t instances in LISTENING state */
 static smartlist_t *finished_listeners = NULL;
 
-/* Counter for ID numbers */
-static uint64_t n_channels_allocated = 0;
-/*
- * Channel global byte/cell counters, for statistics and for scheduler high
- * /low-water marks.
- */
-
-/*
- * Total number of cells ever given to any channel with the
- * channel_write_*_cell() functions.
- */
-
-static uint64_t n_channel_cells_queued = 0;
-
-/*
- * Total number of cells ever passed to a channel lower layer with the
- * write_*_cell() methods.
- */
-
-static uint64_t n_channel_cells_passed_to_lower_layer = 0;
-
-/*
- * Current number of cells in all channel queues; should be
- * n_channel_cells_queued - n_channel_cells_passed_to_lower_layer.
- */
-
-static uint64_t n_channel_cells_in_queues = 0;
-
-/*
- * Total number of bytes for all cells ever queued to a channel and
- * counted in n_channel_cells_queued.
- */
-
-static uint64_t n_channel_bytes_queued = 0;
-
-/*
- * Total number of bytes for all cells ever passed to a channel lower layer
- * and counted in n_channel_cells_passed_to_lower_layer.
- */
+/** Map from channel->global_identifier to channel.  Contains the same
+ * elements as all_channels. */
+static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER();
 
-static uint64_t n_channel_bytes_passed_to_lower_layer = 0;
-
-/*
- * Current number of bytes in all channel queues; should be
- * n_channel_bytes_queued - n_channel_bytes_passed_to_lower_layer.
- */
-
-static uint64_t n_channel_bytes_in_queues = 0;
+static unsigned
+channel_id_hash(const channel_t *chan)
+{
+  return (unsigned) chan->global_identifier;
+}
+static int
+channel_id_eq(const channel_t *a, const channel_t *b)
+{
+  return a->global_identifier == b->global_identifier;
+}
+HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node,
+             channel_id_hash, channel_id_eq)
+HT_GENERATE2(channel_gid_map, channel_s, gidmap_node,
+             channel_id_hash, channel_id_eq,
+             0.6, tor_reallocarray_, tor_free_)
 
-/*
- * Current total estimated queue size *including lower layer queues and
- * transmit overhead*
- */
+HANDLE_IMPL(channel, channel_s,)
 
-STATIC uint64_t estimated_total_queue_size = 0;
+/* Counter for ID numbers */
+static uint64_t n_channels_allocated = 0;
 
 /* Digest->channel map
  *
@@ -175,49 +160,23 @@ HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
 HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
              channel_idmap_eq, 0.5,  tor_reallocarray_, tor_free_)
 
-static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q);
-#if 0
-static int cell_queue_entry_is_padding(cell_queue_entry_t *q);
-#endif
-static cell_queue_entry_t *
-cell_queue_entry_new_fixed(cell_t *cell);
-static cell_queue_entry_t *
-cell_queue_entry_new_var(var_cell_t *var_cell);
-static int is_destroy_cell(channel_t *chan,
-                           const cell_queue_entry_t *q, circid_t *circid_out);
-
-static void channel_assert_counter_consistency(void);
-
 /* Functions to maintain the digest map */
-static void channel_add_to_digest_map(channel_t *chan);
 static void channel_remove_from_digest_map(channel_t *chan);
 
-/*
- * Flush cells from just the outgoing queue without trying to get them
- * from circuits; used internall by channel_flush_some_cells().
- */
-static ssize_t
-channel_flush_some_cells_from_outgoing_queue(channel_t *chan,
-                                             ssize_t num_cells);
-static void channel_force_free(channel_t *chan);
-static void
-channel_free_list(smartlist_t *channels, int mark_for_close);
-static void
-channel_listener_free_list(smartlist_t *channels, int mark_for_close);
-static void channel_listener_force_free(channel_listener_t *chan_l);
-static size_t channel_get_cell_queue_entry_size(channel_t *chan,
-                                                cell_queue_entry_t *q);
-static void
-channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q);
+static void channel_force_xfree(channel_t *chan);
+static void channel_free_list(smartlist_t *channels,
+                               int mark_for_close);
+static void channel_listener_free_list(smartlist_t *channels,
+                                        int mark_for_close);
+static void channel_listener_force_xfree(channel_listener_t *chan_l);
 
 /***********************************
  * Channel state utility functions *
  **********************************/
 
 /**
- * Indicate whether a given channel state is valid
+ * Indicate whether a given channel state is valid.
  */
-
 int
 channel_state_is_valid(channel_state_t state)
 {
@@ -241,9 +200,8 @@ channel_state_is_valid(channel_state_t state)
 }
 
 /**
- * Indicate whether a given channel listener state is valid
+ * Indicate whether a given channel listener state is valid.
  */
-
 int
 channel_listener_state_is_valid(channel_listener_state_t state)
 {
@@ -265,13 +223,12 @@ channel_listener_state_is_valid(channel_listener_state_t state)
 }
 
 /**
- * Indicate whether a channel state transition is valid
+ * Indicate whether a channel state transition is valid.
  *
  * This function takes two channel states and indicates whether a
  * transition between them is permitted (see the state definitions and
  * transition table in or.h at the channel_state_t typedef).
  */
-
 int
 channel_state_can_transition(channel_state_t from, channel_state_t to)
 {
@@ -312,13 +269,12 @@ channel_state_can_transition(channel_state_t from, channel_state_t to)
 }
 
 /**
- * Indicate whether a channel listener state transition is valid
+ * Indicate whether a channel listener state transition is valid.
  *
  * This function takes two channel listener states and indicates whether a
  * transition between them is permitted (see the state definitions and
  * transition table in or.h at the channel_listener_state_t typedef).
  */
-
 int
 channel_listener_state_can_transition(channel_listener_state_t from,
                                       channel_listener_state_t to)
@@ -349,9 +305,8 @@ channel_listener_state_can_transition(channel_listener_state_t from,
 }
 
 /**
- * Return a human-readable description for a channel state
+ * Return a human-readable description for a channel state.
  */
-
 const char *
 channel_state_to_string(channel_state_t state)
 {
@@ -385,9 +340,8 @@ channel_state_to_string(channel_state_t state)
 }
 
 /**
- * Return a human-readable description for a channel listenier state
+ * Return a human-readable description for a channel listener state.
  */
-
 const char *
 channel_listener_state_to_string(channel_listener_state_t state)
 {
@@ -419,16 +373,16 @@ channel_listener_state_to_string(channel_listener_state_t state)
  ***************************************/
 
 /**
- * Register a channel
+ * Register a channel.
  *
  * This function registers a newly created channel in the global lists/maps
  * of active channels.
  */
-
 void
 channel_register(channel_t *chan)
 {
   tor_assert(chan);
+  tor_assert(chan->global_identifier);
 
   /* No-op if already registered */
   if (chan->registered) return;
@@ -443,12 +397,15 @@ channel_register(channel_t *chan)
   /* Make sure we have all_channels, then add it */
   if (!all_channels) all_channels = smartlist_new();
   smartlist_add(all_channels, chan);
+  channel_t *oldval = HT_REPLACE(channel_gid_map, &channel_gid_map, chan);
+  tor_assert(! oldval);
 
   /* Is it finished? */
   if (CHANNEL_FINISHED(chan)) {
     /* Put it in the finished list, creating it if necessary */
     if (!finished_channels) finished_channels = smartlist_new();
     smartlist_add(finished_channels, chan);
+    mainloop_schedule_postloop_cleanup();
   } else {
     /* Put it in the active list, creating it if necessary */
     if (!active_channels) active_channels = smartlist_new();
@@ -474,12 +431,11 @@ channel_register(channel_t *chan)
 }
 
 /**
- * Unregister a channel
+ * Unregister a channel.
  *
  * This function removes a channel from the global lists and maps and is used
  * when freeing a closed/errored channel.
  */
-
 void
 channel_unregister(channel_t *chan)
 {
@@ -498,7 +454,9 @@ channel_unregister(channel_t *chan)
   }
 
   /* Get it out of all_channels */
- if (all_channels) smartlist_remove(all_channels, chan);
+  if (all_channels) smartlist_remove(all_channels, chan);
+  channel_t *oldval = HT_REMOVE(channel_gid_map, &channel_gid_map, chan);
+  tor_assert(oldval == NULL || oldval == chan);
 
   /* Mark it as unregistered */
   chan->registered = 0;
@@ -512,12 +470,11 @@ channel_unregister(channel_t *chan)
 }
 
 /**
- * Register a channel listener
+ * Register a channel listener.
  *
- * This function registers a newly created channel listner in the global
+ * This function registers a newly created channel listener in the global
  * lists/maps of active channel listeners.
  */
-
 void
 channel_listener_register(channel_listener_t *chan_l)
 {
@@ -533,7 +490,7 @@ channel_listener_register(channel_listener_t *chan_l)
             channel_listener_state_to_string(chan_l->state),
             chan_l->state);
 
-  /* Make sure we have all_channels, then add it */
+  /* Make sure we have all_listeners, then add it */
   if (!all_listeners) all_listeners = smartlist_new();
   smartlist_add(all_listeners, chan_l);
 
@@ -554,12 +511,11 @@ channel_listener_register(channel_listener_t *chan_l)
 }
 
 /**
- * Unregister a channel listener
+ * Unregister a channel listener.
  *
  * This function removes a channel listener from the global lists and maps
  * and is used when freeing a closed/errored channel listener.
  */
-
 void
 channel_listener_unregister(channel_listener_t *chan_l)
 {
@@ -578,7 +534,7 @@ channel_listener_unregister(channel_listener_t *chan_l)
     if (active_listeners) smartlist_remove(active_listeners, chan_l);
   }
 
-  /* Get it out of all_channels */
+  /* Get it out of all_listeners */
  if (all_listeners) smartlist_remove(all_listeners, chan_l);
 
   /* Mark it as unregistered */
@@ -590,14 +546,13 @@ channel_listener_unregister(channel_listener_t *chan_l)
  *********************************/
 
 /**
- * Add a channel to the digest map
+ * Add a channel to the digest map.
  *
  * This function adds a channel to the digest map and inserts it into the
  * correct linked list if channels with that remote endpoint identity digest
  * already exist.
  */
-
-static void
+STATIC void
 channel_add_to_digest_map(channel_t *chan)
 {
   channel_idmap_entry_t *ent, search;
@@ -629,12 +584,11 @@ channel_add_to_digest_map(channel_t *chan)
 }
 
 /**
- * Remove a channel from the digest map
+ * Remove a channel from the digest map.
  *
  * This function removes a channel from the digest map and the linked list of
  * channels for that digest if more than one exists.
  */
-
 static void
 channel_remove_from_digest_map(channel_t *chan)
 {
@@ -645,33 +599,6 @@ channel_remove_from_digest_map(channel_t *chan)
   /* Assert that there is a digest */
   tor_assert(!tor_digest_is_zero(chan->identity_digest));
 
-#if 0
-  /* Make sure we have a map */
-  if (!channel_identity_map) {
-    /*
-     * No identity map, so we can't find it by definition.  This
-     * case is similar to digestmap_get() failing below.
-     */
-    log_warn(LD_BUG,
-             "Trying to remove channel %p (global ID " U64_FORMAT ") "
-             "with digest %s from identity map, but didn't have any identity "
-             "map",
-             chan, U64_PRINTF_ARG(chan->global_identifier),
-             hex_str(chan->identity_digest, DIGEST_LEN));
-    /* Clear out its next/prev pointers */
-    if (chan->next_with_same_id) {
-      chan->next_with_same_id->prev_with_same_id = chan->prev_with_same_id;
-    }
-    if (chan->prev_with_same_id) {
-      chan->prev_with_same_id->next_with_same_id = chan->next_with_same_id;
-    }
-    chan->next_with_same_id = NULL;
-    chan->prev_with_same_id = NULL;
-
-    return;
-  }
-#endif
-
   /* Pull it out of its list, wherever that list is */
   TOR_LIST_REMOVE(chan, next_with_same_id);
 
@@ -709,64 +636,95 @@ channel_remove_from_digest_map(channel_t *chan)
  ***************************/
 
 /**
- * Find channel by global ID
+ * Find channel by global ID.
  *
  * This function searches for a channel by the global_identifier assigned
  * at initialization time.  This identifier is unique for the lifetime of the
  * Tor process.
  */
-
 channel_t *
 channel_find_by_global_id(uint64_t global_identifier)
 {
+  channel_t lookup;
   channel_t *rv = NULL;
 
-  if (all_channels && smartlist_len(all_channels) > 0) {
-    SMARTLIST_FOREACH_BEGIN(all_channels, channel_t *, curr) {
-      if (curr->global_identifier == global_identifier) {
-        rv = curr;
-        break;
-      }
-    } SMARTLIST_FOREACH_END(curr);
+  lookup.global_identifier = global_identifier;
+  rv = HT_FIND(channel_gid_map, &channel_gid_map, &lookup);
+  if (rv) {
+    tor_assert(rv->global_identifier == global_identifier);
   }
 
   return rv;
 }
 
+/** Return true iff <b>chan</b> matches <b>rsa_id_digest</b> and <b>ed_id</b>.
+ * as its identity keys.  If either is NULL, do not check for a match. */
+static int
+channel_remote_identity_matches(const channel_t *chan,
+                                const char *rsa_id_digest,
+                                const ed25519_public_key_t *ed_id)
+{
+  if (BUG(!chan))
+    return 0;
+  if (rsa_id_digest) {
+    if (tor_memneq(rsa_id_digest, chan->identity_digest, DIGEST_LEN))
+      return 0;
+  }
+  if (ed_id) {
+    if (tor_memneq(ed_id->pubkey, chan->ed25519_identity.pubkey,
+                   ED25519_PUBKEY_LEN))
+      return 0;
+  }
+  return 1;
+}
+
 /**
- * Find channel by digest of the remote endpoint
+ * Find channel by RSA/Ed25519 identity of of the remote endpoint.
+ *
+ * This function looks up a channel by the digest of its remote endpoint's RSA
+ * identity key.  If <b>ed_id</b> is provided and nonzero, only a channel
+ * matching the <b>ed_id</b> will be returned.
  *
- * This function looks up a channel by the digest of its remote endpoint in
- * the channel digest map.  It's possible that more than one channel to a
- * given endpoint exists.  Use channel_next_with_digest() to walk the list.
+ * It's possible that more than one channel to a given endpoint exists.  Use
+ * channel_next_with_rsa_identity() to walk the list of channels; make sure
+ * to test for Ed25519 identity match too (as appropriate)
  */
-
 channel_t *
-channel_find_by_remote_digest(const char *identity_digest)
+channel_find_by_remote_identity(const char *rsa_id_digest,
+                                const ed25519_public_key_t *ed_id)
 {
   channel_t *rv = NULL;
   channel_idmap_entry_t *ent, search;
 
-  tor_assert(identity_digest);
+  tor_assert(rsa_id_digest); /* For now, we require that every channel have
+                              * an RSA identity, and that every lookup
+                              * contain an RSA identity */
+  if (ed_id && ed25519_public_key_is_zero(ed_id)) {
+    /* Treat zero as meaning "We don't care about the presence or absence of
+     * an Ed key", not "There must be no Ed key". */
+    ed_id = NULL;
+  }
 
-  memcpy(search.digest, identity_digest, DIGEST_LEN);
+  memcpy(search.digest, rsa_id_digest, DIGEST_LEN);
   ent = HT_FIND(channel_idmap, &channel_identity_map, &search);
   if (ent) {
     rv = TOR_LIST_FIRST(&ent->channel_list);
   }
+  while (rv && ! channel_remote_identity_matches(rv, rsa_id_digest, ed_id)) {
+    rv = channel_next_with_rsa_identity(rv);
+  }
 
   return rv;
 }
 
 /**
- * Get next channel with digest
+ * Get next channel with digest.
  *
  * This function takes a channel and finds the next channel in the list
  * with the same digest.
  */
-
 channel_t *
-channel_next_with_digest(channel_t *chan)
+channel_next_with_rsa_identity(channel_t *chan)
 {
   tor_assert(chan);
 
@@ -774,20 +732,96 @@ channel_next_with_digest(channel_t *chan)
 }
 
 /**
- * Initialize a channel
+ * Relays run this once an hour to look over our list of channels to other
+ * relays. It prints out some statistics if there are multiple connections
+ * to many relays.
+ *
+ * This function is similar to connection_or_set_bad_connections(),
+ * and probably could be adapted to replace it, if it was modified to actually
+ * take action on any of these connections.
+ */
+void
+channel_check_for_duplicates(void)
+{
+  channel_idmap_entry_t **iter;
+  channel_t *chan;
+  int total_relay_connections = 0, total_relays = 0, total_canonical = 0;
+  int total_half_canonical = 0;
+  int total_gt_one_connection = 0, total_gt_two_connections = 0;
+  int total_gt_four_connections = 0;
+
+  HT_FOREACH(iter, channel_idmap, &channel_identity_map) {
+    int connections_to_relay = 0;
+
+    /* Only consider relay connections */
+    if (!connection_or_digest_is_known_relay((char*)(*iter)->digest))
+      continue;
+
+    total_relays++;
+
+    for (chan = TOR_LIST_FIRST(&(*iter)->channel_list); chan;
+        chan = channel_next_with_rsa_identity(chan)) {
+
+      if (CHANNEL_CONDEMNED(chan) || !CHANNEL_IS_OPEN(chan))
+        continue;
+
+      connections_to_relay++;
+      total_relay_connections++;
+
+      if (chan->is_canonical(chan, 0)) total_canonical++;
+
+      if (!chan->is_canonical_to_peer && chan->is_canonical(chan, 0)
+          && chan->is_canonical(chan, 1)) {
+        total_half_canonical++;
+      }
+    }
+
+    if (connections_to_relay > 1) total_gt_one_connection++;
+    if (connections_to_relay > 2) total_gt_two_connections++;
+    if (connections_to_relay > 4) total_gt_four_connections++;
+  }
+
+#define MIN_RELAY_CONNECTIONS_TO_WARN 5
+
+  /* If we average 1.5 or more connections per relay, something is wrong */
+  if (total_relays > MIN_RELAY_CONNECTIONS_TO_WARN &&
+          total_relay_connections >= 1.5*total_relays) {
+    log_notice(LD_OR,
+        "Your relay has a very large number of connections to other relays. "
+        "Is your outbound address the same as your relay address? "
+        "Found %d connections to %d relays. Found %d current canonical "
+        "connections, in %d of which we were a non-canonical peer. "
+        "%d relays had more than 1 connection, %d had more than 2, and "
+        "%d had more than 4 connections.",
+        total_relay_connections, total_relays, total_canonical,
+        total_half_canonical, total_gt_one_connection,
+        total_gt_two_connections, total_gt_four_connections);
+  } else {
+    log_info(LD_OR, "Performed connection pruning. "
+        "Found %d connections to %d relays. Found %d current canonical "
+        "connections, in %d of which we were a non-canonical peer. "
+        "%d relays had more than 1 connection, %d had more than 2, and "
+        "%d had more than 4 connections.",
+        total_relay_connections, total_relays, total_canonical,
+        total_half_canonical, total_gt_one_connection,
+        total_gt_two_connections, total_gt_four_connections);
+  }
+}
+
+/**
+ * Initialize a channel.
  *
  * This function should be called by subclasses to set up some per-channel
  * variables.  I.e., this is the superclass constructor.  Before this, the
  * channel should be allocated with tor_malloc_zero().
  */
-
 void
 channel_init(channel_t *chan)
 {
   tor_assert(chan);
 
   /* Assign an ID and bump the counter */
-  chan->global_identifier = n_channels_allocated++;
+  chan->global_identifier = ++n_channels_allocated;
 
   /* Init timestamp */
   chan->timestamp_last_had_circuits = time(NULL);
@@ -795,10 +829,6 @@ channel_init(channel_t *chan)
   /* Warn about exhausted circuit IDs no more than hourly. */
   chan->last_warned_circ_ids_exhausted.rate = 3600;
 
-  /* Initialize queues. */
-  TOR_SIMPLEQ_INIT(&chan->incoming_queue);
-  TOR_SIMPLEQ_INIT(&chan->outgoing_queue);
-
   /* Initialize list entries. */
   memset(&chan->next_with_same_id, 0, sizeof(chan->next_with_same_id));
 
@@ -810,23 +840,25 @@ channel_init(channel_t *chan)
 
   /* Scheduler state is idle */
   chan->scheduler_state = SCHED_CHAN_IDLE;
+
+  /* Channel is not in the scheduler heap. */
+  chan->sched_heap_idx = -1;
 }
 
 /**
- * Initialize a channel listener
+ * Initialize a channel listener.
  *
  * This function should be called by subclasses to set up some per-channel
  * variables.  I.e., this is the superclass constructor.  Before this, the
  * channel listener should be allocated with tor_malloc_zero().
  */
-
 void
 channel_init_listener(channel_listener_t *chan_l)
 {
   tor_assert(chan_l);
 
   /* Assign an ID and bump the counter */
-  chan_l->global_identifier = n_channels_allocated++;
+  chan_l->global_identifier = ++n_channels_allocated;
 
   /* Timestamp it */
   channel_listener_timestamp_created(chan_l);
@@ -836,9 +868,8 @@ channel_init_listener(channel_listener_t *chan_l)
  * Free a channel; nothing outside of channel.c and subclasses should call
  * this - it frees channels after they have closed and been unregistered.
  */
-
 void
-channel_free(channel_t *chan)
+channel_free_(channel_t *chan)
 {
   if (!chan) return;
 
@@ -863,6 +894,11 @@ channel_free(channel_t *chan)
     circuitmux_set_policy(chan->cmux, NULL);
   }
 
+  /* Remove all timers and associated handle entries now */
+  timer_free(chan->padding_timer);
+  channel_handle_free(chan->timer_handle);
+  channel_handles_clear(chan);
+
   /* Call a free method if there is one */
   if (chan->free_fn) chan->free_fn(chan);
 
@@ -876,8 +912,6 @@ channel_free(channel_t *chan)
     chan->cmux = NULL;
   }
 
-  /* We're in CLOSED or ERROR, so the cell queue is already empty */
-
   tor_free(chan);
 }
 
@@ -886,9 +920,8 @@ channel_free(channel_t *chan)
  * should call this - it frees channel listeners after they have closed and
  * been unregistered.
  */
-
 void
-channel_listener_free(channel_listener_t *chan_l)
+channel_listener_free_(channel_listener_t *chan_l)
 {
   if (!chan_l) return;
 
@@ -906,11 +939,6 @@ channel_listener_free(channel_listener_t *chan_l)
   /* Call a free method if there is one */
   if (chan_l->free_fn) chan_l->free_fn(chan_l);
 
-  /*
-   * We're in CLOSED or ERROR, so the incoming channel queue is already
-   * empty.
-   */
-
   tor_free(chan_l);
 }
 
@@ -919,11 +947,9 @@ channel_listener_free(channel_listener_t *chan_l)
  * use-only function should be called only from channel_free_all() when
  * shutting down the Tor process.
  */
-
 static void
-channel_force_free(channel_t *chan)
+channel_force_xfree(channel_t *chan)
 {
-  cell_queue_entry_t *cell, *cell_tmp;
   tor_assert(chan);
 
   log_debug(LD_CHANNEL,
@@ -941,6 +967,11 @@ channel_force_free(channel_t *chan)
     circuitmux_set_policy(chan->cmux, NULL);
   }
 
+  /* Remove all timers and associated handle entries now */
+  timer_free(chan->padding_timer);
+  channel_handle_free(chan->timer_handle);
+  channel_handles_clear(chan);
+
   /* Call a free method if there is one */
   if (chan->free_fn) chan->free_fn(chan);
 
@@ -952,29 +983,16 @@ channel_force_free(channel_t *chan)
     chan->cmux = NULL;
   }
 
-  /* We might still have a cell queue; kill it */
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->incoming_queue, next, cell_tmp) {
-      cell_queue_entry_free(cell, 0);
-  }
-  TOR_SIMPLEQ_INIT(&chan->incoming_queue);
-
-  /* Outgoing cell queue is similar, but we can have to free packed cells */
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->outgoing_queue, next, cell_tmp) {
-    cell_queue_entry_free(cell, 0);
-  }
-  TOR_SIMPLEQ_INIT(&chan->outgoing_queue);
-
   tor_free(chan);
 }
 
 /**
- * Free a channel listener and skip the state/reigstration asserts; this
+ * Free a channel listener and skip the state/registration asserts; this
  * internal-use-only function should be called only from channel_free_all()
  * when shutting down the Tor process.
  */
-
 static void
-channel_listener_force_free(channel_listener_t *chan_l)
+channel_listener_force_xfree(channel_listener_t *chan_l)
 {
   tor_assert(chan_l);
 
@@ -1005,30 +1023,11 @@ channel_listener_force_free(channel_listener_t *chan_l)
 }
 
 /**
- * Return the current registered listener for a channel listener
- *
- * This function returns a function pointer to the current registered
- * handler for new incoming channels on a channel listener.
- */
-
-channel_listener_fn_ptr
-channel_listener_get_listener_fn(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  if (chan_l->state == CHANNEL_LISTENER_STATE_LISTENING)
-    return chan_l->listener;
-
-  return NULL;
-}
-
-/**
- * Set the listener for a channel listener
+ * Set the listener for a channel listener.
  *
  * This function sets the handler for new incoming channels on a channel
  * listener.
  */
-
 void
 channel_listener_set_listener_fn(channel_listener_t *chan_l,
                                 channel_listener_fn_ptr listener)
@@ -1047,12 +1046,11 @@ channel_listener_set_listener_fn(channel_listener_t *chan_l,
 }
 
 /**
- * Return the fixed-length cell handler for a channel
+ * Return the fixed-length cell handler for a channel.
  *
  * This function gets the handler for incoming fixed-length cells installed
  * on a channel.
  */
-
 channel_cell_handler_fn_ptr
 channel_get_cell_handler(channel_t *chan)
 {
@@ -1065,12 +1063,11 @@ channel_get_cell_handler(channel_t *chan)
 }
 
 /**
- * Return the variable-length cell handler for a channel
+ * Return the variable-length cell handler for a channel.
  *
  * This function gets the handler for incoming variable-length cells
  * installed on a channel.
  */
-
 channel_var_cell_handler_fn_ptr
 channel_get_var_cell_handler(channel_t *chan)
 {
@@ -1083,21 +1080,17 @@ channel_get_var_cell_handler(channel_t *chan)
 }
 
 /**
- * Set both cell handlers for a channel
+ * Set both cell handlers for a channel.
  *
  * This function sets both the fixed-length and variable length cell handlers
- * for a channel and processes any incoming cells that had been blocked in the
- * queue because none were available.
+ * for a channel.
  */
-
 void
 channel_set_cell_handlers(channel_t *chan,
                           channel_cell_handler_fn_ptr cell_handler,
                           channel_var_cell_handler_fn_ptr
                             var_cell_handler)
 {
-  int try_again = 0;
-
   tor_assert(chan);
   tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan));
 
@@ -1108,21 +1101,9 @@ channel_set_cell_handlers(channel_t *chan,
            "Setting var_cell_handler callback for channel %p to %p",
            chan, var_cell_handler);
 
-  /* Should we try the queue? */
-  if (cell_handler &&
-      cell_handler != chan->cell_handler) try_again = 1;
-  if (var_cell_handler &&
-      var_cell_handler != chan->var_cell_handler) try_again = 1;
-
   /* Change them */
   chan->cell_handler = cell_handler;
   chan->var_cell_handler = var_cell_handler;
-
-  /* Re-run the queue if we have one and there's any reason to */
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue) &&
-      try_again &&
-      (chan->cell_handler ||
-       chan->var_cell_handler)) channel_process_cells(chan);
 }
 
 /*
@@ -1139,13 +1120,12 @@ channel_set_cell_handlers(channel_t *chan,
  */
 
 /**
- * Mark a channel for closure
+ * Mark a channel for closure.
  *
  * This function tries to close a channel_t; it will go into the CLOSING
  * state, and eventually the lower layer should put it into the CLOSED or
  * ERROR state.  Then, channel_run_cleanup() will eventually free it.
  */
-
 void
 channel_mark_for_close(channel_t *chan)
 {
@@ -1179,13 +1159,12 @@ channel_mark_for_close(channel_t *chan)
 }
 
 /**
- * Mark a channel listener for closure
+ * Mark a channel listener for closure.
  *
  * This function tries to close a channel_listener_t; it will go into the
  * CLOSING state, and eventually the lower layer should put it into the CLOSED
  * or ERROR state.  Then, channel_run_cleanup() will eventually free it.
  */
-
 void
 channel_listener_mark_for_close(channel_listener_t *chan_l)
 {
@@ -1220,13 +1199,12 @@ channel_listener_mark_for_close(channel_listener_t *chan_l)
 }
 
 /**
- * Close a channel from the lower layer
+ * Close a channel from the lower layer.
  *
  * Notify the channel code that the channel is being closed due to a non-error
  * condition in the lower layer.  This does not call the close() method, since
  * the lower layer already knows.
  */
-
 void
 channel_close_from_lower_layer(channel_t *chan)
 {
@@ -1249,43 +1227,12 @@ channel_close_from_lower_layer(channel_t *chan)
 }
 
 /**
- * Close a channel listener from the lower layer
- *
- * Notify the channel code that the channel listener is being closed due to a
- * non-error condition in the lower layer.  This does not call the close()
- * method, since the lower layer already knows.
- */
-
-void
-channel_listener_close_from_lower_layer(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l != NULL);
-
-  /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */
-  if (chan_l->state == CHANNEL_LISTENER_STATE_CLOSING ||
-      chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-      chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
-
-  log_debug(LD_CHANNEL,
-            "Closing channel listener %p (global ID " U64_FORMAT ") "
-            "due to lower-layer event",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier));
-
-  /* Note closing by event from below */
-  chan_l->reason_for_closing = CHANNEL_LISTENER_CLOSE_FROM_BELOW;
-
-  /* Change state to CLOSING */
-  channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSING);
-}
-
-/**
- * Notify that the channel is being closed due to an error condition
+ * Notify that the channel is being closed due to an error condition.
  *
  * This function is called by the lower layer implementing the transport
  * when a channel must be closed due to an error condition.  This does not
  * call the channel's close method, since the lower layer already knows.
  */
-
 void
 channel_close_for_error(channel_t *chan)
 {
@@ -1307,44 +1254,12 @@ channel_close_for_error(channel_t *chan)
 }
 
 /**
- * Notify that the channel listener is being closed due to an error condition
- *
- * This function is called by the lower layer implementing the transport
- * when a channel listener must be closed due to an error condition.  This
- * does not call the channel listener's close method, since the lower layer
- * already knows.
- */
-
-void
-channel_listener_close_for_error(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l != NULL);
-
-  /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */
-  if (chan_l->state == CHANNEL_LISTENER_STATE_CLOSING ||
-      chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-      chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
-
-  log_debug(LD_CHANNEL,
-            "Closing channel listener %p (global ID " U64_FORMAT ") "
-            "due to lower-layer error",
-            chan_l, U64_PRINTF_ARG(chan_l->global_identifier));
-
-  /* Note closing by event from below */
-  chan_l->reason_for_closing = CHANNEL_LISTENER_CLOSE_FOR_ERROR;
-
-  /* Change state to CLOSING */
-  channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSING);
-}
-
-/**
- * Notify that the lower layer is finished closing the channel
+ * Notify that the lower layer is finished closing the channel.
  *
  * This function should be called by the lower layer when a channel
  * is finished closing and it should be regarded as inactive and
  * freed by the channel code.
  */
-
 void
 channel_closed(channel_t *chan)
 {
@@ -1371,39 +1286,11 @@ channel_closed(channel_t *chan)
 }
 
 /**
- * Notify that the lower layer is finished closing the channel listener
- *
- * This function should be called by the lower layer when a channel listener
- * is finished closing and it should be regarded as inactive and
- * freed by the channel code.
- */
-
-void
-channel_listener_closed(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-  tor_assert(chan_l->state == CHANNEL_LISTENER_STATE_CLOSING ||
-             chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-             chan_l->state == CHANNEL_LISTENER_STATE_ERROR);
-
-  /* No-op if already inactive */
-  if (chan_l->state == CHANNEL_LISTENER_STATE_CLOSED ||
-      chan_l->state == CHANNEL_LISTENER_STATE_ERROR) return;
-
-  if (chan_l->reason_for_closing != CHANNEL_LISTENER_CLOSE_FOR_ERROR) {
-    channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_CLOSED);
-  } else {
-    channel_listener_change_state(chan_l, CHANNEL_LISTENER_STATE_ERROR);
-  }
-}
-
-/**
- * Clear the identity_digest of a channel
+ * Clear the identity_digest of a channel.
  *
  * This function clears the identity digest of the remote endpoint for a
  * channel; this is intended for use by the lower layer.
  */
-
 void
 channel_clear_identity_digest(channel_t *chan)
 {
@@ -1428,15 +1315,15 @@ channel_clear_identity_digest(channel_t *chan)
 }
 
 /**
- * Set the identity_digest of a channel
+ * Set the identity_digest of a channel.
  *
  * This function sets the identity digest of the remote endpoint for a
  * channel; this is intended for use by the lower layer.
  */
-
 void
 channel_set_identity_digest(channel_t *chan,
-                            const char *identity_digest)
+                            const char *identity_digest,
+                            const ed25519_public_key_t *ed_identity)
 {
   int was_in_digest_map, should_be_in_digest_map, state_not_in_map;
 
@@ -1475,6 +1362,11 @@ channel_set_identity_digest(channel_t *chan,
     memset(chan->identity_digest, 0,
            sizeof(chan->identity_digest));
   }
+  if (ed_identity) {
+    memcpy(&chan->ed25519_identity, ed_identity, sizeof(*ed_identity));
+  } else {
+    memset(&chan->ed25519_identity, 0, sizeof(*ed_identity));
+  }
 
   /* Put it in the digest map if we should */
   if (should_be_in_digest_map)
@@ -1482,12 +1374,11 @@ channel_set_identity_digest(channel_t *chan,
 }
 
 /**
- * Clear the remote end metadata (identity_digest/nickname) of a channel
+ * Clear the remote end metadata (identity_digest) of a channel.
  *
  * This function clears all the remote end info from a channel; this is
  * intended for use by the lower layer.
  */
-
 void
 channel_clear_remote_end(channel_t *chan)
 {
@@ -1509,460 +1400,105 @@ channel_clear_remote_end(channel_t *chan)
 
   memset(chan->identity_digest, 0,
          sizeof(chan->identity_digest));
-  tor_free(chan->nickname);
 }
 
 /**
- * Set the remote end metadata (identity_digest/nickname) of a channel
+ * Write to a channel the given packed cell.
  *
- * This function sets new remote end info on a channel; this is intended
- * for use by the lower layer.
+ * Two possible errors can happen. Either the channel is not opened or the
+ * lower layer (specialized channel) failed to write it. In both cases, it is
+ * the caller responsibility to free the cell.
  */
-
-void
-channel_set_remote_end(channel_t *chan,
-                       const char *identity_digest,
-                       const char *nickname)
-{
-  int was_in_digest_map, should_be_in_digest_map, state_not_in_map;
-
-  tor_assert(chan);
-
-  log_debug(LD_CHANNEL,
-            "Setting remote endpoint identity on channel %p with "
-            "global ID " U64_FORMAT " to nickname %s, digest %s",
-            chan, U64_PRINTF_ARG(chan->global_identifier),
-            nickname ? nickname : "(null)",
-            identity_digest ?
-              hex_str(identity_digest, DIGEST_LEN) : "(null)");
-
-  state_not_in_map = CHANNEL_CONDEMNED(chan);
-
-  was_in_digest_map =
-    !state_not_in_map &&
-    chan->registered &&
-    !tor_digest_is_zero(chan->identity_digest);
-  should_be_in_digest_map =
-    !state_not_in_map &&
-    chan->registered &&
-    (identity_digest &&
-     !tor_digest_is_zero(identity_digest));
-
-  if (was_in_digest_map)
-    /* We should always remove it; we'll add it back if we're writing
-     * in a new digest.
-     */
-    channel_remove_from_digest_map(chan);
-
-  if (identity_digest) {
-    memcpy(chan->identity_digest,
-           identity_digest,
-           sizeof(chan->identity_digest));
-
-  } else {
-    memset(chan->identity_digest, 0,
-           sizeof(chan->identity_digest));
-  }
-
-  tor_free(chan->nickname);
-  if (nickname)
-    chan->nickname = tor_strdup(nickname);
-
-  /* Put it in the digest map if we should */
-  if (should_be_in_digest_map)
-    channel_add_to_digest_map(chan);
-}
-
-/**
- * Duplicate a cell queue entry; this is a shallow copy intended for use
- * in channel_write_cell_queue_entry().
- */
-
-static cell_queue_entry_t *
-cell_queue_entry_dup(cell_queue_entry_t *q)
-{
-  cell_queue_entry_t *rv = NULL;
-
-  tor_assert(q);
-
-  rv = tor_malloc(sizeof(*rv));
-  memcpy(rv, q, sizeof(*rv));
-
-  return rv;
-}
-
-/**
- * Free a cell_queue_entry_t; the handed_off parameter indicates whether
- * the contents were passed to the lower layer (it is responsible for
- * them) or not (we should free).
- */
-
-STATIC void
-cell_queue_entry_free(cell_queue_entry_t *q, int handed_off)
-{
-  if (!q) return;
-
-  if (!handed_off) {
-    /*
-     * If we handed it off, the recipient becomes responsible (or
-     * with packed cells the channel_t subclass calls packed_cell
-     * free after writing out its contents; see, e.g.,
-     * channel_tls_write_packed_cell_method().  Otherwise, we have
-     * to take care of it here if possible.
-     */
-    switch (q->type) {
-      case CELL_QUEUE_FIXED:
-        if (q->u.fixed.cell) {
-          /*
-           * There doesn't seem to be a cell_free() function anywhere in the
-           * pre-channel code; just use tor_free()
-           */
-          tor_free(q->u.fixed.cell);
-        }
-        break;
-      case CELL_QUEUE_PACKED:
-        if (q->u.packed.packed_cell) {
-          packed_cell_free(q->u.packed.packed_cell);
-        }
-        break;
-      case CELL_QUEUE_VAR:
-        if (q->u.var.var_cell) {
-          /*
-           * This one's in connection_or.c; it'd be nice to figure out the
-           * whole flow of cells from one end to the other and factor the
-           * cell memory management functions like this out of the specific
-           * TLS lower layer.
-           */
-          var_cell_free(q->u.var.var_cell);
-        }
-        break;
-      default:
-        /*
-         * Nothing we can do if we don't know the type; this will
-         * have been warned about elsewhere.
-         */
-        break;
-    }
-  }
-  tor_free(q);
-}
-
-#if 0
-/**
- * Check whether a cell queue entry is padding; this is a helper function
- * for channel_write_cell_queue_entry()
- */
-
 static int
-cell_queue_entry_is_padding(cell_queue_entry_t *q)
-{
-  tor_assert(q);
-
-  if (q->type == CELL_QUEUE_FIXED) {
-    if (q->u.fixed.cell) {
-      if (q->u.fixed.cell->command == CELL_PADDING ||
-          q->u.fixed.cell->command == CELL_VPADDING) {
-        return 1;
-      }
-    }
-  } else if (q->type == CELL_QUEUE_VAR) {
-    if (q->u.var.var_cell) {
-      if (q->u.var.var_cell->command == CELL_PADDING ||
-          q->u.var.var_cell->command == CELL_VPADDING) {
-        return 1;
-      }
-    }
-  }
-
-  return 0;
-}
-#endif
-
-/**
- * Allocate a new cell queue entry for a fixed-size cell
- */
-
-static cell_queue_entry_t *
-cell_queue_entry_new_fixed(cell_t *cell)
-{
-  cell_queue_entry_t *q = NULL;
-
-  tor_assert(cell);
-
-  q = tor_malloc(sizeof(*q));
-  q->type = CELL_QUEUE_FIXED;
-  q->u.fixed.cell = cell;
-
-  return q;
-}
-
-/**
- * Allocate a new cell queue entry for a variable-size cell
- */
-
-static cell_queue_entry_t *
-cell_queue_entry_new_var(var_cell_t *var_cell)
-{
-  cell_queue_entry_t *q = NULL;
-
-  tor_assert(var_cell);
-
-  q = tor_malloc(sizeof(*q));
-  q->type = CELL_QUEUE_VAR;
-  q->u.var.var_cell = var_cell;
-
-  return q;
-}
-
-/**
- * Ask how big the cell contained in a cell_queue_entry_t is
- */
-
-static size_t
-channel_get_cell_queue_entry_size(channel_t *chan, cell_queue_entry_t *q)
+write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
-  size_t rv = 0;
-
-  tor_assert(chan);
-  tor_assert(q);
-
-  switch (q->type) {
-    case CELL_QUEUE_FIXED:
-      rv = get_cell_network_size(chan->wide_circ_ids);
-      break;
-    case CELL_QUEUE_VAR:
-      rv = get_var_cell_header_size(chan->wide_circ_ids) +
-           (q->u.var.var_cell ? q->u.var.var_cell->payload_len : 0);
-      break;
-    case CELL_QUEUE_PACKED:
-      rv = get_cell_network_size(chan->wide_circ_ids);
-      break;
-    default:
-      tor_assert(1);
-  }
-
-  return rv;
-}
-
-/**
- * Write to a channel based on a cell_queue_entry_t
- *
- * Given a cell_queue_entry_t filled out by the caller, try to send the cell
- * and queue it if we can't.
- */
-
-static void
-channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q)
-{
-  int result = 0, sent = 0;
-  cell_queue_entry_t *tmp = NULL;
+  int ret = -1;
   size_t cell_bytes;
 
   tor_assert(chan);
-  tor_assert(q);
+  tor_assert(cell);
 
   /* Assert that the state makes sense for a cell write */
   tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan));
 
   {
     circid_t circ_id;
-    if (is_destroy_cell(chan, q, &circ_id)) {
+    if (packed_cell_is_destroy(chan, cell, &circ_id)) {
       channel_note_destroy_not_pending(chan, circ_id);
     }
   }
 
   /* For statistical purposes, figure out how big this cell is */
-  cell_bytes = channel_get_cell_queue_entry_size(chan, q);
+  cell_bytes = get_cell_network_size(chan->wide_circ_ids);
 
   /* Can we send it right out?  If so, try */
-  if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue) &&
-      CHANNEL_IS_OPEN(chan)) {
-    /* Pick the right write function for this cell type and save the result */
-    switch (q->type) {
-      case CELL_QUEUE_FIXED:
-        tor_assert(chan->write_cell);
-        tor_assert(q->u.fixed.cell);
-        result = chan->write_cell(chan, q->u.fixed.cell);
-        break;
-      case CELL_QUEUE_PACKED:
-        tor_assert(chan->write_packed_cell);
-        tor_assert(q->u.packed.packed_cell);
-        result = chan->write_packed_cell(chan, q->u.packed.packed_cell);
-        break;
-      case CELL_QUEUE_VAR:
-        tor_assert(chan->write_var_cell);
-        tor_assert(q->u.var.var_cell);
-        result = chan->write_var_cell(chan, q->u.var.var_cell);
-        break;
-      default:
-        tor_assert(1);
-    }
-
-    /* Check if we got it out */
-    if (result > 0) {
-      sent = 1;
-      /* Timestamp for transmission */
-      channel_timestamp_xmit(chan);
-      /* If we're here the queue is empty, so it's drained too */
-      channel_timestamp_drained(chan);
-      /* Update the counter */
-      ++(chan->n_cells_xmitted);
-      chan->n_bytes_xmitted += cell_bytes;
-      /* Update global counters */
-      ++n_channel_cells_queued;
-      ++n_channel_cells_passed_to_lower_layer;
-      n_channel_bytes_queued += cell_bytes;
-      n_channel_bytes_passed_to_lower_layer += cell_bytes;
-      channel_assert_counter_consistency();
-    }
+  if (!CHANNEL_IS_OPEN(chan)) {
+    goto done;
   }
 
-  if (!sent) {
-    /* Not sent, queue it */
-    /*
-     * We have to copy the queue entry passed in, since the caller probably
-     * used the stack.
-     */
-    tmp = cell_queue_entry_dup(q);
-    TOR_SIMPLEQ_INSERT_TAIL(&chan->outgoing_queue, tmp, next);
-    /* Update global counters */
-    ++n_channel_cells_queued;
-    ++n_channel_cells_in_queues;
-    n_channel_bytes_queued += cell_bytes;
-    n_channel_bytes_in_queues += cell_bytes;
-    channel_assert_counter_consistency();
-    /* Update channel queue size */
-    chan->bytes_in_queue += cell_bytes;
-    /* Try to process the queue? */
-    if (CHANNEL_IS_OPEN(chan)) channel_flush_cells(chan);
-  }
-}
-
-/**
- * Write a cell to a channel
- *
- * Write a fixed-length cell to a channel using the write_cell() method.
- * This is equivalent to the pre-channels connection_or_write_cell_to_buf();
- * it is called by the transport-independent code to deliver a cell to a
- * channel for transmission.
- */
-
-void
-channel_write_cell(channel_t *chan, cell_t *cell)
-{
-  cell_queue_entry_t q;
-
-  tor_assert(chan);
-  tor_assert(cell);
-
-  if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding cell_t %p on closing channel %p with "
-              "global ID "U64_FORMAT, cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    tor_free(cell);
-    return;
+  /* Write the cell on the connection's outbuf. */
+  if (chan->write_packed_cell(chan, cell) < 0) {
+    goto done;
   }
+  /* Timestamp for transmission */
+  channel_timestamp_xmit(chan);
+  /* Update the counter */
+  ++(chan->n_cells_xmitted);
+  chan->n_bytes_xmitted += cell_bytes;
+  /* Successfully sent the cell. */
+  ret = 0;
 
-  log_debug(LD_CHANNEL,
-            "Writing cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            cell, chan, U64_PRINTF_ARG(chan->global_identifier));
-
-  q.type = CELL_QUEUE_FIXED;
-  q.u.fixed.cell = cell;
-  channel_write_cell_queue_entry(chan, &q);
-
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
+ done:
+  return ret;
 }
 
 /**
- * Write a packed cell to a channel
+ * Write a packed cell to a channel.
  *
  * Write a packed cell to a channel using the write_cell() method.  This is
  * called by the transport-independent code to deliver a packed cell to a
  * channel for transmission.
- */
-
-void
-channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell)
-{
-  cell_queue_entry_t q;
-
-  tor_assert(chan);
-  tor_assert(packed_cell);
-
-  if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding packed_cell_t %p on closing channel %p "
-              "with global ID "U64_FORMAT, packed_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    packed_cell_free(packed_cell);
-    return;
-  }
-
-  log_debug(LD_CHANNEL,
-            "Writing packed_cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            packed_cell, chan,
-            U64_PRINTF_ARG(chan->global_identifier));
-
-  q.type = CELL_QUEUE_PACKED;
-  q.u.packed.packed_cell = packed_cell;
-  channel_write_cell_queue_entry(chan, &q);
-
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
-}
-
-/**
- * Write a variable-length cell to a channel
  *
- * Write a variable-length cell to a channel using the write_cell() method.
- * This is equivalent to the pre-channels
- * connection_or_write_var_cell_to_buf(); it's called by the transport-
- * independent code to deliver a var_cell to a channel for transmission.
+ * Return 0 on success else a negative value. In both cases, the caller should
+ * not access the cell anymore, it is freed both on success and error.
  */
-
-void
-channel_write_var_cell(channel_t *chan, var_cell_t *var_cell)
+int
+channel_write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
-  cell_queue_entry_t q;
+  int ret = -1;
 
   tor_assert(chan);
-  tor_assert(var_cell);
+  tor_assert(cell);
 
   if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding var_cell_t %p on closing channel %p "
-              "with global ID "U64_FORMAT, var_cell, chan,
+    log_debug(LD_CHANNEL, "Discarding %p on closing channel %p with "
+              "global ID "U64_FORMAT, cell, chan,
               U64_PRINTF_ARG(chan->global_identifier));
-    var_cell_free(var_cell);
-    return;
+    goto end;
   }
-
   log_debug(LD_CHANNEL,
-            "Writing var_cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            var_cell, chan,
-            U64_PRINTF_ARG(chan->global_identifier));
+            "Writing %p to channel %p with global ID "
+            U64_FORMAT, cell, chan, U64_PRINTF_ARG(chan->global_identifier));
 
-  q.type = CELL_QUEUE_VAR;
-  q.u.var.var_cell = var_cell;
-  channel_write_cell_queue_entry(chan, &q);
+  ret = write_packed_cell(chan, cell);
 
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
+ end:
+  /* Whatever happens, we free the cell. Either an error occurred or the cell
+   * was put on the connection outbuf, both cases we have ownership of the
+   * cell and we free it. */
+  packed_cell_free(cell);
+  return ret;
 }
 
 /**
- * Change channel state
+ * Change channel state.
  *
  * This internal and subclass use only function is used to change channel
  * state, performing all transition validity checks and whatever actions
  * are appropriate to the state transition in question.
  */
-
-void
-channel_change_state(channel_t *chan, channel_state_t to_state)
+static void
+channel_change_state_(channel_t *chan, channel_state_t to_state)
 {
   channel_state_t from_state;
   unsigned char was_active, is_active;
@@ -1992,15 +1528,6 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
     tor_assert(chan->reason_for_closing != CHANNEL_NOT_CLOSING);
   }
 
-  /*
-   * We need to maintain the queues here for some transitions:
-   * when we enter CHANNEL_STATE_OPEN (especially from CHANNEL_STATE_MAINT)
-   * we may have a backlog of cells to transmit, so drain the queues in
-   * that case, and when going to CHANNEL_STATE_CLOSED the subclass
-   * should have made sure to finish sending things (or gone to
-   * CHANNEL_STATE_ERROR if not possible), so we assert for that here.
-   */
-
   log_debug(LD_CHANNEL,
             "Changing state of channel %p (global ID " U64_FORMAT
             ") from \"%s\" to \"%s\"",
@@ -2023,6 +1550,7 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
       if (active_channels) smartlist_remove(active_channels, chan);
       if (!finished_channels) finished_channels = smartlist_new();
       smartlist_add(finished_channels, chan);
+      mainloop_schedule_postloop_cleanup();
     }
     /* Need to put on active list? */
     else if (!was_active && is_active) {
@@ -2057,56 +1585,38 @@ channel_change_state(channel_t *chan, channel_state_t to_state)
   } else if (to_state == CHANNEL_STATE_MAINT) {
     scheduler_channel_doesnt_want_writes(chan);
   }
+}
 
-  /*
-   * If we're closing, this channel no longer counts toward the global
-   * estimated queue size; if we're open, it now does.
-   */
-  if ((to_state == CHANNEL_STATE_CLOSING ||
-       to_state == CHANNEL_STATE_CLOSED ||
-       to_state == CHANNEL_STATE_ERROR) &&
-      (from_state == CHANNEL_STATE_OPEN ||
-       from_state == CHANNEL_STATE_MAINT)) {
-    estimated_total_queue_size -= chan->bytes_in_queue;
-  }
+/**
+ * As channel_change_state_, but change the state to any state but open.
+ */
+void
+channel_change_state(channel_t *chan, channel_state_t to_state)
+{
+  tor_assert(to_state != CHANNEL_STATE_OPEN);
+  channel_change_state_(chan, to_state);
+}
 
-  /*
-   * If we're opening, this channel now does count toward the global
-   * estimated queue size.
-   */
-  if ((to_state == CHANNEL_STATE_OPEN ||
-       to_state == CHANNEL_STATE_MAINT) &&
-      !(from_state == CHANNEL_STATE_OPEN ||
-        from_state == CHANNEL_STATE_MAINT)) {
-    estimated_total_queue_size += chan->bytes_in_queue;
-  }
+/**
+ * As channel_change_state, but change the state to open.
+ */
+void
+channel_change_state_open(channel_t *chan)
+{
+  channel_change_state_(chan, CHANNEL_STATE_OPEN);
 
   /* Tell circuits if we opened and stuff */
-  if (to_state == CHANNEL_STATE_OPEN) {
-    channel_do_open_actions(chan);
-    chan->has_been_open = 1;
-
-    /* Check for queued cells to process */
-    if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-      channel_process_cells(chan);
-    if (! TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue))
-      channel_flush_cells(chan);
-  } else if (to_state == CHANNEL_STATE_CLOSED ||
-             to_state == CHANNEL_STATE_ERROR) {
-    /* Assert that all queues are empty */
-    tor_assert(TOR_SIMPLEQ_EMPTY(&chan->incoming_queue));
-    tor_assert(TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue));
-  }
+  channel_do_open_actions(chan);
+  chan->has_been_open = 1;
 }
 
 /**
- * Change channel listener state
+ * Change channel listener state.
  *
  * This internal and subclass use only function is used to change channel
  * listener state, performing all transition validity checks and whatever
  * actions are appropriate to the state transition in question.
  */
-
 void
 channel_listener_change_state(channel_listener_t *chan_l,
                               channel_listener_state_t to_state)
@@ -2138,15 +1648,6 @@ channel_listener_change_state(channel_listener_t *chan_l,
     tor_assert(chan_l->reason_for_closing != CHANNEL_LISTENER_NOT_CLOSING);
   }
 
-  /*
-   * We need to maintain the queues here for some transitions:
-   * when we enter CHANNEL_STATE_OPEN (especially from CHANNEL_STATE_MAINT)
-   * we may have a backlog of cells to transmit, so drain the queues in
-   * that case, and when going to CHANNEL_STATE_CLOSED the subclass
-   * should have made sure to finish sending things (or gone to
-   * CHANNEL_STATE_ERROR if not possible), so we assert for that here.
-   */
-
   log_debug(LD_CHANNEL,
             "Changing state of channel listener %p (global ID " U64_FORMAT
             "from \"%s\" to \"%s\"",
@@ -2168,6 +1669,7 @@ channel_listener_change_state(channel_listener_t *chan_l,
       if (active_listeners) smartlist_remove(active_listeners, chan_l);
       if (!finished_listeners) finished_listeners = smartlist_new();
       smartlist_add(finished_listeners, chan_l);
+      mainloop_schedule_postloop_cleanup();
     }
     /* Need to put on active list? */
     else if (!was_active && is_active) {
@@ -2179,30 +1681,39 @@ channel_listener_change_state(channel_listener_t *chan_l,
 
   if (to_state == CHANNEL_LISTENER_STATE_CLOSED ||
       to_state == CHANNEL_LISTENER_STATE_ERROR) {
-    /* Assert that the queue is empty */
     tor_assert(!(chan_l->incoming_list) ||
                 smartlist_len(chan_l->incoming_list) == 0);
   }
 }
 
-/**
- * Try to flush cells to the lower layer
- *
- * this is called by the lower layer to indicate that it wants more cells;
- * it will try to write up to num_cells cells from the channel's cell queue or
- * from circuits active on that channel, or as many as it has available if
- * num_cells == -1.
- */
-
+/* Maximum number of cells that is allowed to flush at once within
+ * channel_flush_some_cells(). */
 #define MAX_CELLS_TO_GET_FROM_CIRCUITS_FOR_UNLIMITED 256
 
+/**
+ * Try to flush cells of the given channel chan up to a maximum of num_cells.
+ *
+ * This is called by the scheduler when it wants to flush cells from the
+ * channel's circuit queue(s) to the connection outbuf (not yet on the wire).
+ *
+ * If the channel is not in state CHANNEL_STATE_OPEN, this does nothing and
+ * will return 0 meaning no cells were flushed.
+ *
+ * If num_cells is -1, we'll try to flush up to the maximum cells allowed
+ * defined in MAX_CELLS_TO_GET_FROM_CIRCUITS_FOR_UNLIMITED.
+ *
+ * On success, the number of flushed cells are returned and it can never be
+ * above num_cells. If 0 is returned, no cells were flushed either because the
+ * channel was not opened or we had no cells on the channel. A negative number
+ * can NOT be sent back.
+ *
+ * This function is part of the fast path. */
 MOCK_IMPL(ssize_t,
 channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
 {
   unsigned int unlimited = 0;
   ssize_t flushed = 0;
-  int num_cells_from_circs, clamped_num_cells;
-  int q_len_before, q_len_after;
+  int clamped_num_cells;
 
   tor_assert(chan);
 
@@ -2211,11 +1722,6 @@ channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
 
   /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */
   if (CHANNEL_IS_OPEN(chan)) {
-    /* Try to flush as much as we can that's already queued */
-    flushed += channel_flush_some_cells_from_outgoing_queue(chan,
-        (unlimited ? -1 : num_cells - flushed));
-    if (!unlimited && num_cells <= flushed) goto done;
-
     if (circuitmux_num_cells(chan->cmux) > 0) {
       /* Calculate number of cells, including clamp */
       if (unlimited) {
@@ -2229,45 +1735,9 @@ channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
         }
       }
 
-      /*
-       * Keep track of the change in queue size; we have to count cells
-       * channel_flush_from_first_active_circuit() writes out directly,
-       * but not double-count ones we might get later in
-       * channel_flush_some_cells_from_outgoing_queue()
-       */
-      q_len_before = chan_cell_queue_len(&(chan->outgoing_queue));
-
       /* Try to get more cells from any active circuits */
-      num_cells_from_circs = channel_flush_from_first_active_circuit(
+      flushed = channel_flush_from_first_active_circuit(
           chan, clamped_num_cells);
-
-      q_len_after = chan_cell_queue_len(&(chan->outgoing_queue));
-
-      /*
-       * If it claims we got some, adjust the flushed counter and consider
-       * processing the queue again
-       */
-      if (num_cells_from_circs > 0) {
-        /*
-         * Adjust flushed by the number of cells counted in
-         * num_cells_from_circs that didn't go to the cell queue.
-         */
-
-        if (q_len_after > q_len_before) {
-          num_cells_from_circs -= (q_len_after - q_len_before);
-          if (num_cells_from_circs < 0) num_cells_from_circs = 0;
-        }
-
-        flushed += num_cells_from_circs;
-
-        /* Now process the queue if necessary */
-
-        if ((q_len_after > q_len_before) &&
-            (unlimited || (flushed < num_cells))) {
-          flushed += channel_flush_some_cells_from_outgoing_queue(chan,
-            (unlimited ? -1 : num_cells - flushed));
-        }
-      }
     }
   }
 
@@ -2276,198 +1746,16 @@ channel_flush_some_cells, (channel_t *chan, ssize_t num_cells))
 }
 
 /**
- * Flush cells from just the channel's outgoing cell queue
+ * Check if any cells are available.
  *
- * This gets called from channel_flush_some_cells() above to flush cells
- * just from the queue without trying for active_circuits.
+ * This is used by the scheduler to know if the channel has more to flush
+ * after a scheduling round.
  */
-
-static ssize_t
-channel_flush_some_cells_from_outgoing_queue(channel_t *chan,
-                                             ssize_t num_cells)
-{
-  unsigned int unlimited = 0;
-  ssize_t flushed = 0;
-  cell_queue_entry_t *q = NULL;
-  size_t cell_size;
-  int free_q = 0, handed_off = 0;
-
-  tor_assert(chan);
-  tor_assert(chan->write_cell);
-  tor_assert(chan->write_packed_cell);
-  tor_assert(chan->write_var_cell);
-
-  if (num_cells < 0) unlimited = 1;
-  if (!unlimited && num_cells <= flushed) return 0;
-
-  /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */
-  if (CHANNEL_IS_OPEN(chan)) {
-    while ((unlimited || num_cells > flushed) &&
-           NULL != (q = TOR_SIMPLEQ_FIRST(&chan->outgoing_queue))) {
-      free_q = 0;
-      handed_off = 0;
-
-      if (1) {
-        /* Figure out how big it is for statistical purposes */
-        cell_size = channel_get_cell_queue_entry_size(chan, q);
-        /*
-         * Okay, we have a good queue entry, try to give it to the lower
-         * layer.
-         */
-        switch (q->type) {
-          case CELL_QUEUE_FIXED:
-            if (q->u.fixed.cell) {
-              if (chan->write_cell(chan,
-                    q->u.fixed.cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_FIXED "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-         case CELL_QUEUE_PACKED:
-            if (q->u.packed.packed_cell) {
-              if (chan->write_packed_cell(chan,
-                    q->u.packed.packed_cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_PACKED "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-         case CELL_QUEUE_VAR:
-            if (q->u.var.var_cell) {
-              if (chan->write_var_cell(chan,
-                    q->u.var.var_cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_VAR "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-          default:
-            /* Unknown type, log and free it */
-            log_info(LD_CHANNEL,
-                     "Saw an unknown cell queue entry type %d on channel %p "
-                     "(global ID " U64_FORMAT "; ignoring it."
-                     "  Someone should fix this.",
-                     q->type, chan, U64_PRINTF_ARG(chan->global_identifier));
-            free_q = 1;
-            handed_off = 0;
-        }
-
-        /*
-         * if free_q is set, we used it and should remove the queue entry;
-         * we have to do the free down here so TOR_SIMPLEQ_REMOVE_HEAD isn't
-         * accessing freed memory
-         */
-        if (free_q) {
-          TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next);
-          /*
-           * ...and we handed a cell off to the lower layer, so we should
-           * update the counters.
-           */
-          ++n_channel_cells_passed_to_lower_layer;
-          --n_channel_cells_in_queues;
-          n_channel_bytes_passed_to_lower_layer += cell_size;
-          n_channel_bytes_in_queues -= cell_size;
-          channel_assert_counter_consistency();
-          /* Update the channel's queue size too */
-          chan->bytes_in_queue -= cell_size;
-          /* Finally, free q */
-          cell_queue_entry_free(q, handed_off);
-          q = NULL;
-        }
-        /* No cell removed from list, so we can't go on any further */
-        else break;
-      }
-    }
-  }
-
-  /* Did we drain the queue? */
-  if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue)) {
-    channel_timestamp_drained(chan);
-  }
-
-  /* Update the estimate queue size */
-  channel_update_xmit_queue_size(chan);
-
-  return flushed;
-}
-
-/**
- * Flush as many cells as we possibly can from the queue
- *
- * This tries to flush as many cells from the queue as the lower layer
- * will take.  It just calls channel_flush_some_cells_from_outgoing_queue()
- * in unlimited mode.
- */
-
-void
-channel_flush_cells(channel_t *chan)
-{
-  channel_flush_some_cells_from_outgoing_queue(chan, -1);
-}
-
-/**
- * Check if any cells are available
- *
- * This gets used from the lower layer to check if any more cells are
- * available.
- */
-
-int
-channel_more_to_flush(channel_t *chan)
+MOCK_IMPL(int,
+channel_more_to_flush, (channel_t *chan))
 {
   tor_assert(chan);
 
-  /* Check if we have any queued */
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-      return 1;
-
-  /* Check if any circuits would like to queue some */
   if (circuitmux_num_cells(chan->cmux) > 0) return 1;
 
   /* Else no */
@@ -2475,12 +1763,11 @@ channel_more_to_flush(channel_t *chan)
 }
 
 /**
- * Notify the channel we're done flushing the output in the lower layer
+ * Notify the channel we're done flushing the output in the lower layer.
  *
  * Connection.c will call this when we've flushed the output; there's some
  * dirreq-related maintenance to do.
  */
-
 void
 channel_notify_flushed(channel_t *chan)
 {
@@ -2493,12 +1780,11 @@ channel_notify_flushed(channel_t *chan)
 }
 
 /**
- * Process the queue of incoming channels on a listener
+ * Process the queue of incoming channels on a listener.
  *
  * Use a listener's registered callback to process as many entries in the
  * queue of incoming channels as possible.
  */
-
 void
 channel_listener_process_incoming(channel_listener_t *listener)
 {
@@ -2540,18 +1826,17 @@ channel_listener_process_incoming(channel_listener_t *listener)
 }
 
 /**
- * Take actions required when a channel becomes open
+ * Take actions required when a channel becomes open.
  *
  * Handle actions we should do when we know a channel is open; a lot of
  * this comes from the old connection_or_set_state_open() of connection_or.c.
  *
  * Because of this mechanism, future channel_t subclasses should take care
- * not to change a channel to from CHANNEL_STATE_OPENING to CHANNEL_STATE_OPEN
+ * not to change a channel from CHANNEL_STATE_OPENING to CHANNEL_STATE_OPEN
  * until there is positive confirmation that the network is operational.
  * In particular, anything UDP-based should not make this transition until a
  * packet is received from the other side.
  */
-
 void
 channel_do_open_actions(channel_t *chan)
 {
@@ -2566,21 +1851,10 @@ channel_do_open_actions(channel_t *chan)
 
   if (started_here) {
     circuit_build_times_network_is_live(get_circuit_build_times_mutable());
-    rep_hist_note_connect_succeeded(chan->identity_digest, now);
-    if (entry_guard_register_connect_status(
-          chan->identity_digest, 1, 0, now) < 0) {
-      /* Close any circuits pending on this channel. We leave it in state
-       * 'open' though, because it didn't actually *fail* -- we just
-       * chose not to use it. */
-      log_debug(LD_OR,
-                "New entry guard was reachable, but closing this "
-                "connection so we can retry the earlier entry guards.");
-      close_origin_circuits = 1;
-    }
     router_set_status(chan->identity_digest, 1);
   } else {
-    /* only report it to the geoip module if it's not a known router */
-    if (!router_get_by_id_digest(chan->identity_digest)) {
+    /* only report it to the geoip module if it's a client */
+    if (channel_is_client(chan)) {
       if (channel_get_addr_if_possible(chan, &remote_addr)) {
         char *transport_name = NULL;
         channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
@@ -2600,17 +1874,42 @@ channel_do_open_actions(channel_t *chan)
     }
   }
 
+  /* Disable or reduce padding according to user prefs. */
+  if (chan->padding_enabled || get_options()->ConnectionPadding == 1) {
+    if (!get_options()->ConnectionPadding) {
+      /* Disable if torrc disabled */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (get_options()->Tor2webMode &&
+            !networkstatus_get_param(NULL,
+                                     CHANNELPADDING_TOR2WEB_PARAM,
+                                     CHANNELPADDING_TOR2WEB_DEFAULT, 0, 1)) {
+      /* Disable if we're using tor2web and the consensus disabled padding
+       * for tor2web */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (rend_service_allow_non_anonymous_connection(get_options()) &&
+               !networkstatus_get_param(NULL,
+                                        CHANNELPADDING_SOS_PARAM,
+                                        CHANNELPADDING_SOS_DEFAULT, 0, 1)) {
+      /* Disable if we're using RSOS and the consensus disabled padding
+       * for RSOS */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (get_options()->ReducedConnectionPadding) {
+      /* Padding can be forced and/or reduced by clients, regardless of if
+       * the channel supports it */
+      channelpadding_reduce_padding_on_channel(chan);
+    }
+  }
+
   circuit_n_chan_done(chan, 1, close_origin_circuits);
 }
 
 /**
- * Queue an incoming channel on a listener
+ * Queue an incoming channel on a listener.
  *
  * Internal and subclass use only function to queue an incoming channel from
  * a listener.  A subclass of channel_listener_t should call this when a new
  * incoming channel is created.
  */
-
 void
 channel_listener_queue_incoming(channel_listener_t *listener,
                                 channel_t *incoming)
@@ -2660,207 +1959,31 @@ channel_listener_queue_incoming(channel_listener_t *listener,
 }
 
 /**
- * Process queued incoming cells
- *
- * Process as many queued cells as we can from the incoming
- * cell queue.
+ * Process a cell from the given channel.
  */
-
 void
-channel_process_cells(channel_t *chan)
+channel_process_cell(channel_t *chan, cell_t *cell)
 {
-  cell_queue_entry_t *q;
   tor_assert(chan);
   tor_assert(CHANNEL_IS_CLOSING(chan) || CHANNEL_IS_MAINT(chan) ||
              CHANNEL_IS_OPEN(chan));
-
-  log_debug(LD_CHANNEL,
-            "Processing as many incoming cells as we can for channel %p",
-            chan);
-
-  /* Nothing we can do if we have no registered cell handlers */
-  if (!(chan->cell_handler ||
-        chan->var_cell_handler)) return;
-  /* Nothing we can do if we have no cells */
-  if (TOR_SIMPLEQ_EMPTY(&chan->incoming_queue)) return;
-
-  /*
-   * Process cells until we're done or find one we have no current handler
-   * for.
-   *
-   * We must free the cells here after calling the handler, since custody
-   * of the buffer was given to the channel layer when they were queued;
-   * see comments on memory management in channel_queue_cell() and in
-   * channel_queue_var_cell() below.
-   */
-  while (NULL != (q = TOR_SIMPLEQ_FIRST(&chan->incoming_queue))) {
-    tor_assert(q);
-    tor_assert(q->type == CELL_QUEUE_FIXED ||
-               q->type == CELL_QUEUE_VAR);
-
-    if (q->type == CELL_QUEUE_FIXED &&
-        chan->cell_handler) {
-      /* Handle a fixed-length cell */
-      TOR_SIMPLEQ_REMOVE_HEAD(&chan->incoming_queue, next);
-      tor_assert(q->u.fixed.cell);
-      log_debug(LD_CHANNEL,
-                "Processing incoming cell_t %p for channel %p (global ID "
-                U64_FORMAT ")",
-                q->u.fixed.cell, chan,
-                U64_PRINTF_ARG(chan->global_identifier));
-      chan->cell_handler(chan, q->u.fixed.cell);
-      tor_free(q->u.fixed.cell);
-      tor_free(q);
-    } else if (q->type == CELL_QUEUE_VAR &&
-               chan->var_cell_handler) {
-      /* Handle a variable-length cell */
-      TOR_SIMPLEQ_REMOVE_HEAD(&chan->incoming_queue, next);
-      tor_assert(q->u.var.var_cell);
-      log_debug(LD_CHANNEL,
-                "Processing incoming var_cell_t %p for channel %p (global ID "
-                U64_FORMAT ")",
-                q->u.var.var_cell, chan,
-                U64_PRINTF_ARG(chan->global_identifier));
-      chan->var_cell_handler(chan, q->u.var.var_cell);
-      tor_free(q->u.var.var_cell);
-      tor_free(q);
-    } else {
-      /* Can't handle this one */
-      break;
-    }
-  }
-}
-
-/**
- * Queue incoming cell
- *
- * This should be called by a channel_t subclass to queue an incoming fixed-
- * length cell for processing, and process it if possible.
- */
-
-void
-channel_queue_cell(channel_t *chan, cell_t *cell)
-{
-  int need_to_queue = 0;
-  cell_queue_entry_t *q;
-  cell_t *cell_copy = NULL;
-
-  tor_assert(chan);
   tor_assert(cell);
-  tor_assert(CHANNEL_IS_OPEN(chan));
 
-  /* Do we need to queue it, or can we just call the handler right away? */
-  if (!(chan->cell_handler)) need_to_queue = 1;
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-    need_to_queue = 1;
+  /* Nothing we can do if we have no registered cell handlers */
+  if (!chan->cell_handler)
+    return;
 
   /* Timestamp for receiving */
   channel_timestamp_recv(chan);
-
-  /* Update the counters */
+  /* Update received counter. */
   ++(chan->n_cells_recved);
   chan->n_bytes_recved += get_cell_network_size(chan->wide_circ_ids);
 
-  /* If we don't need to queue we can just call cell_handler */
-  if (!need_to_queue) {
-    tor_assert(chan->cell_handler);
-    log_debug(LD_CHANNEL,
-              "Directly handling incoming cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    chan->cell_handler(chan, cell);
-  } else {
-    /*
-     * Otherwise queue it and then process the queue if possible.
-     *
-     * We queue a copy, not the original pointer - it might have been on the
-     * stack in connection_or_process_cells_from_inbuf() (or another caller
-     * if we ever have a subclass other than channel_tls_t), or be freed
-     * there after we return.  This is the uncommon case; the non-copying
-     * fast path occurs in the if (!need_to_queue) case above when the
-     * upper layer has installed cell handlers.
-     */
-    cell_copy = tor_malloc_zero(sizeof(cell_t));
-    memcpy(cell_copy, cell, sizeof(cell_t));
-    q = cell_queue_entry_new_fixed(cell_copy);
-    log_debug(LD_CHANNEL,
-              "Queueing incoming cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    TOR_SIMPLEQ_INSERT_TAIL(&chan->incoming_queue, q, next);
-    if (chan->cell_handler ||
-        chan->var_cell_handler) {
-      channel_process_cells(chan);
-    }
-  }
-}
-
-/**
- * Queue incoming variable-length cell
- *
- * This should be called by a channel_t subclass to queue an incoming
- * variable-length cell for processing, and process it if possible.
- */
-
-void
-channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell)
-{
-  int need_to_queue = 0;
-  cell_queue_entry_t *q;
-  var_cell_t *cell_copy = NULL;
-
-  tor_assert(chan);
-  tor_assert(var_cell);
-  tor_assert(CHANNEL_IS_OPEN(chan));
-
-  /* Do we need to queue it, or can we just call the handler right away? */
-  if (!(chan->var_cell_handler)) need_to_queue = 1;
-  if (! TOR_SIMPLEQ_EMPTY(&chan->incoming_queue))
-    need_to_queue = 1;
-
-  /* Timestamp for receiving */
-  channel_timestamp_recv(chan);
-
-  /* Update the counter */
-  ++(chan->n_cells_recved);
-  chan->n_bytes_recved += get_var_cell_header_size(chan->wide_circ_ids) +
-    var_cell->payload_len;
-
-  /* If we don't need to queue we can just call cell_handler */
-  if (!need_to_queue) {
-    tor_assert(chan->var_cell_handler);
-    log_debug(LD_CHANNEL,
-              "Directly handling incoming var_cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              var_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    chan->var_cell_handler(chan, var_cell);
-  } else {
-    /*
-     * Otherwise queue it and then process the queue if possible.
-     *
-     * We queue a copy, not the original pointer - it might have been on the
-     * stack in connection_or_process_cells_from_inbuf() (or another caller
-     * if we ever have a subclass other than channel_tls_t), or be freed
-     * there after we return.  This is the uncommon case; the non-copying
-     * fast path occurs in the if (!need_to_queue) case above when the
-     * upper layer has installed cell handlers.
-     */
-    cell_copy = var_cell_copy(var_cell);
-    q = cell_queue_entry_new_var(cell_copy);
-    log_debug(LD_CHANNEL,
-              "Queueing incoming var_cell_t %p for channel %p "
-              "(global ID " U64_FORMAT ")",
-              var_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    TOR_SIMPLEQ_INSERT_TAIL(&chan->incoming_queue, q, next);
-    if (chan->cell_handler ||
-        chan->var_cell_handler) {
-      channel_process_cells(chan);
-    }
-  }
+  log_debug(LD_CHANNEL,
+            "Processing incoming cell_t %p for channel %p (global ID "
+            U64_FORMAT ")", cell, chan,
+            U64_PRINTF_ARG(chan->global_identifier));
+  chan->cell_handler(chan, cell);
 }
 
 /** If <b>packed_cell</b> on <b>chan</b> is a destroy cell, then set
@@ -2887,51 +2010,12 @@ packed_cell_is_destroy(channel_t *chan,
 }
 
 /**
- * Assert that the global channel stats counters are internally consistent
- */
-
-static void
-channel_assert_counter_consistency(void)
-{
-  tor_assert(n_channel_cells_queued ==
-      (n_channel_cells_in_queues + n_channel_cells_passed_to_lower_layer));
-  tor_assert(n_channel_bytes_queued ==
-      (n_channel_bytes_in_queues + n_channel_bytes_passed_to_lower_layer));
-}
-
-/* DOCDOC */
-static int
-is_destroy_cell(channel_t *chan,
-                const cell_queue_entry_t *q, circid_t *circid_out)
-{
-  *circid_out = 0;
-  switch (q->type) {
-    case CELL_QUEUE_FIXED:
-      if (q->u.fixed.cell->command == CELL_DESTROY) {
-        *circid_out = q->u.fixed.cell->circ_id;
-        return 1;
-      }
-      break;
-    case CELL_QUEUE_VAR:
-      if (q->u.var.var_cell->command == CELL_DESTROY) {
-        *circid_out = q->u.var.var_cell->circ_id;
-        return 1;
-      }
-      break;
-    case CELL_QUEUE_PACKED:
-      return packed_cell_is_destroy(chan, q->u.packed.packed_cell, circid_out);
-  }
-  return 0;
-}
-
-/**
- * Send destroy cell on a channel
+ * Send destroy cell on a channel.
  *
  * Write a destroy cell with circ ID <b>circ_id</b> and reason <b>reason</b>
  * onto channel <b>chan</b>.  Don't perform range-checking on reason:
  * we may want to propagate reasons from other cells.
  */
-
 int
 channel_send_destroy(circid_t circ_id, channel_t *chan, int reason)
 {
@@ -2967,30 +2051,16 @@ channel_send_destroy(circid_t circ_id, channel_t *chan, int reason)
 }
 
 /**
- * Dump channel statistics to the log
+ * Dump channel statistics to the log.
  *
  * This is called from dumpstats() in main.c and spams the log with
  * statistics on channels.
  */
-
 void
 channel_dumpstats(int severity)
 {
   if (all_channels && smartlist_len(all_channels) > 0) {
     tor_log(severity, LD_GENERAL,
-        "Channels have queued " U64_FORMAT " bytes in " U64_FORMAT " cells, "
-        "and handed " U64_FORMAT " bytes in " U64_FORMAT " cells to the lower"
-        " layer.",
-        U64_PRINTF_ARG(n_channel_bytes_queued),
-        U64_PRINTF_ARG(n_channel_cells_queued),
-        U64_PRINTF_ARG(n_channel_bytes_passed_to_lower_layer),
-        U64_PRINTF_ARG(n_channel_cells_passed_to_lower_layer));
-    tor_log(severity, LD_GENERAL,
-        "There are currently " U64_FORMAT " bytes in " U64_FORMAT " cells "
-        "in channel queues.",
-        U64_PRINTF_ARG(n_channel_bytes_in_queues),
-        U64_PRINTF_ARG(n_channel_cells_in_queues));
-    tor_log(severity, LD_GENERAL,
         "Dumping statistics about %d channels:",
         smartlist_len(all_channels));
     tor_log(severity, LD_GENERAL,
@@ -3012,12 +2082,11 @@ channel_dumpstats(int severity)
 }
 
 /**
- * Dump channel listener statistics to the log
+ * Dump channel listener statistics to the log.
  *
  * This is called from dumpstats() in main.c and spams the log with
  * statistics on channel listeners.
  */
-
 void
 channel_listener_dumpstats(int severity)
 {
@@ -3044,28 +2113,11 @@ channel_listener_dumpstats(int severity)
 }
 
 /**
- * Set the cmux policy on all active channels
- */
-
-void
-channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol)
-{
-  if (!active_channels) return;
-
-  SMARTLIST_FOREACH_BEGIN(active_channels, channel_t *, curr) {
-    if (curr->cmux) {
-      circuitmux_set_policy(curr->cmux, pol);
-    }
-  } SMARTLIST_FOREACH_END(curr);
-}
-
-/**
- * Clean up channels
+ * Clean up channels.
  *
  * This gets called periodically from run_scheduled_events() in main.c;
  * it cleans up after closed channels.
  */
-
 void
 channel_run_cleanup(void)
 {
@@ -3087,12 +2139,11 @@ channel_run_cleanup(void)
 }
 
 /**
- * Clean up channel listeners
+ * Clean up channel listeners.
  *
  * This gets called periodically from run_scheduled_events() in main.c;
  * it cleans up after closed channel listeners.
  */
-
 void
 channel_listener_run_cleanup(void)
 {
@@ -3114,9 +2165,8 @@ channel_listener_run_cleanup(void)
 }
 
 /**
- * Free a list of channels for channel_free_all()
+ * Free a list of channels for channel_free_all().
  */
-
 static void
 channel_free_list(smartlist_t *channels, int mark_for_close)
 {
@@ -3140,15 +2190,14 @@ channel_free_list(smartlist_t *channels, int mark_for_close)
       if (!CHANNEL_CONDEMNED(curr)) {
         channel_mark_for_close(curr);
       }
-      channel_force_free(curr);
+      channel_force_xfree(curr);
     } else channel_free(curr);
   } SMARTLIST_FOREACH_END(curr);
 }
 
 /**
- * Free a list of channel listeners for channel_free_all()
+ * Free a list of channel listeners for channel_free_all().
  */
-
 static void
 channel_listener_free_list(smartlist_t *listeners, int mark_for_close)
 {
@@ -3169,20 +2218,19 @@ channel_listener_free_list(smartlist_t *listeners, int mark_for_close)
             curr->state == CHANNEL_LISTENER_STATE_ERROR)) {
         channel_listener_mark_for_close(curr);
       }
-      channel_listener_force_free(curr);
+      channel_listener_force_xfree(curr);
     } else channel_listener_free(curr);
   } SMARTLIST_FOREACH_END(curr);
 }
 
 /**
- * Close all channels and free everything
+ * Close all channels and free everything.
  *
  * This gets called from tor_free_all() in main.c to clean up on exit.
  * It will close all registered channels and free associated storage,
  * then free the all_channels, active_channels, listening_channels and
  * finished_channels lists and also channel_identity_map.
  */
-
 void
 channel_free_all(void)
 {
@@ -3237,12 +2285,17 @@ channel_free_all(void)
   /* Geez, anything still left over just won't die ... let it leak then */
   HT_CLEAR(channel_idmap, &channel_identity_map);
 
+  /* Same with channel_gid_map */
+  log_debug(LD_CHANNEL,
+            "Freeing channel_gid_map");
+  HT_CLEAR(channel_gid_map, &channel_gid_map);
+
   log_debug(LD_CHANNEL,
             "Done cleaning up after channels");
 }
 
 /**
- * Connect to a given addr/port/digest
+ * Connect to a given addr/port/digest.
  *
  * This sets up a new outgoing channel; in the future if multiple
  * channel_t subclasses are available, this is where the selection policy
@@ -3251,16 +2304,16 @@ channel_free_all(void)
  * single abstract object encapsulating all the protocol details of
  * how to contact an OR.
  */
-
 channel_t *
 channel_connect(const tor_addr_t *addr, uint16_t port,
-                const char *id_digest)
+                const char *id_digest,
+                const ed25519_public_key_t *ed_id)
 {
-  return channel_tls_connect(addr, port, id_digest);
+  return channel_tls_connect(addr, port, id_digest, ed_id);
 }
 
 /**
- * Decide which of two channels to prefer for extending a circuit
+ * Decide which of two channels to prefer for extending a circuit.
  *
  * This function is called while extending a circuit and returns true iff
  * a is 'better' than b.  The most important criterion here is that a
@@ -3269,24 +2322,21 @@ channel_connect(const tor_addr_t *addr, uint16_t port,
  *
  * This is based on the former connection_or_is_better() of connection_or.c
  */
-
 int
-channel_is_better(time_t now, channel_t *a, channel_t *b,
-                  int forgive_new_connections)
+channel_is_better(channel_t *a, channel_t *b)
 {
-  int a_grace, b_grace;
   int a_is_canonical, b_is_canonical;
-  int a_has_circs, b_has_circs;
-
-  /*
-   * Do not definitively deprecate a new channel with no circuits on it
-   * until this much time has passed.
-   */
-#define NEW_CHAN_GRACE_PERIOD (15*60)
 
   tor_assert(a);
   tor_assert(b);
 
+  /* If one channel is bad for new circuits, and the other isn't,
+   * use the one that is still good. */
+  if (!channel_is_bad_for_new_circs(a) && channel_is_bad_for_new_circs(b))
+    return 1;
+  if (channel_is_bad_for_new_circs(a) && !channel_is_bad_for_new_circs(b))
+    return 0;
+
   /* Check if one is canonical and the other isn't first */
   a_is_canonical = channel_is_canonical(a);
   b_is_canonical = channel_is_canonical(b);
@@ -3294,31 +2344,36 @@ channel_is_better(time_t now, channel_t *a, channel_t *b,
   if (a_is_canonical && !b_is_canonical) return 1;
   if (!a_is_canonical && b_is_canonical) return 0;
 
+  /* Check if we suspect that one of the channels will be preferred
+   * by the peer */
+  if (a->is_canonical_to_peer && !b->is_canonical_to_peer) return 1;
+  if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0;
+
   /*
-   * Okay, if we're here they tied on canonicity. Next we check if
-   * they have any circuits, and if one does and the other doesn't,
-   * we prefer the one that does, unless we are forgiving and the
-   * one that has no circuits is in its grace period.
+   * Okay, if we're here they tied on canonicity, the prefer the older
+   * connection, so that the adversary can't create a new connection
+   * and try to switch us over to it (which will leak information
+   * about long-lived circuits). Additionally, switching connections
+   * too often makes us more vulnerable to attacks like Torscan and
+   * passive netflow-based equivalents.
+   *
+   * Connections will still only live for at most a week, due to
+   * the check in connection_or_group_set_badness() against
+   * TIME_BEFORE_OR_CONN_IS_TOO_OLD, which marks old connections as
+   * unusable for new circuits after 1 week. That check sets
+   * is_bad_for_new_circs, which is checked in channel_get_for_extend().
+   *
+   * We check channel_is_bad_for_new_circs() above here anyway, for safety.
    */
+  if (channel_when_created(a) < channel_when_created(b)) return 1;
+  else if (channel_when_created(a) > channel_when_created(b)) return 0;
 
-  a_has_circs = (channel_num_circuits(a) > 0);
-  b_has_circs = (channel_num_circuits(b) > 0);
-  a_grace = (forgive_new_connections &&
-             (now < channel_when_created(a) + NEW_CHAN_GRACE_PERIOD));
-  b_grace = (forgive_new_connections &&
-             (now < channel_when_created(b) + NEW_CHAN_GRACE_PERIOD));
-
-  if (a_has_circs && !b_has_circs && !b_grace) return 1;
-  if (!a_has_circs && b_has_circs && !a_grace) return 0;
-
-  /* They tied on circuits too; just prefer whichever is newer */
-
-  if (channel_when_created(a) > channel_when_created(b)) return 1;
+  if (channel_num_circuits(a) > channel_num_circuits(b)) return 1;
   else return 0;
 }
 
 /**
- * Get a channel to extend a circuit
+ * Get a channel to extend a circuit.
  *
  * Pick a suitable channel to extend a circuit to given the desired digest
  * the address we believe is correct for that digest; this tries to see
@@ -3327,29 +2382,28 @@ channel_is_better(time_t now, channel_t *a, channel_t *b,
  * and our next action, and set *launch_out to a boolean indicated whether
  * the caller should try to launch a new channel with channel_connect().
  */
-
 channel_t *
-channel_get_for_extend(const char *digest,
+channel_get_for_extend(const char *rsa_id_digest,
+                       const ed25519_public_key_t *ed_id,
                        const tor_addr_t *target_addr,
                        const char **msg_out,
                        int *launch_out)
 {
   channel_t *chan, *best = NULL;
   int n_inprogress_goodaddr = 0, n_old = 0;
-  int n_noncanonical = 0, n_possible = 0;
-  time_t now = approx_time();
+  int n_noncanonical = 0;
 
   tor_assert(msg_out);
   tor_assert(launch_out);
 
-  chan = channel_find_by_remote_digest(digest);
+  chan = channel_find_by_remote_identity(rsa_id_digest, ed_id);
 
   /* Walk the list, unrefing the old one and refing the new at each
    * iteration.
    */
-  for (; chan; chan = channel_next_with_digest(chan)) {
+  for (; chan; chan = channel_next_with_rsa_identity(chan)) {
     tor_assert(tor_memeq(chan->identity_digest,
-                         digest, DIGEST_LEN));
+                         rsa_id_digest, DIGEST_LEN));
 
    if (CHANNEL_CONDEMNED(chan))
       continue;
@@ -3360,6 +2414,11 @@ channel_get_for_extend(const char *digest,
       continue;
     }
 
+    /* The Ed25519 key has to match too */
+    if (!channel_remote_identity_matches(chan, rsa_id_digest, ed_id)) {
+      continue;
+    }
+
     /* Never return a non-open connection. */
     if (!CHANNEL_IS_OPEN(chan)) {
       /* If the address matches, don't launch a new connection for this
@@ -3395,14 +2454,12 @@ channel_get_for_extend(const char *digest,
       continue;
     }
 
-    ++n_possible;
-
     if (!best) {
       best = chan; /* If we have no 'best' so far, this one is good enough. */
       continue;
     }
 
-    if (channel_is_better(now, chan, best, 0))
+    if (channel_is_better(chan, best))
       best = chan;
   }
 
@@ -3427,12 +2484,11 @@ channel_get_for_extend(const char *digest,
 }
 
 /**
- * Describe the transport subclass for a channel
+ * Describe the transport subclass for a channel.
  *
  * Invoke a method to get a string description of the lower-layer
  * transport for this channel.
  */
-
 const char *
 channel_describe_transport(channel_t *chan)
 {
@@ -3443,12 +2499,11 @@ channel_describe_transport(channel_t *chan)
 }
 
 /**
- * Describe the transport subclass for a channel listener
+ * Describe the transport subclass for a channel listener.
  *
  * Invoke a method to get a string description of the lower-layer
  * transport for this channel listener.
  */
-
 const char *
 channel_listener_describe_transport(channel_listener_t *chan_l)
 {
@@ -3459,24 +2514,10 @@ channel_listener_describe_transport(channel_listener_t *chan_l)
 }
 
 /**
- * Return the number of entries in <b>queue</b>
- */
-STATIC int
-chan_cell_queue_len(const chan_cell_queue_t *queue)
-{
-  int r = 0;
-  cell_queue_entry_t *cell;
-  TOR_SIMPLEQ_FOREACH(cell, queue, next)
-    ++r;
-  return r;
-}
-
-/**
- * Dump channel statistics
+ * Dump channel statistics.
  *
- * Dump statistics for one channel to the log
+ * Dump statistics for one channel to the log.
  */
-
 MOCK_IMPL(void,
 channel_dump_statistics, (channel_t *chan, int severity))
 {
@@ -3506,35 +2547,18 @@ channel_dump_statistics, (channel_t *chan, int severity))
       U64_PRINTF_ARG(chan->timestamp_active),
       U64_PRINTF_ARG(now - chan->timestamp_active));
 
-  /* Handle digest and nickname */
+  /* Handle digest. */
   if (!tor_digest_is_zero(chan->identity_digest)) {
-    if (chan->nickname) {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " says it is connected "
-          "to an OR with digest %s and nickname %s",
-          U64_PRINTF_ARG(chan->global_identifier),
-          hex_str(chan->identity_digest, DIGEST_LEN),
-          chan->nickname);
-    } else {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " says it is connected "
-          "to an OR with digest %s and no known nickname",
-          U64_PRINTF_ARG(chan->global_identifier),
-          hex_str(chan->identity_digest, DIGEST_LEN));
-    }
+    tor_log(severity, LD_GENERAL,
+        " * Channel " U64_FORMAT " says it is connected "
+        "to an OR with digest %s",
+        U64_PRINTF_ARG(chan->global_identifier),
+        hex_str(chan->identity_digest, DIGEST_LEN));
   } else {
-    if (chan->nickname) {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " does not know the digest"
-          " of the OR it is connected to, but reports its nickname is %s",
-          U64_PRINTF_ARG(chan->global_identifier),
-          chan->nickname);
-    } else {
-      tor_log(severity, LD_GENERAL,
-          " * Channel " U64_FORMAT " does not know the digest"
-          " or the nickname of the OR it is connected to",
-          U64_PRINTF_ARG(chan->global_identifier));
-    }
+    tor_log(severity, LD_GENERAL,
+        " * Channel " U64_FORMAT " does not know the digest"
+        " of the OR it is connected to",
+        U64_PRINTF_ARG(chan->global_identifier));
   }
 
   /* Handle remote address and descriptions */
@@ -3583,14 +2607,6 @@ channel_dump_statistics, (channel_t *chan, int severity))
       channel_is_incoming(chan) ?
         "incoming" : "outgoing");
 
-  /* Describe queues */
-  tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " has %d queued incoming cells"
-      " and %d queued outgoing cells",
-      U64_PRINTF_ARG(chan->global_identifier),
-      chan_cell_queue_len(&chan->incoming_queue),
-      chan_cell_queue_len(&chan->outgoing_queue));
-
   /* Describe circuits */
   tor_log(severity, LD_GENERAL,
       " * Channel " U64_FORMAT " has %d active circuits out of"
@@ -3609,12 +2625,6 @@ channel_dump_statistics, (channel_t *chan, int severity))
       U64_PRINTF_ARG(chan->timestamp_client),
       U64_PRINTF_ARG(now - chan->timestamp_client));
   tor_log(severity, LD_GENERAL,
-      " * Channel " U64_FORMAT " was last drained at "
-      U64_FORMAT " (" U64_FORMAT " seconds ago)",
-      U64_PRINTF_ARG(chan->global_identifier),
-      U64_PRINTF_ARG(chan->timestamp_drained),
-      U64_PRINTF_ARG(now - chan->timestamp_drained));
-  tor_log(severity, LD_GENERAL,
       " * Channel " U64_FORMAT " last received a cell "
       "at " U64_FORMAT " (" U64_FORMAT " seconds ago)",
       U64_PRINTF_ARG(chan->global_identifier),
@@ -3690,11 +2700,10 @@ channel_dump_statistics, (channel_t *chan, int severity))
 }
 
 /**
- * Dump channel listener statistics
+ * Dump channel listener statistics.
  *
- * Dump statistics for one channel listener to the log
+ * Dump statistics for one channel listener to the log.
  */
-
 void
 channel_listener_dump_statistics(channel_listener_t *chan_l, int severity)
 {
@@ -3757,11 +2766,10 @@ channel_listener_dump_statistics(channel_listener_t *chan_l, int severity)
 }
 
 /**
- * Invoke transport-specific stats dump for channel
+ * Invoke transport-specific stats dump for channel.
  *
- * If there is a lower-layer statistics dump method, invoke it
+ * If there is a lower-layer statistics dump method, invoke it.
  */
-
 void
 channel_dump_transport_statistics(channel_t *chan, int severity)
 {
@@ -3771,11 +2779,10 @@ channel_dump_transport_statistics(channel_t *chan, int severity)
 }
 
 /**
- * Invoke transport-specific stats dump for channel listener
+ * Invoke transport-specific stats dump for channel listener.
  *
- * If there is a lower-layer statistics dump method, invoke it
+ * If there is a lower-layer statistics dump method, invoke it.
  */
-
 void
 channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
                                            int severity)
@@ -3786,7 +2793,7 @@ channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
 }
 
 /**
- * Return text description of the remote endpoint
+ * Return text description of the remote endpoint.
  *
  * This function return a test provided by the lower layer of the remote
  * endpoint for this channel; it should specify the actual address connected
@@ -3819,7 +2826,7 @@ channel_get_actual_remote_address(channel_t *chan)
 }
 
 /**
- * Return text description of the remote endpoint canonical address
+ * Return text description of the remote endpoint canonical address.
  *
  * This function return a test provided by the lower layer of the remote
  * endpoint for this channel; it should use the known canonical address for
@@ -3858,37 +2865,25 @@ channel_get_addr_if_possible,(channel_t *chan, tor_addr_t *addr_out))
 }
 
 /**
- * Check if there are outgoing queue writes on this channel
- *
- * Indicate if either we have queued cells, or if not, whether the underlying
- * lower-layer transport thinks it has an output queue.
+ * Return true iff the channel has any cells on the connection outbuf waiting
+ * to be sent onto the network.
  */
-
 int
 channel_has_queued_writes(channel_t *chan)
 {
-  int has_writes = 0;
-
   tor_assert(chan);
   tor_assert(chan->has_queued_writes);
 
-  if (! TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue)) {
-    has_writes = 1;
-  } else {
-    /* Check with the lower layer */
-    has_writes = chan->has_queued_writes(chan);
-  }
-
-  return has_writes;
+  /* Check with the lower layer */
+  return chan->has_queued_writes(chan);
 }
 
 /**
- * Check the is_bad_for_new_circs flag
+ * Check the is_bad_for_new_circs flag.
  *
  * This function returns the is_bad_for_new_circs flag of the specified
  * channel.
  */
-
 int
 channel_is_bad_for_new_circs(channel_t *chan)
 {
@@ -3898,11 +2893,10 @@ channel_is_bad_for_new_circs(channel_t *chan)
 }
 
 /**
- * Mark a channel as bad for new circuits
+ * Mark a channel as bad for new circuits.
  *
  * Set the is_bad_for_new_circs_flag on chan.
  */
-
 void
 channel_mark_bad_for_new_circs(channel_t *chan)
 {
@@ -3912,15 +2906,14 @@ channel_mark_bad_for_new_circs(channel_t *chan)
 }
 
 /**
- * Get the client flag
+ * Get the client flag.
  *
  * This returns the client flag of a channel, which will be set if
  * command_process_create_cell() in command.c thinks this is a connection
  * from a client.
  */
-
 int
-channel_is_client(channel_t *chan)
+channel_is_client(const channel_t *chan)
 {
   tor_assert(chan);
 
@@ -3928,11 +2921,10 @@ channel_is_client(channel_t *chan)
 }
 
 /**
- * Set the client flag
+ * Set the client flag.
  *
- * Mark a channel as being from a client
+ * Mark a channel as being from a client.
  */
-
 void
 channel_mark_client(channel_t *chan)
 {
@@ -3942,12 +2934,24 @@ channel_mark_client(channel_t *chan)
 }
 
 /**
- * Get the canonical flag for a channel
+ * Clear the client flag.
+ *
+ * Mark a channel as being _not_ from a client.
+ */
+void
+channel_clear_client(channel_t *chan)
+{
+  tor_assert(chan);
+
+  chan->is_client = 0;
+}
+
+/**
+ * Get the canonical flag for a channel.
  *
  * This returns the is_canonical for a channel; this flag is determined by
  * the lower layer and can't be set in a transport-independent way.
  */
-
 int
 channel_is_canonical(channel_t *chan)
 {
@@ -3958,12 +2962,11 @@ channel_is_canonical(channel_t *chan)
 }
 
 /**
- * Test if the canonical flag is reliable
+ * Test if the canonical flag is reliable.
  *
  * This function asks if the lower layer thinks it's safe to trust the
- * result of channel_is_canonical()
+ * result of channel_is_canonical().
  */
-
 int
 channel_is_canonical_is_reliable(channel_t *chan)
 {
@@ -3974,12 +2977,11 @@ channel_is_canonical_is_reliable(channel_t *chan)
 }
 
 /**
- * Test incoming flag
+ * Test incoming flag.
  *
  * This function gets the incoming flag; this is set when a listener spawns
  * a channel.  If this returns true the channel was remotely initiated.
  */
-
 int
 channel_is_incoming(channel_t *chan)
 {
@@ -3989,12 +2991,11 @@ channel_is_incoming(channel_t *chan)
 }
 
 /**
- * Set the incoming flag
+ * Set the incoming flag.
  *
  * This function is called when a channel arrives on a listening channel
  * to mark it as incoming.
  */
-
 void
 channel_mark_incoming(channel_t *chan)
 {
@@ -4004,7 +3005,7 @@ channel_mark_incoming(channel_t *chan)
 }
 
 /**
- * Test local flag
+ * Test local flag.
  *
  * This function gets the local flag; the lower layer should set this when
  * setting up the channel if is_local_addr() is true for all of the
@@ -4012,7 +3013,6 @@ channel_mark_incoming(channel_t *chan)
  * used to decide whether to declare the network reachable when seeing incoming
  * traffic on the channel.
  */
-
 int
 channel_is_local(channel_t *chan)
 {
@@ -4022,13 +3022,12 @@ channel_is_local(channel_t *chan)
 }
 
 /**
- * Set the local flag
+ * Set the local flag.
  *
  * This internal-only function should be called by the lower layer if the
  * channel is to a local address.  See channel_is_local() above or the
- * description of the is_local bit in channel.h
+ * description of the is_local bit in channel.h.
  */
-
 void
 channel_mark_local(channel_t *chan)
 {
@@ -4038,14 +3037,13 @@ channel_mark_local(channel_t *chan)
 }
 
 /**
- * Mark a channel as remote
+ * Mark a channel as remote.
  *
  * This internal-only function should be called by the lower layer if the
  * channel is not to a local address but has previously been marked local.
  * See channel_is_local() above or the description of the is_local bit in
  * channel.h
  */
-
 void
 channel_mark_remote(channel_t *chan)
 {
@@ -4055,13 +3053,12 @@ channel_mark_remote(channel_t *chan)
 }
 
 /**
- * Test outgoing flag
+ * Test outgoing flag.
  *
  * This function gets the outgoing flag; this is the inverse of the incoming
  * bit set when a listener spawns a channel.  If this returns true the channel
  * was locally initiated.
  */
-
 int
 channel_is_outgoing(channel_t *chan)
 {
@@ -4071,12 +3068,11 @@ channel_is_outgoing(channel_t *chan)
 }
 
 /**
- * Mark a channel as outgoing
+ * Mark a channel as outgoing.
  *
  * This function clears the incoming flag and thus marks a channel as
  * outgoing.
  */
-
 void
 channel_mark_outgoing(channel_t *chan)
 {
@@ -4089,24 +3085,11 @@ channel_mark_outgoing(channel_t *chan)
  * Flow control queries *
  ***********************/
 
-/*
- * Get the latest estimate for the total queue size of all open channels
- */
-
-uint64_t
-channel_get_global_queue_estimate(void)
-{
-  return estimated_total_queue_size;
-}
-
-/*
- * Estimate the number of writeable cells
+/**
+ * Estimate the number of writeable cells.
  *
- * Ask the lower layer for an estimate of how many cells it can accept, and
- * then subtract the length of our outgoing_queue, if any, to produce an
- * estimate of the number of cells this channel can accept for writes.
+ * Ask the lower layer for an estimate of how many cells it can accept.
  */
-
 int
 channel_num_cells_writeable(channel_t *chan)
 {
@@ -4118,8 +3101,6 @@ channel_num_cells_writeable(channel_t *chan)
   if (chan->state == CHANNEL_STATE_OPEN) {
     /* Query lower layer */
     result = chan->num_cells_writeable(chan);
-    /* Subtract cell queue length, if any */
-    result -= chan_cell_queue_len(&chan->outgoing_queue);
     if (result < 0) result = 0;
   } else {
     /* No cells are writeable in any other state */
@@ -4134,12 +3115,11 @@ channel_num_cells_writeable(channel_t *chan)
  ********************/
 
 /**
- * Update the created timestamp for a channel
+ * Update the created timestamp for a channel.
  *
  * This updates the channel's created timestamp and should only be called
  * from channel_init().
  */
-
 void
 channel_timestamp_created(channel_t *chan)
 {
@@ -4151,12 +3131,11 @@ channel_timestamp_created(channel_t *chan)
 }
 
 /**
- * Update the created timestamp for a channel listener
+ * Update the created timestamp for a channel listener.
  *
  * This updates the channel listener's created timestamp and should only be
  * called from channel_init_listener().
  */
-
 void
 channel_listener_timestamp_created(channel_listener_t *chan_l)
 {
@@ -4168,7 +3147,7 @@ channel_listener_timestamp_created(channel_listener_t *chan_l)
 }
 
 /**
- * Update the last active timestamp for a channel
+ * Update the last active timestamp for a channel.
  *
  * This function updates the channel's last active timestamp; it should be
  * called by the lower layer whenever there is activity on the channel which
@@ -4177,21 +3156,23 @@ channel_listener_timestamp_created(channel_listener_t *chan_l)
  * but it should be updated for things like the v3 handshake and stuff that
  * produce activity only visible to the lower layer.
  */
-
 void
 channel_timestamp_active(channel_t *chan)
 {
   time_t now = time(NULL);
 
   tor_assert(chan);
+  monotime_coarse_get(&chan->timestamp_xfer);
 
   chan->timestamp_active = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  monotime_coarse_zero(&chan->next_padding_time);
 }
 
 /**
- * Update the last active timestamp for a channel listener
+ * Update the last active timestamp for a channel listener.
  */
-
 void
 channel_listener_timestamp_active(channel_listener_t *chan_l)
 {
@@ -4209,7 +3190,6 @@ channel_listener_timestamp_active(channel_listener_t *chan_l)
  * should be called whenever a new incoming channel is accepted on a
  * listener.
  */
-
 void
 channel_listener_timestamp_accepted(channel_listener_t *chan_l)
 {
@@ -4222,12 +3202,11 @@ channel_listener_timestamp_accepted(channel_listener_t *chan_l)
 }
 
 /**
- * Update client timestamp
+ * Update client timestamp.
  *
  * This function is called by relay.c to timestamp a channel that appears to
  * be used as a client.
  */
-
 void
 channel_timestamp_client(channel_t *chan)
 {
@@ -4239,57 +3218,44 @@ channel_timestamp_client(channel_t *chan)
 }
 
 /**
- * Update the last drained timestamp
- *
- * This is called whenever we transmit a cell which leaves the outgoing cell
- * queue completely empty.  It also updates the xmit time and the active time.
- */
-
-void
-channel_timestamp_drained(channel_t *chan)
-{
-  time_t now = time(NULL);
-
-  tor_assert(chan);
-
-  chan->timestamp_active = now;
-  chan->timestamp_drained = now;
-  chan->timestamp_xmit = now;
-}
-
-/**
- * Update the recv timestamp
+ * Update the recv timestamp.
  *
  * This is called whenever we get an incoming cell from the lower layer.
  * This also updates the active timestamp.
  */
-
 void
 channel_timestamp_recv(channel_t *chan)
 {
   time_t now = time(NULL);
-
   tor_assert(chan);
+  monotime_coarse_get(&chan->timestamp_xfer);
 
   chan->timestamp_active = now;
   chan->timestamp_recv = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  monotime_coarse_zero(&chan->next_padding_time);
 }
 
 /**
- * Update the xmit timestamp
+ * Update the xmit timestamp.
+ *
  * This is called whenever we pass an outgoing cell to the lower layer.  This
  * also updates the active timestamp.
  */
-
 void
 channel_timestamp_xmit(channel_t *chan)
 {
   time_t now = time(NULL);
-
   tor_assert(chan);
 
+  monotime_coarse_get(&chan->timestamp_xfer);
+
   chan->timestamp_active = now;
   chan->timestamp_xmit = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  monotime_coarse_zero(&chan->next_padding_time);
 }
 
 /***************************************************************
@@ -4297,9 +3263,8 @@ channel_timestamp_xmit(channel_t *chan)
  **************************************************************/
 
 /**
- * Query created timestamp for a channel
+ * Query created timestamp for a channel.
  */
-
 time_t
 channel_when_created(channel_t *chan)
 {
@@ -4309,57 +3274,8 @@ channel_when_created(channel_t *chan)
 }
 
 /**
- * Query created timestamp for a channel listener
- */
-
-time_t
-channel_listener_when_created(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->timestamp_created;
-}
-
-/**
- * Query last active timestamp for a channel
- */
-
-time_t
-channel_when_last_active(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->timestamp_active;
-}
-
-/**
- * Query last active timestamp for a channel listener
+ * Query client timestamp.
  */
-
-time_t
-channel_listener_when_last_active(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->timestamp_active;
-}
-
-/**
- * Query last accepted timestamp for a channel listener
- */
-
-time_t
-channel_listener_when_last_accepted(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->timestamp_accepted;
-}
-
-/**
- * Query client timestamp
- */
-
 time_t
 channel_when_last_client(channel_t *chan)
 {
@@ -4369,33 +3285,8 @@ channel_when_last_client(channel_t *chan)
 }
 
 /**
- * Query drained timestamp
+ * Query xmit timestamp.
  */
-
-time_t
-channel_when_last_drained(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->timestamp_drained;
-}
-
-/**
- * Query recv timestamp
- */
-
-time_t
-channel_when_last_recv(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->timestamp_recv;
-}
-
-/**
- * Query xmit timestamp
- */
-
 time_t
 channel_when_last_xmit(channel_t *chan)
 {
@@ -4405,48 +3296,11 @@ channel_when_last_xmit(channel_t *chan)
 }
 
 /**
- * Query accepted counter
- */
-
-uint64_t
-channel_listener_count_accepted(channel_listener_t *chan_l)
-{
-  tor_assert(chan_l);
-
-  return chan_l->n_accepted;
-}
-
-/**
- * Query received cell counter
- */
-
-uint64_t
-channel_count_recved(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->n_cells_recved;
-}
-
-/**
- * Query transmitted cell counter
- */
-
-uint64_t
-channel_count_xmitted(channel_t *chan)
-{
-  tor_assert(chan);
-
-  return chan->n_cells_xmitted;
-}
-
-/**
- * Check if a channel matches an extend_info_t
+ * Check if a channel matches an extend_info_t.
  *
  * This function calls the lower layer and asks if this channel matches a
  * given extend_info_t.
  */
-
 int
 channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info)
 {
@@ -4463,7 +3317,6 @@ channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info)
  * This function calls into the lower layer and asks if this channel thinks
  * it matches a given target address for circuit extension purposes.
  */
-
 int
 channel_matches_target_addr_for_extend(channel_t *chan,
                                        const tor_addr_t *target)
@@ -4476,12 +3329,11 @@ channel_matches_target_addr_for_extend(channel_t *chan,
 }
 
 /**
- * Return the total number of circuits used by a channel
+ * Return the total number of circuits used by a channel.
  *
  * @param chan Channel to query
  * @return Number of circuits using this as n_chan or p_chan
  */
-
 unsigned int
 channel_num_circuits(channel_t *chan)
 {
@@ -4492,10 +3344,10 @@ channel_num_circuits(channel_t *chan)
 }
 
 /**
- * Set up circuit ID generation
+ * Set up circuit ID generation.
  *
  * This is called when setting up a channel and replaces the old
- * connection_or_set_circid_type()
+ * connection_or_set_circid_type().
  */
 MOCK_IMPL(void,
 channel_set_circid_type,(channel_t *chan,
@@ -4531,87 +3383,96 @@ channel_set_circid_type,(channel_t *chan,
   }
 }
 
-/**
- * Update the estimated number of bytes queued to transmit for this channel,
- * and notify the scheduler.  The estimate includes both the channel queue and
- * the queue size reported by the lower layer, and an overhead estimate
- * optionally provided by the lower layer.
- */
+static int
+channel_sort_by_ed25519_identity(const void **a_, const void **b_)
+{
+  const channel_t *a = *a_,
+                  *b = *b_;
+  return fast_memcmp(&a->ed25519_identity.pubkey,
+                     &b->ed25519_identity.pubkey,
+                     sizeof(a->ed25519_identity.pubkey));
+}
 
-void
-channel_update_xmit_queue_size(channel_t *chan)
+/** Helper for channel_update_bad_for_new_circs(): Perform the
+ * channel_update_bad_for_new_circs operation on all channels in <b>lst</b>,
+ * all of which MUST have the same RSA ID.  (They MAY have different
+ * Ed25519 IDs.) */
+static void
+channel_rsa_id_group_set_badness(struct channel_list_s *lst, int force)
 {
-  uint64_t queued, adj;
-  double overhead;
+  /*XXXX This function should really be about channels. 15056 */
+  channel_t *chan = TOR_LIST_FIRST(lst);
 
-  tor_assert(chan);
-  tor_assert(chan->num_bytes_queued);
+  if (!chan)
+    return;
 
-  /*
-   * First, get the number of bytes we have queued without factoring in
-   * lower-layer overhead.
-   */
-  queued = chan->num_bytes_queued(chan) + chan->bytes_in_queue;
-  /* Next, adjust by the overhead factor, if any is available */
-  if (chan->get_overhead_estimate) {
-    overhead = chan->get_overhead_estimate(chan);
-    if (overhead >= 1.0) {
-      queued = (uint64_t)(queued * overhead);
-    } else {
-      /* Ignore silly overhead factors */
-      log_notice(LD_CHANNEL, "Ignoring silly overhead factor %f", overhead);
+  /* if there is only one channel, don't bother looping */
+  if (PREDICT_LIKELY(!TOR_LIST_NEXT(chan, next_with_same_id))) {
+    connection_or_single_set_badness_(
+            time(NULL), BASE_CHAN_TO_TLS(chan)->conn, force);
+    return;
+  }
+
+  smartlist_t *channels = smartlist_new();
+
+  TOR_LIST_FOREACH(chan, lst, next_with_same_id) {
+    if (BASE_CHAN_TO_TLS(chan)->conn) {
+      smartlist_add(channels, chan);
     }
   }
 
-  /* Now, compare to the previous estimate */
-  if (queued > chan->bytes_queued_for_xmit) {
-    adj = queued - chan->bytes_queued_for_xmit;
-    log_debug(LD_CHANNEL,
-              "Increasing queue size for channel " U64_FORMAT " by " U64_FORMAT
-              " from " U64_FORMAT " to " U64_FORMAT,
-              U64_PRINTF_ARG(chan->global_identifier),
-              U64_PRINTF_ARG(adj),
-              U64_PRINTF_ARG(chan->bytes_queued_for_xmit),
-              U64_PRINTF_ARG(queued));
-    /* Update the channel's estimate */
-    chan->bytes_queued_for_xmit = queued;
-
-    /* Update the global queue size estimate if appropriate */
-    if (chan->state == CHANNEL_STATE_OPEN ||
-        chan->state == CHANNEL_STATE_MAINT) {
-      estimated_total_queue_size += adj;
-      log_debug(LD_CHANNEL,
-                "Increasing global queue size by " U64_FORMAT " for channel "
-                U64_FORMAT ", new size is " U64_FORMAT,
-                U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier),
-                U64_PRINTF_ARG(estimated_total_queue_size));
-      /* Tell the scheduler we're increasing the queue size */
-      scheduler_adjust_queue_size(chan, 1, adj);
+  smartlist_sort(channels, channel_sort_by_ed25519_identity);
+
+  const ed25519_public_key_t *common_ed25519_identity = NULL;
+  /* it would be more efficient to do a slice, but this case is rare */
+  smartlist_t *or_conns = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(channels, channel_t *, channel) {
+    if (!common_ed25519_identity)
+      common_ed25519_identity = &channel->ed25519_identity;
+
+    if (! ed25519_pubkey_eq(&channel->ed25519_identity,
+                            common_ed25519_identity)) {
+        connection_or_group_set_badness_(or_conns, force);
+        smartlist_clear(or_conns);
+        common_ed25519_identity = &channel->ed25519_identity;
     }
-  } else if (queued < chan->bytes_queued_for_xmit) {
-    adj = chan->bytes_queued_for_xmit - queued;
-    log_debug(LD_CHANNEL,
-              "Decreasing queue size for channel " U64_FORMAT " by " U64_FORMAT
-              " from " U64_FORMAT " to " U64_FORMAT,
-              U64_PRINTF_ARG(chan->global_identifier),
-              U64_PRINTF_ARG(adj),
-              U64_PRINTF_ARG(chan->bytes_queued_for_xmit),
-              U64_PRINTF_ARG(queued));
-    /* Update the channel's estimate */
-    chan->bytes_queued_for_xmit = queued;
-
-    /* Update the global queue size estimate if appropriate */
-    if (chan->state == CHANNEL_STATE_OPEN ||
-        chan->state == CHANNEL_STATE_MAINT) {
-      estimated_total_queue_size -= adj;
-      log_debug(LD_CHANNEL,
-                "Decreasing global queue size by " U64_FORMAT " for channel "
-                U64_FORMAT ", new size is " U64_FORMAT,
-                U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier),
-                U64_PRINTF_ARG(estimated_total_queue_size));
-      /* Tell the scheduler we're decreasing the queue size */
-      scheduler_adjust_queue_size(chan, -1, adj);
+
+    smartlist_add(or_conns, BASE_CHAN_TO_TLS(channel)->conn);
+  } SMARTLIST_FOREACH_END(channel);
+
+  connection_or_group_set_badness_(or_conns, force);
+
+  /* XXXX 15056 we may want to do something special with connections that have
+   * no set Ed25519 identity! */
+
+  smartlist_free(or_conns);
+  smartlist_free(channels);
+}
+
+/** Go through all the channels (or if <b>digest</b> is non-NULL, just
+ * the OR connections with that digest), and set the is_bad_for_new_circs
+ * flag based on the rules in connection_or_group_set_badness() (or just
+ * always set it if <b>force</b> is true).
+ */
+void
+channel_update_bad_for_new_circs(const char *digest, int force)
+{
+  if (digest) {
+    channel_idmap_entry_t *ent;
+    channel_idmap_entry_t search;
+    memset(&search, 0, sizeof(search));
+    memcpy(search.digest, digest, DIGEST_LEN);
+    ent = HT_FIND(channel_idmap, &channel_identity_map, &search);
+    if (ent) {
+      channel_rsa_id_group_set_badness(&ent->channel_list, force);
     }
+    return;
+  }
+
+  /* no digest; just look at everything. */
+  channel_idmap_entry_t **iter;
+  HT_FOREACH(iter, channel_idmap, &channel_identity_map) {
+    channel_rsa_id_group_set_badness(&(*iter)->channel_list, force);
   }
 }
 
diff --git a/src/or/channel.h b/src/or/channel.h
index bcd345e8d2..6cf8cd7f72 100644
--- a/src/or/channel.h
+++ b/src/or/channel.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,15 +11,24 @@
 
 #include "or.h"
 #include "circuitmux.h"
+#include "timers.h"
+#include "handles.h"
 
 /* Channel handler function pointer typedefs */
 typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *);
 typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *);
 typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *);
 
-struct cell_queue_entry_s;
-TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s);
-typedef struct chan_cell_queue chan_cell_queue_t;
+/**
+ * This enum is used by channelpadding to decide when to pad channels.
+ * Don't add values to it without updating the checks in
+ * channelpadding_decide_to_pad_channel().
+ */
+typedef enum {
+    CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS = 0,
+    CHANNEL_USED_FOR_FULL_CIRCS,
+    CHANNEL_USED_FOR_USER_TRAFFIC,
+} channel_usage_info_t;
 
 /**
  * Channel struct; see the channel_t typedef in or.h.  A channel is an
@@ -29,16 +38,21 @@ typedef struct chan_cell_queue chan_cell_queue_t;
  * to a particular node, and once constructed support the abstract operations
  * defined below.
  */
-
 struct channel_s {
   /** Magic number for type-checking cast macros */
   uint32_t magic;
 
+  /** List entry for hashtable for global-identifier lookup. */
+  HT_ENTRY(channel_s) gidmap_node;
+
+  /** Handle entry for handle-based lookup */
+  HANDLE_ENTRY(channel, channel_s);
+
   /** Current channel state */
   channel_state_t state;
 
   /** Globally unique ID number for a channel over the lifetime of a Tor
-   * process.
+   * process.  This may not be 0.
    */
   uint64_t global_identifier;
 
@@ -48,6 +62,60 @@ struct channel_s {
   /** has this channel ever been open? */
   unsigned int has_been_open:1;
 
+  /**
+   * This field indicates if the other side has enabled or disabled
+   * padding via either the link protocol version or
+   * channelpadding_negotiate cells.
+   *
+   * Clients can override this with ConnectionPadding in torrc to
+   * disable or force padding to relays, but relays cannot override the
+   * client's request.
+   */
+  unsigned int padding_enabled:1;
+
+  /** Cached value of our decision to pad (to avoid expensive
+   * checks during critical path statistics counting). */
+  unsigned int currently_padding:1;
+
+  /** Is there a pending netflow padding callback? */
+  unsigned int pending_padding_callback:1;
+
+  /** Is our peer likely to consider this channel canonical? */
+  unsigned int is_canonical_to_peer:1;
+
+  /** Has this channel ever been used for non-directory traffic?
+   * Used to decide what channels to pad, and when. */
+  channel_usage_info_t channel_usage;
+
+  /** When should we send a cell for netflow padding? 0 means no padding is
+   *  scheduled. */
+  monotime_coarse_t next_padding_time;
+
+  /** The callback pointer for the padding callbacks */
+  tor_timer_t *padding_timer;
+  /** The handle to this channel (to free on canceled timers) */
+  struct channel_handle_t *timer_handle;
+
+  /**
+   * These two fields specify the minimum and maximum negotiated timeout
+   * values for inactivity (send or receive) before we decide to pad a
+   * channel. These fields can be set either via a PADDING_NEGOTIATE cell,
+   * or the torrc option ReducedConnectionPadding. The consensus parameters
+   * nf_ito_low and nf_ito_high are used to ensure that padding can only be
+   * negotiated to be less frequent than what is specified in the consensus.
+   * (This is done to prevent wingnut clients from requesting excessive
+   * padding).
+   *
+   * The actual timeout value is randomly chosen between these two values
+   * as per the table in channelpadding_get_netflow_inactive_timeout_ms(),
+   * after ensuring that these values do not specify lower timeouts than
+   * the consensus parameters.
+   *
+   * If these are 0, we have not negotiated or specified custom padding
+   * times, and instead use consensus defaults. */
+  uint16_t padding_timeout_low_ms;
+  uint16_t padding_timeout_high_ms;
+
   /** Why did we close?
    */
   enum {
@@ -87,6 +155,18 @@ struct channel_s {
   time_t timestamp_created; /* Channel created */
   time_t timestamp_active; /* Any activity */
 
+  /**
+   * This is a monotonic timestamp that marks when we
+   * believe the channel has actually sent or received data to/from
+   * the wire. Right now, it is used to determine when we should send
+   * a padding cell for channelpadding.
+   *
+   * XXX: Are we setting timestamp_xfer_ms in the right places to
+   * accurately reflect actual network data transfer? Or might this be
+   * very wrong wrt when bytes actually go on the wire?
+   */
+  monotime_coarse_t timestamp_xfer;
+
   /* Methods implemented by the lower layer */
 
   /** Free a channel */
@@ -153,25 +233,32 @@ struct channel_s {
   int (*write_var_cell)(channel_t *, var_cell_t *);
 
   /**
-   * Hash of the public RSA key for the other side's identity key, or
-   * zeroes if the other side hasn't shown us a valid identity key.
+   * Hash of the public RSA key for the other side's RSA identity key -- or
+   * zeroes if we don't have an RSA identity in mind for the other side, and
+   * it hasn't shown us one.
+   *
+   * Note that this is the RSA identity that we hope the other side has -- not
+   * necessarily its true identity.  Don't believe this identity unless
+   * authentication has happened.
    */
   char identity_digest[DIGEST_LEN];
-  /** Nickname of the OR on the other side, or NULL if none. */
-  char *nickname;
+  /**
+   * Ed25519 key for the other side of this channel -- or zeroes if we don't
+   * have an Ed25519 identity in mind for the other side, and it hasn't shown
+   * us one.
+   *
+   * Note that this is the identity that we hope the other side has -- not
+   * necessarily its true identity.  Don't believe this identity unless
+   * authentication has happened.
+   */
+  ed25519_public_key_t ed25519_identity;
 
   /**
-   * Linked list of channels with the same identity digest, for the
-   * digest->channel map
+   * Linked list of channels with the same RSA identity digest, for use with
+   * the digest->channel map
    */
   TOR_LIST_ENTRY(channel_s) next_with_same_id;
 
-  /** List of incoming cells to handle */
-  chan_cell_queue_t incoming_queue;
-
-  /** List of queued outgoing cells */
-  chan_cell_queue_t outgoing_queue;
-
   /** Circuit mux for circuits sending on this channel */
   circuitmux_t *cmux;
 
@@ -198,8 +285,8 @@ struct channel_s {
   unsigned int is_bad_for_new_circs:1;
 
   /** True iff we have decided that the other end of this connection
-   * is a client.  Channels with this flag set should never be used
-   * to satisfy an EXTEND request.  */
+   * is a client or bridge relay.  Connections with this flag set should never
+   * be used to satisfy an EXTEND request. */
   unsigned int is_client:1;
 
   /** Set if the channel was initiated remotely (came from a listener) */
@@ -218,7 +305,6 @@ struct channel_s {
 
   /** Channel timestamps for cell channels */
   time_t timestamp_client; /* Client used this, according to relay.c */
-  time_t timestamp_drained; /* Output queue empty */
   time_t timestamp_recv; /* Cell received from lower layer */
   time_t timestamp_xmit; /* Cell sent to lower layer */
 
@@ -235,14 +321,6 @@ struct channel_s {
   /** Channel counters for cell channels */
   uint64_t n_cells_recved, n_bytes_recved;
   uint64_t n_cells_xmitted, n_bytes_xmitted;
-
-  /** Our current contribution to the scheduler's total xmit queue */
-  uint64_t bytes_queued_for_xmit;
-
-  /** Number of bytes in this channel's cell queue; does not include
-   * lower-layer queueing.
-   */
-  uint64_t bytes_in_queue;
 };
 
 struct channel_listener_s {
@@ -310,18 +388,13 @@ channel_listener_state_to_string(channel_listener_state_t state);
 /* Abstract channel operations */
 
 void channel_mark_for_close(channel_t *chan);
-void channel_write_cell(channel_t *chan, cell_t *cell);
-void channel_write_packed_cell(channel_t *chan, packed_cell_t *cell);
-void channel_write_var_cell(channel_t *chan, var_cell_t *cell);
+int channel_write_packed_cell(channel_t *chan, packed_cell_t *cell);
 
 void channel_listener_mark_for_close(channel_listener_t *chan_l);
 
 /* Channel callback registrations */
 
 /* Listener callback */
-channel_listener_fn_ptr
-channel_listener_get_listener_fn(channel_listener_t *chan);
-
 void channel_listener_set_listener_fn(channel_listener_t *chan,
                                       channel_listener_fn_ptr listener);
 
@@ -349,40 +422,13 @@ void channel_free_all(void);
 void channel_dumpstats(int severity);
 void channel_listener_dumpstats(int severity);
 
-/* Set the cmux policy on all active channels */
-void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol);
-
 #ifdef TOR_CHANNEL_INTERNAL_
 
 #ifdef CHANNEL_PRIVATE_
-/* Cell queue structure (here rather than channel.c for test suite use) */
-
-typedef struct cell_queue_entry_s cell_queue_entry_t;
-struct cell_queue_entry_s {
-  TOR_SIMPLEQ_ENTRY(cell_queue_entry_s) next;
-  enum {
-    CELL_QUEUE_FIXED,
-    CELL_QUEUE_VAR,
-    CELL_QUEUE_PACKED
-  } type;
-  union {
-    struct {
-      cell_t *cell;
-    } fixed;
-    struct {
-      var_cell_t *var_cell;
-    } var;
-    struct {
-      packed_cell_t *packed_cell;
-    } packed;
-  } u;
-};
 
-/* Cell queue functions for benefit of test suite */
-STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue);
+STATIC void channel_add_to_digest_map(channel_t *chan);
 
-STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off);
-#endif
+#endif /* defined(CHANNEL_PRIVATE_) */
 
 /* Channel operations for subclasses and internal use only */
 
@@ -406,17 +452,17 @@ void channel_close_from_lower_layer(channel_t *chan);
 void channel_close_for_error(channel_t *chan);
 void channel_closed(channel_t *chan);
 
-void channel_listener_close_from_lower_layer(channel_listener_t *chan_l);
-void channel_listener_close_for_error(channel_listener_t *chan_l);
-void channel_listener_closed(channel_listener_t *chan_l);
-
 /* Free a channel */
-void channel_free(channel_t *chan);
-void channel_listener_free(channel_listener_t *chan_l);
+void channel_free_(channel_t *chan);
+#define channel_free(chan) FREE_AND_NULL(channel_t, channel_free_, (chan))
+void channel_listener_free_(channel_listener_t *chan_l);
+#define channel_listener_free(chan_l) \
+  FREE_AND_NULL(channel_listener_t, channel_listener_free_, (chan_l))
 
 /* State/metadata setters */
 
 void channel_change_state(channel_t *chan, channel_state_t to_state);
+void channel_change_state_open(channel_t *chan);
 void channel_clear_identity_digest(channel_t *chan);
 void channel_clear_remote_end(channel_t *chan);
 void channel_mark_local(channel_t *chan);
@@ -424,10 +470,8 @@ void channel_mark_incoming(channel_t *chan);
 void channel_mark_outgoing(channel_t *chan);
 void channel_mark_remote(channel_t *chan);
 void channel_set_identity_digest(channel_t *chan,
-                                 const char *identity_digest);
-void channel_set_remote_end(channel_t *chan,
-                            const char *identity_digest,
-                            const char *nickname);
+                                 const char *identity_digest,
+                                 const ed25519_public_key_t *ed_identity);
 
 void channel_listener_change_state(channel_listener_t *chan_l,
                                    channel_listener_state_t to_state);
@@ -435,7 +479,6 @@ void channel_listener_change_state(channel_listener_t *chan_l,
 /* Timestamp updates */
 void channel_timestamp_created(channel_t *chan);
 void channel_timestamp_active(channel_t *chan);
-void channel_timestamp_drained(channel_t *chan);
 void channel_timestamp_recv(channel_t *chan);
 void channel_timestamp_xmit(channel_t *chan);
 
@@ -449,19 +492,14 @@ void channel_listener_queue_incoming(channel_listener_t *listener,
                                      channel_t *incoming);
 
 /* Incoming cell handling */
-void channel_process_cells(channel_t *chan);
-void channel_queue_cell(channel_t *chan, cell_t *cell);
-void channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell);
-
-/* Outgoing cell handling */
-void channel_flush_cells(channel_t *chan);
+void channel_process_cell(channel_t *chan, cell_t *cell);
 
 /* Request from lower layer for more cells if available */
 MOCK_DECL(ssize_t, channel_flush_some_cells,
           (channel_t *chan, ssize_t num_cells));
 
 /* Query if data available on this channel */
-int channel_more_to_flush(channel_t *chan);
+MOCK_DECL(int, channel_more_to_flush, (channel_t *chan));
 
 /* Notify flushed outgoing for dirreq handling */
 void channel_notify_flushed(channel_t *chan);
@@ -469,11 +507,7 @@ void channel_notify_flushed(channel_t *chan);
 /* Handle stuff we need to do on open like notifying circuits */
 void channel_do_open_actions(channel_t *chan);
 
-#ifdef TOR_UNIT_TESTS
-extern uint64_t estimated_total_queue_size;
-#endif
-
-#endif
+#endif /* defined(TOR_CHANNEL_INTERNAL_) */
 
 /* Helper functions to perform operations on channels */
 
@@ -486,27 +520,29 @@ int channel_send_destroy(circid_t circ_id, channel_t *chan,
  */
 
 channel_t * channel_connect(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest);
+                            const char *rsa_id_digest,
+                            const ed25519_public_key_t *ed_id);
 
-channel_t * channel_get_for_extend(const char *digest,
+channel_t * channel_get_for_extend(const char *rsa_id_digest,
+                                   const ed25519_public_key_t *ed_id,
                                    const tor_addr_t *target_addr,
                                    const char **msg_out,
                                    int *launch_out);
 
 /* Ask which of two channels is better for circuit-extension purposes */
-int channel_is_better(time_t now,
-                      channel_t *a, channel_t *b,
-                      int forgive_new_connections);
+int channel_is_better(channel_t *a, channel_t *b);
 
 /** Channel lookups
  */
 
 channel_t * channel_find_by_global_id(uint64_t global_identifier);
-channel_t * channel_find_by_remote_digest(const char *identity_digest);
+channel_t * channel_find_by_remote_identity(const char *rsa_id_digest,
+                                            const ed25519_public_key_t *ed_id);
 
 /** For things returned by channel_find_by_remote_digest(), walk the list.
+ * The RSA key will match for all returned elements; the Ed25519 key might not.
  */
-channel_t * channel_next_with_digest(channel_t *chan);
+channel_t * channel_next_with_rsa_identity(channel_t *chan);
 
 /*
  * Helper macros to lookup state of given channel.
@@ -558,11 +594,12 @@ int channel_is_bad_for_new_circs(channel_t *chan);
 void channel_mark_bad_for_new_circs(channel_t *chan);
 int channel_is_canonical(channel_t *chan);
 int channel_is_canonical_is_reliable(channel_t *chan);
-int channel_is_client(channel_t *chan);
+int channel_is_client(const channel_t *chan);
 int channel_is_local(channel_t *chan);
 int channel_is_incoming(channel_t *chan);
 int channel_is_outgoing(channel_t *chan);
 void channel_mark_client(channel_t *chan);
+void channel_clear_client(channel_t *chan);
 int channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info);
 int channel_matches_target_addr_for_extend(channel_t *chan,
                                            const tor_addr_t *target);
@@ -571,39 +608,33 @@ MOCK_DECL(void,channel_set_circid_type,(channel_t *chan,
                                         crypto_pk_t *identity_rcvd,
                                         int consider_identity));
 void channel_timestamp_client(channel_t *chan);
-void channel_update_xmit_queue_size(channel_t *chan);
 
 const char * channel_listener_describe_transport(channel_listener_t *chan_l);
 void channel_listener_dump_statistics(channel_listener_t *chan_l,
                                       int severity);
 void channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
                                                 int severity);
+void channel_check_for_duplicates(void);
+
+void channel_update_bad_for_new_circs(const char *digest, int force);
 
 /* Flow control queries */
-uint64_t channel_get_global_queue_estimate(void);
 int channel_num_cells_writeable(channel_t *chan);
 
 /* Timestamp queries */
 time_t channel_when_created(channel_t *chan);
-time_t channel_when_last_active(channel_t *chan);
 time_t channel_when_last_client(channel_t *chan);
-time_t channel_when_last_drained(channel_t *chan);
-time_t channel_when_last_recv(channel_t *chan);
 time_t channel_when_last_xmit(channel_t *chan);
 
-time_t channel_listener_when_created(channel_listener_t *chan_l);
-time_t channel_listener_when_last_active(channel_listener_t *chan_l);
-time_t channel_listener_when_last_accepted(channel_listener_t *chan_l);
-
 /* Counter queries */
-uint64_t channel_count_recved(channel_t *chan);
-uint64_t channel_count_xmitted(channel_t *chan);
-
-uint64_t channel_listener_count_accepted(channel_listener_t *chan_l);
-
 int packed_cell_is_destroy(channel_t *chan,
                            const packed_cell_t *packed_cell,
                            circid_t *circid_out);
 
-#endif
+/* Declare the handle helpers */
+HANDLE_DECL(channel, channel_s,)
+#define channel_handle_free(h)    \
+  FREE_AND_NULL(channel_handle_t, channel_handle_free_, (h))
+
+#endif /* !defined(TOR_CHANNEL_H) */
 
diff --git a/src/or/channelpadding.c b/src/or/channelpadding.c
new file mode 100644
index 0000000000..3dbdd019ab
--- /dev/null
+++ b/src/or/channelpadding.c
@@ -0,0 +1,802 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of
+ * channelpadding_channel_to_channelinfo() */
+#define TOR_CHANNEL_INTERNAL_
+
+#include "or.h"
+#include "channel.h"
+#include "channelpadding.h"
+#include "channeltls.h"
+#include "config.h"
+#include "networkstatus.h"
+#include "connection.h"
+#include "connection_or.h"
+#include "crypto_rand.h"
+#include "main.h"
+#include "rephist.h"
+#include "router.h"
+#include "compat_time.h"
+#include "rendservice.h"
+
+STATIC int32_t channelpadding_get_netflow_inactive_timeout_ms(
+                                                           const channel_t *);
+STATIC int channelpadding_send_disable_command(channel_t *);
+STATIC int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *);
+
+/** The total number of pending channelpadding timers */
+static uint64_t total_timers_pending;
+
+/** These are cached consensus parameters for netflow */
+/** The timeout lower bound that is allowed before sending padding */
+static int consensus_nf_ito_low;
+/** The timeout upper bound that is allowed before sending padding */
+static int consensus_nf_ito_high;
+/** The timeout lower bound that is allowed before sending reduced padding */
+static int consensus_nf_ito_low_reduced;
+/** The timeout upper bound that is allowed before sending reduced padding */
+static int consensus_nf_ito_high_reduced;
+/** The connection timeout between relays */
+static int consensus_nf_conntimeout_relays;
+/** The connection timeout for client connections */
+static int consensus_nf_conntimeout_clients;
+/** Should we pad before circuits are actually used for client data? */
+static int consensus_nf_pad_before_usage;
+/** Should we pad relay-to-relay connections? */
+static int consensus_nf_pad_relays;
+/** Should we pad tor2web connections? */
+static int consensus_nf_pad_tor2web;
+/** Should we pad rosos connections? */
+static int consensus_nf_pad_single_onion;
+
+#define TOR_MSEC_PER_SEC 1000
+#define TOR_USEC_PER_MSEC 1000
+
+/**
+ * How often do we get called by the connection housekeeping (ie: once
+ * per second) */
+#define TOR_HOUSEKEEPING_CALLBACK_MSEC 1000
+/**
+ * Additional extra time buffer on the housekeeping callback, since
+ * it can be delayed. This extra slack is used to decide if we should
+ * schedule a timer or wait for the next callback. */
+#define TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC 100
+
+/**
+ * This macro tells us if either end of the channel is connected to a client.
+ * (If we're not a server, we're definitely a client. If the channel thinks
+ *  it's a client, use that. Then finally verify in the consensus).
+ */
+#define CHANNEL_IS_CLIENT(chan, options) \
+  (!public_server_mode((options)) || channel_is_client(chan) || \
+      !connection_or_digest_is_known_relay((chan)->identity_digest))
+
+/**
+ * This function is called to update cached consensus parameters every time
+ * there is a consensus update. This allows us to move the consensus param
+ * search off of the critical path, so it does not need to be evaluated
+ * for every single connection, every second.
+ */
+void
+channelpadding_new_consensus_params(networkstatus_t *ns)
+{
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW 1500
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH 9500
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN 0
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX 60000
+  consensus_nf_ito_low = networkstatus_get_param(ns, "nf_ito_low",
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX);
+  consensus_nf_ito_high = networkstatus_get_param(ns, "nf_ito_high",
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH,
+      consensus_nf_ito_low,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX);
+
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW 9000
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH 14000
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN 0
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX 60000
+  consensus_nf_ito_low_reduced =
+    networkstatus_get_param(ns, "nf_ito_low_reduced",
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX);
+
+  consensus_nf_ito_high_reduced =
+    networkstatus_get_param(ns, "nf_ito_high_reduced",
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH,
+        consensus_nf_ito_low_reduced,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX);
+
+#define CONNTIMEOUT_RELAYS_DFLT (60*60) // 1 hour
+#define CONNTIMEOUT_RELAYS_MIN 60
+#define CONNTIMEOUT_RELAYS_MAX (7*24*60*60) // 1 week
+  consensus_nf_conntimeout_relays =
+    networkstatus_get_param(ns, "nf_conntimeout_relays",
+        CONNTIMEOUT_RELAYS_DFLT,
+        CONNTIMEOUT_RELAYS_MIN,
+        CONNTIMEOUT_RELAYS_MAX);
+
+#define CIRCTIMEOUT_CLIENTS_DFLT (30*60) // 30 minutes
+#define CIRCTIMEOUT_CLIENTS_MIN 60
+#define CIRCTIMEOUT_CLIENTS_MAX (24*60*60) // 24 hours
+  consensus_nf_conntimeout_clients =
+    networkstatus_get_param(ns, "nf_conntimeout_clients",
+        CIRCTIMEOUT_CLIENTS_DFLT,
+        CIRCTIMEOUT_CLIENTS_MIN,
+        CIRCTIMEOUT_CLIENTS_MAX);
+
+  consensus_nf_pad_before_usage =
+    networkstatus_get_param(ns, "nf_pad_before_usage", 1, 0, 1);
+
+  consensus_nf_pad_relays =
+    networkstatus_get_param(ns, "nf_pad_relays", 0, 0, 1);
+
+  consensus_nf_pad_tor2web =
+    networkstatus_get_param(ns,
+                            CHANNELPADDING_TOR2WEB_PARAM,
+                            CHANNELPADDING_TOR2WEB_DEFAULT, 0, 1);
+
+  consensus_nf_pad_single_onion =
+    networkstatus_get_param(ns,
+                            CHANNELPADDING_SOS_PARAM,
+                            CHANNELPADDING_SOS_DEFAULT, 0, 1);
+}
+
+/**
+ * Get a random netflow inactive timeout keepalive period in milliseconds,
+ * the range for which is determined by consensus parameters, negotiation,
+ * configuration, or default values. The consensus parameters enforce the
+ * minimum possible value, to avoid excessively frequent padding.
+ *
+ * The ranges for this value were chosen to be low enough to ensure that
+ * routers do not emit a new netflow record for a connection due to it
+ * being idle.
+ *
+ * Specific timeout values for major routers are listed in Proposal 251.
+ * No major router appeared capable of setting an inactive timeout below 10
+ * seconds, so we set the defaults below that value, since we can always
+ * scale back if it ends up being too much padding.
+ *
+ * Returns the next timeout period (in milliseconds) after which we should
+ * send a padding packet, or 0 if padding is disabled.
+ */
+STATIC int32_t
+channelpadding_get_netflow_inactive_timeout_ms(const channel_t *chan)
+{
+  int low_timeout = consensus_nf_ito_low;
+  int high_timeout = consensus_nf_ito_high;
+  int X1, X2;
+
+  if (low_timeout == 0 && low_timeout == high_timeout)
+    return 0; // No padding
+
+  /* If we have negotiated different timeout values, use those, but
+   * don't allow them to be lower than the consensus ones */
+  if (chan->padding_timeout_low_ms && chan->padding_timeout_high_ms) {
+    low_timeout = MAX(low_timeout, chan->padding_timeout_low_ms);
+    high_timeout = MAX(high_timeout, chan->padding_timeout_high_ms);
+  }
+
+  if (low_timeout == high_timeout)
+    return low_timeout; // No randomization
+
+  /*
+   * This MAX() hack is here because we apply the timeout on both the client
+   * and the server. This creates the situation where the total time before
+   * sending a packet in either direction is actually
+   * min(client_timeout,server_timeout).
+   *
+   * If X is a random variable uniform from 0..R-1 (where R=high-low),
+   * then Y=max(X,X) has Prob(Y == i) = (2.0*i + 1)/(R*R).
+   *
+   * If we create a third random variable Z=min(Y,Y), then it turns out that
+   * Exp[Z] ~= Exp[X]. Here's a table:
+   *
+   *    R     Exp[X]    Exp[Z]    Exp[min(X,X)]   Exp[max(X,X)]
+   *  2000     999.5    1066        666.2           1332.8
+   *  3000    1499.5    1599.5      999.5           1999.5
+   *  5000    2499.5    2666       1666.2           3332.8
+   *  6000    2999.5    3199.5     1999.5           3999.5
+   *  7000    3499.5    3732.8     2332.8           4666.2
+   *  8000    3999.5    4266.2     2666.2           5332.8
+   *  10000   4999.5    5328       3332.8           6666.2
+   *  15000   7499.5    7995       4999.5           9999.5
+   *  20000   9900.5    10661      6666.2           13332.8
+   *
+   * In other words, this hack makes it so that when both the client and
+   * the guard are sending this padding, then the averages work out closer
+   * to the midpoint of the range, making the overhead easier to tune.
+   * If only one endpoint is padding (for example: if the relay does not
+   * support padding, but the client has set ConnectionPadding 1; or
+   * if the relay does support padding, but the client has set
+   * ReducedConnectionPadding 1), then the defense will still prevent
+   * record splitting, but with less overhead than the midpoint
+   * (as seen by the Exp[max(X,X)] column).
+   *
+   * To calculate average padding packet frequency (and thus overhead),
+   * index into the table by picking a row based on R = high-low. Then,
+   * use the appropriate column (Exp[Z] for two-sided padding, and
+   * Exp[max(X,X)] for one-sided padding). Finally, take this value
+   * and add it to the low timeout value. This value is the average
+   * frequency which padding packets will be sent.
+   */
+
+  X1 = crypto_rand_int(high_timeout - low_timeout);
+  X2 = crypto_rand_int(high_timeout - low_timeout);
+  return low_timeout + MAX(X1, X2);
+}
+
+/**
+ * Update this channel's padding settings based on the PADDING_NEGOTIATE
+ * contents.
+ *
+ * Returns -1 on error; 1 on success.
+ */
+int
+channelpadding_update_padding_for_channel(channel_t *chan,
+                const channelpadding_negotiate_t *pad_vars)
+{
+  if (pad_vars->version != 0) {
+    static ratelim_t version_limit = RATELIM_INIT(600);
+
+    log_fn_ratelim(&version_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL,
+           "Got a PADDING_NEGOTIATE cell with an unknown version. Ignoring.");
+    return -1;
+  }
+
+  // We should not allow malicious relays to disable or reduce padding for
+  // us as clients. In fact, we should only accept this cell at all if we're
+  // operating as a relay. Bridges should not accept it from relays, either
+  // (only from their clients).
+  if ((get_options()->BridgeRelay &&
+       connection_or_digest_is_known_relay(chan->identity_digest)) ||
+      !get_options()->ORPort_set) {
+    static ratelim_t relay_limit = RATELIM_INIT(600);
+
+    log_fn_ratelim(&relay_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL,
+           "Got a PADDING_NEGOTIATE from relay at %s (%s). "
+           "This should not happen.",
+           chan->get_remote_descr(chan, 0),
+           hex_str(chan->identity_digest, DIGEST_LEN));
+    return -1;
+  }
+
+  chan->padding_enabled = (pad_vars->command == CHANNELPADDING_COMMAND_START);
+
+  /* Min must not be lower than the current consensus parameter
+     nf_ito_low. */
+  chan->padding_timeout_low_ms = MAX(consensus_nf_ito_low,
+                                     pad_vars->ito_low_ms);
+
+  /* Max must not be lower than ito_low_ms */
+  chan->padding_timeout_high_ms = MAX(chan->padding_timeout_low_ms,
+                                      pad_vars->ito_high_ms);
+
+  log_fn(LOG_INFO,LD_OR,
+         "Negotiated padding=%d, lo=%d, hi=%d on "U64_FORMAT,
+         chan->padding_enabled, chan->padding_timeout_low_ms,
+         chan->padding_timeout_high_ms,
+         U64_PRINTF_ARG(chan->global_identifier));
+
+  return 1;
+}
+
+/**
+ * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side not
+ * to send padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+STATIC int
+channelpadding_send_disable_command(channel_t *chan)
+{
+  channelpadding_negotiate_t disable;
+  cell_t cell;
+
+  tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >=
+             MIN_LINK_PROTO_FOR_CHANNEL_PADDING);
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&disable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP);
+
+  if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                                      &disable) < 0)
+    return -1;
+
+  if (chan->write_cell(chan, &cell) == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side to
+ * resume sending padding at some rate.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+int
+channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout,
+                                   uint16_t high_timeout)
+{
+  channelpadding_negotiate_t enable;
+  cell_t cell;
+
+  tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >=
+             MIN_LINK_PROTO_FOR_CHANNEL_PADDING);
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&enable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&enable, CHANNELPADDING_COMMAND_START);
+  channelpadding_negotiate_set_ito_low_ms(&enable, low_timeout);
+  channelpadding_negotiate_set_ito_high_ms(&enable, high_timeout);
+
+  if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                                      &enable) < 0)
+    return -1;
+
+  if (chan->write_cell(chan, &cell) == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Sends a CELL_PADDING cell on a channel if it has been idle since
+ * our callback was scheduled.
+ *
+ * This function also clears the pending padding timer and the callback
+ * flags.
+ */
+static void
+channelpadding_send_padding_cell_for_callback(channel_t *chan)
+{
+  cell_t cell;
+
+  /* Check that the channel is still valid and open */
+  if (!chan || chan->state != CHANNEL_STATE_OPEN) {
+    if (chan) chan->pending_padding_callback = 0;
+    log_fn(LOG_INFO,LD_OR,
+           "Scheduled a netflow padding cell, but connection already closed.");
+    return;
+  }
+
+  /* We should have a pending callback flag set. */
+  if (BUG(chan->pending_padding_callback == 0))
+    return;
+
+  chan->pending_padding_callback = 0;
+
+  if (monotime_coarse_is_zero(&chan->next_padding_time) ||
+      chan->has_queued_writes(chan) ||
+      (chan->cmux && circuitmux_num_cells(chan->cmux))) {
+    /* We must have been active before the timer fired */
+    monotime_coarse_zero(&chan->next_padding_time);
+    return;
+  }
+
+  {
+    monotime_coarse_t now;
+    monotime_coarse_get(&now);
+
+    log_fn(LOG_INFO,LD_OR,
+        "Sending netflow keepalive on "U64_FORMAT" to %s (%s) after "
+        I64_FORMAT" ms. Delta "I64_FORMAT"ms",
+        U64_PRINTF_ARG(chan->global_identifier),
+        safe_str_client(chan->get_remote_descr(chan, 0)),
+        safe_str_client(hex_str(chan->identity_digest, DIGEST_LEN)),
+        I64_PRINTF_ARG(monotime_coarse_diff_msec(&chan->timestamp_xfer,&now)),
+        I64_PRINTF_ARG(
+                   monotime_coarse_diff_msec(&chan->next_padding_time,&now)));
+  }
+
+  /* Clear the timer */
+  monotime_coarse_zero(&chan->next_padding_time);
+
+  /* Send the padding cell. This will cause the channel to get a
+   * fresh timestamp_active */
+  memset(&cell, 0, sizeof(cell));
+  cell.command = CELL_PADDING;
+  chan->write_cell(chan, &cell);
+}
+
+/**
+ * tor_timer callback function for us to send padding on an idle channel.
+ *
+ * This function just obtains the channel from the callback handle, ensures
+ * it is still valid, and then hands it off to
+ * channelpadding_send_padding_cell_for_callback(), which checks if
+ * the channel is still idle before sending padding.
+ */
+static void
+channelpadding_send_padding_callback(tor_timer_t *timer, void *args,
+                                     const struct monotime_t *when)
+{
+  channel_t *chan = channel_handle_get((struct channel_handle_t*)args);
+  (void)timer; (void)when;
+
+  if (chan && CHANNEL_CAN_HANDLE_CELLS(chan)) {
+    /* Hrmm.. It might be nice to have an equivalent to assert_connection_ok
+     * for channels. Then we could get rid of the channeltls dependency */
+    tor_assert(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->magic ==
+               OR_CONNECTION_MAGIC);
+    assert_connection_ok(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), approx_time());
+
+    channelpadding_send_padding_cell_for_callback(chan);
+  } else {
+     log_fn(LOG_INFO,LD_OR,
+            "Channel closed while waiting for timer.");
+  }
+
+  total_timers_pending--;
+}
+
+/**
+ * Schedules a callback to send padding on a channel in_ms milliseconds from
+ * now.
+ *
+ * Returns CHANNELPADDING_WONTPAD on error, CHANNELPADDING_PADDING_SENT if we
+ * sent the packet immediately without a timer, and
+ * CHANNELPADDING_PADDING_SCHEDULED if we decided to schedule a timer.
+ */
+static channelpadding_decision_t
+channelpadding_schedule_padding(channel_t *chan, int in_ms)
+{
+  struct timeval timeout;
+  tor_assert(!chan->pending_padding_callback);
+
+  if (in_ms <= 0) {
+    chan->pending_padding_callback = 1;
+    channelpadding_send_padding_cell_for_callback(chan);
+    return CHANNELPADDING_PADDING_SENT;
+  }
+
+  timeout.tv_sec = in_ms/TOR_MSEC_PER_SEC;
+  timeout.tv_usec = (in_ms%TOR_USEC_PER_MSEC)*TOR_USEC_PER_MSEC;
+
+  if (!chan->timer_handle) {
+    chan->timer_handle = channel_handle_new(chan);
+  }
+
+  if (chan->padding_timer) {
+    timer_set_cb(chan->padding_timer,
+                 channelpadding_send_padding_callback,
+                 chan->timer_handle);
+  } else {
+    chan->padding_timer = timer_new(channelpadding_send_padding_callback,
+                                    chan->timer_handle);
+  }
+  timer_schedule(chan->padding_timer, &timeout);
+
+  rep_hist_padding_count_timers(++total_timers_pending);
+
+  chan->pending_padding_callback = 1;
+  return CHANNELPADDING_PADDING_SCHEDULED;
+}
+
+/**
+ * Calculates the number of milliseconds from now to schedule a padding cell.
+ *
+ * Returns the number of milliseconds from now (relative) to schedule the
+ * padding callback. If the padding timer is more than 1.1 seconds in the
+ * future, we return -1, to avoid scheduling excessive callbacks. If padding
+ * is disabled in the consensus, we return -2.
+ *
+ * Side-effects: Updates chan->next_padding_time_ms, storing an (absolute, not
+ * relative) millisecond representation of when we should send padding, unless
+ * other activity happens first. This side-effect allows us to avoid
+ * scheduling a libevent callback until we're within 1.1 seconds of the padding
+ * time.
+ */
+#define CHANNELPADDING_TIME_LATER -1
+#define CHANNELPADDING_TIME_DISABLED -2
+STATIC int64_t
+channelpadding_compute_time_until_pad_for_netflow(channel_t *chan)
+{
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+
+  if (monotime_coarse_is_zero(&chan->next_padding_time)) {
+    /* If the below line or crypto_rand_int() shows up on a profile,
+     * we can avoid getting a timeout until we're at least nf_ito_lo
+     * from a timeout window. That will prevent us from setting timers
+     * on connections that were active up to 1.5 seconds ago.
+     * Idle connections should only call this once every 5.5s on average
+     * though, so that might be a micro-optimization for little gain. */
+    int32_t padding_timeout =
+        channelpadding_get_netflow_inactive_timeout_ms(chan);
+
+    if (!padding_timeout)
+      return CHANNELPADDING_TIME_DISABLED;
+
+    monotime_coarse_add_msec(&chan->next_padding_time,
+                             &chan->timestamp_xfer,
+                             padding_timeout);
+  }
+
+  const int64_t ms_till_pad =
+    monotime_coarse_diff_msec(&now, &chan->next_padding_time);
+
+  /* If the next padding time is beyond the maximum possible consensus value,
+   * then this indicates a clock jump, so just send padding now. This is
+   * better than using monotonic time because we want to avoid the situation
+   * where we wait around forever for monotonic time to move forward after
+   * a clock jump far into the past.
+   */
+  if (ms_till_pad > DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX) {
+    tor_fragile_assert();
+    log_warn(LD_BUG,
+        "Channel padding timeout scheduled "I64_FORMAT"ms in the future. "
+        "Did the monotonic clock just jump?",
+        I64_PRINTF_ARG(ms_till_pad));
+    return 0; /* Clock jumped: Send padding now */
+  }
+
+  /* If the timeout will expire before the next time we're called (1000ms
+     from now, plus some slack), then calculate the number of milliseconds
+     from now which we should send padding, so we can schedule a callback
+     then.
+   */
+  if (ms_till_pad < (TOR_HOUSEKEEPING_CALLBACK_MSEC +
+                       TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC)) {
+    /* If the padding time is in the past, that means that libevent delayed
+     * calling the once-per-second callback due to other work taking too long.
+     * See https://bugs.torproject.org/22212 and
+     * https://bugs.torproject.org/16585. This is a systemic problem
+     * with being single-threaded, but let's emit a notice if this
+     * is long enough in the past that we might have missed a netflow window,
+     * and allowed a router to emit a netflow frame, just so we don't forget
+     * about it entirely.. */
+#define NETFLOW_MISSED_WINDOW (150000 - DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH)
+    if (ms_till_pad < 0) {
+      int severity = (ms_till_pad < -NETFLOW_MISSED_WINDOW)
+                      ? LOG_NOTICE : LOG_INFO;
+      log_fn(severity, LD_OR,
+              "Channel padding timeout scheduled "I64_FORMAT"ms in the past. ",
+             I64_PRINTF_ARG(-ms_till_pad));
+      return 0; /* Clock jumped: Send padding now */
+    }
+
+    return ms_till_pad;
+  }
+  return CHANNELPADDING_TIME_LATER;
+}
+
+/**
+ * Returns a randomized value for channel idle timeout in seconds.
+ * The channel idle timeout governs how quickly we close a channel
+ * after its last circuit has disappeared.
+ *
+ * There are three classes of channels:
+ *  1. Client+non-canonical. These live for 3-4.5 minutes
+ *  2. relay to relay. These live for 45-75 min by default
+ *  3. Reduced padding clients. These live for 1.5-2.25 minutes.
+ *
+ * Also allows the default relay-to-relay value to be controlled by the
+ * consensus.
+ */
+unsigned int
+channelpadding_get_channel_idle_timeout(const channel_t *chan,
+                                        int is_canonical)
+{
+  const or_options_t *options = get_options();
+  unsigned int timeout;
+
+  /* Non-canonical and client channels only last for 3-4.5 min when idle */
+  if (!is_canonical || CHANNEL_IS_CLIENT(chan, options)) {
+#define CONNTIMEOUT_CLIENTS_BASE 180 // 3 to 4.5 min
+    timeout = CONNTIMEOUT_CLIENTS_BASE
+        + crypto_rand_int(CONNTIMEOUT_CLIENTS_BASE/2);
+  } else { // Canonical relay-to-relay channels
+    // 45..75min or consensus +/- 25%
+    timeout = consensus_nf_conntimeout_relays;
+    timeout = 3*timeout/4 + crypto_rand_int(timeout/2);
+  }
+
+  /* If ReducedConnectionPadding is set, we want to halve the duration of
+   * the channel idle timeout, since reducing the additional time that
+   * a channel stays open will reduce the total overhead for making
+   * new channels. This reduction in overhead/channel expense
+   * is important for mobile users. The option cannot be set by relays.
+   *
+   * We also don't reduce any values for timeout that the user explicitly
+   * set.
+   */
+  if (options->ReducedConnectionPadding
+      && !options->CircuitsAvailableTimeout) {
+    timeout /= 2;
+  }
+
+  return timeout;
+}
+
+/**
+ * This function controls how long we keep idle circuits open,
+ * and how long we build predicted circuits. This behavior is under
+ * the control of channelpadding because circuit availability is the
+ * dominant factor in channel lifespan, which influences total padding
+ * overhead.
+ *
+ * Returns a randomized number of seconds in a range from
+ * CircuitsAvailableTimeout to 2*CircuitsAvailableTimeout. This value is halved
+ * if ReducedConnectionPadding is set. The default value of
+ * CircuitsAvailableTimeout can be controlled by the consensus.
+ */
+int
+channelpadding_get_circuits_available_timeout(void)
+{
+  const or_options_t *options = get_options();
+  int timeout = options->CircuitsAvailableTimeout;
+
+  if (!timeout) {
+    timeout = consensus_nf_conntimeout_clients;
+
+    /* If ReducedConnectionPadding is set, we want to halve the duration of
+     * the channel idle timeout, since reducing the additional time that
+     * a channel stays open will reduce the total overhead for making
+     * new connections. This reduction in overhead/connection expense
+     * is important for mobile users. The option cannot be set by relays.
+     *
+     * We also don't reduce any values for timeout that the user explicitly
+     * set.
+     */
+    if (options->ReducedConnectionPadding) {
+      // half the value to 15..30min by default
+      timeout /= 2;
+    }
+  }
+
+  // 30..60min by default
+  timeout = timeout + crypto_rand_int(timeout);
+
+  return timeout;
+}
+
+/**
+ * Calling this function on a channel causes it to tell the other side
+ * not to send padding, and disables sending padding from this side as well.
+ */
+void
+channelpadding_disable_padding_on_channel(channel_t *chan)
+{
+  chan->padding_enabled = 0;
+
+  // Send cell to disable padding on the other end
+  channelpadding_send_disable_command(chan);
+}
+
+/**
+ * Calling this function on a channel causes it to tell the other side
+ * not to send padding, and reduces the rate that padding is sent from
+ * this side.
+ */
+void
+channelpadding_reduce_padding_on_channel(channel_t *chan)
+{
+  /* Padding can be forced and reduced by clients, regardless of if
+   * the channel supports it. So we check for support here before
+   * sending any commands. */
+  if (chan->padding_enabled) {
+    channelpadding_send_disable_command(chan);
+  }
+
+  chan->padding_timeout_low_ms = consensus_nf_ito_low_reduced;
+  chan->padding_timeout_high_ms = consensus_nf_ito_high_reduced;
+
+  log_fn(LOG_INFO,LD_OR,
+         "Reduced padding on channel "U64_FORMAT": lo=%d, hi=%d",
+         U64_PRINTF_ARG(chan->global_identifier),
+         chan->padding_timeout_low_ms, chan->padding_timeout_high_ms);
+}
+
+/**
+ * This function is called once per second by run_connection_housekeeping(),
+ * but only if the channel is still open, valid, and non-wedged.
+ *
+ * It decides if and when we should send a padding cell, and if needed,
+ * schedules a callback to send that cell at the appropriate time.
+ *
+ * Returns an enum that represents the current padding decision state.
+ * Return value is currently used only by unit tests.
+ */
+channelpadding_decision_t
+channelpadding_decide_to_pad_channel(channel_t *chan)
+{
+  const or_options_t *options = get_options();
+
+  /* Only pad open channels */
+  if (chan->state != CHANNEL_STATE_OPEN)
+    return CHANNELPADDING_WONTPAD;
+
+  if (chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS) {
+    if (!consensus_nf_pad_before_usage)
+      return CHANNELPADDING_WONTPAD;
+  } else if (chan->channel_usage != CHANNEL_USED_FOR_USER_TRAFFIC) {
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (chan->pending_padding_callback)
+    return CHANNELPADDING_PADDING_ALREADY_SCHEDULED;
+
+  /* Don't pad the channel if we didn't negotiate it, but still
+   * allow clients to force padding if options->ChannelPadding is
+   * explicitly set to 1.
+   */
+  if (!chan->padding_enabled && options->ConnectionPadding != 1) {
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (options->Tor2webMode && !consensus_nf_pad_tor2web) {
+    /* If the consensus just changed values, this channel may still
+     * think padding is enabled. Negotiate it off. */
+    if (chan->padding_enabled)
+      channelpadding_disable_padding_on_channel(chan);
+
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (rend_service_allow_non_anonymous_connection(options) &&
+      !consensus_nf_pad_single_onion) {
+    /* If the consensus just changed values, this channel may still
+     * think padding is enabled. Negotiate it off. */
+    if (chan->padding_enabled)
+      channelpadding_disable_padding_on_channel(chan);
+
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  /* There should always be a cmux on the circuit. After that,
+   * only schedule padding if there are no queued writes and no
+   * queued cells in circuitmux queues. */
+  if (chan->cmux && !chan->has_queued_writes(chan) &&
+      !circuitmux_num_cells(chan->cmux)) {
+    int is_client_channel = 0;
+
+    if (CHANNEL_IS_CLIENT(chan, options)) {
+       is_client_channel = 1;
+    }
+
+    /* If nf_pad_relays=1 is set in the consensus, we pad
+     * on *all* idle connections, relay-relay or relay-client.
+     * Otherwise pad only for client+bridge cons */
+    if (is_client_channel || consensus_nf_pad_relays) {
+      int64_t pad_time_ms =
+          channelpadding_compute_time_until_pad_for_netflow(chan);
+
+      if (pad_time_ms == CHANNELPADDING_TIME_DISABLED) {
+        return CHANNELPADDING_WONTPAD;
+      } else if (pad_time_ms == CHANNELPADDING_TIME_LATER) {
+        chan->currently_padding = 1;
+        return CHANNELPADDING_PADLATER;
+      } else {
+        if (BUG(pad_time_ms > INT_MAX)) {
+          pad_time_ms = INT_MAX;
+        }
+       /* We have to schedule a callback because we're called exactly once per
+        * second, but we don't want padding packets to go out exactly on an
+        * integer multiple of seconds. This callback will only be scheduled
+        * if we're within 1.1 seconds of the padding time.
+        */
+        chan->currently_padding = 1;
+        return channelpadding_schedule_padding(chan, (int)pad_time_ms);
+      }
+    } else {
+      chan->currently_padding = 0;
+      return CHANNELPADDING_WONTPAD;
+    }
+  } else {
+    return CHANNELPADDING_PADLATER;
+  }
+}
+
diff --git a/src/or/channelpadding.h b/src/or/channelpadding.h
new file mode 100644
index 0000000000..58bf741d5c
--- /dev/null
+++ b/src/or/channelpadding.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitbuild.h
+ * \brief Header file for circuitbuild.c.
+ **/
+#ifndef TOR_CHANNELPADDING_H
+#define TOR_CHANNELPADDING_H
+
+#include "channelpadding_negotiation.h"
+
+#define CHANNELPADDING_TOR2WEB_PARAM  "nf_pad_tor2web"
+#define CHANNELPADDING_TOR2WEB_DEFAULT 1
+#define CHANNELPADDING_SOS_PARAM  "nf_pad_single_onion"
+#define CHANNELPADDING_SOS_DEFAULT 1
+
+typedef enum {
+  CHANNELPADDING_WONTPAD,
+  CHANNELPADDING_PADLATER,
+  CHANNELPADDING_PADDING_SCHEDULED,
+  CHANNELPADDING_PADDING_ALREADY_SCHEDULED,
+  CHANNELPADDING_PADDING_SENT,
+} channelpadding_decision_t;
+
+channelpadding_decision_t channelpadding_decide_to_pad_channel(channel_t
+                                                               *chan);
+int channelpadding_update_padding_for_channel(channel_t *,
+                                              const channelpadding_negotiate_t
+                                              *chan);
+
+void channelpadding_disable_padding_on_channel(channel_t *chan);
+void channelpadding_reduce_padding_on_channel(channel_t *chan);
+int channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout,
+                                       uint16_t high_timeout);
+
+int channelpadding_get_circuits_available_timeout(void);
+unsigned int channelpadding_get_channel_idle_timeout(const channel_t *, int);
+void channelpadding_new_consensus_params(networkstatus_t *ns);
+
+#endif /* !defined(TOR_CHANNELPADDING_H) */
+
diff --git a/src/or/channeltls.c b/src/or/channeltls.c
index 3a352d47fe..54d94f6109 100644
--- a/src/or/channeltls.c
+++ b/src/or/channeltls.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,6 @@
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-
 #define TOR_CHANNEL_INTERNAL_
 
 #define CHANNELTLS_PRIVATE
@@ -49,12 +48,17 @@
 #include "connection.h"
 #include "connection_or.h"
 #include "control.h"
+#include "entrynodes.h"
 #include "link_handshake.h"
 #include "relay.h"
 #include "rephist.h"
 #include "router.h"
 #include "routerlist.h"
 #include "scheduler.h"
+#include "torcert.h"
+#include "networkstatus.h"
+#include "channelpadding_negotiation.h"
+#include "channelpadding.h"
 
 /** How many CELL_PADDING cells have we received, ever? */
 uint64_t stats_n_padding_cells_processed = 0;
@@ -120,12 +124,13 @@ static void channel_tls_process_netinfo_cell(cell_t *cell,
 static int command_allowed_before_handshake(uint8_t command);
 static int enter_v3_handshake_with_cell(var_cell_t *cell,
                                         channel_tls_t *tlschan);
+static void channel_tls_process_padding_negotiate_cell(cell_t *cell,
+                                                       channel_tls_t *chan);
 
 /**
  * Do parts of channel_tls_t initialization common to channel_tls_connect()
  * and channel_tls_handle_incoming().
  */
-
 STATIC void
 channel_tls_common_init(channel_tls_t *tlschan)
 {
@@ -155,22 +160,21 @@ channel_tls_common_init(channel_tls_t *tlschan)
   chan->write_var_cell = channel_tls_write_var_cell_method;
 
   chan->cmux = circuitmux_alloc();
-  if (cell_ewma_enabled()) {
-    circuitmux_set_policy(chan->cmux, &ewma_policy);
-  }
+  /* We only have one policy for now so always set it to EWMA. */
+  circuitmux_set_policy(chan->cmux, &ewma_policy);
 }
 
 /**
- * Start a new TLS channel
+ * Start a new TLS channel.
  *
  * Launch a new OR connection to <b>addr</b>:<b>port</b> and expect to
  * handshake with an OR with identity digest <b>id_digest</b>, and wrap
  * it in a channel_tls_t.
  */
-
 channel_t *
 channel_tls_connect(const tor_addr_t *addr, uint16_t port,
-                    const char *id_digest)
+                    const char *id_digest,
+                    const ed25519_public_key_t *ed_id)
 {
   channel_tls_t *tlschan = tor_malloc_zero(sizeof(*tlschan));
   channel_t *chan = &(tlschan->base_);
@@ -198,7 +202,7 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port,
   channel_mark_outgoing(chan);
 
   /* Set up or_connection stuff */
-  tlschan->conn = connection_or_connect(addr, port, id_digest, tlschan);
+  tlschan->conn = connection_or_connect(addr, port, id_digest, ed_id, tlschan);
   /* connection_or_connect() will fill in tlschan->conn */
   if (!(tlschan->conn)) {
     chan->reason_for_closing = CHANNEL_CLOSE_FOR_ERROR;
@@ -225,12 +229,11 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port,
 }
 
 /**
- * Return the current channel_tls_t listener
+ * Return the current channel_tls_t listener.
  *
  * Returns the current channel listener for incoming TLS connections, or
  * NULL if none has been established
  */
-
 channel_listener_t *
 channel_tls_get_listener(void)
 {
@@ -238,12 +241,11 @@ channel_tls_get_listener(void)
 }
 
 /**
- * Start a channel_tls_t listener if necessary
+ * Start a channel_tls_t listener if necessary.
  *
  * Return the current channel_tls_t listener, or start one if we haven't yet,
  * and return that.
  */
-
 channel_listener_t *
 channel_tls_start_listener(void)
 {
@@ -270,12 +272,11 @@ channel_tls_start_listener(void)
 }
 
 /**
- * Free everything on shutdown
+ * Free everything on shutdown.
  *
  * Not much to do here, since channel_free_all() takes care of a lot, but let's
  * get rid of the listener.
  */
-
 void
 channel_tls_free_all(void)
 {
@@ -306,9 +307,8 @@ channel_tls_free_all(void)
 }
 
 /**
- * Create a new channel around an incoming or_connection_t
+ * Create a new channel around an incoming or_connection_t.
  */
-
 channel_t *
 channel_tls_handle_incoming(or_connection_t *orconn)
 {
@@ -351,7 +351,6 @@ channel_tls_handle_incoming(or_connection_t *orconn)
 /**
  * Cast a channel_tls_t to a channel_t.
  */
-
 channel_t *
 channel_tls_to_base(channel_tls_t *tlschan)
 {
@@ -364,7 +363,6 @@ channel_tls_to_base(channel_tls_t *tlschan)
  * Cast a channel_t to a channel_tls_t, with appropriate type-checking
  * asserts.
  */
-
 channel_tls_t *
 channel_tls_from_base(channel_t *chan)
 {
@@ -380,11 +378,10 @@ channel_tls_from_base(channel_t *chan)
  *******************************************/
 
 /**
- * Close a channel_tls_t
+ * Close a channel_tls_t.
  *
- * This implements the close method for channel_tls_t
+ * This implements the close method for channel_tls_t.
  */
-
 static void
 channel_tls_close_method(channel_t *chan)
 {
@@ -403,12 +400,11 @@ channel_tls_close_method(channel_t *chan)
 }
 
 /**
- * Describe the transport for a channel_tls_t
+ * Describe the transport for a channel_tls_t.
  *
  * This returns the string "TLS channel on connection <id>" to the upper
  * layer.
  */
-
 static const char *
 channel_tls_describe_transport_method(channel_t *chan)
 {
@@ -438,7 +434,7 @@ channel_tls_describe_transport_method(channel_t *chan)
 }
 
 /**
- * Free a channel_tls_t
+ * Free a channel_tls_t.
  *
  * This is called by the generic channel layer when freeing a channel_tls_t;
  * this happens either on a channel which has already reached
@@ -447,7 +443,6 @@ channel_tls_describe_transport_method(channel_t *chan)
  * have an orconn active (which connection_free_all() will get to later),
  * so we should null out its channel pointer now.
  */
-
 static void
 channel_tls_free_method(channel_t *chan)
 {
@@ -462,9 +457,8 @@ channel_tls_free_method(channel_t *chan)
 }
 
 /**
- * Get an estimate of the average TLS overhead for the upper layer
+ * Get an estimate of the average TLS overhead for the upper layer.
  */
-
 static double
 channel_tls_get_overhead_estimate_method(channel_t *chan)
 {
@@ -497,13 +491,12 @@ channel_tls_get_overhead_estimate_method(channel_t *chan)
 }
 
 /**
- * Get the remote address of a channel_tls_t
+ * Get the remote address of a channel_tls_t.
  *
  * This implements the get_remote_addr method for channel_tls_t; copy the
  * remote endpoint of the channel to addr_out and return 1 (always
  * succeeds for this transport).
  */
-
 static int
 channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out)
 {
@@ -527,8 +520,8 @@ channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out)
  * This implements the get_transport_name for channel_tls_t. If the
  * channel uses a pluggable transport, copy its name to
  * <b>transport_out</b> and return 0. If the channel did not use a
- * pluggable transport, return -1. */
-
+ * pluggable transport, return -1.
+ */
 static int
 channel_tls_get_transport_name_method(channel_t *chan, char **transport_out)
 {
@@ -546,14 +539,13 @@ channel_tls_get_transport_name_method(channel_t *chan, char **transport_out)
 }
 
 /**
- * Get endpoint description of a channel_tls_t
+ * Get endpoint description of a channel_tls_t.
  *
  * This implements the get_remote_descr method for channel_tls_t; it returns
  * a text description of the remote endpoint of the channel suitable for use
- * in log messages.  The req parameter is 0 for the canonical address or 1 for
+ * in log messages. The req parameter is 0 for the canonical address or 1 for
  * the actual address seen.
  */
-
 static const char *
 channel_tls_get_remote_descr_method(channel_t *chan, int flags)
 {
@@ -598,7 +590,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
         break;
       default:
         /* Something's broken in channel.c */
-        tor_assert(1);
+        tor_assert_nonfatal_unreached_once();
     }
   } else {
     strlcpy(buf, "(No connection)", sizeof(buf));
@@ -609,12 +601,11 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
 }
 
 /**
- * Tell the upper layer if we have queued writes
+ * Tell the upper layer if we have queued writes.
  *
  * This implements the has_queued_writes method for channel_tls t_; it returns
  * 1 iff we have queued writes on the outbuf of the underlying or_connection_t.
  */
-
 static int
 channel_tls_has_queued_writes_method(channel_t *chan)
 {
@@ -637,13 +628,12 @@ channel_tls_has_queued_writes_method(channel_t *chan)
 }
 
 /**
- * Tell the upper layer if we're canonical
+ * Tell the upper layer if we're canonical.
  *
  * This implements the is_canonical method for channel_tls_t; if req is zero,
  * it returns whether this is a canonical channel, and if it is one it returns
  * whether that can be relied upon.
  */
-
 static int
 channel_tls_is_canonical_method(channel_t *chan, int req)
 {
@@ -667,7 +657,7 @@ channel_tls_is_canonical_method(channel_t *chan, int req)
         break;
       default:
         /* This shouldn't happen; channel.c is broken if it does */
-        tor_assert(1);
+        tor_assert_nonfatal_unreached_once();
     }
   }
   /* else return 0 for tlschan->conn == NULL */
@@ -676,12 +666,11 @@ channel_tls_is_canonical_method(channel_t *chan, int req)
 }
 
 /**
- * Check if we match an extend_info_t
+ * Check if we match an extend_info_t.
  *
  * This implements the matches_extend_info method for channel_tls_t; the upper
  * layer wants to know if this channel matches an extend_info_t.
  */
-
 static int
 channel_tls_matches_extend_info_method(channel_t *chan,
                                        extend_info_t *extend_info)
@@ -712,7 +701,6 @@ channel_tls_matches_extend_info_method(channel_t *chan,
  * layer wants to know if this channel matches a target address when extending
  * a circuit.
  */
-
 static int
 channel_tls_matches_target_method(channel_t *chan,
                                   const tor_addr_t *target)
@@ -731,6 +719,15 @@ channel_tls_matches_target_method(channel_t *chan,
     return 0;
   }
 
+  /* real_addr is the address this connection came from.
+   * base_.addr is updated by connection_or_init_conn_from_address()
+   * to be the address in the descriptor. It may be tempting to
+   * allow either address to be allowed, but if we did so, it would
+   * enable someone who steals a relay's keys to impersonate/MITM it
+   * from anywhere on the Internet! (Because they could make long-lived
+   * TLS connections from anywhere to all relays, and wait for them to
+   * be used for extends).
+   */
   return tor_addr_eq(&(tlschan->conn->real_addr), target);
 }
 
@@ -738,7 +735,6 @@ channel_tls_matches_target_method(channel_t *chan,
  * Tell the upper layer how many bytes we have queued and not yet
  * sent.
  */
-
 static size_t
 channel_tls_num_bytes_queued_method(channel_t *chan)
 {
@@ -751,13 +747,12 @@ channel_tls_num_bytes_queued_method(channel_t *chan)
 }
 
 /**
- * Tell the upper layer how many cells we can accept to write
+ * Tell the upper layer how many cells we can accept to write.
  *
  * This implements the num_cells_writeable method for channel_tls_t; it
  * returns an estimate of the number of cells we can accept with
  * channel_tls_write_*_cell().
  */
-
 static int
 channel_tls_num_cells_writeable_method(channel_t *chan)
 {
@@ -782,12 +777,11 @@ channel_tls_num_cells_writeable_method(channel_t *chan)
 }
 
 /**
- * Write a cell to a channel_tls_t
+ * Write a cell to a channel_tls_t.
  *
  * This implements the write_cell method for channel_tls_t; given a
  * channel_tls_t and a cell_t, transmit the cell_t.
  */
-
 static int
 channel_tls_write_cell_method(channel_t *chan, cell_t *cell)
 {
@@ -811,12 +805,14 @@ channel_tls_write_cell_method(channel_t *chan, cell_t *cell)
 }
 
 /**
- * Write a packed cell to a channel_tls_t
+ * Write a packed cell to a channel_tls_t.
  *
  * This implements the write_packed_cell method for channel_tls_t; given a
  * channel_tls_t and a packed_cell_t, transmit the packed_cell_t.
+ *
+ * Return 0 on success or negative value on error. The caller must free the
+ * packed cell.
  */
-
 static int
 channel_tls_write_packed_cell_method(channel_t *chan,
                                      packed_cell_t *packed_cell)
@@ -824,35 +820,30 @@ channel_tls_write_packed_cell_method(channel_t *chan,
   tor_assert(chan);
   channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
   size_t cell_network_size = get_cell_network_size(chan->wide_circ_ids);
-  int written = 0;
 
   tor_assert(tlschan);
   tor_assert(packed_cell);
 
   if (tlschan->conn) {
-    connection_write_to_buf(packed_cell->body, cell_network_size,
+    connection_buf_add(packed_cell->body, cell_network_size,
                             TO_CONN(tlschan->conn));
-
-    /* This is where the cell is finished; used to be done from relay.c */
-    packed_cell_free(packed_cell);
-    ++written;
   } else {
     log_info(LD_CHANNEL,
              "something called write_packed_cell on a tlschan "
              "(%p with ID " U64_FORMAT " but no conn",
              chan, U64_PRINTF_ARG(chan->global_identifier));
+    return -1;
   }
 
-  return written;
+  return 0;
 }
 
 /**
- * Write a variable-length cell to a channel_tls_t
+ * Write a variable-length cell to a channel_tls_t.
  *
  * This implements the write_var_cell method for channel_tls_t; given a
  * channel_tls_t and a var_cell_t, transmit the var_cell_t.
  */
-
 static int
 channel_tls_write_var_cell_method(channel_t *chan, var_cell_t *var_cell)
 {
@@ -880,11 +871,10 @@ channel_tls_write_var_cell_method(channel_t *chan, var_cell_t *var_cell)
  ************************************************/
 
 /**
- * Close a channel_listener_t
+ * Close a channel_listener_t.
  *
- * This implements the close method for channel_listener_t
+ * This implements the close method for channel_listener_t.
  */
-
 static void
 channel_tls_listener_close_method(channel_listener_t *chan_l)
 {
@@ -920,12 +910,11 @@ channel_tls_listener_close_method(channel_listener_t *chan_l)
 }
 
 /**
- * Describe the transport for a channel_listener_t
+ * Describe the transport for a channel_listener_t.
  *
  * This returns the string "TLS channel (listening)" to the upper
  * layer.
  */
-
 static const char *
 channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
 {
@@ -939,12 +928,11 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
  ******************************************************/
 
 /**
- * Handle an orconn state change
+ * Handle an orconn state change.
  *
  * This function will be called by connection_or.c when the or_connection_t
  * associated with this channel_tls_t changes state.
  */
-
 void
 channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                           or_connection_t *conn,
@@ -976,7 +964,7 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
      * We can go to CHANNEL_STATE_OPEN from CHANNEL_STATE_OPENING or
      * CHANNEL_STATE_MAINT on this.
      */
-    channel_change_state(base_chan, CHANNEL_STATE_OPEN);
+    channel_change_state_open(base_chan);
     /* We might have just become writeable; check and tell the scheduler */
     if (connection_or_num_cells_writeable(conn) > 0) {
       scheduler_channel_wants_writes(base_chan);
@@ -995,13 +983,12 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
 #ifdef KEEP_TIMING_STATS
 
 /**
- * Timing states wrapper
+ * Timing states wrapper.
  *
  * This is a wrapper function around the actual function that processes the
  * <b>cell</b> that just arrived on <b>chan</b>. Increment <b>*time</b>
  * by the number of microseconds used by the call to <b>*func(cell, chan)</b>.
  */
-
 static void
 channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
                               void (*func)(cell_t *, channel_tls_t *))
@@ -1027,10 +1014,10 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
 
   *time += time_passed;
 }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
 /**
- * Handle an incoming cell on a channel_tls_t
+ * Handle an incoming cell on a channel_tls_t.
  *
  * This is called from connection_or.c to handle an arriving cell; it checks
  * for cell types specific to the handshake for this transport protocol and
@@ -1042,7 +1029,6 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
  * for copying in the case that it queues; we merely pass pointers through
  * which we get from connection_or_process_cells_from_inbuf().
  */
-
 void
 channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
 {
@@ -1055,9 +1041,9 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
     channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
                              channel_tls_process_ ## tp ## _cell);  \
     } STMT_END
-#else
+#else /* !(defined(KEEP_TIMING_STATS)) */
 #define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
   tor_assert(cell);
   tor_assert(conn);
@@ -1092,8 +1078,19 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn, conn->handshake_state, cell, 1);
 
+  /* We note that we're on the internet whenever we read a cell. This is
+   * a fast operation. */
+  entry_guards_note_internet_connectivity(get_guard_selection_info());
+  rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
+
+  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+    rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
+
   switch (cell->command) {
     case CELL_PADDING:
+      rep_hist_padding_count_read(PADDING_TYPE_CELL);
+      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+        rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
       break;
@@ -1104,6 +1101,10 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
       ++stats_n_netinfo_cells_processed;
       PROCESS_CELL(netinfo, cell, chan);
       break;
+    case CELL_PADDING_NEGOTIATE:
+      ++stats_n_netinfo_cells_processed;
+      PROCESS_CELL(padding_negotiate, cell, chan);
+      break;
     case CELL_CREATE:
     case CELL_CREATE_FAST:
     case CELL_CREATED:
@@ -1117,7 +1118,7 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
        * These are all transport independent and we pass them up through the
        * channel_t mechanism.  They are ultimately handled in command.c.
        */
-      channel_queue_cell(TLS_CHAN_TO_BASE(chan), cell);
+      channel_process_cell(TLS_CHAN_TO_BASE(chan), cell);
       break;
     default:
       log_fn(LOG_INFO, LD_PROTOCOL,
@@ -1129,7 +1130,7 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
 }
 
 /**
- * Handle an incoming variable-length cell on a channel_tls_t
+ * Handle an incoming variable-length cell on a channel_tls_t.
  *
  * Process a <b>var_cell</b> that was just received on <b>conn</b>. Keep
  * internal statistics about how many of each cell we've processed so far
@@ -1145,7 +1146,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
  * caller always frees them after this function returns, so this function
  * should never free var_cell.
  */
-
 void
 channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
 {
@@ -1172,7 +1172,7 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
     /* remember which second it is, for next time */
     current_second = now;
   }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
   tor_assert(var_cell);
   tor_assert(conn);
@@ -1270,6 +1270,10 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
       return;
   }
 
+  /* We note that we're on the internet whenever we read a cell. This is
+   * a fast operation. */
+  entry_guards_note_internet_connectivity(get_guard_selection_info());
+
   /* Now handle the cell */
 
   switch (var_cell->command) {
@@ -1306,7 +1310,7 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
 }
 
 /**
- * Update channel marks after connection_or.c has changed an address
+ * Update channel marks after connection_or.c has changed an address.
  *
  * This is called from connection_or_init_conn_from_address() after the
  * connection's _base.addr or real_addr fields have potentially been changed
@@ -1315,7 +1319,6 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
  * remote router by looking it up in the consensus after we finish the
  * handshake and know an authenticated identity digest.
  */
-
 void
 channel_tls_update_marks(or_connection_t *conn)
 {
@@ -1344,12 +1347,11 @@ channel_tls_update_marks(or_connection_t *conn)
 }
 
 /**
- * Check if this cell type is allowed before the handshake is finished
+ * Check if this cell type is allowed before the handshake is finished.
  *
  * Return true if <b>command</b> is a cell command that's allowed to start a
  * V3 handshake.
  */
-
 static int
 command_allowed_before_handshake(uint8_t command)
 {
@@ -1364,14 +1366,13 @@ command_allowed_before_handshake(uint8_t command)
 }
 
 /**
- * Start a V3 handshake on an incoming connection
+ * Start a V3 handshake on an incoming connection.
  *
  * Called when we as a server receive an appropriate cell while waiting
  * either for a cell or a TLS handshake.  Set the connection's state to
  * "handshaking_v3', initializes the or_handshake_state field as needed,
  * and add the cell to the hash of incoming cells.)
  */
-
 static int
 enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *chan)
 {
@@ -1412,7 +1413,6 @@ enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *chan)
  * we support, pick the highest version we have in common, and continue the
  * negotiation from there.
  */
-
 static void
 channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
 {
@@ -1543,7 +1543,7 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
       connection_or_close_normally(chan->conn, 1);
       return;
     }
-#endif
+#endif /* defined(DISABLE_V3_LINKPROTO_SERVERSIDE) */
 
     if (send_versions) {
       if (connection_or_send_versions(chan->conn, 1) < 0) {
@@ -1553,11 +1553,14 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
       }
     }
 
-    /* We set this after sending the verions cell. */
+    /* We set this after sending the versions cell. */
     /*XXXXX symbolic const.*/
-    chan->base_.wide_circ_ids =
+    TLS_CHAN_TO_BASE(chan)->wide_circ_ids =
       chan->conn->link_proto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
-    chan->conn->wide_circ_ids = chan->base_.wide_circ_ids;
+    chan->conn->wide_circ_ids = TLS_CHAN_TO_BASE(chan)->wide_circ_ids;
+
+    TLS_CHAN_TO_BASE(chan)->padding_enabled =
+      chan->conn->link_proto >= MIN_LINK_PROTO_FOR_CHANNEL_PADDING;
 
     if (send_certs) {
       if (connection_or_send_certs_cell(chan->conn) < 0) {
@@ -1584,12 +1587,48 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
 }
 
 /**
- * Process a 'netinfo' cell
+ * Process a 'padding_negotiate' cell.
+ *
+ * This function is called to handle an incoming PADDING_NEGOTIATE cell;
+ * enable or disable padding accordingly, and read and act on its timeout
+ * value contents.
+ */
+static void
+channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
+{
+  channelpadding_negotiate_t *negotiation;
+  tor_assert(cell);
+  tor_assert(chan);
+  tor_assert(chan->conn);
+
+  if (chan->conn->link_proto < MIN_LINK_PROTO_FOR_CHANNEL_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Received a PADDING_NEGOTIATE cell on v%d connection; dropping.",
+           chan->conn->link_proto);
+    return;
+  }
+
+  if (channelpadding_negotiate_parse(&negotiation, cell->payload,
+                                     CELL_PAYLOAD_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+          "Received malformed PADDING_NEGOTIATE cell on v%d connection; "
+          "dropping.", chan->conn->link_proto);
+
+    return;
+  }
+
+  channelpadding_update_padding_for_channel(TLS_CHAN_TO_BASE(chan),
+                                            negotiation);
+
+  channelpadding_negotiate_free(negotiation);
+}
+
+/**
+ * Process a 'netinfo' cell.
  *
  * This function is called to handle an incoming NETINFO cell; read and act
  * on its contents, and set the connection state to "open".
  */
-
 static void
 channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
 {
@@ -1600,9 +1639,12 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
+  const routerinfo_t *me = router_get_my_routerinfo();
 
   long apparent_skew = 0;
   tor_addr_t my_apparent_addr = TOR_ADDR_NULL;
+  int started_here = 0;
+  const char *identity_digest = NULL;
 
   tor_assert(cell);
   tor_assert(chan);
@@ -1622,10 +1664,12 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   }
   tor_assert(chan->conn->handshake_state &&
              chan->conn->handshake_state->received_versions);
+  started_here = connection_or_nonopen_was_started_here(chan->conn);
+  identity_digest = chan->conn->identity_digest;
 
   if (chan->conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) {
     tor_assert(chan->conn->link_proto >= 3);
-    if (chan->conn->handshake_state->started_here) {
+    if (started_here) {
       if (!(chan->conn->handshake_state->authenticated)) {
         log_fn(LOG_PROTOCOL_WARN, LD_OR,
                "Got a NETINFO cell from server, "
@@ -1639,7 +1683,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       if (!(chan->conn->handshake_state->authenticated)) {
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
-                      authenticated_peer_id)));
+                      authenticated_rsa_peer_id)));
+        tor_assert(tor_mem_is_zero(
+                  (const char*)(chan->conn->handshake_state->
+                                authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
          * relay, and we must not use it for EXTEND requests (nor could we, as
          * there are no authenticated peer IDs) */
@@ -1650,8 +1697,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         connection_or_init_conn_from_address(chan->conn,
                   &(chan->conn->base_.addr),
                   chan->conn->base_.port,
+                  /* zero, checked above */
                   (const char*)(chan->conn->handshake_state->
-                   authenticated_peer_id),
+                                authenticated_rsa_peer_id),
+                  NULL, /* Ed25519 ID: Also checked as zero */
                   0);
       }
     }
@@ -1677,8 +1726,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
 
   if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
     tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+
+    if (!get_options()->BridgeRelay && me &&
+        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+      TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
+    }
+
   } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
     tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
+
+    if (!get_options()->BridgeRelay && me &&
+        !tor_addr_is_null(&me->ipv6_addr) &&
+        tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
+      TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
+    }
   }
 
   n_other_addrs = (uint8_t) *cp++;
@@ -1694,6 +1755,13 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_close_for_error(chan->conn, 0);
       return;
     }
+    /* A relay can connect from anywhere and be canonical, so
+     * long as it tells you from where it came. This may sound a bit
+     * concerning... but that's what "canonical" means: that the
+     * address is one that the relay itself has claimed.  The relay
+     * might be doing something funny, but nobody else is doing a MITM
+     * on the relay's TCP.
+     */
     if (tor_addr_eq(&addr, &(chan->conn->real_addr))) {
       connection_or_set_canonical(chan->conn, 1);
       break;
@@ -1702,11 +1770,27 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     --n_other_addrs;
   }
 
+  if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
+      channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
+    const char *descr =
+      TLS_CHAN_TO_BASE(chan)->get_remote_descr(TLS_CHAN_TO_BASE(chan), 0);
+    log_info(LD_OR,
+             "We made a connection to a relay at %s (fp=%s) but we think "
+             "they will not consider this connection canonical. They "
+             "think we are at %s, but we think its %s.",
+             safe_str(descr),
+             safe_str(hex_str(identity_digest, DIGEST_LEN)),
+             safe_str(tor_addr_is_null(&my_apparent_addr) ?
+             "<none>" : fmt_and_decorate_addr(&my_apparent_addr)),
+             safe_str(fmt_addr32(me->addr)));
+  }
+
   /* Act on apparent skew. */
   /** Warn when we get a netinfo skew with at least this value. */
 #define NETINFO_NOTICE_SKEW 3600
   if (labs(apparent_skew) > NETINFO_NOTICE_SKEW &&
-      router_get_by_id_digest(chan->conn->identity_digest)) {
+      (started_here ||
+       connection_or_digest_is_known_relay(chan->conn->identity_digest))) {
     int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest);
     clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL,
                        "NETINFO cell", "OR");
@@ -1740,14 +1824,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
              safe_str_client(chan->conn->base_.address),
              chan->conn->base_.port,
              (int)(chan->conn->link_proto),
-             hex_str(TLS_CHAN_TO_BASE(chan)->identity_digest,
-                     DIGEST_LEN),
+             hex_str(identity_digest, DIGEST_LEN),
              tor_addr_is_null(&my_apparent_addr) ?
              "<none>" : fmt_and_decorate_addr(&my_apparent_addr));
   }
   assert_connection_ok(TO_CONN(chan->conn),time(NULL));
 }
 
+/** Types of certificates that we know how to parse from CERTS cells.  Each
+ * type corresponds to a different encoding format. */
+typedef enum cert_encoding_t {
+  CERT_ENCODING_UNKNOWN, /**< We don't recognize this. */
+  CERT_ENCODING_X509, /**< It's an RSA key, signed with RSA, encoded in x509.
+                   * (Actually, it might not be RSA. We test that later.) */
+  CERT_ENCODING_ED25519, /**< It's something signed with an Ed25519 key,
+                      * encoded asa a tor_cert_t.*/
+  CERT_ENCODING_RSA_CROSSCERT, /**< It's an Ed key signed with an RSA key. */
+} cert_encoding_t;
+
+/**
+ * Given one of the certificate type codes used in a CERTS cell,
+ * return the corresponding cert_encoding_t that we should use to parse
+ * the certificate.
+ */
+static cert_encoding_t
+certs_cell_typenum_to_cert_type(int typenum)
+{
+  switch (typenum) {
+  case CERTTYPE_RSA1024_ID_LINK:
+  case CERTTYPE_RSA1024_ID_ID:
+  case CERTTYPE_RSA1024_ID_AUTH:
+    return CERT_ENCODING_X509;
+  case CERTTYPE_ED_ID_SIGN:
+  case CERTTYPE_ED_SIGN_LINK:
+  case CERTTYPE_ED_SIGN_AUTH:
+    return CERT_ENCODING_ED25519;
+  case CERTTYPE_RSA1024_ID_EDID:
+    return CERT_ENCODING_RSA_CROSSCERT;
+  default:
+    return CERT_ENCODING_UNKNOWN;
+  }
+}
+
 /**
  * Process a CERTS cell from a channel.
  *
@@ -1763,18 +1881,24 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
  * of the connection, we then authenticate the server or mark the connection.
  * If it's the server side, wait for an AUTHENTICATE cell.
  */
-
 STATIC void
 channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
 {
-#define MAX_CERT_TYPE_WANTED OR_CERT_TYPE_AUTH_1024
-  tor_x509_cert_t *certs[MAX_CERT_TYPE_WANTED + 1];
+#define MAX_CERT_TYPE_WANTED CERTTYPE_RSA1024_ID_EDID
+  /* These arrays will be sparse, since a cert type can be at most one
+   * of ed/x509 */
+  tor_x509_cert_t *x509_certs[MAX_CERT_TYPE_WANTED + 1];
+  tor_cert_t *ed_certs[MAX_CERT_TYPE_WANTED + 1];
+  uint8_t *rsa_ed_cc_cert = NULL;
+  size_t rsa_ed_cc_cert_len = 0;
+
   int n_certs, i;
   certs_cell_t *cc = NULL;
 
-  int send_netinfo = 0;
+  int send_netinfo = 0, started_here = 0;
 
-  memset(certs, 0, sizeof(certs));
+  memset(x509_certs, 0, sizeof(x509_certs));
+  memset(ed_certs, 0, sizeof(ed_certs));
   tor_assert(cell);
   tor_assert(chan);
   tor_assert(chan->conn);
@@ -1789,6 +1913,11 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
     goto err;                                                   \
   } while (0)
 
+  /* Can't use connection_or_nonopen_was_started_here(); its conn->tls
+   * check looks like it breaks
+   * test_link_handshake_recv_certs_ok_server().  */
+  started_here = chan->conn->handshake_state->started_here;
+
   if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3)
     ERR("We're not doing a v3 handshake!");
   if (chan->conn->link_proto < 3)
@@ -1818,77 +1947,149 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
 
     if (cert_type > MAX_CERT_TYPE_WANTED)
       continue;
+    const cert_encoding_t ct = certs_cell_typenum_to_cert_type(cert_type);
+    switch (ct) {
+      default:
+      case CERT_ENCODING_UNKNOWN:
+        break;
+      case CERT_ENCODING_X509: {
+        tor_x509_cert_t *x509_cert = tor_x509_cert_decode(cert_body, cert_len);
+        if (!x509_cert) {
+          log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                 "Received undecodable certificate in CERTS cell from %s:%d",
+                 safe_str(chan->conn->base_.address),
+               chan->conn->base_.port);
+        } else {
+          if (x509_certs[cert_type]) {
+            tor_x509_cert_free(x509_cert);
+            ERR("Duplicate x509 certificate");
+          } else {
+            x509_certs[cert_type] = x509_cert;
+          }
+        }
+        break;
+      }
+      case CERT_ENCODING_ED25519: {
+        tor_cert_t *ed_cert = tor_cert_parse(cert_body, cert_len);
+        if (!ed_cert) {
+          log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                 "Received undecodable Ed certificate "
+                 "in CERTS cell from %s:%d",
+                 safe_str(chan->conn->base_.address),
+               chan->conn->base_.port);
+        } else {
+          if (ed_certs[cert_type]) {
+            tor_cert_free(ed_cert);
+            ERR("Duplicate Ed25519 certificate");
+          } else {
+            ed_certs[cert_type] = ed_cert;
+          }
+        }
+        break;
+      }
 
-    tor_x509_cert_t *cert = tor_x509_cert_decode(cert_body, cert_len);
-    if (!cert) {
-      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-             "Received undecodable certificate in CERTS cell from %s:%d",
-             safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
-    } else {
-      if (certs[cert_type]) {
-        tor_x509_cert_free(cert);
-        ERR("Duplicate x509 certificate");
-      } else {
-        certs[cert_type] = cert;
+     case CERT_ENCODING_RSA_CROSSCERT: {
+        if (rsa_ed_cc_cert) {
+          ERR("Duplicate RSA->Ed25519 crosscert");
+        } else {
+          rsa_ed_cc_cert = tor_memdup(cert_body, cert_len);
+          rsa_ed_cc_cert_len = cert_len;
+        }
+        break;
       }
     }
   }
 
-  tor_x509_cert_t *id_cert = certs[OR_CERT_TYPE_ID_1024];
-  tor_x509_cert_t *auth_cert = certs[OR_CERT_TYPE_AUTH_1024];
-  tor_x509_cert_t *link_cert = certs[OR_CERT_TYPE_TLS_LINK];
+  /* Move the certificates we (might) want into the handshake_state->certs
+   * structure. */
+  tor_x509_cert_t *id_cert = x509_certs[CERTTYPE_RSA1024_ID_ID];
+  tor_x509_cert_t *auth_cert = x509_certs[CERTTYPE_RSA1024_ID_AUTH];
+  tor_x509_cert_t *link_cert = x509_certs[CERTTYPE_RSA1024_ID_LINK];
+  chan->conn->handshake_state->certs->auth_cert = auth_cert;
+  chan->conn->handshake_state->certs->link_cert = link_cert;
+  chan->conn->handshake_state->certs->id_cert = id_cert;
+  x509_certs[CERTTYPE_RSA1024_ID_ID] =
+    x509_certs[CERTTYPE_RSA1024_ID_AUTH] =
+    x509_certs[CERTTYPE_RSA1024_ID_LINK] = NULL;
+
+  tor_cert_t *ed_id_sign = ed_certs[CERTTYPE_ED_ID_SIGN];
+  tor_cert_t *ed_sign_link = ed_certs[CERTTYPE_ED_SIGN_LINK];
+  tor_cert_t *ed_sign_auth = ed_certs[CERTTYPE_ED_SIGN_AUTH];
+  chan->conn->handshake_state->certs->ed_id_sign = ed_id_sign;
+  chan->conn->handshake_state->certs->ed_sign_link = ed_sign_link;
+  chan->conn->handshake_state->certs->ed_sign_auth = ed_sign_auth;
+  ed_certs[CERTTYPE_ED_ID_SIGN] =
+    ed_certs[CERTTYPE_ED_SIGN_LINK] =
+    ed_certs[CERTTYPE_ED_SIGN_AUTH] = NULL;
+
+  chan->conn->handshake_state->certs->ed_rsa_crosscert = rsa_ed_cc_cert;
+  chan->conn->handshake_state->certs->ed_rsa_crosscert_len =
+    rsa_ed_cc_cert_len;
+  rsa_ed_cc_cert = NULL;
+
+  int severity;
+  /* Note that this warns more loudly about time and validity if we were
+   * _trying_ to connect to an authority, not necessarily if we _did_ connect
+   * to one. */
+  if (started_here &&
+      router_digest_is_trusted_dir(TLS_CHAN_TO_BASE(chan)->identity_digest))
+    severity = LOG_WARN;
+  else
+    severity = LOG_PROTOCOL_WARN;
+
+  const ed25519_public_key_t *checked_ed_id = NULL;
+  const common_digests_t *checked_rsa_id = NULL;
+  or_handshake_certs_check_both(severity,
+                                chan->conn->handshake_state->certs,
+                                chan->conn->tls,
+                                time(NULL),
+                                &checked_ed_id,
+                                &checked_rsa_id);
+
+  if (!checked_rsa_id)
+    ERR("Invalid certificate chain!");
 
-  if (chan->conn->handshake_state->started_here) {
-    int severity;
-    if (! (id_cert && link_cert))
-      ERR("The certs we wanted were missing");
-    /* Okay. We should be able to check the certificates now. */
-    if (! tor_tls_cert_matches_key(chan->conn->tls, link_cert)) {
-      ERR("The link certificate didn't match the TLS public key");
-    }
-    /* Note that this warns more loudly about time and validity if we were
-    * _trying_ to connect to an authority, not necessarily if we _did_ connect
-    * to one. */
-    if (router_digest_is_trusted_dir(
-          TLS_CHAN_TO_BASE(chan)->identity_digest))
-      severity = LOG_WARN;
-    else
-      severity = LOG_PROTOCOL_WARN;
-
-    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, 0))
-      ERR("The link certificate was not valid");
-    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, 1))
-      ERR("The ID certificate was not valid");
+  if (started_here) {
+    /* No more information is needed. */
 
     chan->conn->handshake_state->authenticated = 1;
+    chan->conn->handshake_state->authenticated_rsa = 1;
     {
-      const common_digests_t *id_digests =
-        tor_x509_cert_get_id_digests(id_cert);
+      const common_digests_t *id_digests = checked_rsa_id;
       crypto_pk_t *identity_rcvd;
       if (!id_digests)
         ERR("Couldn't compute digests for key in ID cert");
 
       identity_rcvd = tor_tls_cert_get_key(id_cert);
-      if (!identity_rcvd)
-        ERR("Internal error: Couldn't get RSA key from ID cert.");
-      memcpy(chan->conn->handshake_state->authenticated_peer_id,
+      if (!identity_rcvd) {
+        ERR("Couldn't get RSA key from ID cert.");
+      }
+      memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id,
              id_digests->d[DIGEST_SHA1], DIGEST_LEN);
       channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd,
                 chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS);
       crypto_pk_free(identity_rcvd);
     }
 
+    if (checked_ed_id) {
+      chan->conn->handshake_state->authenticated_ed25519 = 1;
+      memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id,
+             checked_ed_id, sizeof(ed25519_public_key_t));
+    }
+
+    log_debug(LD_HANDSHAKE, "calling client_learned_peer_id from "
+              "process_certs_cell");
+
     if (connection_or_client_learned_peer_id(chan->conn,
-            chan->conn->handshake_state->authenticated_peer_id) < 0)
+                  chan->conn->handshake_state->authenticated_rsa_peer_id,
+                  checked_ed_id) < 0)
       ERR("Problem setting or checking peer id");
 
-    log_info(LD_OR,
-             "Got some good certificates from %s:%d: Authenticated it.",
-             safe_str(chan->conn->base_.address), chan->conn->base_.port);
-
-    chan->conn->handshake_state->id_cert = id_cert;
-    certs[OR_CERT_TYPE_ID_1024] = NULL;
+    log_info(LD_HANDSHAKE,
+             "Got some good certificates from %s:%d: Authenticated it with "
+             "RSA%s",
+             safe_str(chan->conn->base_.address), chan->conn->base_.port,
+             checked_ed_id ? " and Ed25519" : "");
 
     if (!public_server_mode(get_options())) {
       /* If we initiated the connection and we are not a public server, we
@@ -1897,25 +2098,14 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
       send_netinfo = 1;
     }
   } else {
-    if (! (id_cert && auth_cert))
-      ERR("The certs we wanted were missing");
-
-    /* Remember these certificates so we can check an AUTHENTICATE cell */
-    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, 1))
-      ERR("The authentication certificate was not valid");
-    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, 1))
-      ERR("The ID certificate was not valid");
-
+    /* We can't call it authenticated till we see an AUTHENTICATE cell. */
     log_info(LD_OR,
-             "Got some good certificates from %s:%d: "
+             "Got some good RSA%s certificates from %s:%d. "
              "Waiting for AUTHENTICATE.",
+             checked_ed_id ? " and Ed25519" : "",
              safe_str(chan->conn->base_.address),
              chan->conn->base_.port);
     /* XXXX check more stuff? */
-
-    chan->conn->handshake_state->id_cert = id_cert;
-    chan->conn->handshake_state->auth_cert = auth_cert;
-    certs[OR_CERT_TYPE_ID_1024] = certs[OR_CERT_TYPE_AUTH_1024] = NULL;
   }
 
   chan->conn->handshake_state->received_certs_cell = 1;
@@ -1929,15 +2119,19 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
   }
 
  err:
-  for (unsigned u = 0; u < ARRAY_LENGTH(certs); ++u) {
-    tor_x509_cert_free(certs[u]);
+  for (unsigned u = 0; u < ARRAY_LENGTH(x509_certs); ++u) {
+    tor_x509_cert_free(x509_certs[u]);
+  }
+  for (unsigned u = 0; u < ARRAY_LENGTH(ed_certs); ++u) {
+    tor_cert_free(ed_certs[u]);
   }
+  tor_free(rsa_ed_cc_cert);
   certs_cell_free(cc);
 #undef ERR
 }
 
 /**
- * Process an AUTH_CHALLENGE cell from a channel_tls_t
+ * Process an AUTH_CHALLENGE cell from a channel_tls_t.
  *
  * This function is called to handle an incoming AUTH_CHALLENGE cell on a
  * channel_tls_t; if we weren't supposed to get one (for example, because we're
@@ -1946,7 +2140,6 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
  * want to authenticate, just drop it.  If the cell is well-formed *and* we
  * want to authenticate, send an AUTHENTICATE cell and then a NETINFO cell.
  */
-
 STATIC void
 channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
 {
@@ -1988,8 +2181,12 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
   /* Now see if there is an authentication type we can use */
   for (i = 0; i < n_types; ++i) {
     uint16_t authtype = auth_challenge_cell_get_methods(ac, i);
-    if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET)
-      use_type = authtype;
+    if (authchallenge_type_is_supported(authtype)) {
+      if (use_type == -1 ||
+          authchallenge_type_is_better(authtype, use_type)) {
+        use_type = authtype;
+      }
+    }
   }
 
   chan->conn->handshake_state->received_auth_challenge = 1;
@@ -2004,9 +2201,10 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
   if (use_type >= 0) {
     log_info(LD_OR,
              "Got an AUTH_CHALLENGE cell from %s:%d: Sending "
-             "authentication",
+             "authentication type %d",
              safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
+             chan->conn->base_.port,
+             use_type);
 
     if (connection_or_send_authenticate_cell(chan->conn, use_type) < 0) {
       log_warn(LD_OR,
@@ -2035,7 +2233,7 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
 }
 
 /**
- * Process an AUTHENTICATE cell from a channel_tls_t
+ * Process an AUTHENTICATE cell from a channel_tls_t.
  *
  * If it's ill-formed or we weren't supposed to get one or we're not doing a
  * v3 handshake, then mark the connection.  If it does not authenticate the
@@ -2043,13 +2241,14 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
  * we didn't get a CERTS cell, etc) mark the connection.  Otherwise, accept
  * the identity of the router on the other side of the connection.
  */
-
 STATIC void
 channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
 {
-  uint8_t expected[V3_AUTH_FIXED_PART_LEN+256];
+  var_cell_t *expected_cell = NULL;
   const uint8_t *auth;
   int authlen;
+  int authtype;
+  int bodylen;
 
   tor_assert(cell);
   tor_assert(chan);
@@ -2062,6 +2261,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
            safe_str(chan->conn->base_.address),                 \
            chan->conn->base_.port, (s));                        \
     connection_or_close_for_error(chan->conn, 0);               \
+    var_cell_free(expected_cell);                               \
     return;                                                     \
   } while (0)
 
@@ -2079,9 +2279,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
   }
   if (!(chan->conn->handshake_state->received_certs_cell))
     ERR("We never got a certs cell");
-  if (chan->conn->handshake_state->auth_cert == NULL)
-    ERR("We never got an authentication certificate");
-  if (chan->conn->handshake_state->id_cert == NULL)
+  if (chan->conn->handshake_state->certs->id_cert == NULL)
     ERR("We never got an identity certificate");
   if (cell->payload_len < 4)
     ERR("Cell was way too short");
@@ -2093,8 +2291,9 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
     if (4 + len > cell->payload_len)
       ERR("Authenticator was truncated");
 
-    if (type != AUTHTYPE_RSA_SHA256_TLSSECRET)
+    if (! authchallenge_type_is_supported(type))
       ERR("Authenticator type was not recognized");
+    authtype = type;
 
     auth += 4;
     authlen = len;
@@ -2103,25 +2302,55 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
   if (authlen < V3_AUTH_BODY_LEN + 1)
     ERR("Authenticator was too short");
 
-  ssize_t bodylen =
-    connection_or_compute_authenticate_cell_body(
-                        chan->conn, expected, sizeof(expected), NULL, 1);
-  if (bodylen < 0 || bodylen != V3_AUTH_FIXED_PART_LEN)
+  expected_cell = connection_or_compute_authenticate_cell_body(
+                chan->conn, authtype, NULL, NULL, 1);
+  if (! expected_cell)
     ERR("Couldn't compute expected AUTHENTICATE cell body");
 
-  if (tor_memneq(expected, auth, bodylen))
+  int sig_is_rsa;
+  if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET ||
+      authtype == AUTHTYPE_RSA_SHA256_RFC5705) {
+    bodylen = V3_AUTH_BODY_LEN;
+    sig_is_rsa = 1;
+  } else {
+    tor_assert(authtype == AUTHTYPE_ED25519_SHA256_RFC5705);
+    /* Our earlier check had better have made sure we had room
+     * for an ed25519 sig (inadvertently) */
+    tor_assert(V3_AUTH_BODY_LEN > ED25519_SIG_LEN);
+    bodylen = authlen - ED25519_SIG_LEN;
+    sig_is_rsa = 0;
+  }
+  if (expected_cell->payload_len != bodylen+4) {
+    ERR("Expected AUTHENTICATE cell body len not as expected.");
+  }
+
+  /* Length of random part. */
+  if (BUG(bodylen < 24)) {
+    // LCOV_EXCL_START
+    ERR("Bodylen is somehow less than 24, which should really be impossible");
+    // LCOV_EXCL_STOP
+  }
+
+  if (tor_memneq(expected_cell->payload+4, auth, bodylen-24))
     ERR("Some field in the AUTHENTICATE cell body was not as expected");
 
-  {
+  if (sig_is_rsa) {
+    if (chan->conn->handshake_state->certs->ed_id_sign != NULL)
+      ERR("RSA-signed AUTHENTICATE response provided with an ED25519 cert");
+
+    if (chan->conn->handshake_state->certs->auth_cert == NULL)
+      ERR("We never got an RSA authentication certificate");
+
     crypto_pk_t *pk = tor_tls_cert_get_key(
-                                   chan->conn->handshake_state->auth_cert);
+                             chan->conn->handshake_state->certs->auth_cert);
     char d[DIGEST256_LEN];
     char *signed_data;
     size_t keysize;
     int signed_len;
 
-    if (!pk)
-      ERR("Internal error: couldn't get RSA key from AUTH cert.");
+    if (! pk) {
+      ERR("Couldn't get RSA key from AUTH cert.");
+    }
     crypto_digest256(d, (char*)auth, V3_AUTH_BODY_LEN, DIGEST_SHA256);
 
     keysize = crypto_pk_keysize(pk);
@@ -2132,7 +2361,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
     crypto_pk_free(pk);
     if (signed_len < 0) {
       tor_free(signed_data);
-      ERR("Signature wasn't valid");
+      ERR("RSA signature wasn't valid");
     }
     if (signed_len < DIGEST256_LEN) {
       tor_free(signed_data);
@@ -2145,41 +2374,75 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
       ERR("Signature did not match data to be signed.");
     }
     tor_free(signed_data);
+  } else {
+    if (chan->conn->handshake_state->certs->ed_id_sign == NULL)
+      ERR("We never got an Ed25519 identity certificate.");
+    if (chan->conn->handshake_state->certs->ed_sign_auth == NULL)
+      ERR("We never got an Ed25519 authentication certificate.");
+
+    const ed25519_public_key_t *authkey =
+      &chan->conn->handshake_state->certs->ed_sign_auth->signed_key;
+    ed25519_signature_t sig;
+    tor_assert(authlen > ED25519_SIG_LEN);
+    memcpy(&sig.sig, auth + authlen - ED25519_SIG_LEN, ED25519_SIG_LEN);
+    if (ed25519_checksig(&sig, auth, authlen - ED25519_SIG_LEN, authkey)<0) {
+      ERR("Ed25519 signature wasn't valid.");
+    }
   }
 
   /* Okay, we are authenticated. */
   chan->conn->handshake_state->received_authenticate = 1;
   chan->conn->handshake_state->authenticated = 1;
+  chan->conn->handshake_state->authenticated_rsa = 1;
   chan->conn->handshake_state->digest_received_data = 0;
   {
-    crypto_pk_t *identity_rcvd =
-      tor_tls_cert_get_key(chan->conn->handshake_state->id_cert);
-    const common_digests_t *id_digests =
-      tor_x509_cert_get_id_digests(chan->conn->handshake_state->id_cert);
+    tor_x509_cert_t *id_cert = chan->conn->handshake_state->certs->id_cert;
+    crypto_pk_t *identity_rcvd = tor_tls_cert_get_key(id_cert);
+    const common_digests_t *id_digests = tor_x509_cert_get_id_digests(id_cert);
+    const ed25519_public_key_t *ed_identity_received = NULL;
+
+    if (! sig_is_rsa) {
+      chan->conn->handshake_state->authenticated_ed25519 = 1;
+      ed_identity_received =
+        &chan->conn->handshake_state->certs->ed_id_sign->signing_key;
+      memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id,
+             ed_identity_received, sizeof(ed25519_public_key_t));
+    }
 
     /* This must exist; we checked key type when reading the cert. */
     tor_assert(id_digests);
 
-    memcpy(chan->conn->handshake_state->authenticated_peer_id,
+    memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id,
            id_digests->d[DIGEST_SHA1], DIGEST_LEN);
 
     channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd,
                chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS);
     crypto_pk_free(identity_rcvd);
 
+    log_debug(LD_HANDSHAKE,
+              "Calling connection_or_init_conn_from_address for %s "
+              " from %s, with%s ed25519 id.",
+              safe_str(chan->conn->base_.address),
+              __func__,
+              ed_identity_received ? "" : "out");
+
     connection_or_init_conn_from_address(chan->conn,
                   &(chan->conn->base_.addr),
                   chan->conn->base_.port,
                   (const char*)(chan->conn->handshake_state->
-                    authenticated_peer_id),
+                    authenticated_rsa_peer_id),
+                  ed_identity_received,
                   0);
 
-    log_info(LD_OR,
-             "Got an AUTHENTICATE cell from %s:%d: Looks good.",
+    log_debug(LD_HANDSHAKE,
+             "Got an AUTHENTICATE cell from %s:%d, type %d: Looks good.",
              safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
+             chan->conn->base_.port,
+             authtype);
   }
 
+  var_cell_free(expected_cell);
+
 #undef ERR
 }
 
diff --git a/src/or/channeltls.h b/src/or/channeltls.h
index 463f7d928c..65ce387a9a 100644
--- a/src/or/channeltls.h
+++ b/src/or/channeltls.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,10 +28,11 @@ struct channel_tls_s {
   or_connection_t *conn;
 };
 
-#endif /* TOR_CHANNEL_INTERNAL_ */
+#endif /* defined(TOR_CHANNEL_INTERNAL_) */
 
 channel_t * channel_tls_connect(const tor_addr_t *addr, uint16_t port,
-                                const char *id_digest);
+                                const char *id_digest,
+                                const ed25519_public_key_t *ed_id);
 channel_listener_t * channel_tls_get_listener(void);
 channel_listener_t * channel_tls_start_listener(void);
 channel_t * channel_tls_handle_incoming(or_connection_t *orconn);
@@ -70,7 +71,7 @@ STATIC void channel_tls_process_auth_challenge_cell(var_cell_t *cell,
 STATIC void channel_tls_common_init(channel_tls_t *tlschan);
 STATIC void channel_tls_process_authenticate_cell(var_cell_t *cell,
                                                   channel_tls_t *tlschan);
-#endif
+#endif /* defined(CHANNELTLS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CHANNELTLS_H) */
 
diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c
index 9f93e737f7..3cdd16261a 100644
--- a/src/or/circpathbias.c
+++ b/src/or/circpathbias.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,14 @@
  * different tor nodes, in an attempt to detect attacks where
  * an attacker deliberately causes circuits to fail until the client
  * choses a path they like.
+ *
+ * This code is currently configured in a warning-only mode, though false
+ * positives appear to be rare in practice.  There is also support for
+ * disabling really bad guards, but it's quite experimental and may have bad
+ * anonymity effects.
+ *
+ * The information here is associated with the entry_guard_t object for
+ * each guard, and stored persistently in the state file.
  */
 
 #include "or.h"
@@ -22,6 +30,7 @@
 #include "circuitstats.h"
 #include "connection_edge.h"
 #include "config.h"
+#include "crypto_rand.h"
 #include "entrynodes.h"
 #include "networkstatus.h"
 #include "relay.h"
@@ -43,19 +52,21 @@ static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard);
 static int
 entry_guard_inc_circ_attempt_count(entry_guard_t *guard)
 {
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
   entry_guards_changed();
 
   pathbias_measure_close_rate(guard);
 
-  if (guard->path_bias_disabled)
+  if (pb->path_bias_disabled)
     return -1;
 
   pathbias_scale_close_rates(guard);
-  guard->circ_attempts++;
+  pb->circ_attempts++;
 
-  log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)",
-           guard->circ_successes, guard->circ_attempts, guard->nickname,
-           hex_str(guard->identity, DIGEST_LEN));
+  log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+           pb->circ_successes, pb->circ_attempts,
+           entry_guard_describe(guard));
   return 0;
 }
 
@@ -282,7 +293,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
   return circ->cpath &&
          circ->cpath->next != circ->cpath &&
          circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
-#else
+#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
   /* If tagging attacks are no longer possible, we probably want to
    * count bias from the first hop. However, one could argue that
    * timing-based tagging is still more useful than per-hop failure.
@@ -290,7 +301,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
    */
   return circ->cpath &&
          circ->cpath->state == CPATH_STATE_AWAITING_KEYS;
-#endif
+#endif /* defined(N2N_TAGGING_IS_POSSIBLE) */
 }
 
 /**
@@ -505,14 +516,16 @@ pathbias_count_build_success(origin_circuit_t *circ)
     }
 
     if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
       if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) {
         circ->path_state = PATH_STATE_BUILD_SUCCEEDED;
-        guard->circ_successes++;
+        pb->circ_successes++;
         entry_guards_changed();
 
-        log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)",
-                 guard->circ_successes, guard->circ_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+        log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
       } else {
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
@@ -527,11 +540,11 @@ pathbias_count_build_success(origin_circuit_t *circ)
         }
       }
 
-      if (guard->circ_attempts < guard->circ_successes) {
+      if (pb->circ_attempts < pb->circ_successes) {
         log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) "
-                 "for guard %s ($%s)",
-                 guard->circ_successes, guard->circ_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+                 "for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
       }
     /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
      * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
@@ -574,8 +587,6 @@ pathbias_count_build_success(origin_circuit_t *circ)
 void
 pathbias_count_use_attempt(origin_circuit_t *circ)
 {
-  entry_guard_t *guard;
-
   if (!pathbias_should_count(circ)) {
     return;
   }
@@ -588,19 +599,21 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
   } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) {
-    guard = entry_guard_get_by_id_digest(
+    entry_guard_t *guard = entry_guard_get_by_id_digest(
                 circ->cpath->extend_info->identity_digest);
     if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
       pathbias_measure_use_rate(guard);
       pathbias_scale_use_rates(guard);
-      guard->use_attempts++;
+      pb->use_attempts++;
       entry_guards_changed();
 
       log_debug(LD_CIRC,
-               "Marked circuit %d (%f/%f) as used for guard %s ($%s).",
+               "Marked circuit %d (%f/%f) as used for guard %s.",
                circ->global_identifier,
-               guard->use_successes, guard->use_attempts,
-               guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+               pb->use_successes, pb->use_attempts,
+               entry_guard_describe(guard));
     }
 
     circ->path_state = PATH_STATE_USE_ATTEMPTED;
@@ -702,22 +715,23 @@ pathbias_count_use_success(origin_circuit_t *circ)
     guard = entry_guard_get_by_id_digest(
                 circ->cpath->extend_info->identity_digest);
     if (guard) {
-      guard->use_successes++;
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+      pb->use_successes++;
       entry_guards_changed();
 
-      if (guard->use_attempts < guard->use_successes) {
+      if (pb->use_attempts < pb->use_successes) {
         log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) "
-                 "for guard %s=%s",
-                 guard->use_successes, guard->use_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+                 "for guard %s",
+                 pb->use_successes, pb->use_attempts,
+                 entry_guard_describe(guard));
       }
 
       log_debug(LD_CIRC,
-                "Marked circuit %d (%f/%f) as used successfully for guard "
-                "%s ($%s).",
-                circ->global_identifier, guard->use_successes,
-                guard->use_attempts, guard->nickname,
-                hex_str(guard->identity, DIGEST_LEN));
+                "Marked circuit %d (%f/%f) as used successfully for guard %s",
+                circ->global_identifier, pb->use_successes,
+                pb->use_attempts,
+                entry_guard_describe(guard));
     }
   }
 
@@ -879,6 +893,7 @@ pathbias_check_probe_response(circuit_t *circ, const cell_t *cell)
     /* Check nonce */
     if (ipv4_host == ocirc->pathbias_probe_nonce) {
       pathbias_mark_use_success(ocirc);
+      circuit_read_valid_data(ocirc, rh.length);
       circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
       log_info(LD_CIRC,
                "Got valid path bias probe back for circ %d, stream %d.",
@@ -900,6 +915,68 @@ pathbias_check_probe_response(circuit_t *circ, const cell_t *cell)
 }
 
 /**
+ * Check if a cell is counts as valid data for a circuit,
+ * and if so, count it as valid.
+ */
+void
+pathbias_count_valid_cells(circuit_t *circ, const cell_t *cell)
+{
+  origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+  relay_header_t rh;
+
+  relay_header_unpack(&rh, cell->payload);
+
+  /* Check to see if this is a cell from a previous connection,
+   * or is a request to close the circuit. */
+  switch (rh.command) {
+    case RELAY_COMMAND_TRUNCATED:
+      /* Truncated cells can arrive on path bias circs. When they do,
+       * just process them. This closes the circ, but it was junk anyway.
+       * No reason to wait for the probe. */
+      circuit_read_valid_data(ocirc, rh.length);
+      circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
+                        get_uint8(cell->payload + RELAY_HEADER_SIZE));
+
+      break;
+
+    case RELAY_COMMAND_END:
+      if (connection_half_edge_is_valid_end(ocirc->half_streams,
+                                             rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_DATA:
+      if (connection_half_edge_is_valid_data(ocirc->half_streams,
+                                             rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_SENDME:
+      if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                             rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_CONNECTED:
+      if (connection_half_edge_is_valid_connected(ocirc->half_streams,
+                                                  rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+
+    case RELAY_COMMAND_RESOLVED:
+      if (connection_half_edge_is_valid_resolved(ocirc->half_streams,
+                                                 rh.stream_id)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
+      break;
+  }
+}
+
+/**
  * Check if a circuit was used and/or closed successfully.
  *
  * If we attempted to use the circuit to carry a stream but failed
@@ -1018,9 +1095,11 @@ pathbias_count_successful_close(origin_circuit_t *circ)
   }
 
   if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
     /* In the long run: circuit_success ~= successful_circuit_close +
      *                                     circ_failure + stream_failure */
-    guard->successful_circuits_closed++;
+    pb->successful_circuits_closed++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1057,7 +1136,9 @@ pathbias_count_collapse(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->collapsed_circuits++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->collapsed_circuits++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1090,7 +1171,9 @@ pathbias_count_use_failed(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->unusable_circuits++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->unusable_circuits++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1133,7 +1216,9 @@ pathbias_count_timeout(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->timeouts++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->timeouts++;
     entry_guards_changed();
   }
 }
@@ -1165,7 +1250,7 @@ pathbias_count_circs_in_states(entry_guard_t *guard,
     if (ocirc->path_state >= from &&
         ocirc->path_state <= to &&
         pathbias_should_count(ocirc) &&
-        fast_memeq(guard->identity,
+        fast_memeq(entry_guard_get_rsa_id_digest(guard),
                    ocirc->cpath->extend_info->identity_digest,
                    DIGEST_LEN)) {
       log_debug(LD_CIRC, "Found opened circuit %d in path_state %s",
@@ -1189,7 +1274,9 @@ pathbias_count_circs_in_states(entry_guard_t *guard,
 double
 pathbias_get_close_success_count(entry_guard_t *guard)
 {
-  return guard->successful_circuits_closed +
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->successful_circuits_closed +
          pathbias_count_circs_in_states(guard,
                        PATH_STATE_BUILD_SUCCEEDED,
                        PATH_STATE_USE_SUCCEEDED);
@@ -1205,7 +1292,9 @@ pathbias_get_close_success_count(entry_guard_t *guard)
 double
 pathbias_get_use_success_count(entry_guard_t *guard)
 {
-  return guard->use_successes +
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->use_successes +
          pathbias_count_circs_in_states(guard,
                        PATH_STATE_USE_ATTEMPTED,
                        PATH_STATE_USE_SUCCEEDED);
@@ -1223,18 +1312,19 @@ static void
 pathbias_measure_use_rate(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
-  if (guard->use_attempts > pathbias_get_min_use(options)) {
+  if (pb->use_attempts > pathbias_get_min_use(options)) {
     /* Note: We rely on the < comparison here to allow us to set a 0
      * rate and disable the feature entirely. If refactoring, don't
      * change to <= */
-    if (pathbias_get_use_success_count(guard)/guard->use_attempts
+    if (pathbias_get_use_success_count(guard)/pb->use_attempts
         < pathbias_get_extreme_use_rate(options)) {
       /* Dropping is currently disabled by default. */
       if (pathbias_get_dropguards(options)) {
-        if (!guard->path_bias_disabled) {
+        if (!pb->path_bias_disabled) {
           log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry an extremely large "
+                 "Your Guard %s is failing to carry an extremely large "
                  "amount of stream on its circuits. "
                  "To avoid potential route manipulation attacks, Tor has "
                  "disabled use of this guard. "
@@ -1242,25 +1332,23 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
-          guard->path_bias_disabled = 1;
-          guard->bad_since = approx_time();
-          entry_guards_changed();
+          pb->path_bias_disabled = 1;
           return;
         }
-      } else if (!guard->path_bias_use_extreme) {
-        guard->path_bias_use_extreme = 1;
+      } else if (!pb->path_bias_use_extreme) {
+        pb->path_bias_use_extreme = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry an extremely large "
+                 "Your Guard %s is failing to carry an extremely large "
                  "amount of streams on its circuits. "
                  "This could indicate a route manipulation attack, network "
                  "overload, bad local network connectivity, or a bug. "
@@ -1268,23 +1356,23 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_use_success_count(guard)/guard->use_attempts
+    } else if (pathbias_get_use_success_count(guard)/pb->use_attempts
                < pathbias_get_notice_use_rate(options)) {
-      if (!guard->path_bias_use_noticed) {
-        guard->path_bias_use_noticed = 1;
+      if (!pb->path_bias_use_noticed) {
+        pb->path_bias_use_noticed = 1;
         log_notice(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry more streams on its "
+                 "Your Guard %s is failing to carry more streams on its "
                  "circuits than usual. "
                  "Most likely this means the Tor network is overloaded "
                  "or your network connection is poor. "
@@ -1292,15 +1380,15 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
     }
@@ -1329,18 +1417,19 @@ static void
 pathbias_measure_close_rate(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
-  if (guard->circ_attempts > pathbias_get_min_circs(options)) {
+  if (pb->circ_attempts > pathbias_get_min_circs(options)) {
     /* Note: We rely on the < comparison here to allow us to set a 0
      * rate and disable the feature entirely. If refactoring, don't
      * change to <= */
-    if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    if (pathbias_get_close_success_count(guard)/pb->circ_attempts
         < pathbias_get_extreme_rate(options)) {
       /* Dropping is currently disabled by default. */
       if (pathbias_get_dropguards(options)) {
-        if (!guard->path_bias_disabled) {
+        if (!pb->path_bias_disabled) {
           log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing an extremely large "
+                 "Your Guard %s is failing an extremely large "
                  "amount of circuits. "
                  "To avoid potential route manipulation attacks, Tor has "
                  "disabled use of this guard. "
@@ -1348,25 +1437,23 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
-          guard->path_bias_disabled = 1;
-          guard->bad_since = approx_time();
-          entry_guards_changed();
+          pb->path_bias_disabled = 1;
           return;
         }
-      } else if (!guard->path_bias_extreme) {
-        guard->path_bias_extreme = 1;
+      } else if (!pb->path_bias_extreme) {
+        pb->path_bias_extreme = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing an extremely large "
+                 "Your Guard %s is failing an extremely large "
                  "amount of circuits. "
                  "This could indicate a route manipulation attack, "
                  "extreme network overload, or a bug. "
@@ -1374,23 +1461,23 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
                 < pathbias_get_warn_rate(options)) {
-      if (!guard->path_bias_warned) {
-        guard->path_bias_warned = 1;
+      if (!pb->path_bias_warned) {
+        pb->path_bias_warned = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing a very large "
+                 "Your Guard %s is failing a very large "
                  "amount of circuits. "
                  "Most likely this means the Tor network is "
                  "overloaded, but it could also mean an attack against "
@@ -1399,38 +1486,38 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
                < pathbias_get_notice_rate(options)) {
-      if (!guard->path_bias_noticed) {
-        guard->path_bias_noticed = 1;
+      if (!pb->path_bias_noticed) {
+        pb->path_bias_noticed = 1;
         log_notice(LD_CIRC,
-                 "Your Guard %s ($%s) is failing more circuits than "
+                 "Your Guard %s is failing more circuits than "
                  "usual. "
                  "Most likely this means the Tor network is overloaded. "
                  "Success counts are %ld/%ld. Use counts are %ld/%ld. "
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
     }
@@ -1444,15 +1531,16 @@ pathbias_measure_close_rate(entry_guard_t *guard)
  *
  * XXX: The attempt count transfer stuff here might be done
  * better by keeping separate pending counters that get
- * transfered at circuit close. See ticket #8160.
+ * transferred at circuit close. See ticket #8160.
  */
 static void
 pathbias_scale_close_rates(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
   /* If we get a ton of circuits, just scale everything down */
-  if (guard->circ_attempts > pathbias_get_scale_threshold(options)) {
+  if (pb->circ_attempts > pathbias_get_scale_threshold(options)) {
     double scale_ratio = pathbias_get_scale_ratio(options);
     int opened_attempts = pathbias_count_circs_in_states(guard,
             PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED);
@@ -1460,38 +1548,38 @@ pathbias_scale_close_rates(entry_guard_t *guard)
                         PATH_STATE_BUILD_SUCCEEDED,
                         PATH_STATE_USE_FAILED);
     /* Verify that the counts are sane before and after scaling */
-    int counts_are_sane = (guard->circ_attempts >= guard->circ_successes);
+    int counts_are_sane = (pb->circ_attempts >= pb->circ_successes);
 
-    guard->circ_attempts -= (opened_attempts+opened_built);
-    guard->circ_successes -= opened_built;
+    pb->circ_attempts -= (opened_attempts+opened_built);
+    pb->circ_successes -= opened_built;
 
-    guard->circ_attempts *= scale_ratio;
-    guard->circ_successes *= scale_ratio;
-    guard->timeouts *= scale_ratio;
-    guard->successful_circuits_closed *= scale_ratio;
-    guard->collapsed_circuits *= scale_ratio;
-    guard->unusable_circuits *= scale_ratio;
+    pb->circ_attempts *= scale_ratio;
+    pb->circ_successes *= scale_ratio;
+    pb->timeouts *= scale_ratio;
+    pb->successful_circuits_closed *= scale_ratio;
+    pb->collapsed_circuits *= scale_ratio;
+    pb->unusable_circuits *= scale_ratio;
 
-    guard->circ_attempts += (opened_attempts+opened_built);
-    guard->circ_successes += opened_built;
+    pb->circ_attempts += (opened_attempts+opened_built);
+    pb->circ_successes += opened_built;
 
     entry_guards_changed();
 
     log_info(LD_CIRC,
              "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard "
-             "%s ($%s)",
-             guard->circ_successes, guard->successful_circuits_closed,
-             guard->circ_attempts, opened_built, opened_attempts,
-             guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+             "%s",
+             pb->circ_successes, pb->successful_circuits_closed,
+             pb->circ_attempts, opened_built, opened_attempts,
+             entry_guard_describe(guard));
 
     /* Have the counts just become invalid by this scaling attempt? */
-    if (counts_are_sane && guard->circ_attempts < guard->circ_successes) {
+    if (counts_are_sane && pb->circ_attempts < pb->circ_successes) {
       log_notice(LD_BUG,
                "Scaling has mangled pathbias counts to %f/%f (%d/%d open) "
-               "for guard %s ($%s)",
-               guard->circ_successes, guard->circ_attempts, opened_built,
-               opened_attempts, guard->nickname,
-               hex_str(guard->identity, DIGEST_LEN));
+               "for guard %s",
+               pb->circ_successes, pb->circ_attempts, opened_built,
+               opened_attempts,
+               entry_guard_describe(guard));
     }
   }
 }
@@ -1503,41 +1591,41 @@ pathbias_scale_close_rates(entry_guard_t *guard)
  *
  * XXX: The attempt count transfer stuff here might be done
  * better by keeping separate pending counters that get
- * transfered at circuit close. See ticket #8160.
+ * transferred at circuit close. See ticket #8160.
  */
 void
 pathbias_scale_use_rates(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
   /* If we get a ton of circuits, just scale everything down */
-  if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) {
+  if (pb->use_attempts > pathbias_get_scale_use_threshold(options)) {
     double scale_ratio = pathbias_get_scale_ratio(options);
     int opened_attempts = pathbias_count_circs_in_states(guard,
             PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED);
     /* Verify that the counts are sane before and after scaling */
-    int counts_are_sane = (guard->use_attempts >= guard->use_successes);
+    int counts_are_sane = (pb->use_attempts >= pb->use_successes);
 
-    guard->use_attempts -= opened_attempts;
+    pb->use_attempts -= opened_attempts;
 
-    guard->use_attempts *= scale_ratio;
-    guard->use_successes *= scale_ratio;
+    pb->use_attempts *= scale_ratio;
+    pb->use_successes *= scale_ratio;
 
-    guard->use_attempts += opened_attempts;
+    pb->use_attempts += opened_attempts;
 
     log_info(LD_CIRC,
-           "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)",
-           guard->use_successes, guard->use_attempts, opened_attempts,
-           guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+           "Scaled pathbias use counts to %f/%f (%d open) for guard %s",
+           pb->use_successes, pb->use_attempts, opened_attempts,
+           entry_guard_describe(guard));
 
     /* Have the counts just become invalid by this scaling attempt? */
-    if (counts_are_sane && guard->use_attempts < guard->use_successes) {
+    if (counts_are_sane && pb->use_attempts < pb->use_successes) {
       log_notice(LD_BUG,
                "Scaling has mangled pathbias usage counts to %f/%f "
-               "(%d open) for guard %s ($%s)",
-               guard->circ_successes, guard->circ_attempts,
-               opened_attempts, guard->nickname,
-               hex_str(guard->identity, DIGEST_LEN));
+               "(%d open) for guard %s",
+               pb->circ_successes, pb->circ_attempts,
+               opened_attempts, entry_guard_describe(guard));
     }
 
     entry_guards_changed();
diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h
index ce76689d5f..689b2a6207 100644
--- a/src/or/circpathbias.h
+++ b/src/or/circpathbias.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,10 +20,11 @@ void pathbias_count_build_success(origin_circuit_t *circ);
 int pathbias_count_build_attempt(origin_circuit_t *circ);
 int pathbias_check_close(origin_circuit_t *circ, int reason);
 int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell);
+void pathbias_count_valid_cells(circuit_t *circ, const cell_t *cell);
 void pathbias_count_use_attempt(origin_circuit_t *circ);
 void pathbias_mark_use_success(origin_circuit_t *circ);
 void pathbias_mark_use_rollback(origin_circuit_t *circ);
 const char *pathbias_state_to_string(path_state_t state);
 
-#endif
+#endif /* !defined(TOR_CIRCPATHBIAS_H) */
 
diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c
index cb9c146fb7..8f17f27868 100644
--- a/src/or/circuitbuild.c
+++ b/src/or/circuitbuild.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,11 +9,26 @@
  *
  * \brief Implements the details of building circuits (by chosing paths,
  * constructing/sending create/extend cells, and so on).
+ *
+ * On the client side, this module handles launching circuits. Circuit
+ * launches are srtarted from circuit_establish_circuit(), called from
+ * circuit_launch_by_extend_info()).  To choose the path the circuit will
+ * take, onion_extend_cpath() calls into a maze of node selection functions.
+ *
+ * Once the circuit is ready to be launched, the first hop is treated as a
+ * special case with circuit_handle_first_hop(), since it might need to open a
+ * channel.  As the channel opens, and later as CREATED and RELAY_EXTENDED
+ * cells arrive, the client will invoke circuit_send_next_onion_skin() to send
+ * CREATE or RELAY_EXTEND cells.
+ *
+ * On the server side, this module also handles the logic of responding to
+ * RELAY_EXTEND requests, using circuit_extend().
  **/
 
 #define CIRCUITBUILD_PRIVATE
 
 #include "or.h"
+#include "bridges.h"
 #include "channel.h"
 #include "circpathbias.h"
 #define CIRCUITBUILD_PRIVATE
@@ -28,9 +43,10 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
-#include "crypto.h"
+#include "crypto_rand.h"
 #include "directory.h"
 #include "entrynodes.h"
+#include "hs_ntor.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
@@ -40,6 +56,7 @@
 #include "onion_fast.h"
 #include "policies.h"
 #include "relay.h"
+#include "relay_crypto.h"
 #include "rendcommon.h"
 #include "rephist.h"
 #include "router.h"
@@ -49,16 +66,22 @@
 #include "transports.h"
 
 static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
-                                               uint16_t port,
-                                               const char *id_digest);
+                                            uint16_t port,
+                                            const char *id_digest,
+                                            const ed25519_public_key_t *ed_id);
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        const create_cell_t *create_cell,
                                        int relayed);
-static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit);
 static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
-static int onion_extend_cpath(origin_circuit_t *circ);
-static int count_acceptable_nodes(smartlist_t *routers);
-static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+static int circuit_send_first_onion_skin(origin_circuit_t *circ);
+static int circuit_build_no_more_hops(origin_circuit_t *circ);
+static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
+                                                crypt_path_t *hop);
+static const node_t *choose_good_middle_server(uint8_t purpose,
+                          cpath_build_state_t *state,
+                          crypt_path_t *head,
+                          int cur_len);
 
 /** This function tries to get a channel to the specified endpoint,
  * and then calls command_setup_channel() to give it the right
@@ -66,11 +89,12 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
  */
 static channel_t *
 channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest)
+                            const char *id_digest,
+                            const ed25519_public_key_t *ed_id)
 {
   channel_t *chan;
 
-  chan = channel_connect(addr, port, id_digest);
+  chan = channel_connect(addr, port, id_digest, ed_id);
   if (chan) command_setup_channel(chan);
 
   return chan;
@@ -268,14 +292,9 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names)
         base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN);
       }
     } else { /* ! verbose_names */
-      node = node_get_by_id(id);
-      if (node && node_is_named(node)) {
-        elt = tor_strdup(node_get_nickname(node));
-      } else {
-        elt = tor_malloc(HEX_DIGEST_LEN+2);
-        elt[0] = '$';
-        base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN);
-      }
+      elt = tor_malloc(HEX_DIGEST_LEN+2);
+      elt[0] = '$';
+      base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN);
     }
     tor_assert(elt);
     if (verbose) {
@@ -326,45 +345,6 @@ circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ)
   tor_free(s);
 }
 
-/** Tell the rep(utation)hist(ory) module about the status of the links
- * in <b>circ</b>.  Hops that have become OPEN are marked as successfully
- * extended; the _first_ hop that isn't open (if any) is marked as
- * unable to extend.
- */
-/* XXXX Someday we should learn from OR circuits too. */
-void
-circuit_rep_hist_note_result(origin_circuit_t *circ)
-{
-  crypt_path_t *hop;
-  const char *prev_digest = NULL;
-  hop = circ->cpath;
-  if (!hop) /* circuit hasn't started building yet. */
-    return;
-  if (server_mode(get_options())) {
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (!me)
-      return;
-    prev_digest = me->cache_info.identity_digest;
-  }
-  do {
-    const node_t *node = node_get_by_id(hop->extend_info->identity_digest);
-    if (node) { /* Why do we check this?  We know the identity. -NM XXXX */
-      if (prev_digest) {
-        if (hop->state == CPATH_STATE_OPEN)
-          rep_hist_note_extend_succeeded(prev_digest, node->identity);
-        else {
-          rep_hist_note_extend_failed(prev_digest, node->identity);
-          break;
-        }
-      }
-      prev_digest = node->identity;
-    } else {
-      prev_digest = NULL;
-    }
-    hop=hop->next;
-  } while (hop!=circ->cpath);
-}
-
 /** Return 1 iff every node in circ's cpath definitely supports ntor. */
 static int
 circuit_cpath_supports_ntor(const origin_circuit_t *circ)
@@ -435,7 +415,7 @@ onion_populate_cpath(origin_circuit_t *circ)
                                     circ->cpath->extend_info->identity_digest);
     /* If we don't know the node and its descriptor, we must be bootstrapping.
      */
-    if (!node || !node_has_descriptor(node)) {
+    if (!node || !node_has_preferred_descriptor(node, 1)) {
       return 0;
     }
   }
@@ -484,10 +464,15 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
 {
   origin_circuit_t *circ;
   int err_reason = 0;
+  int is_hs_v3_rp_circuit = 0;
+
+  if (flags & CIRCLAUNCH_IS_V3_RP) {
+    is_hs_v3_rp_circuit = 1;
+  }
 
   circ = origin_circuit_init(purpose, flags);
 
-  if (onion_pick_cpath_exit(circ, exit_ei) < 0 ||
+  if (onion_pick_cpath_exit(circ, exit_ei, is_hs_v3_rp_circuit) < 0 ||
       onion_populate_cpath(circ) < 0) {
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOPATH);
     return NULL;
@@ -502,6 +487,13 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
   return circ;
 }
 
+/** Return the guard state associated with <b>circ</b>, which may be NULL. */
+circuit_guard_state_t *
+origin_circuit_get_guard_state(origin_circuit_t *circ)
+{
+  return circ->guard_state;
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -540,6 +532,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
                          firsthop->extend_info->port));
 
   n_chan = channel_get_for_extend(firsthop->extend_info->identity_digest,
+                                  &firsthop->extend_info->ed_identity,
                                   &firsthop->extend_info->addr,
                                   &msg,
                                   &should_launch);
@@ -557,7 +550,8 @@ circuit_handle_first_hop(origin_circuit_t *circ)
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
-          firsthop->extend_info->identity_digest);
+          firsthop->extend_info->identity_digest,
+          &firsthop->extend_info->ed_identity);
       if (!n_chan) { /* connect failed, forget the whole thing */
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
@@ -600,8 +594,7 @@ circuit_n_chan_done(channel_t *chan, int status, int close_origin_circuits)
 
   tor_assert(chan);
 
-  log_debug(LD_CIRC,"chan to %s/%s, status=%d",
-            chan->nickname ? chan->nickname : "NULL",
+  log_debug(LD_CIRC,"chan to %s, status=%d",
             channel_get_canonical_remote_descr(chan), status);
 
   pending_circs = smartlist_new();
@@ -791,24 +784,28 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ)
      * creating on behalf of others. */
     return 0;
   }
-  if (options->FastFirstHopPK == -1) {
-    /* option is "auto", so look at the consensus. */
-    return networkstatus_get_param(NULL, "usecreatefast", 1, 0, 1);
-  }
-
-  return options->FastFirstHopPK;
+  return networkstatus_get_param(NULL, "usecreatefast", 0, 0, 1);
 }
 
-/** Return true if <b>circ</b> is the type of circuit we want to count
- * timeouts from. In particular, we want it to have not completed yet
- * (already completing indicates we cannibalized it), and we want it to
- * have exactly three hops.
+/**
+ * Return true if <b>circ</b> is the type of circuit we want to count
+ * timeouts from.
+ *
+ * In particular, we want to consider any circuit that plans to build
+ * at least 3 hops (but maybe more), but has 3 or fewer hops built
+ * so far.
+ *
+ * We still want to consider circuits before 3 hops, because we need
+ * to decide if we should convert them to a measurement circuit in
+ * circuit_build_times_handle_completed_hop(), rather than letting
+ * slow circuits get killed right away.
  */
 int
-circuit_timeout_want_to_count_circ(origin_circuit_t *circ)
+circuit_timeout_want_to_count_circ(const origin_circuit_t *circ)
 {
   return !circ->has_opened
-          && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN;
+          && circ->build_state->desired_path_len >= DEFAULT_ROUTE_LEN
+          && circuit_get_cpath_opened_len(circ) <= DEFAULT_ROUTE_LEN;
 }
 
 /** Decide whether to use a TAP or ntor handshake for connecting to <b>ei</b>
@@ -866,212 +863,296 @@ circuit_pick_extend_handshake(uint8_t *cell_type_out,
   }
 }
 
+/**
+ * Return true iff <b>purpose</b> is a purpose for a circuit which is
+ * allowed to have no guard configured, even if the circuit is multihop
+ * and guards are enabled.
+ */
+static int
+circuit_purpose_may_omit_guard(int purpose)
+{
+  switch (purpose) {
+    case CIRCUIT_PURPOSE_TESTING:
+    case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT:
+      /* Testing circuits may omit guards because they're measuring
+       * liveness or performance, and don't want guards to interfere. */
+      return 1;
+    default:
+      /* All other multihop circuits should use guards if guards are
+       * enabled. */
+      return 0;
+  }
+}
+
 /** This is the backbone function for building circuits.
  *
  * If circ's first hop is closed, then we need to build a create
  * cell and send it forward.
  *
- * Otherwise, we need to build a relay extend cell and send it
- * forward.
+ * Otherwise, if circ's cpath still has any non-open hops, we need to
+ * build a relay extend cell and send it forward to the next non-open hop.
+ *
+ * If all hops on the cpath are open, we're done building the circuit
+ * and we should do housekeeping for the newly opened circuit.
  *
  * Return -reason if we want to tear down circ, else return 0.
  */
 int
 circuit_send_next_onion_skin(origin_circuit_t *circ)
 {
-  crypt_path_t *hop;
-  const node_t *node;
-
   tor_assert(circ);
 
   if (circ->cpath->state == CPATH_STATE_CLOSED) {
-    /* This is the first hop. */
-    create_cell_t cc;
-    int fast;
-    int len;
-    log_debug(LD_CIRC,"First skin; sending create cell.");
-    memset(&cc, 0, sizeof(cc));
-    if (circ->build_state->onehop_tunnel)
-      control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0);
-    else
-      control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0);
+    /* Case one: we're on the first hop. */
+    return circuit_send_first_onion_skin(circ);
+  }
 
-    node = node_get_by_id(circ->base_.n_chan->identity_digest);
-    fast = should_use_create_fast_for_circuit(circ);
-    if (!fast) {
-      /* We are an OR and we know the right onion key: we should
-       * send a create cell.
-       */
-      circuit_pick_create_handshake(&cc.cell_type, &cc.handshake_type,
-                                    circ->cpath->extend_info);
-    } else {
-      /* We are not an OR, and we're building the first hop of a circuit to a
-       * new OR: we can be speedy and use CREATE_FAST to save an RSA operation
-       * and a DH operation. */
-      cc.cell_type = CELL_CREATE_FAST;
-      cc.handshake_type = ONION_HANDSHAKE_TYPE_FAST;
-    }
+  tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
+  tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
 
-    len = onion_skin_create(cc.handshake_type,
-                            circ->cpath->extend_info,
-                            &circ->cpath->handshake_state,
-                            cc.onionskin);
-    if (len < 0) {
-      log_warn(LD_CIRC,"onion_skin_create (first hop) failed.");
-      return - END_CIRC_REASON_INTERNAL;
-    }
-    cc.handshake_len = len;
+  crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
+  circuit_build_times_handle_completed_hop(circ);
+
+  if (hop) {
+    /* Case two: we're on a hop after the first. */
+    return circuit_send_intermediate_onion_skin(circ, hop);
+  }
+
+  /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  return circuit_build_no_more_hops(circ);
+}
+
+/**
+ * Called from circuit_send_next_onion_skin() when we find ourselves connected
+ * to the first hop in <b>circ</b>: Send a CREATE or CREATE2 or CREATE_FAST
+ * cell to that hop.  Return 0 on success; -reason on failure (if the circuit
+ * should be torn down).
+ */
+static int
+circuit_send_first_onion_skin(origin_circuit_t *circ)
+{
+  int fast;
+  int len;
+  const node_t *node;
+  create_cell_t cc;
+  memset(&cc, 0, sizeof(cc));
 
-    if (circuit_deliver_create_cell(TO_CIRCUIT(circ), &cc, 0) < 0)
-      return - END_CIRC_REASON_RESOURCELIMIT;
+  log_debug(LD_CIRC,"First skin; sending create cell.");
 
-    circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
-    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
-    log_info(LD_CIRC,"First hop: finished sending %s cell to '%s'",
-             fast ? "CREATE_FAST" : "CREATE",
-             node ? node_describe(node) : "<unnamed>");
+  if (circ->build_state->onehop_tunnel) {
+    control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0);
   } else {
-    extend_cell_t ec;
-    int len;
-    tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
-    tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
-    log_debug(LD_CIRC,"starting to send subsequent skin.");
-    hop = onion_next_hop_in_cpath(circ->cpath);
-    memset(&ec, 0, sizeof(ec));
-    if (!hop) {
-      /* done building the circuit. whew. */
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
-      if (circuit_timeout_want_to_count_circ(circ)) {
-        struct timeval end;
-        long timediff;
-        tor_gettimeofday(&end);
-        timediff = tv_mdiff(&circ->base_.timestamp_began, &end);
-
-        /*
-         * If the circuit build time is much greater than we would have cut
-         * it off at, we probably had a suspend event along this codepath,
-         * and we should discard the value.
-         */
-        if (timediff < 0 ||
-            timediff > 2*get_circuit_build_close_time_ms()+1000) {
-          log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. "
-                              "Assuming clock jump. Purpose %d (%s)", timediff,
-                     circ->base_.purpose,
-                     circuit_purpose_to_string(circ->base_.purpose));
-        } else if (!circuit_build_times_disabled()) {
-          /* Only count circuit times if the network is live */
-          if (circuit_build_times_network_check_live(
-              get_circuit_build_times())) {
-            circuit_build_times_add_time(get_circuit_build_times_mutable(),
-                (build_time_t)timediff);
-            circuit_build_times_set_timeout(get_circuit_build_times_mutable());
-          }
-
-          if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
-            circuit_build_times_network_circ_success(
-                get_circuit_build_times_mutable());
-          }
-        }
-      }
-      log_info(LD_CIRC,"circuit built!");
-      circuit_reset_failure_count(0);
+    control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0);
 
-      if (circ->build_state->onehop_tunnel || circ->has_opened) {
-        control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0);
-      }
+    /* If this is not a one-hop tunnel, the channel is being used
+     * for traffic that wants anonymity and protection from traffic
+     * analysis (such as netflow record retention). That means we want
+     * to pad it.
+     */
+    if (circ->base_.n_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS)
+      circ->base_.n_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+  }
 
-      pathbias_count_build_success(circ);
-      circuit_rep_hist_note_result(circ);
-      circuit_has_opened(circ); /* do other actions as necessary */
-
-      if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) {
-        const or_options_t *options = get_options();
-        note_that_we_completed_a_circuit();
-        /* FFFF Log a count of known routers here */
-        log_notice(LD_GENERAL,
-            "Tor has successfully opened a circuit. "
-            "Looks like client functionality is working.");
-        if (control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0) == 0) {
-          log_notice(LD_GENERAL,
-                     "Tor has successfully opened a circuit. "
-                     "Looks like client functionality is working.");
-        }
-        control_event_client_status(LOG_NOTICE, "CIRCUIT_ESTABLISHED");
-        clear_broken_connection_map(1);
-        if (server_mode(options) && !check_whether_orport_reachable(options)) {
-          inform_testing_reachability();
-          consider_testing_reachability(1, 1);
-        }
-      }
+  node = node_get_by_id(circ->base_.n_chan->identity_digest);
+  fast = should_use_create_fast_for_circuit(circ);
+  if (!fast) {
+    /* We know the right onion key: we should send a create cell. */
+    circuit_pick_create_handshake(&cc.cell_type, &cc.handshake_type,
+                                  circ->cpath->extend_info);
+  } else {
+    /* We don't know an onion key, so we need to fall back to CREATE_FAST. */
+    cc.cell_type = CELL_CREATE_FAST;
+    cc.handshake_type = ONION_HANDSHAKE_TYPE_FAST;
+  }
 
-      /* We're done with measurement circuits here. Just close them */
-      if (circ->base_.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
-        circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
-      }
-      return 0;
-    }
+  len = onion_skin_create(cc.handshake_type,
+                          circ->cpath->extend_info,
+                          &circ->cpath->handshake_state,
+                          cc.onionskin);
+  if (len < 0) {
+    log_warn(LD_CIRC,"onion_skin_create (first hop) failed.");
+    return - END_CIRC_REASON_INTERNAL;
+  }
+  cc.handshake_len = len;
+
+  if (circuit_deliver_create_cell(TO_CIRCUIT(circ), &cc, 0) < 0)
+    return - END_CIRC_REASON_RESOURCELIMIT;
 
-    if (tor_addr_family(&hop->extend_info->addr) != AF_INET) {
-      log_warn(LD_BUG, "Trying to extend to a non-IPv4 address.");
-      return - END_CIRC_REASON_INTERNAL;
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+  log_info(LD_CIRC,"First hop: finished sending %s cell to '%s'",
+           fast ? "CREATE_FAST" : "CREATE",
+           node ? node_describe(node) : "<unnamed>");
+  return 0;
+}
+
+/**
+ * Called from circuit_send_next_onion_skin() when we find that we have no
+ * more hops: mark the circuit as finished, and perform the necessary
+ * bookkeeping.  Return 0 on success; -reason on failure (if the circuit
+ * should be torn down).
+ */
+static int
+circuit_build_no_more_hops(origin_circuit_t *circ)
+{
+  guard_usable_t r;
+  if (! circ->guard_state) {
+    if (circuit_get_cpath_len(circ) != 1 &&
+        ! circuit_purpose_may_omit_guard(circ->base_.purpose) &&
+        get_options()->UseEntryGuards) {
+      log_warn(LD_BUG, "%d-hop circuit %p with purpose %d has no "
+               "guard state",
+               circuit_get_cpath_len(circ), circ, circ->base_.purpose);
     }
+    r = GUARD_USABLE_NOW;
+  } else {
+    r = entry_guard_succeeded(&circ->guard_state);
+  }
+  const int is_usable_for_streams = (r == GUARD_USABLE_NOW);
+  if (r == GUARD_USABLE_NOW) {
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+  } else if (r == GUARD_MAYBE_USABLE_LATER) {
+    // Wait till either a better guard succeeds, or till
+    // all better guards fail.
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_GUARD_WAIT);
+  } else {
+    tor_assert_nonfatal(r == GUARD_USABLE_NEVER);
+    return - END_CIRC_REASON_INTERNAL;
+  }
+
+  /* XXXX #21422 -- the rest of this branch needs careful thought!
+   * Some of the things here need to happen when a circuit becomes
+   * mechanically open; some need to happen when it is actually usable.
+   * I think I got them right, but more checking would be wise. -NM
+   */
+
+  log_info(LD_CIRC,"circuit built!");
+  circuit_reset_failure_count(0);
+
+  if (circ->build_state->onehop_tunnel || circ->has_opened) {
+    control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0);
+  }
 
-    circuit_pick_extend_handshake(&ec.cell_type,
-                                  &ec.create_cell.cell_type,
-                                  &ec.create_cell.handshake_type,
-                                  hop->extend_info);
-
-    tor_addr_copy(&ec.orport_ipv4.addr, &hop->extend_info->addr);
-    ec.orport_ipv4.port = hop->extend_info->port;
-    tor_addr_make_unspec(&ec.orport_ipv6.addr);
-    memcpy(ec.node_id, hop->extend_info->identity_digest, DIGEST_LEN);
-
-    len = onion_skin_create(ec.create_cell.handshake_type,
-                            hop->extend_info,
-                            &hop->handshake_state,
-                            ec.create_cell.onionskin);
-    if (len < 0) {
-      log_warn(LD_CIRC,"onion_skin_create failed.");
-      return - END_CIRC_REASON_INTERNAL;
+  pathbias_count_build_success(circ);
+  if (is_usable_for_streams)
+    circuit_has_opened(circ); /* do other actions as necessary */
+
+  if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) {
+    const or_options_t *options = get_options();
+    note_that_we_completed_a_circuit();
+    /* FFFF Log a count of known routers here */
+    log_notice(LD_GENERAL,
+               "Tor has successfully opened a circuit. "
+               "Looks like client functionality is working.");
+    if (control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0) == 0) {
+      log_notice(LD_GENERAL,
+                 "Tor has successfully opened a circuit. "
+                 "Looks like client functionality is working.");
+    }
+    control_event_client_status(LOG_NOTICE, "CIRCUIT_ESTABLISHED");
+    clear_broken_connection_map(1);
+    if (server_mode(options) && !check_whether_orport_reachable(options)) {
+      inform_testing_reachability();
+      router_do_reachability_checks(1, 1);
     }
-    ec.create_cell.handshake_len = len;
+  }
 
-    log_info(LD_CIRC,"Sending extend relay cell.");
-    {
-      uint8_t command = 0;
-      uint16_t payload_len=0;
-      uint8_t payload[RELAY_PAYLOAD_SIZE];
-      if (extend_cell_format(&command, &payload_len, payload, &ec)<0) {
-        log_warn(LD_CIRC,"Couldn't format extend cell");
-        return -END_CIRC_REASON_INTERNAL;
-      }
+  /* We're done with measurement circuits here. Just close them */
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+  }
+  return 0;
+}
+
+/**
+ * Called from circuit_send_next_onion_skin() when we find that we have a hop
+ * other than the first that we need to extend to: use <b>hop</b>'s
+ * information to extend the circuit another step. Return 0 on success;
+ * -reason on failure (if the circuit should be torn down).
+ */
+static int
+circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
+                                     crypt_path_t *hop)
+{
+  int len;
+  extend_cell_t ec;
+  memset(&ec, 0, sizeof(ec));
+
+  log_debug(LD_CIRC,"starting to send subsequent skin.");
+
+  if (tor_addr_family(&hop->extend_info->addr) != AF_INET) {
+    log_warn(LD_BUG, "Trying to extend to a non-IPv4 address.");
+    return - END_CIRC_REASON_INTERNAL;
+  }
 
-      /* send it to hop->prev, because it will transfer
-       * it to a create cell and then send to hop */
-      if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
-                                       command,
-                                       (char*)payload, payload_len,
-                                       hop->prev) < 0)
-        return 0; /* circuit is closed */
+  circuit_pick_extend_handshake(&ec.cell_type,
+                                &ec.create_cell.cell_type,
+                                &ec.create_cell.handshake_type,
+                                hop->extend_info);
+
+  tor_addr_copy(&ec.orport_ipv4.addr, &hop->extend_info->addr);
+  ec.orport_ipv4.port = hop->extend_info->port;
+  tor_addr_make_unspec(&ec.orport_ipv6.addr);
+  memcpy(ec.node_id, hop->extend_info->identity_digest, DIGEST_LEN);
+  /* Set the ED25519 identity too -- it will only get included
+   * in the extend2 cell if we're configured to use it, though. */
+  ed25519_pubkey_copy(&ec.ed_pubkey, &hop->extend_info->ed_identity);
+
+  len = onion_skin_create(ec.create_cell.handshake_type,
+                          hop->extend_info,
+                          &hop->handshake_state,
+                          ec.create_cell.onionskin);
+  if (len < 0) {
+    log_warn(LD_CIRC,"onion_skin_create failed.");
+    return - END_CIRC_REASON_INTERNAL;
+  }
+  ec.create_cell.handshake_len = len;
+
+  log_info(LD_CIRC,"Sending extend relay cell.");
+  {
+    uint8_t command = 0;
+    uint16_t payload_len=0;
+    uint8_t payload[RELAY_PAYLOAD_SIZE];
+    if (extend_cell_format(&command, &payload_len, payload, &ec)<0) {
+      log_warn(LD_CIRC,"Couldn't format extend cell");
+      return -END_CIRC_REASON_INTERNAL;
     }
-    hop->state = CPATH_STATE_AWAITING_KEYS;
+
+    /* send it to hop->prev, because that relay will transfer
+     * it to a create cell and then send to hop */
+    if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                     command,
+                                     (char*)payload, payload_len,
+                                     hop->prev) < 0)
+      return 0; /* circuit is closed */
   }
+  hop->state = CPATH_STATE_AWAITING_KEYS;
   return 0;
 }
 
-/** Our clock just jumped by <b>seconds_elapsed</b>. Assume
- * something has also gone wrong with our network: notify the user,
- * and abandon all not-yet-used circuits. */
+/** Our clock just jumped by <b>seconds_elapsed</b>. If <b>was_idle</b> is
+ * true, then the monotonic time matches; otherwise it doesn't. Assume
+ * something has also gone wrong with our network: notify the user, and
+ * abandon all not-yet-used circuits. */
 void
-circuit_note_clock_jumped(int seconds_elapsed)
+circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle)
 {
   int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE;
-  tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; "
-      "assuming established circuits no longer work.",
-      seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed,
-      seconds_elapsed >=0 ? "forward" : "backward");
-  control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d",
-                               seconds_elapsed);
+  if (was_idle) {
+    tor_log(severity, LD_GENERAL, "Tor has been idle for "I64_FORMAT
+            " seconds; assuming established circuits no longer work.",
+            I64_PRINTF_ARG(seconds_elapsed));
+  } else {
+    tor_log(severity, LD_GENERAL,
+            "Your system clock just jumped "I64_FORMAT" seconds %s; "
+            "assuming established circuits no longer work.",
+            I64_PRINTF_ARG(
+                 seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed),
+            seconds_elapsed >=0 ? "forward" : "backward");
+  }
+  control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME="I64_FORMAT
+                               " IDLE=%d",
+                               I64_PRINTF_ARG(seconds_elapsed), was_idle?1:0);
   /* so we log when it works again */
   note_that_we_maybe_cant_complete_circuits();
   control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s",
@@ -1143,7 +1224,7 @@ circuit_extend(cell_t *cell, circuit_t *circ)
 
   /* Check if they asked us for 0000..0000. We support using
    * an empty fingerprint for the first hop (e.g. for a bridge relay),
-   * but we don't want to let people send us extend cells for empty
+   * but we don't want to let clients send us extend cells for empty
    * fingerprints -- a) because it opens the user up to a mitm attack,
    * and b) because it lets an attacker force the relay to hold open a
    * new TLS connection for each extend request. */
@@ -1153,6 +1234,18 @@ circuit_extend(cell_t *cell, circuit_t *circ)
     return -1;
   }
 
+  /* Fill in ed_pubkey if it was not provided and we can infer it from
+   * our networkstatus */
+  if (ed25519_public_key_is_zero(&ec.ed_pubkey)) {
+    const node_t *node = node_get_by_id((const char*)ec.node_id);
+    const ed25519_public_key_t *node_ed_id = NULL;
+    if (node &&
+        node_supports_ed25519_link_authentication(node, 1) &&
+        (node_ed_id = node_get_ed25519_id(node))) {
+      ed25519_pubkey_copy(&ec.ed_pubkey, node_ed_id);
+    }
+  }
+
   /* Next, check if we're being asked to connect to the hop that the
    * extend cell came from. There isn't any reason for that, and it can
    * assist circular-path attacks. */
@@ -1164,7 +1257,18 @@ circuit_extend(cell_t *cell, circuit_t *circ)
     return -1;
   }
 
+  /* Check the previous hop Ed25519 ID too */
+  if (! ed25519_public_key_is_zero(&ec.ed_pubkey) &&
+      ed25519_pubkey_eq(&ec.ed_pubkey,
+                        &TO_OR_CIRCUIT(circ)->p_chan->ed25519_identity)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop "
+           "(by Ed25519 ID).");
+    return -1;
+  }
+
   n_chan = channel_get_for_extend((const char*)ec.node_id,
+                                  &ec.ed_pubkey,
                                   &ec.orport_ipv4.addr,
                                   &msg,
                                   &should_launch);
@@ -1176,8 +1280,9 @@ circuit_extend(cell_t *cell, circuit_t *circ)
 
     circ->n_hop = extend_info_new(NULL /*nickname*/,
                                   (const char*)ec.node_id,
-                                  NULL /*onion_key*/,
-                                  NULL /*curve25519_key*/,
+                                  &ec.ed_pubkey,
+                                  NULL, /*onion_key*/
+                                  NULL, /*curve25519_key*/
                                   &ec.orport_ipv4.addr,
                                   ec.orport_ipv4.port);
 
@@ -1190,7 +1295,8 @@ circuit_extend(cell_t *cell, circuit_t *circ)
       /* we should try to open a connection */
       n_chan = channel_connect_for_circuit(&ec.orport_ipv4.addr,
                                            ec.orport_ipv4.port,
-                                           (const char*)ec.node_id);
+                                           (const char*)ec.node_id,
+                                           &ec.ed_pubkey);
       if (!n_chan) {
         log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
         circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
@@ -1217,54 +1323,32 @@ circuit_extend(cell_t *cell, circuit_t *circ)
   return 0;
 }
 
-/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in
- * key_data.  key_data must contain CPATH_KEY_MATERIAL bytes, which are
- * used as follows:
+/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
  *   - 20 to initialize f_digest
  *   - 20 to initialize b_digest
  *   - 16 to key f_crypto
  *   - 16 to key b_crypto
  *
  * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
  */
 int
-circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data,
-                          int reverse)
+circuit_init_cpath_crypto(crypt_path_t *cpath,
+                          const char *key_data, size_t key_data_len,
+                          int reverse, int is_hs_v3)
 {
-  crypto_digest_t *tmp_digest;
-  crypto_cipher_t *tmp_crypto;
 
   tor_assert(cpath);
-  tor_assert(key_data);
-  tor_assert(!(cpath->f_crypto || cpath->b_crypto ||
-             cpath->f_digest || cpath->b_digest));
-
-  cpath->f_digest = crypto_digest_new();
-  crypto_digest_add_bytes(cpath->f_digest, key_data, DIGEST_LEN);
-  cpath->b_digest = crypto_digest_new();
-  crypto_digest_add_bytes(cpath->b_digest, key_data+DIGEST_LEN, DIGEST_LEN);
-
-  if (!(cpath->f_crypto =
-        crypto_cipher_new(key_data+(2*DIGEST_LEN)))) {
-    log_warn(LD_BUG,"Forward cipher initialization failed.");
-    return -1;
-  }
-  if (!(cpath->b_crypto =
-        crypto_cipher_new(key_data+(2*DIGEST_LEN)+CIPHER_KEY_LEN))) {
-    log_warn(LD_BUG,"Backward cipher initialization failed.");
-    return -1;
-  }
-
-  if (reverse) {
-    tmp_digest = cpath->f_digest;
-    cpath->f_digest = cpath->b_digest;
-    cpath->b_digest = tmp_digest;
-    tmp_crypto = cpath->f_crypto;
-    cpath->f_crypto = cpath->b_crypto;
-    cpath->b_crypto = tmp_crypto;
-  }
-
-  return 0;
+  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
+                           is_hs_v3);
 }
 
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
@@ -1316,7 +1400,7 @@ circuit_finish_handshake(origin_circuit_t *circ,
 
   onion_handshake_state_release(&hop->handshake_state);
 
-  if (circuit_init_cpath_crypto(hop, keys, 0)<0) {
+  if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
     return -END_CIRC_REASON_TORPROTOCOL;
   }
 
@@ -1335,13 +1419,12 @@ circuit_finish_handshake(origin_circuit_t *circ,
  * just give up: force circ to close, and return 0.
  */
 int
-circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
+circuit_truncated(origin_circuit_t *circ, int reason)
 {
 //  crypt_path_t *victim;
 //  connection_t *stream;
 
   tor_assert(circ);
-  tor_assert(layer);
 
   /* XXX Since we don't send truncates currently, getting a truncated
    *     means that a connection broke or an extend failed. For now,
@@ -1374,7 +1457,7 @@ circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
 
   log_info(LD_CIRC, "finished");
   return 0;
-#endif
+#endif /* 0 */
 }
 
 /** Given a response payload and keys, initialize, then send a created
@@ -1383,11 +1466,12 @@ circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason)
 int
 onionskin_answer(or_circuit_t *circ,
                  const created_cell_t *created_cell,
-                 const char *keys,
+                 const char *keys, size_t keys_len,
                  const uint8_t *rend_circ_nonce)
 {
   cell_t cell;
-  crypt_path_t *tmp_cpath;
+
+  tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
 
   if (created_cell_format(&cell, created_cell) < 0) {
     log_warn(LD_BUG,"couldn't format created cell (type=%d, len=%d)",
@@ -1396,34 +1480,24 @@ onionskin_answer(or_circuit_t *circ,
   }
   cell.circ_id = circ->p_circ_id;
 
-  tmp_cpath = tor_malloc_zero(sizeof(crypt_path_t));
-  tmp_cpath->magic = CRYPT_PATH_MAGIC;
-
   circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
 
   log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
             (unsigned int)get_uint32(keys),
             (unsigned int)get_uint32(keys+20));
-  if (circuit_init_cpath_crypto(tmp_cpath, keys, 0)<0) {
+  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
     log_warn(LD_BUG,"Circuit initialization failed");
-    tor_free(tmp_cpath);
     return -1;
   }
-  circ->n_digest = tmp_cpath->f_digest;
-  circ->n_crypto = tmp_cpath->f_crypto;
-  circ->p_digest = tmp_cpath->b_digest;
-  circ->p_crypto = tmp_cpath->b_crypto;
-  tmp_cpath->magic = 0;
-  tor_free(tmp_cpath);
 
   memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
 
-  circ->is_first_hop = (created_cell->cell_type == CELL_CREATED_FAST);
+  int used_create_fast = (created_cell->cell_type == CELL_CREATED_FAST);
 
   append_cell_to_circuit_queue(TO_CIRCUIT(circ),
                                circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
   log_debug(LD_CIRC,"Finished sending '%s' cell.",
-            circ->is_first_hop ? "created_fast" : "created");
+            used_create_fast ? "created_fast" : "created");
 
   /* Ignore the local bit when ExtendAllowPrivateAddresses is set:
    * it violates the assumption that private addresses are local.
@@ -1441,13 +1515,137 @@ onionskin_answer(or_circuit_t *circ,
   return 0;
 }
 
-/** Choose a length for a circuit of purpose <b>purpose</b>: three + the
- * number of endpoints that would give something away about our destination.
+/** Helper for new_route_len().  Choose a circuit length for purpose
+ * <b>purpose</b>: DEFAULT_ROUTE_LEN (+ 1 if someone else chose the
+ * exit).  If someone else chose the exit, they could be colluding
+ * with the exit, so add a randomly selected node to preserve
+ * anonymity.
+ *
+ * Here, "exit node" sometimes means an OR acting as an internal
+ * endpoint, rather than as a relay to an external endpoint.  This
+ * means there need to be at least DEFAULT_ROUTE_LEN routers between
+ * us and the internal endpoint to preserve the same anonymity
+ * properties that we would get when connecting to an external
+ * endpoint.  These internal endpoints can include:
+ *
+ *   - Connections to a directory of hidden services
+ *     (CIRCUIT_PURPOSE_C_GENERAL)
+ *
+ *   - A client connecting to an introduction point, which the hidden
+ *     service picked (CIRCUIT_PURPOSE_C_INTRODUCING, via
+ *     circuit_get_open_circ_or_launch() which rewrites it from
+ *     CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)
+ *
+ *   - A hidden service connecting to a rendezvous point, which the
+ *     client picked (CIRCUIT_PURPOSE_S_CONNECT_REND, via
+ *     rend_service_receive_introduction() and
+ *     rend_service_relaunch_rendezvous)
+ *
+ * There are currently two situations where we picked the exit node
+ * ourselves, making DEFAULT_ROUTE_LEN a safe circuit length:
+ *
+ *   - We are a hidden service connecting to an introduction point
+ *     (CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, via
+ *     rend_service_launch_establish_intro())
+ *
+ *   - We are a router testing its own reachabiity
+ *     (CIRCUIT_PURPOSE_TESTING, via router_do_reachability_checks())
+ *
+ * onion_pick_cpath_exit() bypasses us (by not calling
+ * new_route_len()) in the one-hop tunnel case, so we don't need to
+ * handle that.
+ */
+int
+route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
+{
+  int routelen = DEFAULT_ROUTE_LEN;
+  int known_purpose = 0;
+
+  if (circuit_should_use_vanguards(purpose)) {
+    /* Clients want an extra hop for rends to avoid linkability.
+     * Services want it for intro points to avoid publishing their
+     * layer3 guards. They want it for hsdir posts to use
+     * their full layer3 guard set for those connections.
+     * Ex: C - G - L2 - L3 - R
+     *     S - G - L2 - L3 - HSDIR
+     *     S - G - L2 - L3 - I
+     */
+    if (purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND ||
+        purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+        purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
+        purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)
+      return routelen+1;
+
+    /* If we only have Layer2 vanguards, then we do not need
+     * the extra hop for linkabilty reasons (see below).
+     * This means all hops can be of the form:
+     *   S/C - G - L2 - M - R/HSDir/I
+     */
+    if (get_options()->HSLayer2Nodes && !get_options()->HSLayer3Nodes)
+      return routelen+1;
+
+    /* For connections to hsdirs, clients want two extra hops
+     * when using layer3 guards, to avoid linkability.
+     * Same goes for intro points. Note that the route len
+     * includes the intro point or hsdir, hence the +2.
+     * Ex: C - G - L2 - L3 - M - I
+     *     C - G - L2 - L3 - M - HSDIR
+     *     S - G - L2 - L3 - M - R
+     */
+    if (purpose == CIRCUIT_PURPOSE_S_CONNECT_REND ||
+        purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+        purpose == CIRCUIT_PURPOSE_C_INTRODUCING)
+      return routelen+2;
+  }
+
+  if (!exit_ei)
+    return routelen;
+
+  switch (purpose) {
+    /* These two purposes connect to a router that we chose, so
+     * DEFAULT_ROUTE_LEN is safe. */
+  case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
+    /* hidden service connecting to introduction point */
+  case CIRCUIT_PURPOSE_TESTING:
+    /* router reachability testing */
+    known_purpose = 1;
+    break;
+
+    /* These three purposes connect to a router that someone else
+     * might have chosen, so add an extra hop to protect anonymity. */
+  case CIRCUIT_PURPOSE_C_GENERAL:
+  case CIRCUIT_PURPOSE_C_HSDIR_GET:
+  case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    /* connecting to hidden service directory */
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    /* client connecting to introduction point */
+  case CIRCUIT_PURPOSE_S_CONNECT_REND:
+    /* hidden service connecting to rendezvous point */
+    known_purpose = 1;
+    routelen++;
+    break;
+
+  default:
+    /* Got a purpose not listed above along with a chosen exit.
+     * Increase the circuit length by one anyway for safety. */
+    routelen++;
+    break;
+  }
+
+  if (BUG(exit_ei && !known_purpose)) {
+    log_warn(LD_BUG, "Unhandled purpose %d with a chosen exit; "
+             "assuming routelen %d.", purpose, routelen);
+  }
+  return routelen;
+}
+
+/** Choose a length for a circuit of purpose <b>purpose</b> and check
+ * if enough routers are available.
  *
  * If the routerlist <b>nodes</b> doesn't have enough routers
  * to handle the desired path length, return -1.
  */
-static int
+STATIC int
 new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
 {
   int num_acceptable_routers;
@@ -1455,11 +1653,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
 
   tor_assert(nodes);
 
-  routelen = DEFAULT_ROUTE_LEN;
-  if (exit_ei &&
-      purpose != CIRCUIT_PURPOSE_TESTING &&
-      purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)
-    routelen++;
+  routelen = route_len_for_purpose(purpose, exit_ei);
 
   num_acceptable_routers = count_acceptable_nodes(nodes);
 
@@ -1492,9 +1686,9 @@ circuit_get_unhandled_ports(time_t now)
  * If we're returning 0, set need_uptime and need_capacity to
  * indicate any requirements that the unhandled ports have.
  */
-int
-circuit_all_predicted_ports_handled(time_t now, int *need_uptime,
-                                    int *need_capacity)
+MOCK_IMPL(int,
+circuit_all_predicted_ports_handled, (time_t now, int *need_uptime,
+                                      int *need_capacity))
 {
   int i, enough;
   uint16_t *port;
@@ -1571,7 +1765,7 @@ ap_stream_wants_exit_attention(connection_t *conn)
  * Return NULL if we can't find any suitable routers.
  */
 static const node_t *
-choose_good_exit_server_general(int need_uptime, int need_capacity)
+choose_good_exit_server_general(router_crn_flags_t flags)
 {
   int *n_supported;
   int n_pending_connections = 0;
@@ -1581,6 +1775,9 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
   const or_options_t *options = get_options();
   const smartlist_t *the_nodes;
   const node_t *selected_node=NULL;
+  const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
+  const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
+  const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
 
   connections = get_connection_array();
 
@@ -1613,7 +1810,7 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
        */
       continue;
     }
-    if (!node_has_descriptor(node)) {
+    if (!node_has_preferred_descriptor(node, direct_conn)) {
       n_supported[i] = -1;
       continue;
     }
@@ -1643,15 +1840,16 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
                  * we'll retry later in this function with need_update and
                  * need_capacity set to 0. */
     }
-    if (!(node->is_valid || options->AllowInvalid_ & ALLOW_INVALID_EXIT)) {
+    if (!(node->is_valid)) {
       /* if it's invalid and we don't want it */
       n_supported[i] = -1;
 //      log_fn(LOG_DEBUG,"Skipping node %s (index %d) -- invalid router.",
 //             router->nickname, i);
       continue; /* skip invalid routers */
     }
-    if (options->ExcludeSingleHopRelays &&
-        node_allows_single_hop_exits(node)) {
+    /* We do not allow relays that allow single hop exits by default. Option
+     * was deprecated in 0.2.9.2-alpha and removed in 0.3.1.0-alpha. */
+    if (node_allows_single_hop_exits(node)) {
       n_supported[i] = -1;
       continue;
     }
@@ -1725,7 +1923,8 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
                  need_capacity?", fast":"",
                  need_uptime?", stable":"");
         tor_free(n_supported);
-        return choose_good_exit_server_general(0, 0);
+        flags &= ~(CRN_NEED_UPTIME|CRN_NEED_CAPACITY);
+        return choose_good_exit_server_general(flags);
       }
       log_notice(LD_CIRC, "All routers are down or won't exit%s -- "
                  "choosing a doomed exit at random.",
@@ -1766,9 +1965,10 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
   }
   if (options->ExitNodes) {
     log_warn(LD_CIRC,
-             "No specified %sexit routers seem to be running: "
+             "No exits in ExitNodes%s seem to be running: "
              "can't choose an exit.",
-             options->ExcludeExitNodesUnion_ ? "non-excluded " : "");
+             options->ExcludeExitNodesUnion_ ?
+             ", except possibly those excluded by your configuration, " : "");
   }
   return NULL;
 }
@@ -1783,7 +1983,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags,
                              const or_options_t *options)
 {
   const node_t *rp_node = NULL;
-  const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0;
   const int need_desc = (flags & CRN_NEED_DESC) != 0;
   const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
@@ -1795,7 +1994,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags,
 
   /* Add all running nodes to all_live_nodes */
   router_add_running_nodes_to_smartlist(all_live_nodes,
-                                        allow_invalid,
                                         0, 0, 0,
                                         need_desc,
                                         pref_addr,
@@ -1829,7 +2027,7 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags,
 
   return rp_node;
 }
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) */
 
 /* Pick a Rendezvous Point for our HS circuits according to <b>flags</b>. */
 static const node_t *
@@ -1837,9 +2035,6 @@ pick_rendezvous_node(router_crn_flags_t flags)
 {
   const or_options_t *options = get_options();
 
-  if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS)
-    flags |= CRN_ALLOW_INVALID;
-
 #ifdef ENABLE_TOR2WEB_MODE
   /* We want to connect directly to the node if we can */
   router_crn_flags_t direct_flags = flags;
@@ -1868,11 +2063,103 @@ pick_rendezvous_node(router_crn_flags_t flags)
              "Unable to find a random rendezvous point that is reachable via "
              "a direct connection, falling back to a 3-hop path.");
   }
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
 
   return router_choose_random_node(NULL, options->ExcludeNodes, flags);
 }
 
+/*
+ * Helper function to pick a configured restricted middle node
+ * (either HSLayer2Nodes or HSLayer3Nodes).
+ *
+ * Make sure that the node we chose is alive, and not excluded,
+ * and return it.
+ *
+ * The exclude_set is a routerset of nodes that the selected node
+ * must not match, and the exclude_list is a simple list of nodes
+ * that the selected node must not be in. Either or both may be
+ * NULL.
+ *
+ * Return NULL if no usable nodes could be found. */
+static const node_t *
+pick_restricted_middle_node(router_crn_flags_t flags,
+                            const routerset_t *pick_from,
+                            const routerset_t *exclude_set,
+                            const smartlist_t *exclude_list,
+                            int position_hint)
+{
+  const node_t *middle_node = NULL;
+
+  smartlist_t *whitelisted_live_middles = smartlist_new();
+  smartlist_t *all_live_nodes = smartlist_new();
+
+  tor_assert(pick_from);
+
+  /* Add all running nodes to all_live_nodes */
+  router_add_running_nodes_to_smartlist(all_live_nodes,
+                                        (flags & CRN_NEED_UPTIME) != 0,
+                                        (flags & CRN_NEED_CAPACITY) != 0,
+                                        (flags & CRN_NEED_GUARD) != 0,
+                                        (flags & CRN_NEED_DESC) != 0,
+                                        (flags & CRN_PREF_ADDR) != 0,
+                                        (flags & CRN_DIRECT_CONN) != 0);
+
+  /* Filter all_live_nodes to only add live *and* whitelisted middles
+   * to the list whitelisted_live_middles. */
+  SMARTLIST_FOREACH_BEGIN(all_live_nodes, node_t *, live_node) {
+    if (routerset_contains_node(pick_from, live_node)) {
+      smartlist_add(whitelisted_live_middles, live_node);
+    }
+  } SMARTLIST_FOREACH_END(live_node);
+
+  /* Honor ExcludeNodes */
+  if (exclude_set) {
+    routerset_subtract_nodes(whitelisted_live_middles, exclude_set);
+  }
+
+  if (exclude_list) {
+    smartlist_subtract(whitelisted_live_middles, exclude_list);
+  }
+
+  /**
+   * Max number of restricted nodes before we alert the user and try
+   * to load balance for them.
+   *
+   * The most aggressive vanguard design had 16 nodes at layer3.
+   * Let's give a small ceiling above that. */
+#define MAX_SANE_RESTRICTED_NODES 20
+  /* If the user (or associated tor controller) selected only a few nodes,
+   * assume they took load balancing into account and don't do it for them.
+   *
+   * If there are a lot of nodes in here, assume they did not load balance
+   * and do it for them, but also warn them that they may be Doing It Wrong.
+   */
+  if (smartlist_len(whitelisted_live_middles) <=
+          MAX_SANE_RESTRICTED_NODES) {
+    middle_node = smartlist_choose(whitelisted_live_middles);
+  } else {
+    static ratelim_t pinned_notice_limit = RATELIM_INIT(24*3600);
+    log_fn_ratelim(&pinned_notice_limit, LOG_NOTICE, LD_CIRC,
+            "Your _HSLayer%dNodes setting has resulted "
+            "in %d total nodes. This is a lot of nodes. "
+            "You may want to consider using a Tor controller "
+            "to select and update a smaller set of nodes instead.",
+            position_hint, smartlist_len(whitelisted_live_middles));
+
+    /* NO_WEIGHTING here just means don't take node flags into account
+     * (ie: use consensus measurement only). This is done so that
+     * we don't further surprise the user by not using Exits that they
+     * specified at all */
+    middle_node = node_sl_choose_by_bandwidth(whitelisted_live_middles,
+                                              NO_WEIGHTING);
+  }
+
+  smartlist_free(whitelisted_live_middles);
+  smartlist_free(all_live_nodes);
+
+  return middle_node;
+}
+
 /** Return a pointer to a suitable router to be the exit node for the
  * circuit of purpose <b>purpose</b> that we're about to build (or NULL
  * if no router is suitable).
@@ -1884,24 +2171,25 @@ pick_rendezvous_node(router_crn_flags_t flags)
  * toward the preferences in 'options'.
  */
 static const node_t *
-choose_good_exit_server(uint8_t purpose,
-                        int need_uptime, int need_capacity, int is_internal)
+choose_good_exit_server(origin_circuit_t *circ,
+                        router_crn_flags_t flags, int is_internal)
 {
   const or_options_t *options = get_options();
-  router_crn_flags_t flags = CRN_NEED_DESC;
-  if (need_uptime)
-    flags |= CRN_NEED_UPTIME;
-  if (need_capacity)
-    flags |= CRN_NEED_CAPACITY;
-
-  switch (purpose) {
+  flags |= CRN_NEED_DESC;
+
+  switch (TO_CIRCUIT(circ)->purpose) {
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
+      /* For these three, we want to pick the exit like a middle hop,
+       * since it should be random. */
+      tor_assert_nonfatal(is_internal);
+      /* Falls through */
     case CIRCUIT_PURPOSE_C_GENERAL:
-      if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE)
-        flags |= CRN_ALLOW_INVALID;
       if (is_internal) /* pick it like a middle hop */
         return router_choose_random_node(NULL, options->ExcludeNodes, flags);
       else
-        return choose_good_exit_server_general(need_uptime,need_capacity);
+        return choose_good_exit_server_general(flags);
     case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
       {
         /* Pick a new RP */
@@ -1911,7 +2199,7 @@ choose_good_exit_server(uint8_t purpose,
         return rendezvous_node;
       }
   }
-  log_warn(LD_BUG,"Unhandled purpose %d", purpose);
+  log_warn(LD_BUG,"Unhandled purpose %d", TO_CIRCUIT(circ)->purpose);
   tor_fragile_assert();
   return NULL;
 }
@@ -1941,6 +2229,8 @@ warn_if_last_router_excluded(origin_circuit_t *circ,
                (int)purpose,
                circuit_purpose_to_string(purpose));
       return;
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
     case CIRCUIT_PURPOSE_C_GENERAL:
       if (circ->build_state->is_internal)
         return;
@@ -1994,9 +2284,15 @@ warn_if_last_router_excluded(origin_circuit_t *circ,
 
 /** Decide a suitable length for circ's cpath, and pick an exit
  * router (or use <b>exit</b> if provided). Store these in the
- * cpath. Return 0 if ok, -1 if circuit should be closed. */
-static int
-onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
+ * cpath.
+ *
+ * If <b>is_hs_v3_rp_circuit</b> is set, then this exit should be suitable to
+ * be used as an HS v3 rendezvous point.
+ *
+ * Return 0 if ok, -1 if circuit should be closed. */
+STATIC int
+onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
+                      int is_hs_v3_rp_circuit)
 {
   cpath_build_state_t *state = circ->build_state;
 
@@ -2018,15 +2314,24 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
              extend_info_describe(exit_ei));
     exit_ei = extend_info_dup(exit_ei);
   } else { /* we have to decide one */
+    router_crn_flags_t flags = CRN_NEED_DESC;
+    if (state->need_uptime)
+      flags |= CRN_NEED_UPTIME;
+    if (state->need_capacity)
+      flags |= CRN_NEED_CAPACITY;
+    if (is_hs_v3_rp_circuit)
+      flags |= CRN_RENDEZVOUS_V3;
+    if (state->onehop_tunnel)
+      flags |= CRN_DIRECT_CONN;
     const node_t *node =
-      choose_good_exit_server(circ->base_.purpose, state->need_uptime,
-                              state->need_capacity, state->is_internal);
+      choose_good_exit_server(circ, flags, state->is_internal);
     if (!node) {
       log_warn(LD_CIRC,"Failed to choose an exit server");
       return -1;
     }
-    exit_ei = extend_info_from_node(node, 0);
-    tor_assert(exit_ei);
+    exit_ei = extend_info_from_node(node, state->onehop_tunnel);
+    if (BUG(exit_ei == NULL))
+      return -1;
   }
   state->chosen_exit = exit_ei;
   return 0;
@@ -2081,9 +2386,13 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
 
 /** Return the number of routers in <b>routers</b> that are currently up
  * and available for building circuits through.
+ *
+ * (Note that this function may overcount or undercount, if we have
+ * descriptors that are not the type we would prefer to use for some
+ * particular router. See bug #25885.)
  */
-static int
-count_acceptable_nodes(smartlist_t *nodes)
+MOCK_IMPL(STATIC int,
+count_acceptable_nodes, (smartlist_t *nodes))
 {
   int num=0;
 
@@ -2094,14 +2403,10 @@ count_acceptable_nodes(smartlist_t *nodes)
     if (! node->is_running)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not running.",i);
       continue;
-    /* XXX This clause makes us count incorrectly: if AllowInvalidRouters
-     * allows this node in some places, then we're getting an inaccurate
-     * count. For now, be conservative and don't count it. But later we
-     * should try to be smarter. */
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
-    if (! node_has_descriptor(node))
+    if (! node_has_any_descriptor(node))
       continue;
     /* The node has a descriptor, so we can just check the ntor key directly */
     if (!node_has_curve25519_onion_key(node))
@@ -2131,6 +2436,190 @@ onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop)
   }
 }
 
+#ifdef TOR_UNIT_TESTS
+
+/** Unittest helper function: Count number of hops in cpath linked list. */
+unsigned int
+cpath_get_n_hops(crypt_path_t **head_ptr)
+{
+  unsigned int n_hops = 0;
+  crypt_path_t *tmp;
+
+  if (!*head_ptr) {
+    return 0;
+  }
+
+  tmp = *head_ptr;
+  do {
+    n_hops++;
+    tmp = tmp->next;
+  } while (tmp != *head_ptr);
+
+  return n_hops;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Build the exclude list for vanguard circuits.
+ *
+ * For vanguard circuits we exclude all the already chosen nodes (including the
+ * exit) from being middle hops to prevent the creation of A - B - A subpaths.
+ * We also allow the 4th hop to be the same as the guard node so as to not leak
+ * guard information to RP/IP/HSDirs.
+ *
+ * For vanguard circuits, we don't apply any subnet or family restrictions.
+ * This is to avoid impossible-to-build circuit paths, or just situations where
+ * our earlier guards prevent us from using most of our later ones.
+ *
+ * The alternative is building the circuit in reverse. Reverse calls to
+ * onion_extend_cpath() (ie: select outer hops first) would then have the
+ * property that you don't gain information about inner hops by observing
+ * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487
+ * for this.
+ *
+ * (Note further that we still exclude the exit to prevent A - B - A
+ * at the end of the path. */
+static smartlist_t *
+build_vanguard_middle_exclude_list(uint8_t purpose,
+                                   cpath_build_state_t *state,
+                                   crypt_path_t *head,
+                                   int cur_len)
+{
+  smartlist_t *excluded;
+  const node_t *r;
+  crypt_path_t *cpath;
+  int i;
+
+  (void) purpose;
+
+  excluded = smartlist_new();
+
+  /* Add the exit to the exclude list (note that the exit/last hop is always
+   * chosen first in circuit_establish_circuit()). */
+  if ((r = build_state_get_exit_node(state))) {
+    smartlist_add(excluded, (node_t*)r);
+  }
+
+  /* If we are picking the 4th hop, allow that node to be the guard too.
+   * This prevents us from avoiding the Guard for those hops, which
+   * gives the adversary information about our guard if they control
+   * the RP, IP, or HSDIR. We don't do this check based on purpose
+   * because we also want to allow HS_VANGUARDS pre-build circuits
+   * to use the guard for that last hop.
+   */
+  if (cur_len == DEFAULT_ROUTE_LEN+1) {
+    /* Skip the first hop for the exclude list below */
+    head = head->next;
+    cur_len--;
+  }
+
+  for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) {
+    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
+      smartlist_add(excluded, (node_t*)r);
+    }
+  }
+
+  return excluded;
+}
+
+/**
+ * Build a list of nodes to exclude from the choice of this middle
+ * hop, based on already chosen nodes.
+ */
+static smartlist_t *
+build_middle_exclude_list(uint8_t purpose,
+                          cpath_build_state_t *state,
+                          crypt_path_t *head,
+                          int cur_len)
+{
+  smartlist_t *excluded;
+  const node_t *r;
+  crypt_path_t *cpath;
+  int i;
+
+  /** Vanguard circuits have their own path selection rules */
+  if (circuit_should_use_vanguards(purpose)) {
+    return build_vanguard_middle_exclude_list(purpose, state, head, cur_len);
+  }
+
+  excluded = smartlist_new();
+
+  /* For non-vanguard circuits, add the exit and its family to the exclude list
+   * (note that the exit/last hop is always chosen first in
+   * circuit_establish_circuit()). */
+  if ((r = build_state_get_exit_node(state))) {
+    nodelist_add_node_and_family(excluded, r);
+  }
+
+  /* also exclude all other already chosen nodes and their family */
+  for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) {
+    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
+      nodelist_add_node_and_family(excluded, r);
+    }
+  }
+
+  return excluded;
+}
+
+/** Return true if we MUST use vanguards for picking this middle node. */
+static int
+middle_node_must_be_vanguard(const or_options_t *options,
+                             uint8_t purpose, int cur_len)
+{
+  /* If this is not a hidden service circuit, don't use vanguards */
+  if (!circuit_purpose_is_hidden_service(purpose)) {
+    return 0;
+  }
+
+  /* If we have sticky L2 nodes, and this is an L2 pick, use vanguards */
+  if (options->HSLayer2Nodes && cur_len == 1) {
+    return 1;
+  }
+
+  /* If we have sticky L3 nodes, and this is an L3 pick, use vanguards */
+  if (options->HSLayer3Nodes && cur_len == 2) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Pick a sticky vanguard middle node or return NULL if not found.
+ *  See doc of pick_restricted_middle_node() for argument details. */
+static const node_t *
+pick_vanguard_middle_node(const or_options_t *options,
+                          router_crn_flags_t flags, int cur_len,
+                          const smartlist_t *excluded)
+{
+  const routerset_t *vanguard_routerset = NULL;
+  const node_t *node = NULL;
+
+  /* Pick the right routerset based on the current hop */
+  if (cur_len == 1) {
+    vanguard_routerset = options->HSLayer2Nodes;
+  } else if (cur_len == 2) {
+    vanguard_routerset = options->HSLayer3Nodes;
+  } else {
+    /* guaranteed by middle_node_should_be_vanguard() */
+    tor_assert_nonfatal_unreached();
+    return NULL;
+  }
+
+  node = pick_restricted_middle_node(flags, vanguard_routerset,
+                                     options->ExcludeNodes, excluded,
+                                     cur_len+1);
+
+  if (!node) {
+    static ratelim_t pinned_warning_limit = RATELIM_INIT(300);
+    log_fn_ratelim(&pinned_warning_limit, LOG_WARN, LD_CIRC,
+            "Could not find a node that matches the configured "
+            "_HSLayer%dNodes set", cur_len+1);
+  }
+
+  return node;
+}
+
 /** A helper function used by onion_extend_cpath(). Use <b>purpose</b>
  * and <b>state</b> and the cpath <b>head</b> (currently populated only
  * to length <b>cur_len</b> to decide a suitable middle hop for a
@@ -2143,33 +2632,31 @@ choose_good_middle_server(uint8_t purpose,
                           crypt_path_t *head,
                           int cur_len)
 {
-  int i;
-  const node_t *r, *choice;
-  crypt_path_t *cpath;
+  const node_t *choice;
   smartlist_t *excluded;
   const or_options_t *options = get_options();
   router_crn_flags_t flags = CRN_NEED_DESC;
   tor_assert(CIRCUIT_PURPOSE_MIN_ <= purpose &&
              purpose <= CIRCUIT_PURPOSE_MAX_);
 
-  log_debug(LD_CIRC, "Contemplating intermediate hop %d: random choice.",
-            cur_len);
-  excluded = smartlist_new();
-  if ((r = build_state_get_exit_node(state))) {
-    nodelist_add_node_and_family(excluded, r);
-  }
-  for (i = 0, cpath = head; i < cur_len; ++i, cpath=cpath->next) {
-    if ((r = node_get_by_id(cpath->extend_info->identity_digest))) {
-      nodelist_add_node_and_family(excluded, r);
-    }
-  }
+  log_debug(LD_CIRC, "Contemplating intermediate hop #%d: random choice.",
+            cur_len+1);
+
+  excluded = build_middle_exclude_list(purpose, state, head, cur_len);
 
   if (state->need_uptime)
     flags |= CRN_NEED_UPTIME;
   if (state->need_capacity)
     flags |= CRN_NEED_CAPACITY;
-  if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE)
-    flags |= CRN_ALLOW_INVALID;
+
+  /** If a hidden service circuit wants a specific middle node, pin it. */
+  if (middle_node_must_be_vanguard(options, purpose, cur_len)) {
+    log_debug(LD_GENERAL, "Picking a sticky node (cur_len = %d)", cur_len);
+    choice = pick_vanguard_middle_node(options, flags, cur_len, excluded);
+    smartlist_free(excluded);
+    return choice;
+  }
+
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
   smartlist_free(excluded);
   return choice;
@@ -2180,11 +2667,13 @@ choose_good_middle_server(uint8_t purpose,
  * router (if we're an OR), and respect firewall settings; if we're
  * configured to use entry guards, return one.
  *
- * If <b>state</b> is NULL, we're choosing a router to serve as an entry
- * guard, not for any particular circuit.
+ * Set *<b>guard_state_out</b> to information about the guard that
+ * we're selecting, which we'll use later to remember whether the
+ * guard worked or not.
  */
 const node_t *
-choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
+choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
+                         circuit_guard_state_t **guard_state_out)
 {
   const node_t *choice;
   smartlist_t *excluded;
@@ -2195,11 +2684,17 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
                               CRN_DIRECT_CONN);
   const node_t *node;
 
+  /* Once we used this function to select a node to be a guard.  We had
+   * 'state == NULL' be the signal for that.  But we don't do that any more.
+   */
+  tor_assert_nonfatal(state);
+
   if (state && options->UseEntryGuards &&
       (purpose != CIRCUIT_PURPOSE_TESTING || options->BridgeRelay)) {
     /* This request is for an entry server to use for a regular circuit,
      * and we use entry guard nodes.  Just return one of the guard nodes.  */
-    return choose_random_entry(state);
+    tor_assert(guard_state_out);
+    return guards_choose_guard(state, purpose, guard_state_out);
   }
 
   excluded = smartlist_new();
@@ -2209,25 +2704,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
      * family. */
     nodelist_add_node_and_family(excluded, node);
   }
-  /* and exclude current entry guards and their families,
-   * unless we're in a test network, and excluding guards
-   * would exclude all nodes (i.e. we're in an incredibly small tor network,
-   * or we're using TestingAuthVoteGuard *).
-   * This is an incomplete fix, but is no worse than the previous behaviour,
-   * and only applies to minimal, testing tor networks
-   * (so it's no less secure) */
-  /*XXXX++ use the using_as_guard flag to accomplish this.*/
-  if (options->UseEntryGuards
-      && (!options->TestingTorNetwork ||
-         smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards())
-     )) {
-    SMARTLIST_FOREACH(get_entry_guards(), const entry_guard_t *, entry,
-      {
-        if ((node = node_get_by_id(entry->identity))) {
-          nodelist_add_node_and_family(excluded, node);
-        }
-      });
-  }
 
   if (state) {
     if (state->need_uptime)
@@ -2235,8 +2711,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
     if (state->need_capacity)
       flags |= CRN_NEED_CAPACITY;
   }
-  if (options->AllowInvalid_ & ALLOW_INVALID_ENTRY)
-    flags |= CRN_ALLOW_INVALID;
 
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
   smartlist_free(excluded);
@@ -2263,7 +2737,7 @@ onion_next_hop_in_cpath(crypt_path_t *cpath)
  * Return 1 if the path is complete, 0 if we successfully added a hop,
  * and -1 on error.
  */
-static int
+STATIC int
 onion_extend_cpath(origin_circuit_t *circ)
 {
   uint8_t purpose = circ->base_.purpose;
@@ -2283,7 +2757,8 @@ onion_extend_cpath(origin_circuit_t *circ)
   if (cur_len == state->desired_path_len - 1) { /* Picking last node */
     info = extend_info_dup(state->chosen_exit);
   } else if (cur_len == 0) { /* picking first node */
-    const node_t *r = choose_good_entry_server(purpose, state);
+    const node_t *r = choose_good_entry_server(purpose, state,
+                                               &circ->guard_state);
     if (r) {
       /* If we're a client, use the preferred address rather than the
          primary address, for potentially connecting to an IPv6 OR
@@ -2291,24 +2766,24 @@ onion_extend_cpath(origin_circuit_t *circ)
       int client = (server_mode(get_options()) == 0);
       info = extend_info_from_node(r, client);
       /* Clients can fail to find an allowed address */
-      tor_assert(info || client);
+      tor_assert_nonfatal(info || client);
     }
   } else {
     const node_t *r =
       choose_good_middle_server(purpose, state, circ->cpath, cur_len);
     if (r) {
       info = extend_info_from_node(r, 0);
-      tor_assert(info);
+      tor_assert_nonfatal(info);
     }
   }
 
   if (!info) {
-    log_warn(LD_CIRC,"Failed to find node for hop %d of our path. Discarding "
-             "this circuit.", cur_len);
+    log_warn(LD_CIRC,"Failed to find node for hop #%d of our path. Discarding "
+             "this circuit.", cur_len+1);
     return -1;
   }
 
-  log_debug(LD_CIRC,"Chose router %s for hop %d (exit is %s)",
+  log_debug(LD_CIRC,"Chose router %s for hop #%d (exit is %s)",
             extend_info_describe(info),
             cur_len+1, build_state_get_exit_nickname(state));
 
@@ -2320,7 +2795,7 @@ onion_extend_cpath(origin_circuit_t *circ)
 /** Create a new hop, annotate it with information about its
  * corresponding router <b>choice</b>, and append it to the
  * end of the cpath <b>head_ptr</b>. */
-static int
+STATIC int
 onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
 {
   crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
@@ -2341,19 +2816,23 @@ onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
 
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
-extend_info_new(const char *nickname, const char *digest,
+extend_info_new(const char *nickname,
+                const char *rsa_id_digest,
+                const ed25519_public_key_t *ed_id,
                 crypto_pk_t *onion_key,
-                const curve25519_public_key_t *curve25519_key,
+                const curve25519_public_key_t *ntor_key,
                 const tor_addr_t *addr, uint16_t port)
 {
   extend_info_t *info = tor_malloc_zero(sizeof(extend_info_t));
-  memcpy(info->identity_digest, digest, DIGEST_LEN);
+  memcpy(info->identity_digest, rsa_id_digest, DIGEST_LEN);
+  if (ed_id && !ed25519_public_key_is_zero(ed_id))
+    memcpy(&info->ed_identity, ed_id, sizeof(ed25519_public_key_t));
   if (nickname)
     strlcpy(info->nickname, nickname, sizeof(info->nickname));
   if (onion_key)
     info->onion_key = crypto_pk_dup_key(onion_key);
-  if (curve25519_key)
-    memcpy(&info->curve25519_onion_key, curve25519_key,
+  if (ntor_key)
+    memcpy(&info->curve25519_onion_key, ntor_key,
            sizeof(curve25519_public_key_t));
   tor_addr_copy(&info->addr, addr);
   info->port = port;
@@ -2365,9 +2844,10 @@ extend_info_new(const char *nickname, const char *digest,
  * of the node (i.e. its IPv4 address) unless
  * <b>for_direct_connect</b> is true, in which case the preferred
  * address is used instead. May return NULL if there is not enough
- * info about <b>node</b> to extend to it--for example, if there is no
- * routerinfo_t or microdesc_t, or if for_direct_connect is true and none of
- * the node's addresses are allowed by tor's firewall and IP version config.
+ * info about <b>node</b> to extend to it--for example, if the preferred
+ * routerinfo_t or microdesc_t is missing, or if for_direct_connect is
+ * true and none of the node's addresses is allowed by tor's firewall
+ * and IP version config.
  **/
 extend_info_t *
 extend_info_from_node(const node_t *node, int for_direct_connect)
@@ -2375,17 +2855,17 @@ extend_info_from_node(const node_t *node, int for_direct_connect)
   tor_addr_port_t ap;
   int valid_addr = 0;
 
-  if (node->ri == NULL && (node->rs == NULL || node->md == NULL))
+  if (!node_has_preferred_descriptor(node, for_direct_connect)) {
     return NULL;
+  }
 
-  /* Choose a preferred address first, but fall back to an allowed address.
-   * choose_address returns 1 on success, but get_prim_orport returns 0. */
+  /* Choose a preferred address first, but fall back to an allowed address. */
   if (for_direct_connect)
-    valid_addr = fascist_firewall_choose_address_node(node,
-                                                      FIREWALL_OR_CONNECTION,
-                                                      0, &ap);
-  else
-    valid_addr = !node_get_prim_orport(node, &ap);
+    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &ap);
+  else {
+    node_get_prim_orport(node, &ap);
+  }
+  valid_addr = tor_addr_port_is_valid_ap(&ap, 0);
 
   if (valid_addr)
     log_debug(LD_CIRC, "using %s for %s",
@@ -2403,27 +2883,46 @@ extend_info_from_node(const node_t *node, int for_direct_connect)
     return NULL;
   }
 
+  const ed25519_public_key_t *ed_pubkey = NULL;
+
+  /* Don't send the ed25519 pubkey unless the target node actually supports
+   * authenticating with it. */
+  if (node_supports_ed25519_link_authentication(node, 0)) {
+    log_info(LD_CIRC, "Including Ed25519 ID for %s", node_describe(node));
+    ed_pubkey = node_get_ed25519_id(node);
+  } else if (node_get_ed25519_id(node)) {
+    log_info(LD_CIRC, "Not including the ed25519 ID for %s, since it won't "
+             "be able to authenticate it.",
+             node_describe(node));
+  }
+
+  /* Retrieve the curve25519 pubkey. */
+  const curve25519_public_key_t *curve_pubkey =
+    node_get_curve25519_onion_key(node);
+
   if (valid_addr && node->ri)
     return extend_info_new(node->ri->nickname,
-                             node->identity,
-                             node->ri->onion_pkey,
-                             node->ri->onion_curve25519_pkey,
-                             &ap.addr,
-                             ap.port);
+                           node->identity,
+                           ed_pubkey,
+                           node->ri->onion_pkey,
+                           curve_pubkey,
+                           &ap.addr,
+                           ap.port);
   else if (valid_addr && node->rs && node->md)
     return extend_info_new(node->rs->nickname,
-                             node->identity,
-                             node->md->onion_pkey,
-                             node->md->onion_curve25519_pkey,
-                             &ap.addr,
-                             ap.port);
+                           node->identity,
+                           ed_pubkey,
+                           node->md->onion_pkey,
+                           curve_pubkey,
+                           &ap.addr,
+                           ap.port);
   else
     return NULL;
 }
 
 /** Release storage held by an extend_info_t struct. */
 void
-extend_info_free(extend_info_t *info)
+extend_info_free_(extend_info_t *info)
 {
   if (!info)
     return;
@@ -2447,8 +2946,8 @@ extend_info_dup(extend_info_t *info)
   return newinfo;
 }
 
-/** Return the routerinfo_t for the chosen exit router in <b>state</b>.
- * If there is no chosen exit, or if we don't know the routerinfo_t for
+/** Return the node_t for the chosen exit router in <b>state</b>.
+ * If there is no chosen exit, or if we don't know the node_t for
  * the chosen exit, return NULL.
  */
 const node_t *
@@ -2459,6 +2958,17 @@ build_state_get_exit_node(cpath_build_state_t *state)
   return node_get_by_id(state->chosen_exit->identity_digest);
 }
 
+/** Return the RSA ID digest for the chosen exit router in <b>state</b>.
+ * If there is no chosen exit, return NULL.
+ */
+const uint8_t *
+build_state_get_exit_rsa_id(cpath_build_state_t *state)
+{
+  if (!state || !state->chosen_exit)
+    return NULL;
+  return (const uint8_t *) state->chosen_exit->identity_digest;
+}
+
 /** Return the nickname for the chosen exit router in <b>state</b>. If
  * there is no chosen exit, or if we don't know the routerinfo_t for the
  * chosen exit, return NULL.
@@ -2551,3 +3061,26 @@ extend_info_has_preferred_onion_key(const extend_info_t* ei)
   return extend_info_supports_ntor(ei);
 }
 
+/** Find the circuits that are waiting to find out whether their guards are
+ * usable, and if any are ready to become usable, mark them open and try
+ * attaching streams as appropriate. */
+void
+circuit_upgrade_circuits_from_guard_wait(void)
+{
+  smartlist_t *to_upgrade =
+    circuit_find_circuits_to_upgrade_from_guard_wait();
+
+  if (to_upgrade == NULL)
+    return;
+
+  log_info(LD_GUARD, "Upgrading %d circuits from 'waiting for better guard' "
+           "to 'open'.", smartlist_len(to_upgrade));
+
+  SMARTLIST_FOREACH_BEGIN(to_upgrade, origin_circuit_t *, circ) {
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+    circuit_has_opened(circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(to_upgrade);
+}
+
diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h
index 1244601f71..e3d30c1104 100644
--- a/src/or/circuitbuild.h
+++ b/src/or/circuitbuild.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,67 +12,91 @@
 #ifndef TOR_CIRCUITBUILD_H
 #define TOR_CIRCUITBUILD_H
 
+int route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei);
 char *circuit_list_path(origin_circuit_t *circ, int verbose);
 char *circuit_list_path_for_controller(origin_circuit_t *circ);
 void circuit_log_path(int severity, unsigned int domain,
                       origin_circuit_t *circ);
-void circuit_rep_hist_note_result(origin_circuit_t *circ);
 origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags);
 origin_circuit_t *circuit_establish_circuit(uint8_t purpose,
                                             extend_info_t *exit,
                                             int flags);
+struct circuit_guard_state_t *origin_circuit_get_guard_state(
+                                            origin_circuit_t *circ);
 int circuit_handle_first_hop(origin_circuit_t *circ);
 void circuit_n_chan_done(channel_t *chan, int status,
                          int close_origin_circuits);
 int inform_testing_reachability(void);
-int circuit_timeout_want_to_count_circ(origin_circuit_t *circ);
+int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
-void circuit_note_clock_jumped(int seconds_elapsed);
+void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
 int circuit_extend(cell_t *cell, circuit_t *circ);
-int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data,
-                              int reverse);
+int circuit_init_cpath_crypto(crypt_path_t *cpath,
+                              const char *key_data, size_t key_data_len,
+                              int reverse, int is_hs_v3);
 struct created_cell_t;
 int circuit_finish_handshake(origin_circuit_t *circ,
                              const struct created_cell_t *created_cell);
-int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer,
-                      int reason);
+int circuit_truncated(origin_circuit_t *circ, int reason);
 int onionskin_answer(or_circuit_t *circ,
                      const struct created_cell_t *created_cell,
-                     const char *keys,
+                     const char *keys, size_t keys_len,
                      const uint8_t *rend_circ_nonce);
-int circuit_all_predicted_ports_handled(time_t now, int *need_uptime,
-                                        int *need_capacity);
+MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
+                                                     int *need_uptime,
+                                                     int *need_capacity));
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
 void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
-extend_info_t *extend_info_new(const char *nickname, const char *digest,
+extend_info_t *extend_info_new(const char *nickname,
+                               const char *rsa_id_digest,
+                               const ed25519_public_key_t *ed_id,
                                crypto_pk_t *onion_key,
-                               const curve25519_public_key_t *curve25519_key,
+                               const curve25519_public_key_t *ntor_key,
                                const tor_addr_t *addr, uint16_t port);
 extend_info_t *extend_info_from_node(const node_t *r, int for_direct_connect);
 extend_info_t *extend_info_dup(extend_info_t *info);
-void extend_info_free(extend_info_t *info);
+void extend_info_free_(extend_info_t *info);
+#define extend_info_free(info) \
+  FREE_AND_NULL(extend_info_t, extend_info_free_, (info))
 int extend_info_addr_is_allowed(const tor_addr_t *addr);
 int extend_info_supports_tap(const extend_info_t* ei);
 int extend_info_supports_ntor(const extend_info_t* ei);
 int circuit_can_use_tap(const origin_circuit_t *circ);
 int circuit_has_usable_onion_key(const origin_circuit_t *circ);
 int extend_info_has_preferred_onion_key(const extend_info_t* ei);
+const uint8_t *build_state_get_exit_rsa_id(cpath_build_state_t *state);
 const node_t *build_state_get_exit_node(cpath_build_state_t *state);
 const char *build_state_get_exit_nickname(cpath_build_state_t *state);
 
+struct circuit_guard_state_t;
+
 const node_t *choose_good_entry_server(uint8_t purpose,
-                                       cpath_build_state_t *state);
+                           cpath_build_state_t *state,
+                           struct circuit_guard_state_t **guard_state_out);
+void circuit_upgrade_circuits_from_guard_wait(void);
 
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
+STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+                         smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
+
+STATIC int onion_extend_cpath(origin_circuit_t *circ);
+
+STATIC int
+onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
+                      int is_hs_v3_rp_circuit);
+
 #if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS)
 STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags,
                                                   const or_options_t *options);
-#endif
+unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
+
+#endif /* defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* defined(CIRCUITBUILD_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CIRCUITBUILD_H) */
 
diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c
index 8cfdd3bb94..fdb039ef0d 100644
--- a/src/or/circuitlist.c
+++ b/src/or/circuitlist.c
@@ -1,15 +1,58 @@
 /* Copyright 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file circuitlist.c
  *
- * \brief Manage the global circuit list, and looking up circuits within it.
+ * \brief Manage global structures that list and index circuits, and
+ *   look up circuits within them.
+ *
+ * One of the most frequent operations in Tor occurs every time that
+ * a relay cell arrives on a channel.  When that happens, we need to
+ * find which circuit it is associated with, based on the channel and the
+ * circuit ID in the relay cell.
+ *
+ * To handle that, we maintain a global list of circuits, and a hashtable
+ * mapping [channel,circID] pairs to circuits.  Circuits are added to and
+ * removed from this mapping using circuit_set_p_circid_chan() and
+ * circuit_set_n_circid_chan().  To look up a circuit from this map, most
+ * callers should use circuit_get_by_circid_channel(), though
+ * circuit_get_by_circid_channel_even_if_marked() is appropriate under some
+ * circumstances.
+ *
+ * We also need to allow for the possibility that we have blocked use of a
+ * circuit ID (because we are waiting to send a DESTROY cell), but the
+ * circuit is not there any more.  For that case, we allow placeholder
+ * entries in the table, using channel_mark_circid_unusable().
+ *
+ * To efficiently handle a channel that has just opened, we also maintain a
+ * list of the circuits waiting for channels, so we can attach them as
+ * needed without iterating through the whole list of circuits, using
+ * circuit_get_all_pending_on_channel().
+ *
+ * In this module, we also handle the list of circuits that have been
+ * marked for close elsewhere, and close them as needed.  (We use this
+ * "mark now, close later" pattern here and elsewhere to avoid
+ * unpredictable recursion if we closed every circuit immediately upon
+ * realizing it needed to close.)  See circuit_mark_for_close() for the
+ * mark function, and circuit_close_all_marked() for the close function.
+ *
+ * For hidden services, we need to be able to look up introduction point
+ * circuits and rendezvous circuits by cookie, key, etc.  These are
+ * currently handled with linear searches in
+ * circuit_get_ready_rend_circuit_by_rend_data(),
+ * circuit_get_next_by_pk_and_purpose(), and with hash lookups in
+ * circuit_get_rendezvous() and circuit_get_intro_point().
+ *
+ * This module is also the entry point for our out-of-memory handler
+ * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
+#include "torint.h"  /* TOR_PRIuSZ */
+
 #include "or.h"
 #include "channel.h"
 #include "channeltls.h"
@@ -23,18 +66,29 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "entrynodes.h"
 #include "main.h"
+#include "hs_circuit.h"
+#include "hs_circuitmap.h"
+#include "hs_ident.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "onion.h"
 #include "onion_fast.h"
 #include "policies.h"
 #include "relay.h"
+#include "relay_crypto.h"
 #include "rendclient.h"
 #include "rendcommon.h"
 #include "rephist.h"
 #include "routerlist.h"
 #include "routerset.h"
+#include "channelpadding.h"
+#include "compress_lzma.h"
+#include "compress_zlib.h"
+#include "compress_zstd.h"
 
 #include "ht.h"
 
@@ -43,21 +97,34 @@
 /** A global list of all circuits at this hop. */
 static smartlist_t *global_circuitlist = NULL;
 
+/** A global list of all origin circuits. Every element of this is also
+ * an element of global_circuitlist. */
+static smartlist_t *global_origin_circuit_list = NULL;
+
 /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */
 static smartlist_t *circuits_pending_chans = NULL;
 
+/** List of all the (origin) circuits whose state is
+ * CIRCUIT_STATE_GUARD_WAIT. */
+static smartlist_t *circuits_pending_other_guards = NULL;
+
 /** A list of all the circuits that have been marked with
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
 static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
-//static void circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ,
-//                                   const uint8_t *token);
-static void circuit_clear_rend_token(or_circuit_t *circ);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
 
+/**
+ * A cached value of the current state of the origin circuit list.  Has the
+ * value 1 if we saw any opened circuits recently (since the last call to
+ * circuit_any_opened_circuits(), which gets called around once a second by
+ * circuit_expire_building). 0 otherwise.
+ */
+static int any_opened_circs_cached_val = 0;
+
 /********* END VARIABLES ************/
 
 /** A map from channel and circuit ID to circuit.  (Lookup performance is
@@ -310,8 +377,8 @@ channel_note_destroy_pending(channel_t *chan, circid_t id)
 
 /** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with
  * circuit ID <b>id</b> -- typically, because it has been sent. */
-MOCK_IMPL(void, channel_note_destroy_not_pending,
-          (channel_t *chan, circid_t id))
+MOCK_IMPL(void,
+channel_note_destroy_not_pending,(channel_t *chan, circid_t id))
 {
   circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan);
   if (circ) {
@@ -343,9 +410,6 @@ circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id,
   circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan);
 
   if (chan) {
-    tor_assert(bool_eq(or_circ->p_chan_cells.n,
-                       or_circ->next_active_on_p_chan));
-
     chan->timestamp_last_had_circuits = approx_time();
   }
 
@@ -368,8 +432,6 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan);
 
   if (chan) {
-    tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan));
-
     chan->timestamp_last_had_circuits = approx_time();
   }
 
@@ -387,8 +449,10 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   tor_assert(circ);
   if (state == circ->state)
     return;
-  if (!circuits_pending_chans)
+  if (PREDICT_UNLIKELY(!circuits_pending_chans))
     circuits_pending_chans = smartlist_new();
+  if (PREDICT_UNLIKELY(!circuits_pending_other_guards))
+    circuits_pending_other_guards = smartlist_new();
   if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
     /* remove from waiting-circuit list. */
     smartlist_remove(circuits_pending_chans, circ);
@@ -397,7 +461,13 @@ circuit_set_state(circuit_t *circ, uint8_t state)
     /* add to waiting-circuit list. */
     smartlist_add(circuits_pending_chans, circ);
   }
-  if (state == CIRCUIT_STATE_OPEN)
+  if (circ->state == CIRCUIT_STATE_GUARD_WAIT) {
+    smartlist_remove(circuits_pending_other_guards, circ);
+  }
+  if (state == CIRCUIT_STATE_GUARD_WAIT) {
+    smartlist_add(circuits_pending_other_guards, circ);
+  }
+  if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
 }
@@ -446,13 +516,45 @@ circuit_count_pending_on_channel(channel_t *chan)
   circuit_get_all_pending_on_channel(sl, chan);
   cnt = smartlist_len(sl);
   smartlist_free(sl);
-  log_debug(LD_CIRC,"or_conn to %s at %s, %d pending circs",
-            chan->nickname ? chan->nickname : "NULL",
+  log_debug(LD_CIRC,"or_conn to %s, %d pending circs",
             channel_get_canonical_remote_descr(chan),
             cnt);
   return cnt;
 }
 
+/** Remove <b>origin_circ</b> from the global list of origin circuits.
+ * Called when we are freeing a circuit.
+ */
+static void
+circuit_remove_from_origin_circuit_list(origin_circuit_t *origin_circ)
+{
+  int origin_idx = origin_circ->global_origin_circuit_list_idx;
+  if (origin_idx < 0)
+    return;
+  origin_circuit_t *c2;
+  tor_assert(origin_idx <= smartlist_len(global_origin_circuit_list));
+  c2 = smartlist_get(global_origin_circuit_list, origin_idx);
+  tor_assert(origin_circ == c2);
+  smartlist_del(global_origin_circuit_list, origin_idx);
+  if (origin_idx < smartlist_len(global_origin_circuit_list)) {
+    origin_circuit_t *replacement =
+      smartlist_get(global_origin_circuit_list, origin_idx);
+    replacement->global_origin_circuit_list_idx = origin_idx;
+  }
+  origin_circ->global_origin_circuit_list_idx = -1;
+}
+
+/** Add <b>origin_circ</b> to the global list of origin circuits. Called
+ * when creating the circuit. */
+static void
+circuit_add_to_origin_circuit_list(origin_circuit_t *origin_circ)
+{
+  tor_assert(origin_circ->global_origin_circuit_list_idx == -1);
+  smartlist_t *lst = circuit_get_global_origin_circuit_list();
+  smartlist_add(lst, origin_circ);
+  origin_circ->global_origin_circuit_list_idx = smartlist_len(lst) - 1;
+}
+
 /** Detach from the global circuit list, and deallocate, all
  * circuits that have been marked for close.
  */
@@ -475,6 +577,11 @@ circuit_close_all_marked(void)
     }
     circ->global_circuitlist_idx = -1;
 
+    /* Remove it from the origin circuit list, if appropriate. */
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      circuit_remove_from_origin_circuit_list(TO_ORIGIN_CIRCUIT(circ));
+    }
+
     circuit_about_to_free(circ);
     circuit_free(circ);
   } SMARTLIST_FOREACH_END(circ);
@@ -482,7 +589,7 @@ circuit_close_all_marked(void)
   smartlist_clear(circuits_pending_close);
 }
 
-/** Return the head of the global linked list of circuits. */
+/** Return a pointer to the global list of circuits. */
 MOCK_IMPL(smartlist_t *,
 circuit_get_global_list,(void))
 {
@@ -491,6 +598,65 @@ circuit_get_global_list,(void))
   return global_circuitlist;
 }
 
+/** Return a pointer to the global list of origin circuits. */
+smartlist_t *
+circuit_get_global_origin_circuit_list(void)
+{
+  if (NULL == global_origin_circuit_list)
+    global_origin_circuit_list = smartlist_new();
+  return global_origin_circuit_list;
+}
+
+/**
+ * Return true if we have any opened general-purpose 3 hop
+ * origin circuits.
+ *
+ * The result from this function is cached for use by
+ * circuit_any_opened_circuits_cached().
+ */
+int
+circuit_any_opened_circuits(void)
+{
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_origin_circuit_list(),
+          const origin_circuit_t *, next_circ) {
+    if (!TO_CIRCUIT(next_circ)->marked_for_close &&
+        next_circ->has_opened &&
+        TO_CIRCUIT(next_circ)->state == CIRCUIT_STATE_OPEN &&
+        TO_CIRCUIT(next_circ)->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT &&
+        next_circ->build_state &&
+        next_circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN) {
+      circuit_cache_opened_circuit_state(1);
+      return 1;
+    }
+  } SMARTLIST_FOREACH_END(next_circ);
+
+  circuit_cache_opened_circuit_state(0);
+  return 0;
+}
+
+/**
+ * Cache the "any circuits opened" state, as specified in param
+ * circuits_are_opened. This is a helper function to update
+ * the circuit opened status whenever we happen to look at the
+ * circuit list.
+ */
+void
+circuit_cache_opened_circuit_state(int circuits_are_opened)
+{
+  any_opened_circs_cached_val = circuits_are_opened;
+}
+
+/**
+ * Return true if there were any opened circuits since the last call to
+ * circuit_any_opened_circuits(), or since circuit_expire_building() last
+ * ran (it runs roughly once per second).
+ */
+int
+circuit_any_opened_circuits_cached(void)
+{
+  return any_opened_circs_cached_val;
+}
+
 /** Function to make circ-\>state human-readable */
 const char *
 circuit_state_to_string(int state)
@@ -500,6 +666,8 @@ circuit_state_to_string(int state)
     case CIRCUIT_STATE_BUILDING: return "doing handshakes";
     case CIRCUIT_STATE_ONIONSKIN_PENDING: return "processing the onion";
     case CIRCUIT_STATE_CHAN_WAIT: return "connecting to server";
+    case CIRCUIT_STATE_GUARD_WAIT: return "waiting to see how other "
+      "guards perform";
     case CIRCUIT_STATE_OPEN: return "open";
     default:
       log_warn(LD_BUG, "Unknown circuit state %d", state);
@@ -523,6 +691,10 @@ circuit_purpose_to_controller_string(uint8_t purpose)
 
     case CIRCUIT_PURPOSE_C_GENERAL:
       return "GENERAL";
+
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+      return "HS_CLIENT_HSDIR";
+
     case CIRCUIT_PURPOSE_C_INTRODUCING:
     case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
     case CIRCUIT_PURPOSE_C_INTRODUCE_ACKED:
@@ -534,6 +706,9 @@ circuit_purpose_to_controller_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_C_REND_JOINED:
       return "HS_CLIENT_REND";
 
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+      return "HS_SERVICE_HSDIR";
+
     case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
     case CIRCUIT_PURPOSE_S_INTRO:
       return "HS_SERVICE_INTRO";
@@ -550,6 +725,8 @@ circuit_purpose_to_controller_string(uint8_t purpose)
       return "CONTROLLER";
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
       return "PATH_BIAS_TESTING";
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
+      return "HS_VANGUARDS";
 
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
@@ -578,6 +755,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_TESTING:
     case CIRCUIT_PURPOSE_CONTROLLER:
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return NULL;
 
     case CIRCUIT_PURPOSE_INTRO_POINT:
@@ -587,6 +765,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_REND_ESTABLISHED:
       return "OR_HS_R_JOINED";
 
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
     case CIRCUIT_PURPOSE_C_INTRODUCING:
       return "HSCI_CONNECTING";
     case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
@@ -603,6 +782,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_C_REND_JOINED:
       return "HSCR_JOINED";
 
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
     case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
       return "HSSI_CONNECTING";
     case CIRCUIT_PURPOSE_S_INTRO:
@@ -628,9 +808,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_INTRO_POINT:
       return "Acting as intro point";
     case CIRCUIT_PURPOSE_REND_POINT_WAITING:
-      return "Acting as rendevous (pending)";
+      return "Acting as rendezvous (pending)";
     case CIRCUIT_PURPOSE_REND_ESTABLISHED:
-      return "Acting as rendevous (established)";
+      return "Acting as rendezvous (established)";
     case CIRCUIT_PURPOSE_C_GENERAL:
       return "General-purpose client";
     case CIRCUIT_PURPOSE_C_INTRODUCING:
@@ -647,6 +827,9 @@ circuit_purpose_to_string(uint8_t purpose)
       return "Hidden service client: Pending rendezvous point (ack received)";
     case CIRCUIT_PURPOSE_C_REND_JOINED:
       return "Hidden service client: Active rendezvous point";
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+      return "Hidden service client: Fetching HS descriptor";
+
     case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT:
       return "Measuring circuit timeout";
 
@@ -658,6 +841,8 @@ circuit_purpose_to_string(uint8_t purpose)
       return "Hidden service: Connecting to rendezvous point";
     case CIRCUIT_PURPOSE_S_REND_JOINED:
       return "Hidden service: Active rendezvous point";
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+      return "Hidden service: Uploading HS descriptor";
 
     case CIRCUIT_PURPOSE_TESTING:
       return "Testing circuit";
@@ -668,6 +853,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
       return "Path-bias testing circuit";
 
+    case CIRCUIT_PURPOSE_HS_VANGUARDS:
+      return "Hidden service: Pre-built vanguard circuit";
+
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
       return buf;
@@ -709,6 +897,13 @@ init_circuit_base(circuit_t *circ)
   circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1;
 }
 
+/** If we haven't yet decided on a good timeout value for circuit
+ * building, we close idle circuits aggressively so we can get more
+ * data points. These are the default, min, and max consensus values */
+#define DFLT_IDLE_TIMEOUT_WHILE_LEARNING (3*60)
+#define MIN_IDLE_TIMEOUT_WHILE_LEARNING (10)
+#define MAX_IDLE_TIMEOUT_WHILE_LEARNING (1000*60)
+
 /** Allocate space for a new circuit, initializing with <b>p_circ_id</b>
  * and <b>p_conn</b>. Add it to the global circuit list.
  */
@@ -730,8 +925,55 @@ origin_circuit_new(void)
 
   init_circuit_base(TO_CIRCUIT(circ));
 
+  /* Add to origin-list. */
+  circ->global_origin_circuit_list_idx = -1;
+  circuit_add_to_origin_circuit_list(circ);
+
   circuit_build_times_update_last_circ(get_circuit_build_times_mutable());
 
+  if (! circuit_build_times_disabled(get_options()) &&
+      circuit_build_times_needs_circuits(get_circuit_build_times())) {
+    /* Circuits should be shorter lived if we need more of them
+     * for learning a good build timeout */
+    circ->circuit_idle_timeout =
+      networkstatus_get_param(NULL, "cbtlearntimeout",
+                              DFLT_IDLE_TIMEOUT_WHILE_LEARNING,
+                              MIN_IDLE_TIMEOUT_WHILE_LEARNING,
+                              MAX_IDLE_TIMEOUT_WHILE_LEARNING);
+  } else {
+    // This should always be larger than the current port prediction time
+    // remaining, or else we'll end up with the case where a circuit times out
+    // and another one is built, effectively doubling the timeout window.
+    //
+    // We also randomize it by up to 5% more (ie 5% of 0 to 3600 seconds,
+    // depending on how much circuit prediction time is remaining) so that
+    // we don't close a bunch of unused circuits all at the same time.
+    int prediction_time_remaining =
+      predicted_ports_prediction_time_remaining(time(NULL));
+    circ->circuit_idle_timeout = prediction_time_remaining+1+
+        crypto_rand_int(1+prediction_time_remaining/20);
+
+    if (circ->circuit_idle_timeout <= 0) {
+      log_warn(LD_BUG,
+               "Circuit chose a negative idle timeout of %d based on "
+               "%d seconds of predictive building remaining.",
+               circ->circuit_idle_timeout,
+               prediction_time_remaining);
+      circ->circuit_idle_timeout =
+          networkstatus_get_param(NULL, "cbtlearntimeout",
+                  DFLT_IDLE_TIMEOUT_WHILE_LEARNING,
+                  MIN_IDLE_TIMEOUT_WHILE_LEARNING,
+                  MAX_IDLE_TIMEOUT_WHILE_LEARNING);
+    }
+
+    log_info(LD_CIRC,
+              "Circuit " U64_FORMAT " chose an idle timeout of %d based on "
+              "%d seconds of predictive building remaining.",
+              U64_PRINTF_ARG(circ->global_identifier),
+              circ->circuit_idle_timeout,
+              prediction_time_remaining);
+  }
+
   return circ;
 }
 
@@ -772,21 +1014,42 @@ circuit_clear_testing_cell_stats(circuit_t *circ)
 /** Deallocate space associated with circ.
  */
 STATIC void
-circuit_free(circuit_t *circ)
+circuit_free_(circuit_t *circ)
 {
+  circid_t n_circ_id = 0;
   void *mem;
   size_t memlen;
   int should_free = 1;
   if (!circ)
     return;
 
+  /* We keep a copy of this so we can log its value before it gets unset. */
+  n_circ_id = circ->n_circ_id;
+
   circuit_clear_testing_cell_stats(circ);
 
+  /* Cleanup circuit from anything HS v3 related. We also do this when the
+   * circuit is closed. This is to avoid any code path that free registered
+   * circuits without closing them before. This needs to be done before the
+   * hs identifier is freed. */
+  hs_circ_cleanup(circ);
+
   if (CIRCUIT_IS_ORIGIN(circ)) {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
     mem = ocirc;
     memlen = sizeof(origin_circuit_t);
     tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC);
+
+    circuit_remove_from_origin_circuit_list(ocirc);
+
+    if (ocirc->half_streams) {
+      SMARTLIST_FOREACH_BEGIN(ocirc->half_streams, half_edge_t *,
+                              half_conn) {
+        half_edge_free(half_conn);
+      } SMARTLIST_FOREACH_END(half_conn);
+      smartlist_free(ocirc->half_streams);
+    }
+
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
         circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
@@ -794,11 +1057,22 @@ circuit_free(circuit_t *circ)
     }
     tor_free(ocirc->build_state);
 
+    /* Cancel before freeing, if we haven't already succeeded or failed. */
+    if (ocirc->guard_state) {
+      entry_guard_cancel(&ocirc->guard_state);
+    }
+    circuit_guard_state_free(ocirc->guard_state);
+
     circuit_clear_cpath(ocirc);
 
     crypto_pk_free(ocirc->intro_key);
     rend_data_free(ocirc->rend_data);
 
+    /* Finally, free the identifier of the circuit and nullify it so multiple
+     * cleanup will work. */
+    hs_ident_circuit_free(ocirc->hs_ident);
+    ocirc->hs_ident = NULL;
+
     tor_free(ocirc->dest_address);
     if (ocirc->socks_username) {
       memwipe(ocirc->socks_username, 0x12, ocirc->socks_username_len);
@@ -820,12 +1094,7 @@ circuit_free(circuit_t *circ)
 
     should_free = (ocirc->workqueue_entry == NULL);
 
-    crypto_cipher_free(ocirc->p_crypto);
-    crypto_digest_free(ocirc->p_digest);
-    crypto_cipher_free(ocirc->n_crypto);
-    crypto_digest_free(ocirc->n_digest);
-
-    circuit_clear_rend_token(ocirc);
+    relay_crypto_clear(&ocirc->crypto);
 
     if (ocirc->rend_splice) {
       or_circuit_t *other = ocirc->rend_splice;
@@ -862,6 +1131,11 @@ circuit_free(circuit_t *circ)
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
 
+  log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.",
+           n_circ_id,
+           CIRCUIT_IS_ORIGIN(circ) ?
+              TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);
@@ -913,7 +1187,7 @@ circuit_free_all(void)
       while (or_circ->resolving_streams) {
         edge_connection_t *next_conn;
         next_conn = or_circ->resolving_streams->next_stream;
-        connection_free(TO_CONN(or_circ->resolving_streams));
+        connection_free_(TO_CONN(or_circ->resolving_streams));
         or_circ->resolving_streams = next_conn;
       }
     }
@@ -926,12 +1200,18 @@ circuit_free_all(void)
   smartlist_free(lst);
   global_circuitlist = NULL;
 
+  smartlist_free(global_origin_circuit_list);
+  global_origin_circuit_list = NULL;
+
   smartlist_free(circuits_pending_chans);
   circuits_pending_chans = NULL;
 
   smartlist_free(circuits_pending_close);
   circuits_pending_close = NULL;
 
+  smartlist_free(circuits_pending_other_guards);
+  circuits_pending_other_guards = NULL;
+
   {
     chan_circid_circuit_map_t **elt, **next, *c;
     for (elt = HT_START(chan_circid_map, &chan_circid_map);
@@ -954,10 +1234,7 @@ circuit_free_cpath_node(crypt_path_t *victim)
   if (!victim)
     return;
 
-  crypto_cipher_free(victim->f_crypto);
-  crypto_cipher_free(victim->b_crypto);
-  crypto_digest_free(victim->f_digest);
-  crypto_digest_free(victim->b_digest);
+  relay_crypto_clear(&victim->crypto);
   onion_handshake_state_release(&victim->handshake_state);
   crypto_dh_free(victim->rend_dh_handshake_state);
   extend_info_free(victim->extend_info);
@@ -1263,7 +1540,7 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason)
       smartlist_free(detached_2);
     }
   }
-#endif
+#endif /* defined(DEBUG_CIRCUIT_UNLINK_ALL) */
 
   SMARTLIST_FOREACH_BEGIN(detached, circuit_t *, circ) {
     int mark = 0;
@@ -1312,9 +1589,11 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data)
     if (!circ->marked_for_close &&
         circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) {
       origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-      if (ocirc->rend_data &&
-          !rend_cmp_service_ids(rend_data->onion_address,
-                                ocirc->rend_data->onion_address) &&
+      if (ocirc->rend_data == NULL) {
+        continue;
+      }
+      if (!rend_cmp_service_ids(rend_data_get_address(rend_data),
+                                rend_data_get_address(ocirc->rend_data)) &&
           tor_memeq(ocirc->rend_data->rend_cookie,
                     rend_data->rend_cookie,
                     REND_COOKIE_LEN))
@@ -1325,209 +1604,155 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data)
   return NULL;
 }
 
-/** Return the first circuit originating here in global_circuitlist after
- * <b>start</b> whose purpose is <b>purpose</b>, and where
- * <b>digest</b> (if set) matches the rend_pk_digest field. Return NULL if no
- * circuit is found.  If <b>start</b> is NULL, begin at the start of the list.
- */
+/** Return the first service introduction circuit originating from the global
+ * circuit list after <b>start</b> or at the start of the list if <b>start</b>
+ * is NULL. Return NULL if no circuit is found.
+ *
+ * A service introduction point circuit has a purpose of either
+ * CIRCUIT_PURPOSE_S_ESTABLISH_INTRO or CIRCUIT_PURPOSE_S_INTRO. This does not
+ * return a circuit marked for close and its state must be open. */
 origin_circuit_t *
-circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
-                                   const char *digest, uint8_t purpose)
+circuit_get_next_service_intro_circ(origin_circuit_t *start)
 {
-  int idx;
+  int idx = 0;
   smartlist_t *lst = circuit_get_global_list();
-  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose));
-  if (start == NULL)
-    idx = 0;
-  else
+
+  if (start) {
     idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
+  }
 
   for ( ; idx < smartlist_len(lst); ++idx) {
     circuit_t *circ = smartlist_get(lst, idx);
 
-    if (circ->marked_for_close)
-      continue;
-    if (circ->purpose != purpose)
+    /* Ignore a marked for close circuit or purpose not matching a service
+     * intro point or if the state is not open. */
+    if (circ->marked_for_close || circ->state != CIRCUIT_STATE_OPEN ||
+        (circ->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO &&
+         circ->purpose != CIRCUIT_PURPOSE_S_INTRO)) {
       continue;
-    if (!digest)
-      return TO_ORIGIN_CIRCUIT(circ);
-    else if (TO_ORIGIN_CIRCUIT(circ)->rend_data &&
-             tor_memeq(TO_ORIGIN_CIRCUIT(circ)->rend_data->rend_pk_digest,
-                     digest, DIGEST_LEN))
-      return TO_ORIGIN_CIRCUIT(circ);
+    }
+    /* The purposes we are looking for are only for origin circuits so the
+     * following is valid. */
+    return TO_ORIGIN_CIRCUIT(circ);
   }
+  /* Not found. */
   return NULL;
 }
 
-/** Map from rendezvous cookie to or_circuit_t */
-static digestmap_t *rend_cookie_map = NULL;
-
-/** Map from introduction point digest to or_circuit_t */
-static digestmap_t *intro_digest_map = NULL;
-
-/** Return the OR circuit whose purpose is <b>purpose</b>, and whose
- * rend_token is the REND_TOKEN_LEN-byte <b>token</b>. If <b>is_rend_circ</b>,
- * look for rendezvous point circuits; otherwise look for introduction point
- * circuits. */
-static or_circuit_t *
-circuit_get_by_rend_token_and_purpose(uint8_t purpose, int is_rend_circ,
-                                      const char *token)
+/** Return the first service rendezvous circuit originating from the global
+ * circuit list after <b>start</b> or at the start of the list if <b>start</b>
+ * is NULL. Return NULL if no circuit is found.
+ *
+ * A service rendezvous point circuit has a purpose of either
+ * CIRCUIT_PURPOSE_S_CONNECT_REND or CIRCUIT_PURPOSE_S_REND_JOINED. This does
+ * not return a circuit marked for close and its state must be open. */
+origin_circuit_t *
+circuit_get_next_service_rp_circ(origin_circuit_t *start)
 {
-  or_circuit_t *circ;
-  digestmap_t *map = is_rend_circ ? rend_cookie_map : intro_digest_map;
-
-  if (!map)
-    return NULL;
-
-  circ = digestmap_get(map, token);
-  if (!circ ||
-      circ->base_.purpose != purpose ||
-      circ->base_.marked_for_close)
-    return NULL;
+  int idx = 0;
+  smartlist_t *lst = circuit_get_global_list();
 
-  if (!circ->rendinfo) {
-    char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN));
-    log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned a "
-             "circuit with no rendinfo set.",
-             safe_str(t), is_rend_circ);
-    tor_free(t);
-    return NULL;
+  if (start) {
+    idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
   }
 
-  if (! bool_eq(circ->rendinfo->is_rend_circ, is_rend_circ) ||
-      tor_memneq(circ->rendinfo->rend_token, token, REND_TOKEN_LEN)) {
-    char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN));
-    log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned %s:%d",
-             safe_str(t), is_rend_circ,
-             safe_str(hex_str(circ->rendinfo->rend_token, REND_TOKEN_LEN)),
-             (int)circ->rendinfo->is_rend_circ);
-    tor_free(t);
-    return NULL;
-  }
+  for ( ; idx < smartlist_len(lst); ++idx) {
+    circuit_t *circ = smartlist_get(lst, idx);
 
-  return circ;
+    /* Ignore a marked for close circuit or purpose not matching a service
+     * intro point or if the state is not open. */
+    if (circ->marked_for_close || circ->state != CIRCUIT_STATE_OPEN ||
+        (circ->purpose != CIRCUIT_PURPOSE_S_CONNECT_REND &&
+         circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED)) {
+      continue;
+    }
+    /* The purposes we are looking for are only for origin circuits so the
+     * following is valid. */
+    return TO_ORIGIN_CIRCUIT(circ);
+  }
+  /* Not found. */
+  return NULL;
 }
 
-/** Clear the rendezvous cookie or introduction point key digest that's
- * configured on <b>circ</b>, if any, and remove it from any such maps. */
-static void
-circuit_clear_rend_token(or_circuit_t *circ)
+/** Return the first circuit originating here in global_circuitlist after
+ * <b>start</b> whose purpose is <b>purpose</b>, and where <b>digest</b> (if
+ * set) matches the private key digest of the rend data associated with the
+ * circuit. Return NULL if no circuit is found. If <b>start</b> is NULL,
+ * begin at the start of the list.
+ */
+origin_circuit_t *
+circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
+                                   const uint8_t *digest, uint8_t purpose)
 {
-  or_circuit_t *found_circ;
-  digestmap_t *map;
-
-  if (!circ || !circ->rendinfo)
-    return;
-
-  map = circ->rendinfo->is_rend_circ ? rend_cookie_map : intro_digest_map;
+  int idx;
+  smartlist_t *lst = circuit_get_global_list();
+  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose));
+  if (start == NULL)
+    idx = 0;
+  else
+    idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
 
-  if (!map) {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but found no map");
-    return;
-  }
+  for ( ; idx < smartlist_len(lst); ++idx) {
+    circuit_t *circ = smartlist_get(lst, idx);
+    origin_circuit_t *ocirc;
 
-  found_circ = digestmap_get(map, circ->rendinfo->rend_token);
-  if (found_circ == circ) {
-    /* Great, this is the right one. */
-    digestmap_remove(map, circ->rendinfo->rend_token);
-  } else if (found_circ) {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but "
-             "it was already replaced in the map.");
-  } else {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but "
-             "it not in the map at all.");
+    if (circ->marked_for_close)
+      continue;
+    if (circ->purpose != purpose)
+      continue;
+    /* At this point we should be able to get a valid origin circuit because
+     * the origin purpose we are looking for matches this circuit. */
+    if (BUG(!CIRCUIT_PURPOSE_IS_ORIGIN(circ->purpose))) {
+      break;
+    }
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    if (!digest)
+      return ocirc;
+    if (rend_circuit_pk_digest_eq(ocirc, digest)) {
+      return ocirc;
+    }
   }
-
-  tor_free(circ->rendinfo); /* Sets it to NULL too */
+  return NULL;
 }
 
-/** Set the rendezvous cookie (if is_rend_circ), or the introduction point
- * digest (if ! is_rend_circ) of <b>circ</b> to the REND_TOKEN_LEN-byte value
- * in <b>token</b>, and add it to the appropriate map.  If it previously had a
- * token, clear it.  If another circuit previously had the same
- * cookie/intro-digest, mark that circuit and remove it from the map. */
-static void
-circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ,
-                       const uint8_t *token)
+/** We might cannibalize this circuit: Return true if its last hop can be used
+ *  as a v3 rendezvous point. */
+static int
+circuit_can_be_cannibalized_for_v3_rp(const origin_circuit_t *circ)
 {
-  digestmap_t **map_p, *map;
-  or_circuit_t *found_circ;
-
-  /* Find the right map, creating it as needed */
-  map_p = is_rend_circ ? &rend_cookie_map : &intro_digest_map;
-
-  if (!*map_p)
-    *map_p = digestmap_new();
-
-  map = *map_p;
-
-  /* If this circuit already has a token, we need to remove that. */
-  if (circ->rendinfo)
-    circuit_clear_rend_token(circ);
+  if (!circ->build_state) {
+    return 0;
+  }
 
-  if (token == NULL) {
-    /* We were only trying to remove this token, not set a new one. */
-    return;
+  extend_info_t *chosen_exit = circ->build_state->chosen_exit;
+  if (BUG(!chosen_exit)) {
+    return 0;
   }
 
-  found_circ = digestmap_get(map, (const char *)token);
-  if (found_circ) {
-    tor_assert(found_circ != circ);
-    circuit_clear_rend_token(found_circ);
-    if (! found_circ->base_.marked_for_close) {
-      circuit_mark_for_close(TO_CIRCUIT(found_circ), END_CIRC_REASON_FINISHED);
-      if (is_rend_circ) {
-        log_fn(LOG_PROTOCOL_WARN, LD_REND,
-               "Duplicate rendezvous cookie (%s...) used on two circuits",
-               hex_str((const char*)token, 4)); /* only log first 4 chars */
-      }
+  const node_t *rp_node = node_get_by_id(chosen_exit->identity_digest);
+  if (rp_node) {
+    if (node_supports_v3_rendezvous_point(rp_node)) {
+      return 1;
     }
   }
 
-  /* Now set up the rendinfo */
-  circ->rendinfo = tor_malloc(sizeof(*circ->rendinfo));
-  memcpy(circ->rendinfo->rend_token, token, REND_TOKEN_LEN);
-  circ->rendinfo->is_rend_circ = is_rend_circ ? 1 : 0;
-
-  digestmap_set(map, (const char *)token, circ);
-}
-
-/** Return the circuit waiting for a rendezvous with the provided cookie.
- * Return NULL if no such circuit is found.
- */
-or_circuit_t *
-circuit_get_rendezvous(const uint8_t *cookie)
-{
-  return circuit_get_by_rend_token_and_purpose(
-                                     CIRCUIT_PURPOSE_REND_POINT_WAITING,
-                                     1, (const char*)cookie);
-}
-
-/** Return the circuit waiting for intro cells of the given digest.
- * Return NULL if no such circuit is found.
- */
-or_circuit_t *
-circuit_get_intro_point(const uint8_t *digest)
-{
-  return circuit_get_by_rend_token_and_purpose(
-                                     CIRCUIT_PURPOSE_INTRO_POINT, 0,
-                                     (const char *)digest);
-}
-
-/** Set the rendezvous cookie of <b>circ</b> to <b>cookie</b>.  If another
- * circuit previously had that cookie, mark it. */
-void
-circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie)
-{
-  circuit_set_rend_token(circ, 1, cookie);
+  return 0;
 }
 
-/** Set the intro point key digest of <b>circ</b> to <b>cookie</b>.  If another
- * circuit previously had that intro point digest, mark it. */
-void
-circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest)
+/** We are trying to create a circuit of purpose <b>purpose</b> and we are
+ *  looking for cannibalizable circuits. Return the circuit purpose we would be
+ *  willing to cannibalize. */
+static uint8_t
+get_circuit_purpose_needed_to_cannibalize(uint8_t purpose)
 {
-  circuit_set_rend_token(circ, 0, digest);
+  if (circuit_should_use_vanguards(purpose)) {
+    /* If we are using vanguards, then we should only cannibalize vanguard
+     * circuits so that we get the same path construction logic. */
+    return CIRCUIT_PURPOSE_HS_VANGUARDS;
+  } else {
+    /* If no vanguards are used just get a general circuit! */
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  }
 }
 
 /** Return a circuit that is open, is CIRCUIT_PURPOSE_C_GENERAL,
@@ -1535,14 +1760,21 @@ circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest)
  * flags in <b>flags</b>, and if info is defined, does not already use info
  * as any of its hops; or NULL if no circuit fits this description.
  *
- * The <b>purpose</b> argument (currently ignored) refers to the purpose of
- * the circuit we want to create, not the purpose of the circuit we want to
- * cannibalize.
+ * The <b>purpose</b> argument refers to the purpose of the circuit we want to
+ * create, not the purpose of the circuit we want to cannibalize.
  *
  * If !CIRCLAUNCH_NEED_UPTIME, prefer returning non-uptime circuits.
+ *
+ * To "cannibalize" a circuit means to extend it an extra hop, and use it
+ * for some other purpose than we had originally intended.  We do this when
+ * we want to perform some low-bandwidth task at a specific relay, and we
+ * would like the circuit to complete as soon as possible.  (If we were going
+ * to use a lot of bandwidth, we wouldn't want a circuit with an extra hop.
+ * If we didn't care about circuit completion latency, we would just build
+ * a new circuit.)
  */
 origin_circuit_t *
-circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
+circuit_find_to_cannibalize(uint8_t purpose_to_produce, extend_info_t *info,
                             int flags)
 {
   origin_circuit_t *best=NULL;
@@ -1550,29 +1782,53 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
   int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0;
   int internal = (flags & CIRCLAUNCH_IS_INTERNAL) != 0;
   const or_options_t *options = get_options();
+  /* We want the circuit we are trying to cannibalize to have this purpose */
+  int purpose_to_search_for;
 
   /* Make sure we're not trying to create a onehop circ by
    * cannibalization. */
   tor_assert(!(flags & CIRCLAUNCH_ONEHOP_TUNNEL));
 
+  purpose_to_search_for = get_circuit_purpose_needed_to_cannibalize(
+                                                  purpose_to_produce);
+
+  tor_assert_nonfatal(purpose_to_search_for == CIRCUIT_PURPOSE_C_GENERAL ||
+                      purpose_to_search_for == CIRCUIT_PURPOSE_HS_VANGUARDS);
+
   log_debug(LD_CIRC,
             "Hunting for a circ to cannibalize: purpose %d, uptime %d, "
             "capacity %d, internal %d",
-            purpose, need_uptime, need_capacity, internal);
+            purpose_to_produce, need_uptime, need_capacity, internal);
 
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
     if (CIRCUIT_IS_ORIGIN(circ_) &&
         circ_->state == CIRCUIT_STATE_OPEN &&
         !circ_->marked_for_close &&
-        circ_->purpose == CIRCUIT_PURPOSE_C_GENERAL &&
+        circ_->purpose == purpose_to_search_for &&
         !circ_->timestamp_dirty) {
       origin_circuit_t *circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      /* Only cannibalize from reasonable length circuits. If we
+       * want C_GENERAL, then only choose 3 hop circs. If we want
+       * HS_VANGUARDS, only choose 4 hop circs.
+       */
+      if (circ->build_state->desired_path_len !=
+          route_len_for_purpose(purpose_to_search_for, NULL)) {
+        goto next;
+      }
+
+      /* Ignore any circuits for which we can't use the Guard. It is possible
+       * that the Guard was removed from the samepled set after the circuit
+       * was created so avoid using it. */
+      if (!entry_guard_could_succeed(circ->guard_state)) {
+        goto next;
+      }
+
       if ((!need_uptime || circ->build_state->need_uptime) &&
           (!need_capacity || circ->build_state->need_capacity) &&
           (internal == circ->build_state->is_internal) &&
           !circ->unusable_for_new_conns &&
           circ->remaining_relay_early_cells &&
-          circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN &&
           !circ->build_state->onehop_tunnel &&
           !circ->isolation_values_set) {
         if (info) {
@@ -1604,6 +1860,14 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
             hop = hop->next;
           } while (hop != circ->cpath);
         }
+
+        if ((flags & CIRCLAUNCH_IS_V3_RP) &&
+            !circuit_can_be_cannibalized_for_v3_rp(circ)) {
+          log_debug(LD_GENERAL, "Skipping uncannibalizable circuit for v3 "
+                    "rendezvous point.");
+          goto next;
+        }
+
         if (!best || (best->build_state->need_uptime && !need_uptime))
           best = circ;
       next: ;
@@ -1614,6 +1878,37 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
   return best;
 }
 
+/**
+ * Check whether any of the origin circuits that are waiting to see if
+ * their guard is good enough to use can be upgraded to "ready". If so,
+ * return a new smartlist containing them. Otherwise return NULL.
+ */
+smartlist_t *
+circuit_find_circuits_to_upgrade_from_guard_wait(void)
+{
+  /* Only if some circuit is actually waiting on an upgrade should we
+   * run the algorithm. */
+  if (! circuits_pending_other_guards ||
+      smartlist_len(circuits_pending_other_guards)==0)
+    return NULL;
+  /* Only if we have some origin circuits should we run the algorithm. */
+  if (!global_origin_circuit_list)
+    return NULL;
+
+  /* Okay; we can pass our circuit list to entrynodes.c.*/
+  smartlist_t *result = smartlist_new();
+  int circuits_upgraded  = entry_guards_upgrade_waiting_circuits(
+                                                 get_guard_selection_info(),
+                                                 global_origin_circuit_list,
+                                                 result);
+  if (circuits_upgraded && smartlist_len(result)) {
+    return result;
+  } else {
+    smartlist_free(result);
+    return NULL;
+  }
+}
+
 /** Return the number of hops in circuit's path. If circ has no entries,
  * or is NULL, returns 0. */
 int
@@ -1630,6 +1925,25 @@ circuit_get_cpath_len(origin_circuit_t *circ)
   return n;
 }
 
+/** Return the number of opened hops in circuit's path.
+ * If circ has no entries, or is NULL, returns 0. */
+int
+circuit_get_cpath_opened_len(const origin_circuit_t *circ)
+{
+  int n = 0;
+  if (circ && circ->cpath) {
+    crypt_path_t *cpath, *cpath_next = NULL;
+    for (cpath = circ->cpath;
+         cpath->state == CPATH_STATE_OPEN
+           && cpath_next != circ->cpath;
+         cpath = cpath_next) {
+      cpath_next = cpath->next;
+      ++n;
+    }
+  }
+  return n;
+}
+
 /** Return the <b>hopnum</b>th hop in <b>circ</b>->cpath, or NULL if there
  * aren't that many hops in the list. <b>hopnum</b> starts at 1.
  * Returns NULL if <b>hopnum</b> is 0 or negative. */
@@ -1741,8 +2055,7 @@ circuit_synchronize_written_or_bandwidth(const circuit_t *c,
  *   - If state is onionskin_pending, remove circ from the onion_pending
  *     list.
  *   - If circ isn't open yet: call circuit_build_failed() if we're
- *     the origin, and in either case call circuit_rep_hist_note_result()
- *     to note stats.
+ *     the origin.
  *   - If purpose is C_INTRODUCE_ACK_WAIT, report the intro point
  *     failure we just had to the hidden service client module.
  *   - If purpose is C_INTRODUCING and <b>reason</b> isn't TIMEOUT,
@@ -1816,10 +2129,21 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
     }
   }
 
+  /* Notify the HS subsystem that this circuit is closing. */
+  hs_circ_cleanup(circ);
+
   if (circuits_pending_close == NULL)
     circuits_pending_close = smartlist_new();
 
   smartlist_add(circuits_pending_close, circ);
+  mainloop_schedule_postloop_cleanup();
+
+  log_info(LD_GENERAL, "Circuit %u (id: %" PRIu32 ") marked for close at "
+                       "%s:%d (orig reason: %d, new reason: %d)",
+           circ->n_circ_id,
+           CIRCUIT_IS_ORIGIN(circ) ?
+              TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+           file, line, orig_reason, reason);
 }
 
 /** Called immediately before freeing a marked circuit <b>circ</b> from
@@ -1866,20 +2190,25 @@ circuit_about_to_free(circuit_t *circ)
    * module then.  If it isn't OPEN, we send it there now to remember which
    * links worked and which didn't.
    */
-  if (circ->state != CIRCUIT_STATE_OPEN) {
+  if (circ->state != CIRCUIT_STATE_OPEN &&
+      circ->state != CIRCUIT_STATE_GUARD_WAIT) {
     if (CIRCUIT_IS_ORIGIN(circ)) {
       origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
       circuit_build_failed(ocirc); /* take actions if necessary */
-      circuit_rep_hist_note_result(ocirc);
     }
   }
   if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
     if (circuits_pending_chans)
       smartlist_remove(circuits_pending_chans, circ);
   }
+  if (circuits_pending_other_guards) {
+    smartlist_remove(circuits_pending_other_guards, circ);
+  }
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
-     (circ->state == CIRCUIT_STATE_OPEN)?CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
+     (circ->state == CIRCUIT_STATE_OPEN ||
+      circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
+                                 CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
      orig_reason);
   }
 
@@ -1888,11 +2217,11 @@ circuit_about_to_free(circuit_t *circ)
     int timed_out = (reason == END_CIRC_REASON_TIMEOUT);
     tor_assert(circ->state == CIRCUIT_STATE_OPEN);
     tor_assert(ocirc->build_state->chosen_exit);
-    tor_assert(ocirc->rend_data);
-    if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) {
+    if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
+        ocirc->rend_data) {
       /* treat this like getting a nack from it */
       log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
-          safe_str_client(ocirc->rend_data->onion_address),
+          safe_str_client(rend_data_get_address(ocirc->rend_data)),
           safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
           timed_out ? "Recording timeout." : "Removing from descriptor.");
       rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
@@ -1905,11 +2234,12 @@ circuit_about_to_free(circuit_t *circ)
              reason != END_CIRC_REASON_TIMEOUT) {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
     if (ocirc->build_state->chosen_exit && ocirc->rend_data) {
-      if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) {
+      if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
+          ocirc->rend_data) {
         log_info(LD_REND, "Failed intro circ %s to %s "
             "(building circuit to intro point). "
             "Marking intro point as possibly unreachable.",
-            safe_str_client(ocirc->rend_data->onion_address),
+            safe_str_client(rend_data_get_address(ocirc->rend_data)),
             safe_str_client(build_state_get_exit_nickname(
                                               ocirc->build_state)));
         rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
@@ -2004,10 +2334,10 @@ single_conn_free_bytes(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_DIR) {
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
-    if (dir_conn->zlib_state) {
-      result += tor_zlib_state_size(dir_conn->zlib_state);
-      tor_zlib_free(dir_conn->zlib_state);
-      dir_conn->zlib_state = NULL;
+    if (dir_conn->compress_state) {
+      result += tor_compress_state_size(dir_conn->compress_state);
+      tor_compress_free(dir_conn->compress_state);
+      dir_conn->compress_state = NULL;
     }
   }
   return result;
@@ -2053,13 +2383,27 @@ n_cells_in_circ_queues(const circuit_t *c)
   return n;
 }
 
+/** Return the number of bytes allocated for <b>c</c>'s half-open streams. */
+static size_t
+circuit_alloc_in_half_streams(const circuit_t *c)
+{
+  if (! CIRCUIT_IS_ORIGIN(c)) {
+    return 0;
+  }
+  const origin_circuit_t *ocirc = CONST_TO_ORIGIN_CIRCUIT(c);
+  if (ocirc->half_streams)
+    return smartlist_len(ocirc->half_streams) * sizeof(half_edge_t);
+  else
+    return 0;
+}
+
 /**
- * Return the age of the oldest cell queued on <b>c</b>, in milliseconds.
+ * Return the age of the oldest cell queued on <b>c</b>, in timestamp units.
  * Return 0 if there are no cells queued on c.  Requires that <b>now</b> be
- * the current time in milliseconds since the epoch, truncated.
+ * the current coarse timestamp.
  *
  * This function will return incorrect results if the oldest cell queued on
- * the circuit is older than 2**32 msec (about 49 days) old.
+ * the circuit is older than about 2**32 msec (about 49 days) old.
  */
 STATIC uint32_t
 circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
@@ -2068,12 +2412,12 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
   packed_cell_t *cell;
 
   if (NULL != (cell = TOR_SIMPLEQ_FIRST(&c->n_chan_cells.head)))
-    age = now - cell->inserted_time;
+    age = now - cell->inserted_timestamp;
 
   if (! CIRCUIT_IS_ORIGIN(c)) {
     const or_circuit_t *orcirc = CONST_TO_OR_CIRCUIT(c);
     if (NULL != (cell = TOR_SIMPLEQ_FIRST(&orcirc->p_chan_cells.head))) {
-      uint32_t age2 = now - cell->inserted_time;
+      uint32_t age2 = now - cell->inserted_timestamp;
       if (age2 > age)
         return age2;
     }
@@ -2081,31 +2425,30 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
   return age;
 }
 
-/** Return the age in milliseconds of the oldest buffer chunk on <b>conn</b>,
- * where age is taken in milliseconds before the time <b>now</b> (in truncated
- * absolute monotonic msec).  If the connection has no data, treat
- * it as having age zero.
+/** Return the age of the oldest buffer chunk on <b>conn</b>, where age is
+ * taken in timestamp units before the time <b>now</b>.  If the connection has
+ * no data, treat it as having age zero.
  **/
 static uint32_t
-conn_get_buffer_age(const connection_t *conn, uint32_t now)
+conn_get_buffer_age(const connection_t *conn, uint32_t now_ts)
 {
   uint32_t age = 0, age2;
   if (conn->outbuf) {
-    age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now);
+    age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now_ts);
     if (age2 > age)
       age = age2;
   }
   if (conn->inbuf) {
-    age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now);
+    age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now_ts);
     if (age2 > age)
       age = age2;
   }
   return age;
 }
 
-/** Return the age in milliseconds of the oldest buffer chunk on any stream in
- * the linked list <b>stream</b>, where age is taken in milliseconds before
- * the time <b>now</b> (in truncated milliseconds since the epoch). */
+/** Return the age in timestamp units of the oldest buffer chunk on any stream
+ * in the linked list <b>stream</b>, where age is taken in timestamp units
+ * before the timestamp <b>now</b>. */
 static uint32_t
 circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now)
 {
@@ -2124,9 +2467,9 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now)
   return age;
 }
 
-/** Return the age in milliseconds of the oldest buffer chunk on any stream
- * attached to the circuit <b>c</b>, where age is taken in milliseconds before
- * the time <b>now</b> (in truncated milliseconds since the epoch). */
+/** Return the age in timestamp units of the oldest buffer chunk on any stream
+ * attached to the circuit <b>c</b>, where age is taken before the timestamp
+ * <b>now</b>. */
 STATIC uint32_t
 circuit_max_queued_data_age(const circuit_t *c, uint32_t now)
 {
@@ -2140,8 +2483,8 @@ circuit_max_queued_data_age(const circuit_t *c, uint32_t now)
 }
 
 /** Return the age of the oldest cell or stream buffer chunk on the circuit
- * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in
- * truncated milliseconds since the epoch). */
+ * <b>c</b>, where age is taken in timestamp units before the timestamp
+ * <b>now</b> */
 STATIC uint32_t
 circuit_max_queued_item_age(const circuit_t *c, uint32_t now)
 {
@@ -2171,7 +2514,7 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_)
     return -1;
 }
 
-static uint32_t now_ms_for_buf_cmp;
+static uint32_t now_ts_for_buf_cmp;
 
 /** Helper to sort a list of circuit_t by age of oldest item, in descending
  * order. */
@@ -2180,8 +2523,8 @@ conns_compare_by_buffer_age_(const void **a_, const void **b_)
 {
   const connection_t *a = *a_;
   const connection_t *b = *b_;
-  time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp);
-  time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp);
+  time_t age_a = conn_get_buffer_age(a, now_ts_for_buf_cmp);
+  time_t age_b = conn_get_buffer_age(b, now_ts_for_buf_cmp);
 
   if (age_a < age_b)
     return 1;
@@ -2206,10 +2549,22 @@ circuits_handle_oom(size_t current_allocation)
   size_t mem_recovered=0;
   int n_circuits_killed=0;
   int n_dirconns_killed=0;
-  uint32_t now_ms;
-  log_notice(LD_GENERAL, "We're low on memory.  Killing circuits with "
-             "over-long queues. (This behavior is controlled by "
-             "MaxMemInQueues.)");
+  uint32_t now_ts;
+  log_notice(LD_GENERAL, "We're low on memory (cell queues total alloc:"
+             " %"TOR_PRIuSZ" buffer total alloc: %" TOR_PRIuSZ ","
+             " tor compress total alloc: %" TOR_PRIuSZ
+             " (zlib: %" TOR_PRIuSZ ", zstd: %" TOR_PRIuSZ ","
+             " lzma: %" TOR_PRIuSZ "),"
+             " rendezvous cache total alloc: %" TOR_PRIuSZ "). Killing"
+             " circuits withover-long queues. (This behavior is controlled by"
+             " MaxMemInQueues.)",
+             cell_queues_get_total_allocation(),
+             buf_get_total_allocation(),
+             tor_compress_get_total_allocation(),
+             tor_zlib_get_total_allocation(),
+             tor_zstd_get_total_allocation(),
+             tor_lzma_get_total_allocation(),
+             rend_cache_get_total_allocation());
 
   {
     size_t mem_target = (size_t)(get_options()->MaxMemInQueues *
@@ -2219,11 +2574,11 @@ circuits_handle_oom(size_t current_allocation)
     mem_to_recover = current_allocation - mem_target;
   }
 
-  now_ms = (uint32_t)monotime_coarse_absolute_msec();
+  now_ts = monotime_coarse_get_stamp();
 
   circlist = circuit_get_global_list();
   SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) {
-    circ->age_tmp = circuit_max_queued_item_age(circ, now_ms);
+    circ->age_tmp = circuit_max_queued_item_age(circ, now_ts);
   } SMARTLIST_FOREACH_END(circ);
 
   /* This is O(n log n); there are faster algorithms we could use instead.
@@ -2236,9 +2591,9 @@ circuits_handle_oom(size_t current_allocation)
   } SMARTLIST_FOREACH_END(circ);
 
   /* Now sort the connection array ... */
-  now_ms_for_buf_cmp = now_ms;
+  now_ts_for_buf_cmp = now_ts;
   smartlist_sort(connection_array, conns_compare_by_buffer_age_);
-  now_ms_for_buf_cmp = 0;
+  now_ts_for_buf_cmp = 0;
 
   /* Fix up the connection array to its new order. */
   SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) {
@@ -2257,7 +2612,7 @@ circuits_handle_oom(size_t current_allocation)
      * data older than this circuit. */
     while (conn_idx < smartlist_len(connection_array)) {
       connection_t *conn = smartlist_get(connection_array, conn_idx);
-      uint32_t conn_age = conn_get_buffer_age(conn, now_ms);
+      uint32_t conn_age = conn_get_buffer_age(conn, now_ts);
       if (conn_age < circ->age_tmp) {
         break;
       }
@@ -2276,6 +2631,7 @@ circuits_handle_oom(size_t current_allocation)
 
     /* Now, kill the circuit. */
     n = n_cells_in_circ_queues(circ);
+    const size_t half_stream_alloc = circuit_alloc_in_half_streams(circ);
     if (! circ->marked_for_close) {
       circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
     }
@@ -2285,6 +2641,7 @@ circuits_handle_oom(size_t current_allocation)
     ++n_circuits_killed;
 
     mem_recovered += n * packed_cell_mem_cost();
+    mem_recovered += half_stream_alloc;
     mem_recovered += freed;
 
     if (mem_recovered >= mem_to_recover)
@@ -2315,8 +2672,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp)
   switch (cp->state)
     {
     case CPATH_STATE_OPEN:
-      tor_assert(cp->f_crypto);
-      tor_assert(cp->b_crypto);
+      relay_crypto_assert_ok(&cp->crypto);
       /* fall through */
     case CPATH_STATE_CLOSED:
       /*XXXX Assert that there's no handshake_state either. */
@@ -2359,8 +2715,8 @@ assert_cpath_ok(const crypt_path_t *cp)
 /** Verify that circuit <b>c</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */
-void
-assert_circuit_ok(const circuit_t *c)
+MOCK_IMPL(void,
+assert_circuit_ok,(const circuit_t *c))
 {
   edge_connection_t *conn;
   const or_circuit_t *or_circ = NULL;
@@ -2402,13 +2758,11 @@ assert_circuit_ok(const circuit_t *c)
 
   tor_assert(c->deliver_window >= 0);
   tor_assert(c->package_window >= 0);
-  if (c->state == CIRCUIT_STATE_OPEN) {
+  if (c->state == CIRCUIT_STATE_OPEN ||
+      c->state == CIRCUIT_STATE_GUARD_WAIT) {
     tor_assert(!c->n_chan_create_cell);
     if (or_circ) {
-      tor_assert(or_circ->n_crypto);
-      tor_assert(or_circ->p_crypto);
-      tor_assert(or_circ->n_digest);
-      tor_assert(or_circ->p_digest);
+      relay_crypto_assert_ok(&or_circ->crypto);
     }
   }
   if (c->state == CIRCUIT_STATE_CHAN_WAIT && !c->marked_for_close) {
@@ -2432,4 +2786,3 @@ assert_circuit_ok(const circuit_t *c)
     tor_assert(!or_circ || !or_circ->rend_splice);
   }
 }
-
diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h
index 2ede6f76cd..1d39f36925 100644
--- a/src/or/circuitlist.h
+++ b/src/or/circuitlist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,8 +13,14 @@
 #define TOR_CIRCUITLIST_H
 
 #include "testsupport.h"
+#include "hs_ident.h"
 
 MOCK_DECL(smartlist_t *, circuit_get_global_list, (void));
+smartlist_t *circuit_get_global_origin_circuit_list(void);
+int circuit_any_opened_circuits(void);
+int circuit_any_opened_circuits_cached(void);
+void circuit_cache_opened_circuit_state(int circuits_are_opened);
+
 const char *circuit_state_to_string(int state);
 const char *circuit_purpose_to_controller_string(uint8_t purpose);
 const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose);
@@ -45,11 +51,10 @@ origin_circuit_t *circuit_get_by_global_id(uint32_t id);
 origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data(
   const rend_data_t *rend_data);
 origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
-                                         const char *digest, uint8_t purpose);
-or_circuit_t *circuit_get_rendezvous(const uint8_t *cookie);
-or_circuit_t *circuit_get_intro_point(const uint8_t *digest);
-void circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie);
-void circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest);
+                                     const uint8_t *digest, uint8_t purpose);
+origin_circuit_t *circuit_get_next_service_intro_circ(origin_circuit_t *start);
+origin_circuit_t *circuit_get_next_service_rp_circ(origin_circuit_t *start);
+origin_circuit_t *circuit_get_next_service_hsdir_circ(origin_circuit_t *start);
 origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose,
                                               extend_info_t *info, int flags);
 void circuit_mark_all_unused_circs(void);
@@ -57,6 +62,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void);
 MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
                                           int line, const char *file));
 int circuit_get_cpath_len(origin_circuit_t *circ);
+int circuit_get_cpath_opened_len(const origin_circuit_t *);
 void circuit_clear_cpath(origin_circuit_t *circ);
 crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum);
 void circuit_get_all_pending_on_channel(smartlist_t *out,
@@ -69,7 +75,7 @@ void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
   circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
 
 void assert_cpath_layer_ok(const crypt_path_t *cp);
-void assert_circuit_ok(const circuit_t *c);
+MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c));
 void circuit_free_all(void);
 void circuits_handle_oom(size_t current_allocation);
 
@@ -79,13 +85,16 @@ void channel_note_destroy_pending(channel_t *chan, circid_t id);
 MOCK_DECL(void, channel_note_destroy_not_pending,
           (channel_t *chan, circid_t id));
 
+smartlist_t *circuit_find_circuits_to_upgrade_from_guard_wait(void);
+
 #ifdef CIRCUITLIST_PRIVATE
-STATIC void circuit_free(circuit_t *circ);
+STATIC void circuit_free_(circuit_t *circ);
+#define circuit_free(circ) FREE_AND_NULL(circuit_t, circuit_free_, (circ))
 STATIC size_t n_cells_in_circ_queues(const circuit_t *c);
 STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now);
 STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now);
 STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now);
-#endif
+#endif /* defined(CIRCUITLIST_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CIRCUITLIST_H) */
 
diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c
index 036fb9570d..f9f5faa057 100644
--- a/src/or/circuitmux.c
+++ b/src/or/circuitmux.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -114,13 +114,6 @@ struct circuitmux_s {
    */
   chanid_circid_muxinfo_map_t *chanid_circid_map;
 
-  /*
-   * Double-linked ring of circuits with queued cells waiting for room to
-   * free up on this connection's outbuf.  Every time we pull cells from
-   * a circuit, we advance this pointer to the next circuit in the ring.
-   */
-  struct circuit_t *active_circuits_head, *active_circuits_tail;
-
   /** List of queued destroy cells */
   destroy_cell_queue_t destroy_cell_queue;
   /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
@@ -177,17 +170,6 @@ struct chanid_circid_muxinfo_t {
 };
 
 /*
- * Internal-use #defines
- */
-
-#ifdef CMUX_PARANOIA
-#define circuitmux_assert_okay_paranoid(cmux) \
-  circuitmux_assert_okay(cmux)
-#else
-#define circuitmux_assert_okay_paranoid(cmux)
-#endif
-
-/*
  * Static function declarations
  */
 
@@ -199,21 +181,9 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a);
 static chanid_circid_muxinfo_t *
 circuitmux_find_map_entry(circuitmux_t *cmux, circuit_t *circ);
 static void
-circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
-                               cell_direction_t direction);
+circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ);
 static void
-circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ,
-                                 cell_direction_t direction);
-static inline void
-circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ,
-                                    cell_direction_t direction);
-static inline circuit_t **
-circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ);
-static inline circuit_t **
-circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ);
-static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux);
-static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux);
-static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux);
+circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ);
 
 /* Static global variables */
 
@@ -223,121 +193,6 @@ static int64_t global_destroy_ctr = 0;
 /* Function definitions */
 
 /**
- * Linked list helpers
- */
-
-/**
- * Move an active circuit to the tail of the cmux's active circuits list;
- * used by circuitmux_notify_xmit_cells().
- */
-
-static inline void
-circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ,
-                                    cell_direction_t direction)
-{
-  circuit_t **next_p = NULL, **prev_p = NULL;
-  circuit_t **next_prev = NULL, **prev_next = NULL;
-  circuit_t **tail_next = NULL;
-  or_circuit_t *or_circ = NULL;
-
-  tor_assert(cmux);
-  tor_assert(circ);
-
-  circuitmux_assert_okay_paranoid(cmux);
-
-  /* Figure out our next_p and prev_p for this cmux/direction */
-  if (direction) {
-    if (direction == CELL_DIRECTION_OUT) {
-      tor_assert(circ->n_mux == cmux);
-      next_p = &(circ->next_active_on_n_chan);
-      prev_p = &(circ->prev_active_on_n_chan);
-    } else {
-      or_circ = TO_OR_CIRCUIT(circ);
-      tor_assert(or_circ->p_mux == cmux);
-      next_p = &(or_circ->next_active_on_p_chan);
-      prev_p = &(or_circ->prev_active_on_p_chan);
-    }
-  } else {
-    if (circ->n_mux == cmux) {
-      next_p = &(circ->next_active_on_n_chan);
-      prev_p = &(circ->prev_active_on_n_chan);
-      direction = CELL_DIRECTION_OUT;
-    } else {
-      or_circ = TO_OR_CIRCUIT(circ);
-      tor_assert(or_circ->p_mux == cmux);
-      next_p = &(or_circ->next_active_on_p_chan);
-      prev_p = &(or_circ->prev_active_on_p_chan);
-      direction = CELL_DIRECTION_IN;
-    }
-  }
-  tor_assert(next_p);
-  tor_assert(prev_p);
-
-  /* Check if this really is an active circuit */
-  if ((*next_p == NULL && *prev_p == NULL) &&
-      !(circ == cmux->active_circuits_head ||
-        circ == cmux->active_circuits_tail)) {
-    /* Not active, no-op */
-    return;
-  }
-
-  /* Check if this is already the tail */
-  if (circ == cmux->active_circuits_tail) return;
-
-  /* Okay, we have to move it; figure out next_prev and prev_next */
-  if (*next_p) next_prev = circuitmux_prev_active_circ_p(cmux, *next_p);
-  if (*prev_p) prev_next = circuitmux_next_active_circ_p(cmux, *prev_p);
-  /* Adjust the previous node's next pointer, if any */
-  if (prev_next) *prev_next = *next_p;
-  /* Otherwise, we were the head */
-  else cmux->active_circuits_head = *next_p;
-  /* Adjust the next node's previous pointer, if any */
-  if (next_prev) *next_prev = *prev_p;
-  /* We're out of the list; now re-attach at the tail */
-  /* Adjust our next and prev pointers */
-  *next_p = NULL;
-  *prev_p = cmux->active_circuits_tail;
-  /* Set the next pointer of the tail, or the head if none */
-  if (cmux->active_circuits_tail) {
-    tail_next = circuitmux_next_active_circ_p(cmux,
-                                              cmux->active_circuits_tail);
-    *tail_next = circ;
-  } else {
-    cmux->active_circuits_head = circ;
-  }
-  /* Set the tail to this circuit */
-  cmux->active_circuits_tail = circ;
-
-  circuitmux_assert_okay_paranoid(cmux);
-}
-
-static inline circuit_t **
-circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ)
-{
-  tor_assert(cmux);
-  tor_assert(circ);
-
-  if (circ->n_mux == cmux) return &(circ->next_active_on_n_chan);
-  else {
-    tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
-    return &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
-  }
-}
-
-static inline circuit_t **
-circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ)
-{
-  tor_assert(cmux);
-  tor_assert(circ);
-
-  if (circ->n_mux == cmux) return &(circ->prev_active_on_n_chan);
-  else {
-    tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
-    return &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
-  }
-}
-
-/**
  * Helper for chanid_circid_cell_count_map_t hash table: compare the channel
  * ID and circuit ID for a and b, and return less than, equal to, or greater
  * than zero appropriately.
@@ -408,11 +263,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
   circuit_t *circ = NULL;
 
   tor_assert(cmux);
-  /*
-   * Don't circuitmux_assert_okay_paranoid() here; this gets called when
-   * channels are being freed and have already been unregistered, so
-   * the channel ID lookups it does will fail.
-   */
 
   i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
   while (i) {
@@ -437,7 +287,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
              */
 
             if (to_remove->muxinfo.cell_count > 0) {
-              circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_OUT);
+              circuitmux_make_circuit_inactive(cmux, circ);
             }
 
             /* Clear n_mux */
@@ -452,7 +302,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
              */
 
             if (to_remove->muxinfo.cell_count > 0) {
-              circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_IN);
+              circuitmux_make_circuit_inactive(cmux, circ);
             }
 
             /*
@@ -539,7 +389,7 @@ circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, channel_t *chan)
  */
 
 void
-circuitmux_free(circuitmux_t *cmux)
+circuitmux_free_(circuitmux_t *cmux)
 {
   if (!cmux) return;
 
@@ -608,9 +458,7 @@ circuitmux_clear_policy(circuitmux_t *cmux)
   tor_assert(cmux);
 
   /* Internally, this is just setting policy to NULL */
-  if (cmux->policy) {
-    circuitmux_set_policy(cmux, NULL);
-  }
+  circuitmux_set_policy(cmux, NULL);
 }
 
 /**
@@ -946,7 +794,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
   tor_assert(circ);
   tor_assert(direction == CELL_DIRECTION_IN ||
              direction == CELL_DIRECTION_OUT);
-  circuitmux_assert_okay_paranoid(cmux);
 
   /*
    * Figure out which channel we're using, and get the circuit's current
@@ -1004,10 +851,10 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     if (hashent->muxinfo.cell_count > 0 && cell_count == 0) {
       --(cmux->n_active_circuits);
-      circuitmux_make_circuit_inactive(cmux, circ, direction);
+      circuitmux_make_circuit_inactive(cmux, circ);
     } else if (hashent->muxinfo.cell_count == 0 && cell_count > 0) {
       ++(cmux->n_active_circuits);
-      circuitmux_make_circuit_active(cmux, circ, direction);
+      circuitmux_make_circuit_active(cmux, circ);
     }
     cmux->n_cells -= hashent->muxinfo.cell_count;
     cmux->n_cells += cell_count;
@@ -1035,7 +882,7 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     hashent->muxinfo.cell_count = cell_count;
     hashent->muxinfo.direction = direction;
     /* Allocate policy specific circuit data if we need it */
-    if (cmux->policy && cmux->policy->alloc_circ_data) {
+    if (cmux->policy->alloc_circ_data) {
       /* Assert that we have the means to free policy-specific data */
       tor_assert(cmux->policy->free_circ_data);
       /* Allocate it */
@@ -1055,25 +902,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
     else TO_OR_CIRCUIT(circ)->p_mux = cmux;
 
-    /* Make sure the next/prev pointers are NULL */
-    if (direction == CELL_DIRECTION_OUT) {
-      circ->next_active_on_n_chan = NULL;
-      circ->prev_active_on_n_chan = NULL;
-    } else {
-      TO_OR_CIRCUIT(circ)->next_active_on_p_chan = NULL;
-      TO_OR_CIRCUIT(circ)->prev_active_on_p_chan = NULL;
-    }
-
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
       ++(cmux->n_active_circuits);
-      circuitmux_make_circuit_active(cmux, circ, direction);
+      circuitmux_make_circuit_active(cmux, circ);
     }
     cmux->n_cells += cell_count;
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1097,7 +933,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
   tor_assert(cmux);
   tor_assert(cmux->chanid_circid_map);
   tor_assert(circ);
-  circuitmux_assert_okay_paranoid(cmux);
 
   /* See if we have it for n_chan/n_circ_id */
   if (circ->n_chan) {
@@ -1135,7 +970,7 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
     if (hashent->muxinfo.cell_count > 0) {
       --(cmux->n_active_circuits);
       /* This does policy notifies, so comes before freeing policy data */
-      circuitmux_make_circuit_inactive(cmux, circ, last_searched_direction);
+      circuitmux_make_circuit_inactive(cmux, circ);
     }
     cmux->n_cells -= hashent->muxinfo.cell_count;
 
@@ -1164,8 +999,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
     /* Free the hash entry */
     tor_free(hashent);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1174,94 +1007,22 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
  */
 
 static void
-circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
-                               cell_direction_t direction)
+circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ)
 {
-  circuit_t **next_active = NULL, **prev_active = NULL, **next_prev = NULL;
-  circuitmux_t *circuit_cmux = NULL;
-  chanid_circid_muxinfo_t *hashent = NULL;
-  channel_t *chan = NULL;
-  circid_t circ_id;
-  int already_active;
-
   tor_assert(cmux);
+  tor_assert(cmux->policy);
   tor_assert(circ);
-  tor_assert(direction == CELL_DIRECTION_OUT ||
-             direction == CELL_DIRECTION_IN);
-  /*
-   * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count
-   * already got changed and we have to update the list for it to be consistent
-   * again.
-   */
-
-  /* Get the right set of active list links for this direction */
-  if (direction == CELL_DIRECTION_OUT) {
-    next_active = &(circ->next_active_on_n_chan);
-    prev_active = &(circ->prev_active_on_n_chan);
-    circuit_cmux = circ->n_mux;
-    chan = circ->n_chan;
-    circ_id = circ->n_circ_id;
-  } else {
-    next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
-    prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
-    circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux;
-    chan = TO_OR_CIRCUIT(circ)->p_chan;
-    circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
-  }
-
-  /* Assert that it is attached to this mux and a channel */
-  tor_assert(cmux == circuit_cmux);
-  tor_assert(chan != NULL);
-
-  /*
-   * Check if the circuit really was inactive; if it's active, at least one
-   * of the next_active and prev_active pointers will not be NULL, or this
-   * circuit will be either the head or tail of the list for this cmux.
-   */
-  already_active = (*prev_active != NULL || *next_active != NULL ||
-                    cmux->active_circuits_head == circ ||
-                    cmux->active_circuits_tail == circ);
-
-  /* If we're already active, log a warning and finish */
-  if (already_active) {
-    log_warn(LD_CIRC,
-             "Circuit %u on channel " U64_FORMAT " was already active",
-             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier));
-    return;
-  }
-
-  /*
-   * This is going at the head of the list; if the old head is not NULL,
-   * then its prev pointer should point to this.
-   */
-  *next_active = cmux->active_circuits_head; /* Next is old head */
-  *prev_active = NULL; /* Prev is NULL (this will be the head) */
-  if (cmux->active_circuits_head) {
-    /* The list had an old head; update its prev pointer */
-    next_prev =
-      circuitmux_prev_active_circ_p(cmux, cmux->active_circuits_head);
-    tor_assert(next_prev);
-    *next_prev = circ;
-  } else {
-    /* The list was empty; this becomes the tail as well */
-    cmux->active_circuits_tail = circ;
-  }
-  /* This becomes the new head of the list */
-  cmux->active_circuits_head = circ;
 
   /* Policy-specific notification */
-  if (cmux->policy &&
-      cmux->policy->notify_circ_active) {
+  if (cmux->policy->notify_circ_active) {
     /* Okay, we need to check the circuit for policy data now */
-    hashent = circuitmux_find_map_entry(cmux, circ);
+    chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ);
     /* We should have found something */
     tor_assert(hashent);
     /* Notify */
     cmux->policy->notify_circ_active(cmux, cmux->policy_data,
                                      circ, hashent->muxinfo.policy_data);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1270,112 +1031,22 @@ circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ,
  */
 
 static void
-circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ,
-                                 cell_direction_t direction)
+circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ)
 {
-  circuit_t **next_active = NULL, **prev_active = NULL;
-  circuit_t **next_prev = NULL, **prev_next = NULL;
-  circuitmux_t *circuit_cmux = NULL;
-  chanid_circid_muxinfo_t *hashent = NULL;
-  channel_t *chan = NULL;
-  circid_t circ_id;
-  int already_inactive;
-
   tor_assert(cmux);
+  tor_assert(cmux->policy);
   tor_assert(circ);
-  tor_assert(direction == CELL_DIRECTION_OUT ||
-             direction == CELL_DIRECTION_IN);
-  /*
-   * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count
-   * already got changed and we have to update the list for it to be consistent
-   * again.
-   */
-
-  /* Get the right set of active list links for this direction */
-  if (direction == CELL_DIRECTION_OUT) {
-    next_active = &(circ->next_active_on_n_chan);
-    prev_active = &(circ->prev_active_on_n_chan);
-    circuit_cmux = circ->n_mux;
-    chan = circ->n_chan;
-    circ_id = circ->n_circ_id;
-  } else {
-    next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan);
-    prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan);
-    circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux;
-    chan = TO_OR_CIRCUIT(circ)->p_chan;
-    circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
-  }
-
-  /* Assert that it is attached to this mux and a channel */
-  tor_assert(cmux == circuit_cmux);
-  tor_assert(chan != NULL);
-
-  /*
-   * Check if the circuit really was active; if it's inactive, the
-   * next_active and prev_active pointers will be NULL and this circuit
-   * will not be the head or tail of the list for this cmux.
-   */
-  already_inactive = (*prev_active == NULL && *next_active == NULL &&
-                      cmux->active_circuits_head != circ &&
-                      cmux->active_circuits_tail != circ);
-
-  /* If we're already inactive, log a warning and finish */
-  if (already_inactive) {
-    log_warn(LD_CIRC,
-             "Circuit %d on channel " U64_FORMAT " was already inactive",
-             (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier));
-    return;
-  }
-
-  /* Remove from the list; first get next_prev and prev_next */
-  if (*next_active) {
-    /*
-     * If there's a next circuit, its previous circuit becomes this
-     * circuit's previous circuit.
-     */
-    next_prev = circuitmux_prev_active_circ_p(cmux, *next_active);
-  } else {
-    /* Else, the tail becomes this circuit's previous circuit */
-    next_prev = &(cmux->active_circuits_tail);
-  }
-
-  /* Got next_prev, now prev_next */
-  if (*prev_active) {
-    /*
-     * If there's a previous circuit, its next circuit becomes this circuit's
-     * next circuit.
-     */
-    prev_next = circuitmux_next_active_circ_p(cmux, *prev_active);
-  } else {
-    /* Else, the head becomes this circuit's next circuit */
-    prev_next = &(cmux->active_circuits_head);
-  }
-
-  /* Assert that we got sensible values for the next/prev pointers */
-  tor_assert(next_prev != NULL);
-  tor_assert(prev_next != NULL);
-
-  /* Update the next/prev pointers - this removes circ from the list */
-  *next_prev = *prev_active;
-  *prev_next = *next_active;
-
-  /* Now null out prev_active/next_active */
-  *prev_active = NULL;
-  *next_active = NULL;
 
   /* Policy-specific notification */
-  if (cmux->policy &&
-      cmux->policy->notify_circ_inactive) {
+  if (cmux->policy->notify_circ_inactive) {
     /* Okay, we need to check the circuit for policy data now */
-    hashent = circuitmux_find_map_entry(cmux, circ);
+    chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ);
     /* We should have found something */
     tor_assert(hashent);
     /* Notify */
     cmux->policy->notify_circ_inactive(cmux, cmux->policy_data,
                                        circ, hashent->muxinfo.policy_data);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1402,8 +1073,6 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   tor_assert(cmux);
   tor_assert(circ);
 
-  circuitmux_assert_okay_paranoid(cmux);
-
   /* Search for this circuit's entry */
   hashent = circuitmux_find_map_entry(cmux, circ);
   /* Assert that we found one */
@@ -1414,7 +1083,7 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   cmux->n_cells += n_cells;
 
   /* Do we need to notify a cmux policy? */
-  if (cmux->policy && cmux->policy->notify_set_n_cells) {
+  if (cmux->policy->notify_set_n_cells) {
     /* Call notify_set_n_cells */
     cmux->policy->notify_set_n_cells(cmux,
                                      cmux->policy_data,
@@ -1430,21 +1099,15 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ,
   if (hashent->muxinfo.cell_count > 0 && n_cells == 0) {
     --(cmux->n_active_circuits);
     hashent->muxinfo.cell_count = n_cells;
-    circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction);
+    circuitmux_make_circuit_inactive(cmux, circ);
   /* Is the old cell count == 0 and the new cell count > 0 ? */
   } else if (hashent->muxinfo.cell_count == 0 && n_cells > 0) {
     ++(cmux->n_active_circuits);
     hashent->muxinfo.cell_count = n_cells;
-    circuitmux_make_circuit_active(cmux, circ, hashent->muxinfo.direction);
+    circuitmux_make_circuit_active(cmux, circ);
   } else {
-    /*
-     * Update the entry cell count like this so we can put a
-     * circuitmux_assert_okay_paranoid inside make_circuit_(in)active() too.
-     */
     hashent->muxinfo.cell_count = n_cells;
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /*
@@ -1470,6 +1133,9 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux,
   circuit_t *circ = NULL;
 
   tor_assert(cmux);
+  tor_assert(cmux->policy);
+  /* This callback is mandatory. */
+  tor_assert(cmux->policy->pick_active_circuit);
   tor_assert(destroy_queue_out);
 
   *destroy_queue_out = NULL;
@@ -1488,14 +1154,7 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux,
     /* We also must have a cell available for this to be the case */
     tor_assert(cmux->n_cells > 0);
     /* Do we have a policy-provided circuit selector? */
-    if (cmux->policy && cmux->policy->pick_active_circuit) {
-      circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data);
-    }
-    /* Fall back on the head of the active circuits list */
-    if (!circ) {
-      tor_assert(cmux->active_circuits_head);
-      circ = cmux->active_circuits_head;
-    }
+    circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data);
     cmux->last_cell_was_destroy = 0;
   } else {
     tor_assert(cmux->n_cells == 0);
@@ -1519,7 +1178,6 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
 
   tor_assert(cmux);
   tor_assert(circ);
-  circuitmux_assert_okay_paranoid(cmux);
 
   if (n_cells == 0) return;
 
@@ -1546,17 +1204,11 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
   /* Adjust the mux cell counter */
   cmux->n_cells -= n_cells;
 
-  /* If we aren't making it inactive later, move it to the tail of the list */
-  if (!becomes_inactive) {
-    circuitmux_move_active_circ_to_tail(cmux, circ,
-                                        hashent->muxinfo.direction);
-  }
-
   /*
    * We call notify_xmit_cells() before making the circuit inactive if needed,
    * so the policy can always count on this coming in on an active circuit.
    */
-  if (cmux->policy && cmux->policy->notify_xmit_cells) {
+  if (cmux->policy->notify_xmit_cells) {
     cmux->policy->notify_xmit_cells(cmux, cmux->policy_data, circ,
                                     hashent->muxinfo.policy_data,
                                     n_cells);
@@ -1568,10 +1220,8 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ,
    */
   if (becomes_inactive) {
     --(cmux->n_active_circuits);
-    circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction);
+    circuitmux_make_circuit_inactive(cmux, circ);
   }
-
-  circuitmux_assert_okay_paranoid(cmux);
 }
 
 /**
@@ -1594,286 +1244,6 @@ circuitmux_notify_xmit_destroy(circuitmux_t *cmux)
             I64_PRINTF_ARG(global_destroy_ctr));
 }
 
-/*
- * Circuitmux consistency checking assertions
- */
-
-/**
- * Check that circuitmux data structures are consistent and fail with an
- * assert if not.
- */
-
-void
-circuitmux_assert_okay(circuitmux_t *cmux)
-{
-  tor_assert(cmux);
-
-  /*
-   * Pass 1: iterate the hash table; for each entry:
-   *  a) Check that the circuit has this cmux for n_mux or p_mux
-   *  b) If the cell_count is > 0, set the mark bit; otherwise clear it
-   *  c) Also check activeness (cell_count > 0 should be active)
-   *  d) Count the number of circuits, active circuits and queued cells
-   *     and at the end check that they match the counters in the cmux.
-   *
-   * Pass 2: iterate the active circuits list; for each entry,
-   *         make sure the circuit is attached to this mux and appears
-   *         in the hash table.  Make sure the mark bit is 1, and clear
-   *         it in the hash table entry.  Consistency-check the linked
-   *         list pointers.
-   *
-   * Pass 3: iterate the hash table again; assert if any active circuits
-   *         (mark bit set to 1) are discovered that weren't cleared in pass 2
-   *         (don't appear in the linked list).
-   */
-
-  circuitmux_assert_okay_pass_one(cmux);
-  circuitmux_assert_okay_pass_two(cmux);
-  circuitmux_assert_okay_pass_three(cmux);
-}
-
-/**
- * Do the first pass of circuitmux_assert_okay(); see the comment in that
- * function.
- */
-
-static void
-circuitmux_assert_okay_pass_one(circuitmux_t *cmux)
-{
-  chanid_circid_muxinfo_t **i = NULL;
-  uint64_t chan_id;
-  channel_t *chan;
-  circid_t circ_id;
-  circuit_t *circ;
-  or_circuit_t *or_circ;
-  unsigned int circ_is_active;
-  circuit_t **next_p, **prev_p;
-  unsigned int n_circuits, n_active_circuits, n_cells;
-
-  tor_assert(cmux);
-  tor_assert(cmux->chanid_circid_map);
-
-  /* Reset the counters */
-  n_circuits = n_active_circuits = n_cells = 0;
-  /* Start iterating the hash table */
-  i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
-  while (i) {
-    /* Assert that the hash table entry isn't null */
-    tor_assert(*i);
-
-    /* Get the channel and circuit id */
-    chan_id = (*i)->chan_id;
-    circ_id = (*i)->circ_id;
-
-    /* Find the channel and circuit, assert that they exist */
-    chan = channel_find_by_global_id(chan_id);
-    tor_assert(chan);
-    circ = circuit_get_by_circid_channel_even_if_marked(circ_id, chan);
-    tor_assert(circ);
-    /* Clear the circ_is_active bit to start */
-    circ_is_active = 0;
-
-    /* Assert that we know which direction this is going */
-    tor_assert((*i)->muxinfo.direction == CELL_DIRECTION_OUT ||
-               (*i)->muxinfo.direction == CELL_DIRECTION_IN);
-
-    if ((*i)->muxinfo.direction == CELL_DIRECTION_OUT) {
-      /* We should be n_mux on this circuit */
-      tor_assert(cmux == circ->n_mux);
-      tor_assert(chan == circ->n_chan);
-      /* Get next and prev for next test */
-      next_p = &(circ->next_active_on_n_chan);
-      prev_p = &(circ->prev_active_on_n_chan);
-    } else {
-      /* This should be an or_circuit_t and we should be p_mux */
-      or_circ = TO_OR_CIRCUIT(circ);
-      tor_assert(cmux == or_circ->p_mux);
-      tor_assert(chan == or_circ->p_chan);
-      /* Get next and prev for next test */
-      next_p = &(or_circ->next_active_on_p_chan);
-      prev_p = &(or_circ->prev_active_on_p_chan);
-    }
-
-    /*
-     * Should this circuit be active?  I.e., does the mux know about > 0
-     * cells on it?
-     */
-    circ_is_active = ((*i)->muxinfo.cell_count > 0);
-
-    /* It should be in the linked list iff it's active */
-    if (circ_is_active) {
-      /* Either we have a next link or we are the tail */
-      tor_assert(*next_p || (circ == cmux->active_circuits_tail));
-      /* Either we have a prev link or we are the head */
-      tor_assert(*prev_p || (circ == cmux->active_circuits_head));
-      /* Increment the active circuits counter */
-      ++n_active_circuits;
-    } else {
-      /* Shouldn't be in list, so no next or prev link */
-      tor_assert(!(*next_p));
-      tor_assert(!(*prev_p));
-      /* And can't be head or tail */
-      tor_assert(circ != cmux->active_circuits_head);
-      tor_assert(circ != cmux->active_circuits_tail);
-    }
-
-    /* Increment the circuits counter */
-    ++n_circuits;
-    /* Adjust the cell counter */
-    n_cells += (*i)->muxinfo.cell_count;
-
-    /* Set the mark bit to circ_is_active */
-    (*i)->muxinfo.mark = circ_is_active;
-
-    /* Advance to the next entry */
-    i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i);
-  }
-
-  /* Now check the counters */
-  tor_assert(n_cells == cmux->n_cells);
-  tor_assert(n_circuits == cmux->n_circuits);
-  tor_assert(n_active_circuits == cmux->n_active_circuits);
-}
-
-/**
- * Do the second pass of circuitmux_assert_okay(); see the comment in that
- * function.
- */
-
-static void
-circuitmux_assert_okay_pass_two(circuitmux_t *cmux)
-{
-  circuit_t *curr_circ, *prev_circ = NULL, *next_circ;
-  or_circuit_t *curr_or_circ;
-  uint64_t curr_chan_id;
-  circid_t curr_circ_id;
-  circuit_t **next_p, **prev_p;
-  channel_t *chan;
-  unsigned int n_active_circuits = 0;
-  cell_direction_t direction;
-  chanid_circid_muxinfo_t search, *hashent = NULL;
-
-  tor_assert(cmux);
-  tor_assert(cmux->chanid_circid_map);
-
-  /*
-   * Walk the linked list of active circuits in cmux; keep track of the
-   * previous circuit seen for consistency checking purposes.  Count them
-   * to make sure the number in the linked list matches
-   * cmux->n_active_circuits.
-   */
-  curr_circ = cmux->active_circuits_head;
-  while (curr_circ) {
-    /* Reset some things */
-    chan = NULL;
-    curr_or_circ = NULL;
-    next_circ = NULL;
-    next_p = prev_p = NULL;
-    direction = 0;
-
-    /* Figure out if this is n_mux or p_mux */
-    if (cmux == curr_circ->n_mux) {
-      /* Get next_p and prev_p */
-      next_p = &(curr_circ->next_active_on_n_chan);
-      prev_p = &(curr_circ->prev_active_on_n_chan);
-      /* Get the channel */
-      chan = curr_circ->n_chan;
-      /* Get the circuit id */
-      curr_circ_id = curr_circ->n_circ_id;
-      /* Remember the direction */
-      direction = CELL_DIRECTION_OUT;
-    } else {
-      /* We must be p_mux and this must be an or_circuit_t */
-      curr_or_circ = TO_OR_CIRCUIT(curr_circ);
-      tor_assert(cmux == curr_or_circ->p_mux);
-      /* Get next_p and prev_p */
-      next_p = &(curr_or_circ->next_active_on_p_chan);
-      prev_p = &(curr_or_circ->prev_active_on_p_chan);
-      /* Get the channel */
-      chan = curr_or_circ->p_chan;
-      /* Get the circuit id */
-      curr_circ_id = curr_or_circ->p_circ_id;
-      /* Remember the direction */
-      direction = CELL_DIRECTION_IN;
-    }
-
-    /* Assert that we got a channel and get the channel ID */
-    tor_assert(chan);
-    curr_chan_id = chan->global_identifier;
-
-    /* Assert that prev_p points to last circuit we saw */
-    tor_assert(*prev_p == prev_circ);
-    /* If that's NULL, assert that we are the head */
-    if (!(*prev_p)) tor_assert(curr_circ == cmux->active_circuits_head);
-
-    /* Get the next circuit */
-    next_circ = *next_p;
-    /* If it's NULL, assert that we are the tail */
-    if (!(*next_p)) tor_assert(curr_circ == cmux->active_circuits_tail);
-
-    /* Now find the hash table entry for this circuit */
-    search.chan_id = curr_chan_id;
-    search.circ_id = curr_circ_id;
-    hashent = HT_FIND(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
-                      &search);
-
-    /* Assert that we have one */
-    tor_assert(hashent);
-
-    /* Assert that the direction matches */
-    tor_assert(direction == hashent->muxinfo.direction);
-
-    /* Assert that the hash entry got marked in pass one */
-    tor_assert(hashent->muxinfo.mark);
-
-    /* Clear the mark */
-    hashent->muxinfo.mark = 0;
-
-    /* Increment the counter */
-    ++n_active_circuits;
-
-    /* Advance to the next active circuit and update prev_circ */
-    prev_circ = curr_circ;
-    curr_circ = next_circ;
-  }
-
-  /* Assert that the counter matches the cmux */
-  tor_assert(n_active_circuits == cmux->n_active_circuits);
-}
-
-/**
- * Do the third pass of circuitmux_assert_okay(); see the comment in that
- * function.
- */
-
-static void
-circuitmux_assert_okay_pass_three(circuitmux_t *cmux)
-{
-  chanid_circid_muxinfo_t **i = NULL;
-
-  tor_assert(cmux);
-  tor_assert(cmux->chanid_circid_map);
-
-  /* Start iterating the hash table */
-  i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map);
-
-  /* Advance through each entry */
-  while (i) {
-    /* Assert that it isn't null */
-    tor_assert(*i);
-
-    /*
-     * Assert that this entry is not marked - i.e., that either we didn't
-     * think it should be active in pass one or we saw it in the active
-     * circuits linked list.
-     */
-    tor_assert(!((*i)->muxinfo.mark));
-
-    /* Advance to the next entry */
-    i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i);
-  }
-}
-
 /*DOCDOC */
 void
 circuitmux_append_destroy_cell(channel_t *chan,
diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h
index f53abdd8c8..336e128c76 100644
--- a/src/or/circuitmux.h
+++ b/src/or/circuitmux.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -104,7 +104,9 @@ void circuitmux_assert_okay(circuitmux_t *cmux);
 circuitmux_t * circuitmux_alloc(void);
 void circuitmux_detach_all_circuits(circuitmux_t *cmux,
                                     smartlist_t *detached_out);
-void circuitmux_free(circuitmux_t *cmux);
+void circuitmux_free_(circuitmux_t *cmux);
+#define circuitmux_free(cmux) \
+  FREE_AND_NULL(circuitmux_t, circuitmux_free_, (cmux))
 
 /* Policy control */
 void circuitmux_clear_policy(circuitmux_t *cmux);
@@ -156,5 +158,5 @@ void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux,
 MOCK_DECL(int, circuitmux_compare_muxes,
           (circuitmux_t *cmux_1, circuitmux_t *cmux_2));
 
-#endif /* TOR_CIRCUITMUX_H */
+#endif /* !defined(TOR_CIRCUITMUX_H) */
 
diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c
index 5c2ebde73b..e5d5a14581 100644
--- a/src/or/circuitmux_ewma.c
+++ b/src/or/circuitmux_ewma.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,7 +28,7 @@
  *
  **/
 
-#define TOR_CIRCUITMUX_EWMA_C_
+#define CIRCUITMUX_EWMA_PRIVATE
 
 #include "orconfig.h"
 
@@ -37,6 +37,7 @@
 #include "or.h"
 #include "circuitmux.h"
 #include "circuitmux_ewma.h"
+#include "crypto_rand.h"
 #include "networkstatus.h"
 
 /*** EWMA parameter #defines ***/
@@ -169,8 +170,6 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
 
 static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
 static int compare_cell_ewma_counts(const void *p1, const void *p2);
-static unsigned cell_ewma_tick_from_timeval(const struct timeval *now,
-                                            double *remainder_out);
 static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma);
 static inline double get_scale_factor(unsigned from_tick, unsigned to_tick);
 static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol);
@@ -223,8 +222,6 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
  * has value ewma_scale_factor ** N.)
  */
 static double ewma_scale_factor = 0.1;
-/* DOCDOC ewma_enabled */
-static int ewma_enabled = 0;
 
 /*** EWMA circuitmux_policy_t method table ***/
 
@@ -241,8 +238,26 @@ circuitmux_policy_t ewma_policy = {
   /*.cmp_cmux =*/ ewma_cmp_cmux
 };
 
+/** Have we initialized the ewma tick-counting logic? */
+static int ewma_ticks_initialized = 0;
+/** At what monotime_coarse_t did the current tick begin? */
+static monotime_coarse_t start_of_current_tick;
+/** What is the number of the current tick? */
+static unsigned current_tick_num;
+
 /*** EWMA method implementations using the below EWMA helper functions ***/
 
+/** Compute and return the current cell_ewma tick. */
+static inline unsigned int
+cell_ewma_get_tick(void)
+{
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick,
+                                                  &now);
+  return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN);
+}
+
 /**
  * Allocate an ewma_policy_data_t and upcast it to a circuitmux_policy_data_t;
  * this is called when setting the policy on a circuitmux_t to ewma_policy.
@@ -416,8 +431,6 @@ ewma_notify_xmit_cells(circuitmux_t *cmux,
   ewma_policy_circ_data_t *cdata = NULL;
   unsigned int tick;
   double fractional_tick, ewma_increment;
-  /* The current (hi-res) time */
-  struct timeval now_hires;
   cell_ewma_t *cell_ewma, *tmp;
 
   tor_assert(cmux);
@@ -430,8 +443,7 @@ ewma_notify_xmit_cells(circuitmux_t *cmux,
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
 
   /* Rescale the EWMAs if needed */
-  tor_gettimeofday_cached(&now_hires);
-  tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick);
+  tick = cell_ewma_get_current_tick_and_fraction(&fractional_tick);
 
   if (tick != pol->active_circuit_pqueue_last_recalibrated) {
     scale_active_circuits(pol, tick);
@@ -500,7 +512,7 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
   tor_assert(pol_data_2);
 
   p1 = TO_EWMA_POL_DATA(pol_data_1);
-  p2 = TO_EWMA_POL_DATA(pol_data_1);
+  p2 = TO_EWMA_POL_DATA(pol_data_2);
 
   if (p1 != p2) {
     /* Get the head cell_ewma_t from each queue */
@@ -592,79 +604,122 @@ cell_ewma_to_circuit(cell_ewma_t *ewma)
    rescale.
  */
 
-/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs
- * and the fraction of the tick that has elapsed between the start of the tick
- * and <b>now</b>.  Return the former and store the latter in
- * *<b>remainder_out</b>.
+/**
+ * Initialize the system that tells which ewma tick we are in.
+ */
+STATIC void
+cell_ewma_initialize_ticks(void)
+{
+  if (ewma_ticks_initialized)
+    return;
+  monotime_coarse_get(&start_of_current_tick);
+  crypto_rand((char*)&current_tick_num, sizeof(current_tick_num));
+  ewma_ticks_initialized = 1;
+}
+
+/** Compute the current cell_ewma tick and the fraction of the tick that has
+ * elapsed between the start of the tick and the current time.  Return the
+ * former and store the latter in *<b>remainder_out</b>.
  *
  * These tick values are not meant to be shared between Tor instances, or used
  * for other purposes. */
-
-static unsigned
-cell_ewma_tick_from_timeval(const struct timeval *now,
-                            double *remainder_out)
+STATIC unsigned
+cell_ewma_get_current_tick_and_fraction(double *remainder_out)
 {
-  unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN);
-  /* rem */
-  double rem = (now->tv_sec % EWMA_TICK_LEN) +
-    ((double)(now->tv_usec)) / 1.0e6;
-  *remainder_out = rem / EWMA_TICK_LEN;
-  return res;
+  if (BUG(!ewma_ticks_initialized)) {
+    cell_ewma_initialize_ticks(); // LCOV_EXCL_LINE
+  }
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick,
+                                                  &now);
+  if (msec_diff > (1000*EWMA_TICK_LEN)) {
+    unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN);
+    monotime_coarse_add_msec(&start_of_current_tick,
+                             &start_of_current_tick,
+                             ticks_difference * 1000 * EWMA_TICK_LEN);
+    current_tick_num += ticks_difference;
+    msec_diff %= 1000*EWMA_TICK_LEN;
+  }
+  *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN);
+  return current_tick_num;
 }
 
-/** Tell the caller whether ewma_enabled is set */
-int
-cell_ewma_enabled(void)
+/* Default value for the CircuitPriorityHalflifeMsec consensus parameter in
+ * msec. */
+#define CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT 30000
+/* Minimum and maximum value for the CircuitPriorityHalflifeMsec consensus
+ * parameter. */
+#define CMUX_PRIORITY_HALFLIFE_MSEC_MIN 1
+#define CMUX_PRIORITY_HALFLIFE_MSEC_MAX INT32_MAX
+
+/* Return the value of the circuit priority halflife from the options if
+ * available or else from the consensus (in that order). If none can be found,
+ * a default value is returned.
+ *
+ * The source_msg points to a string describing from where the value was
+ * picked so it can be used for logging. */
+static double
+get_circuit_priority_halflife(const or_options_t *options,
+                              const networkstatus_t *consensus,
+                              const char **source_msg)
 {
-  return ewma_enabled;
-}
+  int32_t halflife_ms;
+  double halflife;
+  /* Compute the default value now. We might need it. */
+  double halflife_default =
+    ((double) CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT) / 1000.0;
 
-/** Compute and return the current cell_ewma tick. */
-unsigned int
-cell_ewma_get_tick(void)
-{
-  return ((unsigned)approx_time() / EWMA_TICK_LEN);
+  /* Try to get it from configuration file first. */
+  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
+    halflife = options->CircuitPriorityHalflife;
+    *source_msg = "CircuitPriorityHalflife in configuration";
+    goto end;
+  }
+
+  /* Try to get the msec value from the consensus. */
+  halflife_ms = networkstatus_get_param(consensus,
+                                        "CircuitPriorityHalflifeMsec",
+                                        CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT,
+                                        CMUX_PRIORITY_HALFLIFE_MSEC_MIN,
+                                        CMUX_PRIORITY_HALFLIFE_MSEC_MAX);
+  halflife = ((double) halflife_ms) / 1000.0;
+  *source_msg = "CircuitPriorityHalflifeMsec in consensus";
+
+ end:
+  /* We should never go below the EPSILON else we would consider it disabled
+   * and we can't have that. */
+  if (halflife < EPSILON) {
+    log_warn(LD_CONFIG, "CircuitPriorityHalflife is too small (%f). "
+                        "Adjusting to the smallest value allowed: %f.",
+             halflife, halflife_default);
+    halflife = halflife_default;
+  }
+  return halflife;
 }
 
 /** Adjust the global cell scale factor based on <b>options</b> */
 void
-cell_ewma_set_scale_factor(const or_options_t *options,
-                           const networkstatus_t *consensus)
+cmux_ewma_set_options(const or_options_t *options,
+                      const networkstatus_t *consensus)
 {
-  int32_t halflife_ms;
   double halflife;
   const char *source;
-  if (options && options->CircuitPriorityHalflife >= -EPSILON) {
-    halflife = options->CircuitPriorityHalflife;
-    source = "CircuitPriorityHalflife in configuration";
-  } else if (consensus && (halflife_ms = networkstatus_get_param(
-                 consensus, "CircuitPriorityHalflifeMsec",
-                 -1, -1, INT32_MAX)) >= 0) {
-    halflife = ((double)halflife_ms)/1000.0;
-    source = "CircuitPriorityHalflifeMsec in consensus";
-  } else {
-    halflife = EWMA_DEFAULT_HALFLIFE;
-    source = "Default value";
-  }
 
-  if (halflife <= EPSILON) {
-    /* The cell EWMA algorithm is disabled. */
-    ewma_scale_factor = 0.1;
-    ewma_enabled = 0;
-    log_info(LD_OR,
-             "Disabled cell_ewma algorithm because of value in %s",
-             source);
-  } else {
-    /* convert halflife into halflife-per-tick. */
-    halflife /= EWMA_TICK_LEN;
-    /* compute per-tick scale factor. */
-    ewma_scale_factor = exp( LOG_ONEHALF / halflife );
-    ewma_enabled = 1;
-    log_info(LD_OR,
-             "Enabled cell_ewma algorithm because of value in %s; "
-             "scale factor is %f per %d seconds",
-             source, ewma_scale_factor, EWMA_TICK_LEN);
-  }
+  cell_ewma_initialize_ticks();
+
+  /* Both options and consensus can be NULL. This assures us to either get a
+   * valid configured value or the default one. */
+  halflife = get_circuit_priority_halflife(options, consensus, &source);
+
+  /* convert halflife into halflife-per-tick. */
+  halflife /= EWMA_TICK_LEN;
+  /* compute per-tick scale factor. */
+  ewma_scale_factor = exp( LOG_ONEHALF / halflife );
+  log_info(LD_OR,
+           "Enabled cell_ewma algorithm because of value in %s; "
+           "scale factor is %f per %d seconds",
+           source, ewma_scale_factor, EWMA_TICK_LEN);
 }
 
 /** Return the multiplier necessary to convert the value of a cell sent in
@@ -731,7 +786,7 @@ add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma)
 
   smartlist_pqueue_add(pol->active_circuit_pqueue,
                        compare_cell_ewma_counts,
-                       STRUCT_OFFSET(cell_ewma_t, heap_index),
+                       offsetof(cell_ewma_t, heap_index),
                        ewma);
 }
 
@@ -746,7 +801,7 @@ remove_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma)
 
   smartlist_pqueue_remove(pol->active_circuit_pqueue,
                           compare_cell_ewma_counts,
-                          STRUCT_OFFSET(cell_ewma_t, heap_index),
+                          offsetof(cell_ewma_t, heap_index),
                           ewma);
 }
 
@@ -760,6 +815,15 @@ pop_first_cell_ewma(ewma_policy_data_t *pol)
 
   return smartlist_pqueue_pop(pol->active_circuit_pqueue,
                               compare_cell_ewma_counts,
-                              STRUCT_OFFSET(cell_ewma_t, heap_index));
+                              offsetof(cell_ewma_t, heap_index));
+}
+
+/**
+ * Drop all resources held by circuitmux_ewma.c, and deinitialize the
+ * module. */
+void
+circuitmux_ewma_free_all(void)
+{
+  ewma_ticks_initialized = 0;
 }
 
diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h
index a7b8961ac6..f0c4c36095 100644
--- a/src/or/circuitmux_ewma.h
+++ b/src/or/circuitmux_ewma.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,19 @@
 #include "or.h"
 #include "circuitmux.h"
 
+/* The public EWMA policy callbacks object. */
 extern circuitmux_policy_t ewma_policy;
 
 /* Externally visible EWMA functions */
-int cell_ewma_enabled(void);
-unsigned int cell_ewma_get_tick(void);
-void cell_ewma_set_scale_factor(const or_options_t *options,
-                                const networkstatus_t *consensus);
+void cmux_ewma_set_options(const or_options_t *options,
+                           const networkstatus_t *consensus);
 
-#endif /* TOR_CIRCUITMUX_EWMA_H */
+void circuitmux_ewma_free_all(void);
+
+#ifdef CIRCUITMUX_EWMA_PRIVATE
+STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out);
+STATIC void cell_ewma_initialize_ticks(void);
+#endif
+
+#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
 
diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c
index 735b3f0fc2..94f75c590f 100644
--- a/src/or/circuitstats.c
+++ b/src/or/circuitstats.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,11 +31,15 @@
 #include "config.h"
 #include "confparse.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "rendclient.h"
 #include "rendservice.h"
+#include "router.h"
 #include "statefile.h"
+#include "circuitlist.h"
+#include "circuituse.h"
 
 #undef log
 #include <math.h>
@@ -43,6 +47,7 @@
 static void cbt_control_event_buildtimeout_set(
                                   const circuit_build_times_t *cbt,
                                   buildtimeout_set_event_t type);
+static void circuit_build_times_scale_circ_counts(circuit_build_times_t *cbt);
 
 #define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2))
 
@@ -60,7 +65,7 @@ static circuit_build_times_t circ_times;
 static int unit_tests = 0;
 #else
 #define unit_tests 0
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Return a pointer to the data structure describing our current circuit
  * build time history and computations. */
@@ -105,16 +110,24 @@ get_circuit_build_timeout_ms(void)
  *  6. If we are configured in Single Onion mode
  */
 int
-circuit_build_times_disabled(void)
+circuit_build_times_disabled(const or_options_t *options)
+{
+  return circuit_build_times_disabled_(options, 0);
+}
+
+/** As circuit_build_times_disabled, but take options as an argument. */
+int
+circuit_build_times_disabled_(const or_options_t *options,
+                              int ignore_consensus)
 {
   if (unit_tests) {
     return 0;
   } else {
-    const or_options_t *options = get_options();
-    int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled",
+    int consensus_disabled =
+      ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled",
                                                      0, 0, 1);
     int config_disabled = !options->LearnCircuitBuildTimeout;
-    int dirauth_disabled = options->AuthoritativeDir;
+    int dirauth_disabled = authdir_mode(options);
     int state_disabled = did_last_state_file_write_fail() ? 1 : 0;
     /* LearnCircuitBuildTimeout and Tor2web/Single Onion Services are
      * incompatible in two ways:
@@ -140,7 +153,7 @@ circuit_build_times_disabled(void)
                "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d",
                consensus_disabled, config_disabled, dirauth_disabled,
                state_disabled);
-#endif
+#endif /* 0 */
       return 1;
     } else {
 #if 0
@@ -149,14 +162,14 @@ circuit_build_times_disabled(void)
                 "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d",
                 consensus_disabled, config_disabled, dirauth_disabled,
                 state_disabled);
-#endif
+#endif /* 0 */
       return 0;
     }
   }
 }
 
 /**
- * Retrieve and bounds-check the cbtmaxtimeouts consensus paramter.
+ * Retrieve and bounds-check the cbtmaxtimeouts consensus parameter.
  *
  * Effect: When this many timeouts happen in the last 'cbtrecentcount'
  * circuit attempts, the client should discard all of its history and
@@ -183,7 +196,7 @@ circuit_build_times_max_timeouts(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtnummodes consensus paramter.
+ * Retrieve and bounds-check the cbtnummodes consensus parameter.
  *
  * Effect: This value governs how many modes to use in the weighted
  * average calculation of Pareto parameter Xm. A value of 3 introduces
@@ -210,7 +223,7 @@ circuit_build_times_default_num_xm_modes(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtmincircs consensus paramter.
+ * Retrieve and bounds-check the cbtmincircs consensus parameter.
  *
  * Effect: This is the minimum number of circuits to build before
  * computing a timeout.
@@ -242,7 +255,7 @@ circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt)
 }
 
 /**
- * Retrieve and bounds-check the cbtquantile consensus paramter.
+ * Retrieve and bounds-check the cbtquantile consensus parameter.
  *
  * Effect: This is the position on the quantile curve to use to set the
  * timeout value. It is a percent (10-99).
@@ -266,7 +279,7 @@ circuit_build_times_quantile_cutoff(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtclosequantile consensus paramter.
+ * Retrieve and bounds-check the cbtclosequantile consensus parameter.
  *
  * Effect: This is the position on the quantile curve to use to set the
  * timeout value to use to actually close circuits. It is a percent
@@ -298,7 +311,7 @@ circuit_build_times_close_quantile(void)
 }
 
 /**
- * Retrieve and bounds-check the cbttestfreq consensus paramter.
+ * Retrieve and bounds-check the cbttestfreq consensus parameter.
  *
  * Effect: Describes how often in seconds to build a test circuit to
  * gather timeout values. Only applies if less than 'cbtmincircs'
@@ -345,7 +358,7 @@ circuit_build_times_min_timeout(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtinitialtimeout consensus paramter.
+ * Retrieve and bounds-check the cbtinitialtimeout consensus parameter.
  *
  * Effect: This is the timeout value to use before computing a timeout,
  * in milliseconds.
@@ -375,7 +388,7 @@ circuit_build_times_initial_timeout(void)
 }
 
 /**
- * Retrieve and bounds-check the cbtrecentcount consensus paramter.
+ * Retrieve and bounds-check the cbtrecentcount consensus parameter.
  *
  * Effect: This is the number of circuit build times to keep track of
  * for deciding if we hit cbtmaxtimeouts and need to reset our state
@@ -417,15 +430,20 @@ circuit_build_times_new_consensus_params(circuit_build_times_t *cbt,
    * update if we aren't.
    */
 
-  if (!circuit_build_times_disabled()) {
+  if (!circuit_build_times_disabled(get_options())) {
     num = circuit_build_times_recent_circuit_count(ns);
 
     if (num > 0) {
       if (num != cbt->liveness.num_recent_circs) {
         int8_t *recent_circs;
-        log_notice(LD_CIRC, "The Tor Directory Consensus has changed how many "
-                   "circuits we must track to detect network failures from %d "
-                   "to %d.", cbt->liveness.num_recent_circs, num);
+        if (cbt->liveness.num_recent_circs > 0) {
+          log_notice(LD_CIRC, "The Tor Directory Consensus has changed how "
+                     "many circuits we must track to detect network failures "
+                     "from %d to %d.", cbt->liveness.num_recent_circs, num);
+        } else {
+          log_notice(LD_CIRC, "Upon receiving a consensus directory, "
+                     "re-enabling circuit-based network failure detection.");
+        }
 
         tor_assert(cbt->liveness.timeouts_after_firsthop ||
                    cbt->liveness.num_recent_circs == 0);
@@ -493,14 +511,15 @@ static double
 circuit_build_times_get_initial_timeout(void)
 {
   double timeout;
+  const or_options_t *options = get_options();
 
   /*
    * Check if we have LearnCircuitBuildTimeout, and if we don't,
    * always use CircuitBuildTimeout, no questions asked.
    */
-  if (!unit_tests && get_options()->CircuitBuildTimeout) {
-    timeout = get_options()->CircuitBuildTimeout*1000;
-    if (!circuit_build_times_disabled() &&
+  if (!unit_tests && options->CircuitBuildTimeout) {
+    timeout = options->CircuitBuildTimeout*1000;
+    if (!circuit_build_times_disabled(options) &&
         timeout < circuit_build_times_min_timeout()) {
       log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds",
                circuit_build_times_min_timeout()/1000);
@@ -526,6 +545,11 @@ circuit_build_times_reset(circuit_build_times_t *cbt)
   cbt->total_build_times = 0;
   cbt->build_times_idx = 0;
   cbt->have_computed_timeout = 0;
+
+  // Reset timeout and close counts
+  cbt->num_circ_succeeded = 0;
+  cbt->num_circ_closed = 0;
+  cbt->num_circ_timeouts = 0;
 }
 
 /**
@@ -542,7 +566,7 @@ circuit_build_times_init(circuit_build_times_t *cbt)
    * Check if we really are using adaptive timeouts, and don't keep
    * track of this stuff if not.
    */
-  if (!circuit_build_times_disabled()) {
+  if (!circuit_build_times_disabled(get_options())) {
     cbt->liveness.num_recent_circs =
       circuit_build_times_recent_circuit_count(NULL);
     cbt->liveness.timeouts_after_firsthop =
@@ -599,7 +623,124 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n)
           "Rewound history by %d places. Current index: %d. "
           "Total: %d", n, cbt->build_times_idx, cbt->total_build_times);
 }
-#endif
+#endif /* 0 */
+
+/**
+ * Mark this circuit as timed out, but change its purpose
+ * so that it continues to build, allowing us to measure
+ * its full build time.
+ */
+void
+circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ)
+{
+  control_event_circuit_status(circ,
+                               CIRC_EVENT_FAILED,
+                               END_CIRC_REASON_TIMEOUT);
+  circuit_change_purpose(TO_CIRCUIT(circ),
+                         CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  /* Record this event to check for too many timeouts
+   * in a row. This function does not record a time value yet
+   * (we do that later); it only counts the fact that we did
+   * have a timeout. We also want to avoid double-counting
+   * already "relaxed" circuits, which are counted in
+   * circuit_expire_building(). */
+  if (!circ->relaxed_timeout) {
+    int first_hop_succeeded = circ->cpath &&
+          circ->cpath->state == CPATH_STATE_OPEN;
+
+    circuit_build_times_count_timeout(
+                                 get_circuit_build_times_mutable(),
+                                 first_hop_succeeded);
+  }
+}
+
+/**
+ * Perform the build time work that needs to be done when a circuit
+ * completes a hop.
+ *
+ * This function decides if we should record a circuit's build time
+ * in our histogram data and other statistics, and if so, records it.
+ * It also will mark circuits that have already timed out as
+ * measurement-only circuits, so they can continue to build but
+ * not get used.
+ *
+ * For this, we want to consider circuits that will eventually make
+ * it to the third hop. For circuits longer than 3 hops, we want to
+ * record their build time when they reach the third hop, but let
+ * them continue (and not count them later). For circuits that are
+ * exactly 3 hops, this will count them when they are completed. We
+ * do this so that CBT is always gathering statistics on circuits
+ * of the same length, regardless of their type.
+ */
+void
+circuit_build_times_handle_completed_hop(origin_circuit_t *circ)
+{
+  struct timeval end;
+  long timediff;
+
+  /* If circuit build times are disabled, let circuit_expire_building()
+   * handle it.. */
+  if (circuit_build_times_disabled(get_options())) {
+    return;
+  }
+
+  /* Is this a circuit for which the timeout applies in a straight-forward
+   * way? If so, handle it below. If not, just return (and let
+   * circuit_expire_building() eventually take care of it).
+   */
+  if (!circuit_timeout_want_to_count_circ(circ)) {
+    return;
+  }
+
+  tor_gettimeofday(&end);
+  timediff = tv_mdiff(&circ->base_.timestamp_began, &end);
+
+  /* Check if we would have timed out already. If so, change the
+   * purpose here. But don't do any timeout handling here if there
+   * are no circuits opened yet. Save it for circuit_expire_building()
+   * (to allow it to handle timeout "relaxing" over there). */
+  if (timediff > get_circuit_build_timeout_ms() &&
+      circuit_any_opened_circuits_cached()) {
+
+    /* Circuits are allowed to last longer for measurement.
+     * Switch their purpose and wait. */
+    if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+      log_info(LD_CIRC,
+               "Deciding to timeout circuit "U64_FORMAT"\n",
+               U64_PRINTF_ARG(circ->global_identifier));
+      circuit_build_times_mark_circ_as_measurement_only(circ);
+    }
+  }
+
+  /* If the circuit is built to exactly the DEFAULT_ROUTE_LEN,
+   * add it to our buildtimes. */
+  if (circuit_get_cpath_opened_len(circ) == DEFAULT_ROUTE_LEN) {
+    /* If the circuit build time is much greater than we would have cut
+     * it off at, we probably had a suspend event along this codepath,
+     * and we should discard the value.
+     */
+    if (timediff < 0 ||
+        timediff > 2*get_circuit_build_close_time_ms()+1000) {
+      log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. "
+                 "Assuming clock jump. Purpose %d (%s)", timediff,
+                 circ->base_.purpose,
+                 circuit_purpose_to_string(circ->base_.purpose));
+    } else {
+      /* Only count circuit times if the network is live */
+      if (circuit_build_times_network_check_live(
+                            get_circuit_build_times())) {
+        circuit_build_times_add_time(get_circuit_build_times_mutable(),
+                                     (build_time_t)timediff);
+        circuit_build_times_set_timeout(get_circuit_build_times_mutable());
+      }
+
+      if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+        circuit_build_times_network_circ_success(
+                                      get_circuit_build_times_mutable());
+      }
+    }
+  }
+}
 
 /**
  * Add a new build time value <b>time</b> to the set of build times. Time
@@ -667,7 +808,7 @@ circuit_build_times_min(circuit_build_times_t *cbt)
   }
   return min_build_time;
 }
-#endif
+#endif /* 0 */
 
 /**
  * Calculate and return a histogram for the set of build times.
@@ -913,12 +1054,12 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
   int tot_values = 0;
   uint32_t loaded_cnt = 0, N = 0;
   config_line_t *line;
-  unsigned int i;
+  int i;
   build_time_t *loaded_times;
   int err = 0;
   circuit_build_times_init(cbt);
 
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     return 0;
   }
 
@@ -942,7 +1083,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
       uint32_t count, k;
       build_time_t ms;
       int ok;
-      ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0,
+      ms = (build_time_t)tor_parse_ulong(ms_str, 10, 0,
                                          CBT_BUILD_TIME_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_GENERAL, "Unable to parse circuit build times: "
@@ -952,7 +1093,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
         smartlist_free(args);
         break;
       }
-      count = (uint32_t)tor_parse_ulong(count_str, 0, 0,
+      count = (uint32_t)tor_parse_ulong(count_str, 10, 0,
                                         UINT32_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_GENERAL, "Unable to parse circuit build times: "
@@ -963,8 +1104,8 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
         break;
       }
 
-      if (loaded_cnt+count+state->CircuitBuildAbandonedCount
-            > state->TotalBuildTimes) {
+      if (loaded_cnt+count+ (unsigned)state->CircuitBuildAbandonedCount
+          > (unsigned) state->TotalBuildTimes) {
         log_warn(LD_CIRC,
                  "Too many build times in state file. "
                  "Stopping short before %d",
@@ -989,7 +1130,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
     loaded_times[loaded_cnt++] = CBT_BUILD_ABANDONED;
   }
 
-  if (loaded_cnt != state->TotalBuildTimes) {
+  if (loaded_cnt != (unsigned)state->TotalBuildTimes) {
     log_warn(LD_CIRC,
             "Corrupt state file? Build times count mismatch. "
             "Read %d times, but file says %d", loaded_cnt,
@@ -1172,7 +1313,7 @@ circuit_build_times_cdf(circuit_build_times_t *cbt, double x)
   tor_assert(0 <= ret && ret <= 1.0);
   return ret;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef TOR_UNIT_TESTS
 /**
@@ -1207,7 +1348,7 @@ circuit_build_times_generate_sample(circuit_build_times_t *cbt,
   tor_assert(ret > 0);
   return ret;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #ifdef TOR_UNIT_TESTS
 /**
@@ -1230,7 +1371,7 @@ circuit_build_times_initial_alpha(circuit_build_times_t *cbt,
     (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms));
   tor_assert(cbt->alpha > 0);
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /**
  * Returns true if we need circuits to be built
@@ -1292,9 +1433,32 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 }
 
 /**
- * Called to indicate that we completed a circuit. Because this circuit
+ * Non-destructively scale all of our circuit success, timeout, and close
+ * counts down by a factor of two. Scaling in this way preserves the
+ * ratios between succeeded vs timed out vs closed circuits, so that
+ * our statistics don't change when we scale.
+ *
+ * This is used only in the rare event that we build more than
+ * INT32_MAX circuits.  Since the num_circ_* variables are
+ * uint32_t, we won't even be close to overflowing them.
+ */
+void
+circuit_build_times_scale_circ_counts(circuit_build_times_t *cbt)
+{
+  cbt->num_circ_succeeded /= 2;
+  cbt->num_circ_timeouts /= 2;
+  cbt->num_circ_closed /= 2;
+}
+
+/**
+ * Called to indicate that we "completed" a circuit. Because this circuit
  * succeeded, it doesn't count as a timeout-after-the-first-hop.
  *
+ * (For the purposes of the cbt code, we consider a circuit "completed" if
+ * it has 3 hops, regardless of its final hop count. We do this because
+ * we're trying to answer the question, "how long should a circuit take to
+ * reach the 3-hop count".)
+ *
  * This is used by circuit_build_times_network_check_changed() to determine
  * if we had too many recent timeouts and need to reset our learned timeout
  * to something higher.
@@ -1302,6 +1466,14 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 void
 circuit_build_times_network_circ_success(circuit_build_times_t *cbt)
 {
+  // Count circuit success
+  cbt->num_circ_succeeded++;
+
+  // If we're going to wrap int32, scale everything
+  if (cbt->num_circ_succeeded >= INT32_MAX) {
+    circuit_build_times_scale_circ_counts(cbt);
+  }
+
   /* Check for NULLness because we might not be using adaptive timeouts */
   if (cbt->liveness.timeouts_after_firsthop &&
       cbt->liveness.num_recent_circs > 0) {
@@ -1324,6 +1496,14 @@ static void
 circuit_build_times_network_timeout(circuit_build_times_t *cbt,
                                     int did_onehop)
 {
+  // Count circuit timeout
+  cbt->num_circ_timeouts++;
+
+  // If we're going to wrap int32, scale everything
+  if (cbt->num_circ_timeouts >= INT32_MAX) {
+    circuit_build_times_scale_circ_counts(cbt);
+  }
+
   /* Check for NULLness because we might not be using adaptive timeouts */
   if (cbt->liveness.timeouts_after_firsthop &&
       cbt->liveness.num_recent_circs > 0) {
@@ -1349,6 +1529,15 @@ circuit_build_times_network_close(circuit_build_times_t *cbt,
                                     int did_onehop, time_t start_time)
 {
   time_t now = time(NULL);
+
+  // Count circuit close
+  cbt->num_circ_closed++;
+
+  // If we're going to wrap int32, scale everything
+  if (cbt->num_circ_closed >= INT32_MAX) {
+    circuit_build_times_scale_circ_counts(cbt);
+  }
+
   /*
    * Check if this is a timeout that was for a circuit that spent its
    * entire existence during a time where we have had no network activity.
@@ -1447,7 +1636,7 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt)
 
 #define MAX_TIMEOUT ((int32_t) (INT32_MAX/2))
   /* Check to see if this has happened before. If so, double the timeout
-   * to give people on abysmally bad network connections a shot at access */
+   * to give clients on abysmally bad network connections a shot at access */
   if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) {
     if (cbt->timeout_ms > MAX_TIMEOUT || cbt->close_ms > MAX_TIMEOUT) {
       log_warn(LD_CIRC, "Insanely large circuit build timeout value. "
@@ -1523,7 +1712,7 @@ circuit_build_times_count_close(circuit_build_times_t *cbt,
                                 int did_onehop,
                                 time_t start_time)
 {
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     cbt->close_ms = cbt->timeout_ms
                   = circuit_build_times_get_initial_timeout();
     return 0;
@@ -1554,7 +1743,7 @@ void
 circuit_build_times_count_timeout(circuit_build_times_t *cbt,
                                   int did_onehop)
 {
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     cbt->close_ms = cbt->timeout_ms
                   = circuit_build_times_get_initial_timeout();
     return;
@@ -1628,7 +1817,7 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt)
   /*
    * Just return if we aren't using adaptive timeouts
    */
-  if (circuit_build_times_disabled())
+  if (circuit_build_times_disabled(get_options()))
     return;
 
   if (!circuit_build_times_set_timeout_worker(cbt))
@@ -1689,7 +1878,7 @@ circuitbuild_running_unit_tests(void)
 {
   unit_tests = 1;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 void
 circuit_build_times_update_last_circ(circuit_build_times_t *cbt)
@@ -1703,6 +1892,8 @@ cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt,
 {
   char *args = NULL;
   double qnt;
+  double timeout_rate = 0.0;
+  double close_rate = 0.0;
 
   switch (type) {
     case BUILDTIMEOUT_SET_EVENT_RESET:
@@ -1717,15 +1908,36 @@ cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt,
       break;
   }
 
+  /* The timeout rate is the ratio of the timeout count over
+   * the total number of circuits attempted. The total number of
+   * circuits is (timeouts+succeeded), since every circuit
+   * either succeeds, or times out. "Closed" circuits are
+   * MEASURE_TIMEOUT circuits whose measurement period expired.
+   * All MEASURE_TIMEOUT circuits are counted in the timeouts stat
+   * before transitioning to MEASURE_TIMEOUT (in
+   * circuit_build_times_mark_circ_as_measurement_only()).
+   * MEASURE_TIMEOUT circuits that succeed are *not* counted as
+   * "succeeded". See circuit_build_times_handle_completed_hop().
+   *
+   * We cast the denominator
+   * to promote it to double before the addition, to avoid int32
+   * overflow. */
+  const double total_circuits =
+    ((double)cbt->num_circ_timeouts) + cbt->num_circ_succeeded;
+  if (total_circuits >= 1.0) {
+    timeout_rate = cbt->num_circ_timeouts / total_circuits;
+    close_rate = cbt->num_circ_closed / total_circuits;
+  }
+
   tor_asprintf(&args, "TOTAL_TIMES=%lu "
                "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f "
                "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f",
                (unsigned long)cbt->total_build_times,
                (unsigned long)cbt->timeout_ms,
                (unsigned long)cbt->Xm, cbt->alpha, qnt,
-               circuit_build_times_timeout_rate(cbt),
+               timeout_rate,
                (unsigned long)cbt->close_ms,
-               circuit_build_times_close_rate(cbt));
+               close_rate);
 
   control_event_buildtimeout_set(type, args);
 
diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h
index 72b160983f..86116cb7f8 100644
--- a/src/or/circuitstats.h
+++ b/src/or/circuitstats.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,7 +17,10 @@ circuit_build_times_t *get_circuit_build_times_mutable(void);
 double get_circuit_build_close_time_ms(void);
 double get_circuit_build_timeout_ms(void);
 
-int circuit_build_times_disabled(void);
+int circuit_build_times_disabled(const or_options_t *options);
+int circuit_build_times_disabled_(const or_options_t *options,
+                                  int ignore_consensus);
+
 int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt);
 void circuit_build_times_update_state(const circuit_build_times_t *cbt,
                                       or_state_t *state);
@@ -31,6 +34,7 @@ void circuit_build_times_set_timeout(circuit_build_times_t *cbt);
 int circuit_build_times_add_time(circuit_build_times_t *cbt,
                                  build_time_t time);
 int circuit_build_times_needs_circuits(const circuit_build_times_t *cbt);
+void circuit_build_times_handle_completed_hop(origin_circuit_t *circ);
 
 int circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt);
 void circuit_build_times_init(circuit_build_times_t *cbt);
@@ -41,6 +45,7 @@ double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt);
 double circuit_build_times_close_rate(const circuit_build_times_t *cbt);
 
 void circuit_build_times_update_last_circ(circuit_build_times_t *cbt);
+void circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ);
 
 #ifdef CIRCUITSTATS_PRIVATE
 STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt,
@@ -51,7 +56,7 @@ STATIC void circuit_build_times_reset(circuit_build_times_t *cbt);
 /* Network liveness functions */
 STATIC int circuit_build_times_network_check_changed(
                                              circuit_build_times_t *cbt);
-#endif
+#endif /* defined(CIRCUITSTATS_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
 build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt,
@@ -60,7 +65,7 @@ double circuit_build_times_cdf(circuit_build_times_t *cbt, double x);
 void circuit_build_times_initial_alpha(circuit_build_times_t *cbt,
                                        double quantile, double time_ms);
 void circuitbuild_running_unit_tests(void);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* Network liveness functions */
 void circuit_build_times_network_is_live(circuit_build_times_t *cbt);
@@ -91,8 +96,18 @@ struct circuit_build_times_s {
   double timeout_ms;
   /** How long we wait before actually closing the circuit. */
   double close_ms;
+  /** Total succeeded counts. Old measurements may be scaled downward if
+   * we've seen a lot of circuits. */
+  uint32_t num_circ_succeeded;
+  /** Total timeout counts.  Old measurements may be scaled downward if
+   * we've seen a lot of circuits. */
+  uint32_t num_circ_timeouts;
+  /** Total closed counts.  Old measurements may be scaled downward if
+   * we've seen a lot of circuits.*/
+  uint32_t num_circ_closed;
+
 };
-#endif
+#endif /* defined(CIRCUITSTATS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CIRCUITSTATS_H) */
 
diff --git a/src/or/circuituse.c b/src/or/circuituse.c
index 03b54fc3d5..03f2ae52cf 100644
--- a/src/or/circuituse.c
+++ b/src/or/circuituse.c
@@ -1,16 +1,35 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file circuituse.c
- * \brief Launch the right sort of circuits and attach streams to them.
+ * \brief Launch the right sort of circuits and attach the right streams to
+ * them.
+ *
+ * As distinct from circuitlist.c, which manages lookups to find circuits, and
+ * circuitbuild.c, which handles the logistics of circuit construction, this
+ * module keeps track of which streams can be attached to which circuits (in
+ * circuit_get_best()), and attaches streams to circuits (with
+ * circuit_try_attaching_streams(), connection_ap_handshake_attach_circuit(),
+ * and connection_ap_handshake_attach_chosen_circuit() ).
+ *
+ * This module also makes sure that we are building circuits for all of the
+ * predicted ports, using circuit_remove_handled_ports(),
+ * circuit_stream_is_being_handled(), and circuit_build_needed_cirs().  It
+ * handles launching circuits for specific targets using
+ * circuit_launch_by_extend_info().
+ *
+ * This is also where we handle expiring circuits that have been around for
+ * too long without actually completing, along with the circuit_build_timeout
+ * logic in circuitstats.c.
  **/
 
 #include "or.h"
 #include "addressmap.h"
+#include "bridges.h"
 #include "channel.h"
 #include "circpathbias.h"
 #include "circuitbuild.h"
@@ -22,6 +41,11 @@
 #include "connection_edge.h"
 #include "control.h"
 #include "entrynodes.h"
+#include "hs_common.h"
+#include "hs_client.h"
+#include "hs_circuit.h"
+#include "hs_ident.h"
+#include "hs_stats.h"
 #include "nodelist.h"
 #include "networkstatus.h"
 #include "policies.h"
@@ -35,6 +59,36 @@
 static void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
 
+/** Check whether the hidden service destination of the stream at
+ *  <b>edge_conn</b> is the same as the destination of the circuit at
+ *  <b>origin_circ</b>. */
+static int
+circuit_matches_with_rend_stream(const edge_connection_t *edge_conn,
+                                 const origin_circuit_t *origin_circ)
+{
+  /* Check if this is a v2 rendezvous circ/stream */
+  if ((edge_conn->rend_data && !origin_circ->rend_data) ||
+      (!edge_conn->rend_data && origin_circ->rend_data) ||
+      (edge_conn->rend_data && origin_circ->rend_data &&
+       rend_cmp_service_ids(rend_data_get_address(edge_conn->rend_data),
+                            rend_data_get_address(origin_circ->rend_data)))) {
+    /* this circ is not for this conn */
+    return 0;
+  }
+
+  /* Check if this is a v3 rendezvous circ/stream */
+  if ((edge_conn->hs_ident && !origin_circ->hs_ident) ||
+      (!edge_conn->hs_ident && origin_circ->hs_ident) ||
+      (edge_conn->hs_ident && origin_circ->hs_ident &&
+       !ed25519_pubkey_eq(&edge_conn->hs_ident->identity_pk,
+                          &origin_circ->hs_ident->identity_pk))) {
+    /* this circ is not for this conn */
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Return 1 if <b>circ</b> could be returned by circuit_get_best().
  * Else return 0.
  */
@@ -80,6 +134,9 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
   }
 
   if (purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+      purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+      purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+      purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
       purpose == CIRCUIT_PURPOSE_C_REND_JOINED) {
     if (circ->timestamp_dirty &&
        circ->timestamp_dirty+get_options()->MaxCircuitDirtiness <= now)
@@ -103,7 +160,9 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
   if (need_internal != build_state->is_internal)
     return 0;
 
-  if (purpose == CIRCUIT_PURPOSE_C_GENERAL) {
+  if (purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+      purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+      purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
     tor_addr_t addr;
     const int family = tor_addr_parse(&addr, conn->socks_request->address);
     if (!exitnode && !build_state->onehop_tunnel) {
@@ -149,14 +208,9 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
       /* can't exit from this router */
       return 0;
     }
-  } else { /* not general */
+  } else { /* not general: this might be a rend circuit */
     const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
-    if ((edge_conn->rend_data && !origin_circ->rend_data) ||
-        (!edge_conn->rend_data && origin_circ->rend_data) ||
-        (edge_conn->rend_data && origin_circ->rend_data &&
-         rend_cmp_service_ids(edge_conn->rend_data->onion_address,
-                              origin_circ->rend_data->onion_address))) {
-      /* this circ is not for this conn */
+    if (!circuit_matches_with_rend_stream(edge_conn, origin_circ)) {
       return 0;
     }
   }
@@ -190,6 +244,8 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob,
     return 1; /* oa is better. It's not relaxed. */
 
   switch (purpose) {
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
     case CIRCUIT_PURPOSE_C_GENERAL:
       /* if it's used but less dirty it's best;
        * else if it's more recently created it's best
@@ -275,6 +331,9 @@ circuit_get_best(const entry_connection_t *conn,
   tor_assert(conn);
 
   tor_assert(purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+             purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
+             purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+             purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
              purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT ||
              purpose == CIRCUIT_PURPOSE_C_REND_JOINED);
 
@@ -289,7 +348,8 @@ circuit_get_best(const entry_connection_t *conn,
     /* Log an info message if we're going to launch a new intro circ in
      * parallel */
     if (purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT &&
-        !must_be_open && origin_circ->hs_circ_has_timed_out) {
+        !must_be_open && origin_circ->hs_circ_has_timed_out &&
+        !circ->marked_for_close) {
         intro_going_on_but_too_old = 1;
         continue;
     }
@@ -323,7 +383,7 @@ count_pending_general_client_circuits(void)
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (circ->marked_for_close ||
         circ->state == CIRCUIT_STATE_OPEN ||
-        circ->purpose != CIRCUIT_PURPOSE_C_GENERAL ||
+        !CIRCUIT_PURPOSE_COUNTS_TOWARDS_MAXPENDING(circ->purpose) ||
         !CIRCUIT_IS_ORIGIN(circ))
       continue;
 
@@ -361,10 +421,19 @@ circuit_conforms_to_options(const origin_circuit_t *circ,
 
   return 1;
 }
-#endif
+#endif /* 0 */
 
-/** Close all circuits that start at us, aren't open, and were born
+/**
+ * Close all circuits that start at us, aren't open, and were born
  * at least CircuitBuildTimeout seconds ago.
+ *
+ * TODO: This function is now partially redundant to
+ * circuit_build_times_handle_completed_hop(), but that function only
+ * covers circuits up to and including 3 hops that are still actually
+ * completing hops. However, circuit_expire_building() also handles longer
+ * circuits, as well as circuits that are completely stalled.
+ * In the future (after prop247/other path selection revamping), we probably
+ * want to eliminate this rats nest in favor of a simpler approach.
  */
 void
 circuit_expire_building(void)
@@ -386,21 +455,7 @@ circuit_expire_building(void)
    * we want to be more lenient with timeouts, in case the
    * user has relocated and/or changed network connections.
    * See bug #3443. */
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, next_circ) {
-    if (!CIRCUIT_IS_ORIGIN(next_circ) || /* didn't originate here */
-        next_circ->marked_for_close) { /* don't mess with marked circs */
-      continue;
-    }
-
-    if (TO_ORIGIN_CIRCUIT(next_circ)->has_opened &&
-        next_circ->state == CIRCUIT_STATE_OPEN &&
-        TO_ORIGIN_CIRCUIT(next_circ)->build_state &&
-        TO_ORIGIN_CIRCUIT(next_circ)->build_state->desired_path_len
-          == DEFAULT_ROUTE_LEN) {
-      any_opened_circs = 1;
-      break;
-    }
-  } SMARTLIST_FOREACH_END(next_circ);
+  any_opened_circs = circuit_any_opened_circuits();
 
 #define SET_CUTOFF(target, msec) do {                       \
     long ms = tor_lround(msec);                             \
@@ -444,6 +499,10 @@ circuit_expire_building(void)
   SET_CUTOFF(general_cutoff, get_circuit_build_timeout_ms());
   SET_CUTOFF(begindir_cutoff, get_circuit_build_timeout_ms());
 
+  // TODO: We should probably use route_len_for_purpose() here instead,
+  // except that does not count the extra round trip for things like server
+  // intros and rends.
+
   /* > 3hop circs seem to have a 1.0 second delay on their cannibalized
    * 4th hop. */
   SET_CUTOFF(fourhop_cutoff, get_circuit_build_timeout_ms() * (10/6.0) + 1000);
@@ -529,16 +588,15 @@ circuit_expire_building(void)
                                       == CPATH_STATE_OPEN;
           log_info(LD_CIRC,
                  "No circuits are opened. Relaxing timeout for circuit %d "
-                 "(a %s %d-hop circuit in state %s with channel state %s). "
-                 "%d guards are live.",
+                 "(a %s %d-hop circuit in state %s with channel state %s).",
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier,
                  circuit_purpose_to_string(victim->purpose),
                  TO_ORIGIN_CIRCUIT(victim)->build_state ?
                    TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len :
                    -1,
                  circuit_state_to_string(victim->state),
-                 channel_state_to_string(victim->n_chan->state),
-                 num_live_entry_guards(0));
+                 victim->n_chan ?
+                    channel_state_to_string(victim->n_chan->state) : "none");
 
           /* We count the timeout here for CBT, because technically this
            * was a timeout, and the timeout value needs to reset if we
@@ -556,16 +614,16 @@ circuit_expire_building(void)
                  "No circuits are opened. Relaxed timeout for circuit %d "
                  "(a %s %d-hop circuit in state %s with channel state %s) to "
                  "%ldms. However, it appears the circuit has timed out "
-                 "anyway. %d guards are live.",
+                 "anyway.",
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier,
                  circuit_purpose_to_string(victim->purpose),
                  TO_ORIGIN_CIRCUIT(victim)->build_state ?
                    TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len :
                    -1,
                  circuit_state_to_string(victim->state),
-                 channel_state_to_string(victim->n_chan->state),
-                 (long)build_close_ms,
-                 num_live_entry_guards(0));
+                 victim->n_chan ?
+                    channel_state_to_string(victim->n_chan->state) : "none",
+                 (long)build_close_ms);
       }
     }
 
@@ -587,7 +645,7 @@ circuit_expire_building(void)
                victim->n_circ_id,
                (int)(now - victim->timestamp_dirty));
     }
-#endif
+#endif /* 0 */
 
     /* if circ is !open, or if it's open but purpose is a non-finished
      * intro or rend, then mark it for close */
@@ -604,6 +662,7 @@ circuit_expire_building(void)
            * because that's set when they switch purposes
            */
           if (TO_ORIGIN_CIRCUIT(victim)->rend_data ||
+              TO_ORIGIN_CIRCUIT(victim)->hs_ident ||
               victim->timestamp_dirty > cutoff.tv_sec)
             continue;
           break;
@@ -614,7 +673,7 @@ circuit_expire_building(void)
           break;
         case CIRCUIT_PURPOSE_C_INTRODUCING:
           /* That purpose means that the intro point circuit has been opened
-           * succesfully but the INTRODUCE1 cell hasn't been sent yet because
+           * successfully but the INTRODUCE1 cell hasn't been sent yet because
            * the client is waiting for the rendezvous point circuit to open.
            * Keep this circuit open while waiting for the rendezvous circuit.
            * We let the circuit idle timeout take care of cleaning this
@@ -647,23 +706,17 @@ circuit_expire_building(void)
 
       if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) &&
           circuit_build_times_enough_to_compute(get_circuit_build_times())) {
+
+        log_info(LD_CIRC,
+                   "Deciding to count the timeout for circuit "U64_FORMAT"\n",
+                   U64_PRINTF_ARG(
+                       TO_ORIGIN_CIRCUIT(victim)->global_identifier));
+
         /* Circuits are allowed to last longer for measurement.
          * Switch their purpose and wait. */
         if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
-          control_event_circuit_status(TO_ORIGIN_CIRCUIT(victim),
-                                       CIRC_EVENT_FAILED,
-                                       END_CIRC_REASON_TIMEOUT);
-          circuit_change_purpose(victim, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
-          /* Record this failure to check for too many timeouts
-           * in a row. This function does not record a time value yet
-           * (we do that later); it only counts the fact that we did
-           * have a timeout. We also want to avoid double-counting
-           * already "relaxed" circuits, which are counted above. */
-          if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) {
-            circuit_build_times_count_timeout(
-                                         get_circuit_build_times_mutable(),
-                                         first_hop_succeeded);
-          }
+          circuit_build_times_mark_circ_as_measurement_only(TO_ORIGIN_CIRCUIT(
+                                                            victim));
           continue;
         }
 
@@ -688,18 +741,15 @@ circuit_expire_building(void)
       }
     }
 
-    /* If this is a hidden service client circuit which is far enough
-     * along in connecting to its destination, and we haven't already
-     * flagged it as 'timed out', and the user has not told us to
-     * close such circs immediately on timeout, flag it as 'timed out'
-     * so we'll launch another intro or rend circ, but don't mark it
-     * for close yet.
+    /* If this is a hidden service client circuit which is far enough along in
+     * connecting to its destination, and we haven't already flagged it as
+     * 'timed out', flag it so we'll launch another intro or rend circ, but
+     * don't mark it for close yet.
      *
      * (Circs flagged as 'timed out' are given a much longer timeout
      * period above, so we won't close them in the next call to
      * circuit_expire_building.) */
-    if (!(options->CloseHSClientCircuitsImmediatelyOnTimeout) &&
-        !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
+    if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
       switch (victim->purpose) {
       case CIRCUIT_PURPOSE_C_REND_READY:
         /* We only want to spare a rend circ if it has been specified in
@@ -731,8 +781,7 @@ circuit_expire_building(void)
     /* If this is a service-side rendezvous circuit which is far
      * enough along in connecting to its destination, consider sparing
      * it. */
-    if (!(options->CloseHSServiceRendCircuitsImmediatelyOnTimeout) &&
-        !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) &&
+    if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) &&
         victim->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
       log_info(LD_CIRC,"Marking circ %u (state %d:%s, purpose %d) "
                "as timed-out HS circ; relaunching rendezvous attempt.",
@@ -740,7 +789,7 @@ circuit_expire_building(void)
                victim->state, circuit_state_to_string(victim->state),
                victim->purpose);
       TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out = 1;
-      rend_service_relaunch_rendezvous(TO_ORIGIN_CIRCUIT(victim));
+      hs_circ_retry_service_rendezvous_point(TO_ORIGIN_CIRCUIT(victim));
       continue;
     }
 
@@ -778,6 +827,25 @@ circuit_expire_building(void)
   } SMARTLIST_FOREACH_END(victim);
 }
 
+/**
+ * Mark for close all circuits that start here, that were built through a
+ * guard we weren't sure if we wanted to use, and that have been waiting
+ * around for way too long.
+ */
+void
+circuit_expire_waiting_for_better_guard(void)
+{
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_origin_circuit_list(),
+                          origin_circuit_t *, circ) {
+    if (TO_CIRCUIT(circ)->marked_for_close)
+      continue;
+    if (circ->guard_state == NULL)
+      continue;
+    if (entry_guard_state_should_expire(circ->guard_state))
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NONE);
+  } SMARTLIST_FOREACH_END(circ);
+}
+
 /** For debugging #8387: track when we last called
  * circuit_expire_old_circuits_clientside. */
 static time_t last_expired_clientside_circuits = 0;
@@ -939,7 +1007,7 @@ circuit_remove_handled_ports(smartlist_t *needed_ports)
     tor_assert(*port);
     if (circuit_stream_is_being_handled(NULL, *port,
                                         MIN_CIRCUITS_HANDLING_STREAM)) {
-//      log_debug(LD_CIRC,"Port %d is already being handled; removing.", port);
+      log_debug(LD_CIRC,"Port %d is already being handled; removing.", *port);
       smartlist_del(needed_ports, i--);
       tor_free(port);
     } else {
@@ -976,6 +1044,10 @@ circuit_stream_is_being_handled(entry_connection_t *conn,
         continue;
       if (origin_circ->unusable_for_new_conns)
         continue;
+      if (origin_circ->isolation_values_set &&
+          (conn == NULL ||
+           !connection_edge_compatible_with_circuit(conn, origin_circ)))
+        continue;
 
       exitnode = build_state_get_exit_node(build_state);
       if (exitnode && (!need_uptime || build_state->need_uptime)) {
@@ -1001,8 +1073,157 @@ circuit_stream_is_being_handled(entry_connection_t *conn,
 /** Don't keep more than this many unused open circuits around. */
 #define MAX_UNUSED_OPEN_CIRCUITS 14
 
-/** Figure out how many circuits we have open that are clean. Make
- * sure it's enough for all the upcoming behaviors we predict we'll have.
+/* Return true if a circuit is available for use, meaning that it is open,
+ * clean, usable for new multi-hop connections, and a general purpose origin
+ * circuit.
+ * Accept any kind of circuit, return false if the above conditions are not
+ * met. */
+STATIC int
+circuit_is_available_for_use(const circuit_t *circ)
+{
+  const origin_circuit_t *origin_circ;
+  cpath_build_state_t *build_state;
+
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0; /* We first filter out only origin circuits before doing the
+                 following checks. */
+  if (circ->marked_for_close)
+    return 0; /* Don't mess with marked circs */
+  if (circ->timestamp_dirty)
+    return 0; /* Only count clean circs */
+  if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL &&
+      circ->purpose != CIRCUIT_PURPOSE_HS_VANGUARDS)
+    return 0; /* We only pay attention to general purpose circuits.
+                 General purpose circuits are always origin circuits. */
+
+  origin_circ = CONST_TO_ORIGIN_CIRCUIT(circ);
+  if (origin_circ->unusable_for_new_conns)
+    return 0;
+
+  build_state = origin_circ->build_state;
+  if (build_state->onehop_tunnel)
+    return 0;
+
+  return 1;
+}
+
+/* Return true if we need any more exit circuits.
+ * needs_uptime and needs_capacity are set only if we need more exit circuits.
+ * Check if we know of a port that's been requested recently and no circuit
+ * is currently available that can handle it. */
+STATIC int
+needs_exit_circuits(time_t now, int *needs_uptime, int *needs_capacity)
+{
+  return (!circuit_all_predicted_ports_handled(now, needs_uptime,
+                                               needs_capacity) &&
+          router_have_consensus_path() == CONSENSUS_PATH_EXIT);
+}
+
+/* Hidden services need at least this many internal circuits */
+#define SUFFICIENT_UPTIME_INTERNAL_HS_SERVERS 3
+
+/* Return true if we need any more hidden service server circuits.
+ * HS servers only need an internal circuit. */
+STATIC int
+needs_hs_server_circuits(time_t now, int num_uptime_internal)
+{
+  if (!rend_num_services() && !hs_service_get_num_services()) {
+    /* No services, we don't need anything. */
+    goto no_need;
+  }
+
+  if (num_uptime_internal >= SUFFICIENT_UPTIME_INTERNAL_HS_SERVERS) {
+    /* We have sufficient amount of internal circuit. */
+    goto no_need;
+  }
+
+  if (router_have_consensus_path() == CONSENSUS_PATH_UNKNOWN) {
+    /* Consensus hasn't been checked or might be invalid so requesting
+     * internal circuits is not wise. */
+    goto no_need;
+  }
+
+  /* At this point, we need a certain amount of circuits and we will most
+   * likely use them for rendezvous so we note down the use of internal
+   * circuit for our prediction for circuit needing uptime and capacity. */
+  rep_hist_note_used_internal(now, 1, 1);
+
+  return 1;
+ no_need:
+  return 0;
+}
+
+/* We need at least this many internal circuits for hidden service clients */
+#define SUFFICIENT_INTERNAL_HS_CLIENTS 3
+
+/* We need at least this much uptime for internal circuits for hidden service
+ * clients */
+#define SUFFICIENT_UPTIME_INTERNAL_HS_CLIENTS 2
+
+/* Return true if we need any more hidden service client circuits.
+ * HS clients only need an internal circuit. */
+STATIC int
+needs_hs_client_circuits(time_t now, int *needs_uptime, int *needs_capacity,
+    int num_internal, int num_uptime_internal)
+{
+  int used_internal_recently = rep_hist_get_predicted_internal(now,
+                                                               needs_uptime,
+                                                               needs_capacity);
+  int requires_uptime = num_uptime_internal <
+                        SUFFICIENT_UPTIME_INTERNAL_HS_CLIENTS &&
+                        needs_uptime;
+
+  return (used_internal_recently &&
+         (requires_uptime || num_internal < SUFFICIENT_INTERNAL_HS_CLIENTS) &&
+          router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN);
+}
+
+/* This is how many circuits can be opened concurrently during the cbt learning
+ * phase. This number cannot exceed the tor-wide MAX_UNUSED_OPEN_CIRCUITS. */
+#define DFLT_CBT_UNUSED_OPEN_CIRCS (10)
+#define MIN_CBT_UNUSED_OPEN_CIRCS 0
+#define MAX_CBT_UNUSED_OPEN_CIRCS MAX_UNUSED_OPEN_CIRCUITS
+
+/* Return true if we need more circuits for a good build timeout.
+ * XXXX make the assumption that build timeout streams should be
+ * created whenever we can build internal circuits. */
+STATIC int
+needs_circuits_for_build(int num)
+{
+  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+    if (num < networkstatus_get_param(NULL, "cbtmaxopencircs",
+                              DFLT_CBT_UNUSED_OPEN_CIRCS,
+                              MIN_CBT_UNUSED_OPEN_CIRCS,
+                              MAX_CBT_UNUSED_OPEN_CIRCS) &&
+        !circuit_build_times_disabled(get_options()) &&
+        circuit_build_times_needs_circuits_now(get_circuit_build_times())) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/**
+ * Launch the appropriate type of predicted circuit for hidden
+ * services, depending on our options.
+ */
+static void
+circuit_launch_predicted_hs_circ(int flags)
+{
+  /* K.I.S.S. implementation of bug #23101: If we are using
+   * vanguards or pinned middles, pre-build a specific purpose
+   * for HS circs. */
+  if (circuit_should_use_vanguards(CIRCUIT_PURPOSE_HS_VANGUARDS)) {
+    circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags);
+  } else {
+    /* If no vanguards, then no HS-specific prebuilt circuits are needed.
+     * Normal GENERAL circs are fine */
+    circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
+  }
+}
+
+/** Determine how many circuits we have open that are clean,
+ * Make sure it's enough for all the upcoming behaviors we predict we'll have.
  * But put an upper bound on the total number of circuits.
  */
 static void
@@ -1014,25 +1235,14 @@ circuit_predict_and_launch_new(void)
   time_t now = time(NULL);
   int flags = 0;
 
-  /* First, count how many of each type of circuit we have already. */
+  /* Count how many of each type of circuit we currently have. */
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    cpath_build_state_t *build_state;
-    origin_circuit_t *origin_circ;
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    if (circ->marked_for_close)
-      continue; /* don't mess with marked circs */
-    if (circ->timestamp_dirty)
-      continue; /* only count clean circs */
-    if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL)
-      continue; /* only pay attention to general-purpose circs */
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-    if (origin_circ->unusable_for_new_conns)
-      continue;
-    build_state = origin_circ->build_state;
-    if (build_state->onehop_tunnel)
+    if (!circuit_is_available_for_use(circ))
       continue;
+
     num++;
+
+    cpath_build_state_t *build_state = TO_ORIGIN_CIRCUIT(circ)->build_state;
     if (build_state->is_internal)
       num_internal++;
     if (build_state->need_uptime && build_state->is_internal)
@@ -1042,19 +1252,14 @@ circuit_predict_and_launch_new(void)
 
   /* If that's enough, then stop now. */
   if (num >= MAX_UNUSED_OPEN_CIRCUITS)
-    return; /* we already have many, making more probably will hurt */
-
-  /* Second, see if we need any more exit circuits. */
-  /* check if we know of a port that's been requested recently
-   * and no circuit is currently available that can handle it.
-   * Exits (obviously) require an exit circuit. */
-  if (!circuit_all_predicted_ports_handled(now, &port_needs_uptime,
-                                           &port_needs_capacity)
-      && router_have_consensus_path() == CONSENSUS_PATH_EXIT) {
+    return;
+
+  if (needs_exit_circuits(now, &port_needs_uptime, &port_needs_capacity)) {
     if (port_needs_uptime)
       flags |= CIRCLAUNCH_NEED_UPTIME;
     if (port_needs_capacity)
       flags |= CIRCLAUNCH_NEED_CAPACITY;
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d internal), need another exit circ.",
              num, num_internal);
@@ -1062,68 +1267,56 @@ circuit_predict_and_launch_new(void)
     return;
   }
 
-  /* Third, see if we need any more hidden service (server) circuits.
-   * HS servers only need an internal circuit. */
-  if (num_rend_services() && num_uptime_internal < 3
-      && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+  if (needs_hs_server_circuits(now, num_uptime_internal)) {
     flags = (CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_NEED_UPTIME |
              CIRCLAUNCH_IS_INTERNAL);
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d internal), need another internal "
              "circ for my hidden service.",
              num, num_internal);
-    circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
+    circuit_launch_predicted_hs_circ(flags);
     return;
   }
 
-  /* Fourth, see if we need any more hidden service (client) circuits.
-   * HS clients only need an internal circuit. */
-  if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime,
-                                      &hidserv_needs_capacity) &&
-      ((num_uptime_internal<2 && hidserv_needs_uptime) ||
-        num_internal<3)
-        && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+  if (needs_hs_client_circuits(now, &hidserv_needs_uptime,
+                               &hidserv_needs_capacity,
+                               num_internal, num_uptime_internal))
+  {
     if (hidserv_needs_uptime)
       flags |= CIRCLAUNCH_NEED_UPTIME;
     if (hidserv_needs_capacity)
       flags |= CIRCLAUNCH_NEED_CAPACITY;
     flags |= CIRCLAUNCH_IS_INTERNAL;
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d uptime-internal, %d internal), need"
              " another hidden service circ.",
              num, num_uptime_internal, num_internal);
-    circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
+
+    circuit_launch_predicted_hs_circ(flags);
     return;
   }
 
-  /* Finally, check to see if we still need more circuits to learn
-   * a good build timeout. But if we're close to our max number we
-   * want, don't do another -- we want to leave a few slots open so
-   * we can still build circuits preemptively as needed.
-   * XXXX make the assumption that build timeout streams should be
-   * created whenever we can build internal circuits. */
-  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
-    if (num < MAX_UNUSED_OPEN_CIRCUITS-2 &&
-        ! circuit_build_times_disabled() &&
-        circuit_build_times_needs_circuits_now(get_circuit_build_times())) {
-      flags = CIRCLAUNCH_NEED_CAPACITY;
-      /* if there are no exits in the consensus, make timeout
-       * circuits internal */
-      if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL)
-        flags |= CIRCLAUNCH_IS_INTERNAL;
+  if (needs_circuits_for_build(num)) {
+    flags = CIRCLAUNCH_NEED_CAPACITY;
+    /* if there are no exits in the consensus, make timeout
+     * circuits internal */
+    if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL)
+      flags |= CIRCLAUNCH_IS_INTERNAL;
+
       log_info(LD_CIRC,
                "Have %d clean circs need another buildtime test circ.", num);
       circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
       return;
-    }
   }
 }
 
 /** Build a new test circuit every 5 minutes */
 #define TESTING_CIRCUIT_INTERVAL 300
 
-/** This function is called once a second, if router_have_min_dir_info() is
- * true. Its job is to make sure all services we offer have enough circuits
+/** This function is called once a second, if router_have_minimum_dir_info()
+ * is true. Its job is to make sure all services we offer have enough circuits
  * available. Some services just want enough circuits for current tasks,
  * whereas others want a minimum set of idle circuits hanging around.
  */
@@ -1139,11 +1332,6 @@ circuit_build_needed_circs(time_t now)
   if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN)
     connection_ap_rescan_and_attach_pending();
 
-  /* make sure any hidden services have enough intro points
-   * HS intro point streams only require an internal circuit */
-  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN)
-    rend_consider_services_intro_points();
-
   circuit_expire_old_circs_as_needed(now);
 
   if (!options->DisablePredictedCircuits)
@@ -1179,7 +1367,7 @@ circuit_expire_old_circs_as_needed(time_t now)
       log_fn(LOG_INFO,"Creating a new testing circuit.");
       circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, 0);
     }
-#endif
+#endif /* 0 */
   }
 }
 
@@ -1225,8 +1413,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
        * number of streams on the circuit associated with the rend service.
        */
       if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
-        tor_assert(origin_circ->rend_data);
-        origin_circ->rend_data->nr_streams--;
+        hs_dec_rdv_stream_counter(origin_circ);
       }
       return;
     }
@@ -1265,11 +1452,6 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
   tor_fragile_assert();
 }
 
-/** If we haven't yet decided on a good timeout value for circuit
- * building, we close idles circuits aggressively so we can get more
- * data points. */
-#define IDLE_TIMEOUT_WHILE_LEARNING (10*60)
-
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
@@ -1279,21 +1461,15 @@ circuit_expire_old_circuits_clientside(void)
   struct timeval cutoff, now;
 
   tor_gettimeofday(&now);
-  cutoff = now;
   last_expired_clientside_circuits = now.tv_sec;
 
-  if (! circuit_build_times_disabled() &&
-      circuit_build_times_needs_circuits(get_circuit_build_times())) {
-    /* Circuits should be shorter lived if we need more of them
-     * for learning a good build timeout */
-    cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING;
-  } else {
-    cutoff.tv_sec -= get_options()->CircuitIdleTimeout;
-  }
-
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ))
       continue;
+
+    cutoff = now;
+    cutoff.tv_sec -= TO_ORIGIN_CIRCUIT(circ)->circuit_idle_timeout;
+
     /* If the circuit has been dirty for too long, and there are no streams
      * on it, mark it for close.
      */
@@ -1313,14 +1489,19 @@ circuit_expire_old_circuits_clientside(void)
     } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) {
       if (timercmp(&circ->timestamp_began, &cutoff, OP_LT)) {
         if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+                circ->purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+                circ->purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+                circ->purpose == CIRCUIT_PURPOSE_HS_VANGUARDS ||
                 circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT ||
                 circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
                 circ->purpose == CIRCUIT_PURPOSE_TESTING ||
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
-          log_debug(LD_CIRC,
-                    "Closing circuit that has been unused for %ld msec.",
+          log_info(LD_CIRC,
+                    "Closing circuit "U64_FORMAT
+                    " that has been unused for %ld msec.",
+                    U64_PRINTF_ARG(TO_ORIGIN_CIRCUIT(circ)->global_identifier),
                     tv_mdiff(&circ->timestamp_began, &now));
           circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
         } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) {
@@ -1364,7 +1545,7 @@ circuit_expire_old_circuits_clientside(void)
 #define IDLE_ONE_HOP_CIRC_TIMEOUT 60
 
 /** Find each non-origin circuit that has been unused for too long,
- * has no streams on it, used a create_fast, and ends here: mark it
+ * has no streams on it, came from a client, and ends here: mark it
  * for close.
  */
 void
@@ -1383,9 +1564,10 @@ circuit_expire_old_circuits_serverside(time_t now)
      * Also if there is a rend_splice on it, it's a single onion service
      * circuit and we should not close it.
      */
-    if (or_circ->is_first_hop && !circ->n_chan &&
+    if (or_circ->p_chan && channel_is_client(or_circ->p_chan) &&
+        !circ->n_chan &&
         !or_circ->n_streams && !or_circ->resolving_streams &&
-        or_circ->p_chan && !or_circ->rend_splice &&
+        !or_circ->rend_splice &&
         channel_when_last_xmit(or_circ->p_chan) <= cutoff) {
       log_info(LD_CIRC, "Closing circ_id %u (empty %d secs ago)",
                (unsigned)or_circ->p_circ_id,
@@ -1453,7 +1635,7 @@ circuit_testing_opened(origin_circuit_t *circ)
     router_perform_bandwidth_test(NUM_PARALLEL_TESTING_CIRCS, time(NULL));
     have_performed_bandwidth_test = 1;
   } else
-    consider_testing_reachability(1, 0);
+    router_do_reachability_checks(1, 0);
 }
 
 /** A testing circuit has failed to build. Take whatever stats we want. */
@@ -1491,7 +1673,7 @@ circuit_has_opened(origin_circuit_t *circ)
 
   switch (TO_CIRCUIT(circ)->purpose) {
     case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
-      rend_client_rendcirc_has_opened(circ);
+      hs_client_circuit_has_opened(circ);
       /* Start building an intro circ if we don't have one yet. */
       connection_ap_attach_pending(1);
       /* This isn't a call to circuit_try_attaching_streams because a
@@ -1503,20 +1685,22 @@ circuit_has_opened(origin_circuit_t *circ)
        * state. */
       break;
     case CIRCUIT_PURPOSE_C_INTRODUCING:
-      rend_client_introcirc_has_opened(circ);
+      hs_client_circuit_has_opened(circ);
       break;
     case CIRCUIT_PURPOSE_C_GENERAL:
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
       /* Tell any AP connections that have been waiting for a new
        * circuit that one is ready. */
       circuit_try_attaching_streams(circ);
       break;
     case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
       /* at the service, waiting for introductions */
-      rend_service_intro_has_opened(circ);
+      hs_service_circuit_has_opened(circ);
       break;
     case CIRCUIT_PURPOSE_S_CONNECT_REND:
       /* at the service, connecting to rend point */
-      rend_service_rendezvous_has_opened(circ);
+      hs_service_circuit_has_opened(circ);
       break;
     case CIRCUIT_PURPOSE_TESTING:
       circuit_testing_opened(circ);
@@ -1576,6 +1760,39 @@ circuit_build_failed(origin_circuit_t *circ)
    * the last hop or an earlier hop. then use this info below.
    */
   int failed_at_last_hop = 0;
+
+  /* First, check to see if this was a path failure, rather than build
+   * failure.
+   *
+   * Note that we deliberately use circuit_get_cpath_len() (and not
+   * circuit_get_cpath_opened_len()) because we only want to ensure
+   * that a full path is *chosen*. This is different than a full path
+   * being *built*. We only want to count *build* failures below.
+   *
+   * Path selection failures can happen spuriously for a number
+   * of reasons (such as aggressive/invalid user-specified path
+   * restrictions in the torrc, insufficient microdescriptors, and
+   * non-user reasons like exitpolicy issues), and so should not be
+   * counted as failures below.
+   */
+  if (circuit_get_cpath_len(circ) < circ->build_state->desired_path_len) {
+    static ratelim_t pathfail_limit = RATELIM_INIT(3600);
+    log_fn_ratelim(&pathfail_limit, LOG_NOTICE, LD_CIRC,
+             "Our circuit %u (id: %" PRIu32 ") died due to an invalid "
+             "selected path, purpose %s. This may be a torrc "
+             "configuration issue, or a bug.",
+              TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier,
+              circuit_purpose_to_string(TO_CIRCUIT(circ)->purpose));
+
+    /* If the path failed on an RP, retry it. */
+    if (TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND)
+      hs_circ_retry_service_rendezvous_point(circ);
+
+    /* In all other cases, just bail. The rest is just failure accounting
+     * that we don't want to do */
+    return;
+  }
+
   /* If the last hop isn't open, and the second-to-last is, we failed
    * at the last hop. */
   if (circ->cpath &&
@@ -1583,13 +1800,16 @@ circuit_build_failed(origin_circuit_t *circ)
       circ->cpath->prev->prev->state == CPATH_STATE_OPEN) {
     failed_at_last_hop = 1;
   }
+
+  /* Check if we failed at first hop */
   if (circ->cpath &&
       circ->cpath->state != CPATH_STATE_OPEN &&
       ! circ->base_.received_destroy) {
     /* We failed at the first hop for some reason other than a DESTROY cell.
      * If there's an OR connection to blame, blame it. Also, avoid this relay
      * for a while, and fail any one-hop directory fetches destined for it. */
-    const char *n_chan_id = circ->cpath->extend_info->identity_digest;
+    const char *n_chan_ident = circ->cpath->extend_info->identity_digest;
+    tor_assert(n_chan_ident);
     int already_marked = 0;
     if (circ->base_.n_chan) {
       n_chan = circ->base_.n_chan;
@@ -1605,23 +1825,35 @@ circuit_build_failed(origin_circuit_t *circ)
         already_marked = 1;
       }
       log_info(LD_OR,
-               "Our circuit failed to get a response from the first hop "
-               "(%s). I'm going to try to rotate to a better connection.",
+               "Our circuit %u (id: %" PRIu32 ") failed to get a response "
+               "from the first hop (%s). I'm going to try to rotate to a "
+               "better connection.",
+               TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier,
                channel_get_canonical_remote_descr(n_chan));
       n_chan->is_bad_for_new_circs = 1;
     } else {
       log_info(LD_OR,
-               "Our circuit died before the first hop with no connection");
+               "Our circuit %u (id: %" PRIu32 ") died before the first hop "
+               "with no connection",
+               TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier);
     }
-    if (n_chan_id && !already_marked) {
-      entry_guard_register_connect_status(n_chan_id, 0, 1, time(NULL));
+    if (!already_marked) {
+      /*
+       * If we have guard state (new guard API) and our path selection
+       * code actually chose a full path, then blame the failure of this
+       * circuit on the guard.
+       */
+      if (circ->guard_state)
+        entry_guard_failed(&circ->guard_state);
       /* if there are any one-hop streams waiting on this circuit, fail
        * them now so they can retry elsewhere. */
-      connection_ap_fail_onehop(n_chan_id, circ->build_state);
+      connection_ap_fail_onehop(n_chan_ident, circ->build_state);
     }
   }
 
   switch (circ->base_.purpose) {
+    case CIRCUIT_PURPOSE_C_HSDIR_GET:
+    case CIRCUIT_PURPOSE_S_HSDIR_POST:
     case CIRCUIT_PURPOSE_C_GENERAL:
       /* If we never built the circuit, note it as a failure. */
       circuit_increment_failure_count();
@@ -1664,7 +1896,7 @@ circuit_build_failed(origin_circuit_t *circ)
                "(%s hop failed).",
                escaped(build_state_get_exit_nickname(circ->build_state)),
                failed_at_last_hop?"last":"non-last");
-      rend_service_relaunch_rendezvous(circ);
+      hs_circ_retry_service_rendezvous_point(circ);
       break;
     /* default:
      * This won't happen in normal operation, but might happen if the
@@ -1706,6 +1938,106 @@ have_enough_path_info(int need_exit)
     return router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN;
 }
 
+/**
+ * Tell us if a circuit is a hidden service circuit.
+ */
+int
+circuit_purpose_is_hidden_service(uint8_t purpose)
+{
+   if (purpose == CIRCUIT_PURPOSE_HS_VANGUARDS) {
+     return 1;
+   }
+
+   /* Client-side purpose */
+   if (purpose >= CIRCUIT_PURPOSE_C_HS_MIN_ &&
+       purpose <= CIRCUIT_PURPOSE_C_HS_MAX_) {
+     return 1;
+   }
+
+   /* Service-side purpose */
+   if (purpose >= CIRCUIT_PURPOSE_S_HS_MIN_ &&
+       purpose <= CIRCUIT_PURPOSE_S_HS_MAX_) {
+     return 1;
+   }
+
+   return 0;
+}
+
+/**
+ * Return true if this circuit purpose should use vanguards
+ * or pinned Layer2 or Layer3 guards.
+ *
+ * This function takes both the circuit purpose and the
+ * torrc options for pinned middles/vanguards into account
+ * (ie: the circuit must be a hidden service circuit and
+ * vanguards/pinned middles must be enabled for it to return
+ * true).
+ */
+int
+circuit_should_use_vanguards(uint8_t purpose)
+{
+  const or_options_t *options = get_options();
+
+  /* Only hidden service circuits use vanguards */
+  if (!circuit_purpose_is_hidden_service(purpose))
+    return 0;
+
+  /* Pinned middles are effectively vanguards */
+  if (options->HSLayer2Nodes || options->HSLayer3Nodes)
+    return 1;
+
+  return 0;
+}
+
+/**
+ * Return true for the set of conditions for which it is OK to use
+ * a cannibalized circuit.
+ *
+ * Don't cannibalize for onehops, or tor2web, or certain purposes.
+ */
+static int
+circuit_should_cannibalize_to_build(uint8_t purpose_to_build,
+                                    int has_extend_info,
+                                    int onehop_tunnel,
+                                    int need_specific_rp)
+{
+
+  /* Do not try to cannibalize if this is a one hop circuit, or
+   * is a tor2web/special rp. */
+  if (onehop_tunnel || need_specific_rp) {
+    return 0;
+  }
+
+  /* Don't try to cannibalize for general purpose circuits that do not
+   * specify a custom exit. */
+  if (purpose_to_build == CIRCUIT_PURPOSE_C_GENERAL && !has_extend_info) {
+    return 0;
+  }
+
+  /* Don't cannibalize for testing circuits. We want to see if they
+   * complete normally. Also don't cannibalize for vanguard-purpose
+   * circuits, since those are specially pre-built for later
+   * cannibalization by the actual specific circuit types that need
+   * vanguards.
+   */
+  if (purpose_to_build == CIRCUIT_PURPOSE_TESTING ||
+      purpose_to_build == CIRCUIT_PURPOSE_HS_VANGUARDS) {
+    return 0;
+  }
+
+  /* For vanguards, the server-side intro circ is not cannibalized
+   * because we pre-build 4 hop HS circuits, and it only needs a 3 hop
+   * circuit. It is also long-lived, so it is more important that
+   * it have lower latency than get built fast.
+   */
+  if (circuit_should_use_vanguards(purpose_to_build) &&
+      purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Launch a new circuit with purpose <b>purpose</b> and exit node
  * <b>extend_info</b> (or NULL to select a random exit node).  If flags
  * contains CIRCLAUNCH_NEED_UPTIME, choose among routers with high uptime.  If
@@ -1723,6 +2055,11 @@ circuit_launch_by_extend_info(uint8_t purpose,
   int have_path = have_enough_path_info(! (flags & CIRCLAUNCH_IS_INTERNAL) );
   int need_specific_rp = 0;
 
+  /* Keep some stats about our attempts to launch HS rendezvous circuits */
+  if (purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
+    hs_stats_note_service_rendezvous_launch();
+  }
+
   if (!onehop_tunnel && (!router_have_minimum_dir_info() || !have_path)) {
     log_debug(LD_CIRC,"Haven't %s yet; canceling "
               "circuit launch.",
@@ -1740,9 +2077,12 @@ circuit_launch_by_extend_info(uint8_t purpose,
     need_specific_rp = 1;
   }
 
-  if ((extend_info || purpose != CIRCUIT_PURPOSE_C_GENERAL) &&
-      purpose != CIRCUIT_PURPOSE_TESTING &&
-      !onehop_tunnel && !need_specific_rp) {
+  /* If we can/should cannibalize another circuit to build this one,
+   * then do so. */
+  if (circuit_should_cannibalize_to_build(purpose,
+                                          extend_info != NULL,
+                                          onehop_tunnel,
+                                          need_specific_rp)) {
     /* see if there are appropriate circs available to cannibalize. */
     /* XXX if we're planning to add a hop, perhaps we want to look for
      * internal circs rather than exit circs? -RD */
@@ -1751,8 +2091,9 @@ circuit_launch_by_extend_info(uint8_t purpose,
       uint8_t old_purpose = circ->base_.purpose;
       struct timeval old_timestamp_began = circ->base_.timestamp_began;
 
-      log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d (%s)",
-               build_state_get_exit_nickname(circ->build_state), purpose,
+      log_info(LD_CIRC, "Cannibalizing circ %u (id: %" PRIu32 ") for "
+                        "purpose %d (%s)",
+               TO_CIRCUIT(circ)->n_circ_id, circ->global_identifier, purpose,
                circuit_purpose_to_string(purpose));
 
       if ((purpose == CIRCUIT_PURPOSE_S_CONNECT_REND ||
@@ -1796,6 +2137,8 @@ circuit_launch_by_extend_info(uint8_t purpose,
         case CIRCUIT_PURPOSE_C_INTRODUCING:
         case CIRCUIT_PURPOSE_S_CONNECT_REND:
         case CIRCUIT_PURPOSE_C_GENERAL:
+        case CIRCUIT_PURPOSE_S_HSDIR_POST:
+        case CIRCUIT_PURPOSE_C_HSDIR_GET:
         case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
           /* need to add a new hop */
           tor_assert(extend_info);
@@ -1875,16 +2218,22 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
             c->state, conn_state_to_string(c->type, c->state));
   }
   tor_assert(ENTRY_TO_CONN(conn)->state == AP_CONN_STATE_CIRCUIT_WAIT);
+
+  /* Will the exit policy of the exit node apply to this stream? */
   check_exit_policy =
       conn->socks_request->command == SOCKS_COMMAND_CONNECT &&
       !conn->use_begindir &&
       !connection_edge_is_rendezvous_stream(ENTRY_TO_EDGE_CONN(conn));
+
+  /* Does this connection want a one-hop circuit? */
   want_onehop = conn->want_onehop;
 
+  /* Do we need a high-uptime circuit? */
   need_uptime = !conn->want_onehop && !conn->use_begindir &&
                 smartlist_contains_int_as_string(options->LongLivedPorts,
                                           conn->socks_request->port);
 
+  /* Do we need an "internal" circuit? */
   if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL)
     need_internal = 1;
   else if (conn->use_begindir || conn->want_onehop)
@@ -1892,23 +2241,37 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
   else
     need_internal = 0;
 
-  circ = circuit_get_best(conn, 1, desired_circuit_purpose,
+  /* We now know what kind of circuit we need.  See if there is an
+   * open circuit that we can use for this stream */
+  circ = circuit_get_best(conn, 1 /* Insist on open circuits */,
+                          desired_circuit_purpose,
                           need_uptime, need_internal);
 
   if (circ) {
+    /* We got a circuit that will work for this stream!  We can return it. */
     *circp = circ;
     return 1; /* we're happy */
   }
 
+  /* Okay, there's no circuit open that will work for this stream. Let's
+   * see if there's an in-progress circuit or if we have to launch one */
+
+  /* Do we know enough directory info to build circuits at all? */
   int have_path = have_enough_path_info(!need_internal);
 
   if (!want_onehop && (!router_have_minimum_dir_info() || !have_path)) {
+    /* If we don't have enough directory information, we can't build
+     * multihop circuits.
+     */
     if (!connection_get_by_type(CONN_TYPE_DIR)) {
       int severity = LOG_NOTICE;
-      /* FFFF if this is a tunneled directory fetch, don't yell
-       * as loudly. the user doesn't even know it's happening. */
-      if (entry_list_is_constrained(options) &&
-          entries_known_but_down(options)) {
+      /* Retry some stuff that might help the connection work. */
+      /* If we are configured with EntryNodes or UseBridges */
+      if (entry_list_is_constrained(options)) {
+        /* Retry all our guards / bridges.
+         * guards_retry_optimistic() always returns true here. */
+        int rv = guards_retry_optimistic(options);
+        tor_assert_nonfatal_once(rv);
         log_fn(severity, LD_APP|LD_DIR,
                "Application request when we haven't %s. "
                "Optimistically trying known %s again.",
@@ -1916,8 +2279,12 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                "used client functionality lately" :
                "received a consensus with exits",
                options->UseBridges ? "bridges" : "entrynodes");
-        entries_retry_all(options);
-      } else if (!options->UseBridges || any_bridge_descriptors_known()) {
+      } else {
+        /* Getting directory documents doesn't help much if we have a limited
+         * number of guards */
+        tor_assert_nonfatal(!options->UseBridges);
+        tor_assert_nonfatal(!options->EntryNodes);
+        /* Retry our directory fetches, so we have a fresh set of guard info */
         log_fn(severity, LD_APP|LD_DIR,
                "Application request when we haven't %s. "
                "Optimistically trying directory fetches again.",
@@ -1927,14 +2294,16 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
         routerlist_retry_directory_downloads(time(NULL));
       }
     }
-    /* the stream will be dealt with when router_have_minimum_dir_info becomes
-     * 1, or when all directory attempts fail and directory_all_unreachable()
+    /* Since we didn't have enough directory info, we can't attach now.  The
+     * stream will be dealt with when router_have_minimum_dir_info becomes 1,
+     * or when all directory attempts fail and directory_all_unreachable()
      * kills it.
      */
     return 0;
   }
 
-  /* Do we need to check exit policy? */
+  /* Check whether the exit policy of the chosen exit, or the exit policies
+   * of _all_ nodes, would forbid this node. */
   if (check_exit_policy) {
     if (!conn->chosen_exit_name) {
       struct in_addr in;
@@ -1955,7 +2324,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     } else {
       /* XXXX Duplicates checks in connection_ap_handshake_attach_circuit:
        * refactor into a single function. */
-      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1);
+      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 0);
       int opt = conn->chosen_exit_optional;
       if (node && !connection_ap_can_use_exit(conn, node)) {
         log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP,
@@ -1975,16 +2344,25 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     }
   }
 
-  /* is one already on the way? */
-  circ = circuit_get_best(conn, 0, desired_circuit_purpose,
+  /* Now, check whether there already a circuit on the way that could handle
+   * this stream. This check matches the one above, but this time we
+   * do not require that the circuit will work. */
+  circ = circuit_get_best(conn, 0 /* don't insist on open circuits */,
+                          desired_circuit_purpose,
                           need_uptime, need_internal);
   if (circ)
     log_debug(LD_CIRC, "one on the way!");
+
   if (!circ) {
+    /* No open or in-progress circuit could handle this stream!  We
+     * will have to launch one!
+     */
+
+    /* The chosen exit node, if there is one. */
     extend_info_t *extend_info=NULL;
-    uint8_t new_circ_purpose;
     const int n_pending = count_pending_general_client_circuits();
 
+    /* Do we have too many pending circuits? */
     if (n_pending >= options->MaxClientCircuitsPending) {
       static ratelim_t delay_limit = RATELIM_INIT(10*60);
       char *m;
@@ -1998,34 +2376,41 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       return 0;
     }
 
+    /* If this is a hidden service trying to start an introduction point,
+     * handle that case. */
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
+      const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
       /* need to pick an intro point */
-      rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
-      tor_assert(rend_data);
-      extend_info = rend_client_get_random_intro(rend_data);
+      extend_info = hs_client_get_random_intro_from_edge(edge_conn);
       if (!extend_info) {
-        log_info(LD_REND,
-                 "No intro points for '%s': re-fetching service descriptor.",
-                 safe_str_client(rend_data->onion_address));
-        rend_client_refetch_v2_renddesc(rend_data);
+        log_info(LD_REND, "No intro points: re-fetching service descriptor.");
+        if (edge_conn->rend_data) {
+          rend_client_refetch_v2_renddesc(edge_conn->rend_data);
+        } else {
+          hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+        }
         connection_ap_mark_as_non_pending_circuit(conn);
         ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_RENDDESC_WAIT;
         return 0;
       }
       log_info(LD_REND,"Chose %s as intro point for '%s'.",
                extend_info_describe(extend_info),
-               safe_str_client(rend_data->onion_address));
+               (edge_conn->rend_data) ?
+               safe_str_client(rend_data_get_address(edge_conn->rend_data)) :
+               "service");
     }
 
     /* If we have specified a particular exit node for our
      * connection, then be sure to open a circuit to that exit node.
      */
-    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) {
+    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
       if (conn->chosen_exit_name) {
         const node_t *r;
         int opt = conn->chosen_exit_optional;
-        r = node_get_by_nickname(conn->chosen_exit_name, 1);
-        if (r && node_has_descriptor(r)) {
+        r = node_get_by_nickname(conn->chosen_exit_name, 0);
+        if (r && node_has_preferred_descriptor(r, conn->want_onehop ? 1 : 0)) {
           /* We might want to connect to an IPv6 bridge for loading
              descriptors so we use the preferred address rather than
              the primary. */
@@ -2035,12 +2420,16 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                      "Discarding this circuit.", conn->chosen_exit_name);
             return -1;
           }
-        } else {
+        } else  { /* ! (r && node_has_preferred_descriptor(...)) */
           log_debug(LD_DIR, "considering %d, %s",
                     want_onehop, conn->chosen_exit_name);
           if (want_onehop && conn->chosen_exit_name[0] == '$') {
             /* We're asking for a one-hop circuit to a router that
              * we don't have a routerinfo about. Make up an extend_info. */
+            /* XXX prop220: we need to make chosen_exit_name able to
+             * encode both key formats. This is not absolutely critical
+             * since this is just for one-hop circuits, but we should
+             * still get it done */
             char digest[DIGEST_LEN];
             char *hexdigest = conn->chosen_exit_name+1;
             tor_addr_t addr;
@@ -2055,10 +2444,13 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                        escaped_safe_str_client(conn->socks_request->address));
               return -1;
             }
+            /* XXXX prop220 add a workaround for ed25519 ID below*/
             extend_info = extend_info_new(conn->chosen_exit_name+1,
-                                          digest, NULL, NULL, &addr,
-                                          conn->socks_request->port);
-          } else {
+                                          digest,
+                                          NULL, /* Ed25519 ID */
+                                          NULL, NULL, /* onion keys */
+                                          &addr, conn->socks_request->port);
+          } else { /* ! (want_onehop && conn->chosen_exit_name[0] == '$') */
             /* We will need an onion key for the router, and we
              * don't have one. Refuse or relax requirements. */
             log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP,
@@ -2076,8 +2468,10 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           }
         }
       }
-    }
+    } /* Done checking for general circutis with chosen exits. */
 
+    /* What purpose do we need to launch this circuit with? */
+    uint8_t new_circ_purpose;
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_REND_JOINED)
       new_circ_purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
     else if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)
@@ -2086,25 +2480,42 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       new_circ_purpose = desired_circuit_purpose;
 
 #ifdef ENABLE_TOR2WEB_MODE
+    /* If tor2Web is on, then hidden service requests should be one-hop.
+     */
     if (options->Tor2webMode &&
         (new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND ||
          new_circ_purpose == CIRCUIT_PURPOSE_C_INTRODUCING)) {
       want_onehop = 1;
     }
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
 
+    /* Determine what kind of a circuit to launch, and actually launch it. */
     {
       int flags = CIRCLAUNCH_NEED_CAPACITY;
       if (want_onehop) flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
       if (need_uptime) flags |= CIRCLAUNCH_NEED_UPTIME;
       if (need_internal) flags |= CIRCLAUNCH_IS_INTERNAL;
+
+      /* If we are about to pick a v3 RP right now, make sure we pick a
+       * rendezvous point that supports the v3 protocol! */
+      if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_REND_JOINED &&
+          new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
+          ENTRY_TO_EDGE_CONN(conn)->hs_ident) {
+        flags |= CIRCLAUNCH_IS_V3_RP;
+        log_info(LD_GENERAL, "Getting rendezvous circuit to v3 service!");
+      }
+
       circ = circuit_launch_by_extend_info(new_circ_purpose, extend_info,
                                            flags);
     }
 
     extend_info_free(extend_info);
 
-    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) {
+    /* Now trigger things that need to happen when we launch circuits */
+
+    if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+        desired_circuit_purpose == CIRCUIT_PURPOSE_S_HSDIR_POST) {
       /* We just caused a circuit to get built because of this stream.
        * If this stream has caused a _lot_ of circuits to be built, that's
        * a bad sign: we should tell the user. */
@@ -2119,14 +2530,25 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       /* help predict this next time */
       rep_hist_note_used_internal(time(NULL), need_uptime, 1);
       if (circ) {
-        /* write the service_id into circ */
-        circ->rend_data = rend_data_dup(ENTRY_TO_EDGE_CONN(conn)->rend_data);
+        const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
+        if (edge_conn->rend_data) {
+          /* write the service_id into circ */
+          circ->rend_data = rend_data_dup(edge_conn->rend_data);
+        } else if (edge_conn->hs_ident) {
+          circ->hs_ident =
+            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk,
+                                 HS_IDENT_CIRCUIT_INTRO);
+        }
         if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
             circ->base_.state == CIRCUIT_STATE_OPEN)
           circuit_has_opened(circ);
       }
     }
   } /* endif (!circ) */
+
+  /* We either found a good circuit, or launched a new circuit, or failed to
+   * do so. Report success, and delay. */
+
   if (circ) {
     /* Mark the circuit with the isolation fields for this connection.
      * When the circuit arrives, we'll clear these flags: this is
@@ -2188,7 +2610,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
   /* reset it, so we can measure circ timeouts */
-  ENTRY_TO_CONN(apconn)->timestamp_lastread = time(NULL);
+  ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
   ENTRY_TO_EDGE_CONN(apconn)->on_circuit = TO_CIRCUIT(circ);
   /* assert_connection_ok(conn, time(NULL)); */
@@ -2198,8 +2620,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
     /* We are attaching a stream to a rendezvous circuit.  That means
      * that an attempt to connect to a hidden service just
      * succeeded.  Tell rendclient.c. */
-    rend_client_note_connection_attempt_ended(
-                    ENTRY_TO_EDGE_CONN(apconn)->rend_data);
+    hs_client_note_connection_attempt_succeeded(ENTRY_TO_EDGE_CONN(apconn));
   }
 
   if (cpath) { /* we were given one; use it */
@@ -2223,6 +2644,8 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   /* See if we can use optimistic data on this circuit */
   if (optimistic_data_enabled() &&
       (circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL ||
+       circ->base_.purpose == CIRCUIT_PURPOSE_C_HSDIR_GET ||
+       circ->base_.purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
        circ->base_.purpose == CIRCUIT_PURPOSE_C_REND_JOINED))
     apconn->may_use_optimistic_data = 1;
   else
@@ -2326,7 +2749,9 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn,
 
   pathbias_count_use_attempt(circ);
 
+  /* Now, actually link the connection. */
   link_apconn_to_circ(conn, circ, cpath);
+
   tor_assert(conn->socks_request);
   if (conn->socks_request->command == SOCKS_COMMAND_CONNECT) {
     if (!conn->use_begindir)
@@ -2341,12 +2766,44 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn,
   return 1;
 }
 
-/** Try to find a safe live circuit for CONN_TYPE_AP connection conn. If
- * we don't find one: if conn cannot be handled by any known nodes,
- * warn and return -1 (conn needs to die, and is maybe already marked);
- * else launch new circuit (if necessary) and return 0.
- * Otherwise, associate conn with a safe live circuit, do the
- * right next step, and return 1.
+/**
+ * Return an appropriate circuit purpose for non-rend streams.
+ * We don't handle rends here because a rend stream triggers two
+ * circuit builds with different purposes, so it is handled elsewhere.
+ *
+ * This function just figures out what type of hsdir activity this is,
+ * and tells us. Everything else is general.
+ */
+static int
+connection_ap_get_nonrend_circ_purpose(const entry_connection_t *conn)
+{
+  const connection_t *base_conn = ENTRY_TO_CONN(conn);
+  tor_assert_nonfatal(!connection_edge_is_rendezvous_stream(
+                      ENTRY_TO_EDGE_CONN(conn)));
+
+  if (base_conn->linked_conn &&
+      base_conn->linked_conn->type == CONN_TYPE_DIR) {
+    /* Set a custom purpose for hsdir activity */
+    if (base_conn->linked_conn->purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||
+       base_conn->linked_conn->purpose == DIR_PURPOSE_UPLOAD_HSDESC) {
+      return CIRCUIT_PURPOSE_S_HSDIR_POST;
+    } else if (base_conn->linked_conn->purpose
+                 == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
+               base_conn->linked_conn->purpose
+                 == DIR_PURPOSE_FETCH_HSDESC) {
+      return CIRCUIT_PURPOSE_C_HSDIR_GET;
+    }
+  }
+
+  /* All other purposes are general for now */
+  return CIRCUIT_PURPOSE_C_GENERAL;
+}
+
+/** Try to find a safe live circuit for stream <b>conn</b>.  If we find one,
+ * attach the stream, send appropriate cells, and return 1.  Otherwise,
+ * try to launch new circuit(s) for the stream.  If we can launch
+ * circuits, return 0.  Otherwise, if we simply can't proceed with
+ * this stream, return -1. (conn needs to die, and is maybe already marked).
  */
 /* XXXX this function should mark for close whenever it returns -1;
  * its callers shouldn't have to worry about that. */
@@ -2365,6 +2822,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
 
   conn_age = (int)(time(NULL) - base_conn->timestamp_created);
 
+  /* Is this connection so old that we should give up on it? */
   if (conn_age >= get_options()->SocksTimeout) {
     int severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port) ?
       LOG_INFO : LOG_NOTICE;
@@ -2375,12 +2833,14 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     return -1;
   }
 
+  /* We handle "general" (non-onion) connections much more straightforwardly.
+   */
   if (!connection_edge_is_rendezvous_stream(ENTRY_TO_EDGE_CONN(conn))) {
     /* we're a general conn */
     origin_circuit_t *circ=NULL;
 
     /* Are we linked to a dir conn that aims to fetch a consensus?
-     * We check here because this conn might no longer be needed. */
+     * We check here because the conn might no longer be needed. */
     if (base_conn->linked_conn &&
         base_conn->linked_conn->type == CONN_TYPE_DIR &&
         base_conn->linked_conn->purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
@@ -2398,8 +2858,11 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
+    /* If we have a chosen exit, we need to use a circuit that's
+     * open to that exit. See what exit we meant, and whether we can use it.
+     */
     if (conn->chosen_exit_name) {
-      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1);
+      const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 0);
       int opt = conn->chosen_exit_optional;
       if (!node && !want_onehop) {
         /* We ran into this warning when trying to extend a circuit to a
@@ -2411,6 +2874,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
                "Requested exit point '%s' is not known. %s.",
                conn->chosen_exit_name, opt ? "Trying others" : "Closing");
         if (opt) {
+          /* If we are allowed to ignore the .exit request, do so */
           conn->chosen_exit_optional = 0;
           tor_free(conn->chosen_exit_name);
           return 0;
@@ -2423,6 +2887,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
                "would refuse request. %s.",
                conn->chosen_exit_name, opt ? "Trying others" : "Closing");
         if (opt) {
+          /* If we are allowed to ignore the .exit request, do so */
           conn->chosen_exit_optional = 0;
           tor_free(conn->chosen_exit_name);
           return 0;
@@ -2431,20 +2896,28 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
-    /* find the circuit that we should use, if there is one. */
-    retval = circuit_get_open_circ_or_launch(
-        conn, CIRCUIT_PURPOSE_C_GENERAL, &circ);
-    if (retval < 1) // XXXX++ if we totally fail, this still returns 0 -RD
+    /* Find the circuit that we should use, if there is one. Otherwise
+     * launch it
+     */
+    retval = circuit_get_open_circ_or_launch(conn,
+            connection_ap_get_nonrend_circ_purpose(conn),
+            &circ);
+
+    if (retval < 1) {
+      /* We were either told "-1" (complete failure) or 0 (circuit in
+       * progress); we can't attach this stream yet. */
       return retval;
+    }
 
     log_debug(LD_APP|LD_CIRC,
               "Attaching apconn to circ %u (stream %d sec old).",
               (unsigned)circ->base_.n_circ_id, conn_age);
-    /* print the circ's path, so people can figure out which circs are
+    /* print the circ's path, so clients can figure out which circs are
      * sucking. */
     circuit_log_path(LOG_INFO,LD_APP|LD_CIRC,circ);
 
-    /* We have found a suitable circuit for our conn. Hurray. */
+    /* We have found a suitable circuit for our conn. Hurray.  Do
+     * the attachment. */
     return connection_ap_handshake_attach_chosen_circuit(conn, circ, NULL);
 
   } else { /* we're a rendezvous conn */
@@ -2462,9 +2935,10 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       tor_assert(rendcirc);
       /* one is already established, attach */
       log_info(LD_REND,
-               "rend joined circ %u already here. attaching. "
-               "(stream %d sec old)",
-               (unsigned)rendcirc->base_.n_circ_id, conn_age);
+               "rend joined circ %u (id: %" PRIu32 ") already here. "
+               "Attaching. (stream %d sec old)",
+               (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+               rendcirc->global_identifier, conn_age);
       /* Mark rendezvous circuits as 'newly dirty' every time you use
        * them, since the process of rebuilding a rendezvous circ is so
        * expensive. There is a tradeoff between linkability and
@@ -2497,9 +2971,10 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     if (rendcirc && (rendcirc->base_.purpose ==
                      CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED)) {
       log_info(LD_REND,
-               "pending-join circ %u already here, with intro ack. "
-               "Stalling. (stream %d sec old)",
-               (unsigned)rendcirc->base_.n_circ_id, conn_age);
+               "pending-join circ %u (id: %" PRIu32 ") already here, with "
+               "intro ack. Stalling. (stream %d sec old)",
+               (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+               rendcirc->global_identifier, conn_age);
       return 0;
     }
 
@@ -2511,10 +2986,13 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     if (retval > 0) {
       /* one has already sent the intro. keep waiting. */
       tor_assert(introcirc);
-      log_info(LD_REND, "Intro circ %u present and awaiting ack (rend %u). "
-               "Stalling. (stream %d sec old)",
-               (unsigned)introcirc->base_.n_circ_id,
-               rendcirc ? (unsigned)rendcirc->base_.n_circ_id : 0,
+      log_info(LD_REND, "Intro circ %u (id: %" PRIu32 ") present and "
+                        "awaiting ACK. Rend circuit %u (id: %" PRIu32 "). "
+                        "Stalling. (stream %d sec old)",
+               (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
+               introcirc->global_identifier,
+               rendcirc ? (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id : 0,
+               rendcirc ? rendcirc->global_identifier : 0,
                conn_age);
       return 0;
     }
@@ -2524,19 +3002,26 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     if (rendcirc && introcirc &&
         rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY) {
       log_info(LD_REND,
-               "ready rend circ %u already here (no intro-ack yet on "
-               "intro %u). (stream %d sec old)",
-               (unsigned)rendcirc->base_.n_circ_id,
-               (unsigned)introcirc->base_.n_circ_id, conn_age);
+               "ready rend circ %u (id: %" PRIu32 ") already here. No"
+               "intro-ack yet on intro %u (id: %" PRIu32 "). "
+               "(stream %d sec old)",
+               (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+               rendcirc->global_identifier,
+               (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
+               introcirc->global_identifier, conn_age);
 
       tor_assert(introcirc->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
       if (introcirc->base_.state == CIRCUIT_STATE_OPEN) {
-        log_info(LD_REND,"found open intro circ %u (rend %u); sending "
-                 "introduction. (stream %d sec old)",
-                 (unsigned)introcirc->base_.n_circ_id,
-                 (unsigned)rendcirc->base_.n_circ_id,
-                 conn_age);
-        switch (rend_client_send_introduction(introcirc, rendcirc)) {
+        int ret;
+        log_info(LD_REND, "Found open intro circ %u (id: %" PRIu32 "). "
+                          "Rend circuit %u (id: %" PRIu32 "); Sending "
+                          "introduction. (stream %d sec old)",
+                 (unsigned) TO_CIRCUIT(introcirc)->n_circ_id,
+                 introcirc->global_identifier,
+                 (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id,
+                 rendcirc->global_identifier, conn_age);
+        ret = hs_client_send_introduce1(introcirc, rendcirc);
+        switch (ret) {
         case 0: /* success */
           rendcirc->base_.timestamp_dirty = time(NULL);
           introcirc->base_.timestamp_dirty = time(NULL);
@@ -2558,10 +3043,13 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
-    log_info(LD_REND, "Intro (%u) and rend (%u) circs are not both ready. "
-             "Stalling conn. (%d sec old)",
-             introcirc ? (unsigned)introcirc->base_.n_circ_id : 0,
-             rendcirc ? (unsigned)rendcirc->base_.n_circ_id : 0, conn_age);
+    log_info(LD_REND, "Intro %u (id: %" PRIu32 ") and rend circuit %u "
+                      "(id: %" PRIu32 ") circuits are not both ready. "
+                      "Stalling conn. (%d sec old)",
+             introcirc ? (unsigned) TO_CIRCUIT(introcirc)->n_circ_id : 0,
+             introcirc ? introcirc->global_identifier : 0,
+             rendcirc ? (unsigned) TO_CIRCUIT(rendcirc)->n_circ_id : 0,
+             rendcirc ? rendcirc->global_identifier : 0, conn_age);
     return 0;
   }
 }
@@ -2622,3 +3110,41 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ)
   circ->unusable_for_new_conns = 1;
 }
 
+/**
+ * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to
+ * the valid delivered written fields and the overhead field,
+ * respectively.
+ */
+void
+circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
+{
+  if (!circ) return;
+
+  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+
+  circ->n_delivered_written_circ_bw =
+      tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
+  circ->n_overhead_written_circ_bw =
+      tor_add_u32_nowrap(circ->n_overhead_written_circ_bw,
+                         RELAY_PAYLOAD_SIZE-relay_body_len);
+}
+
+/**
+ * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to
+ * the valid delivered read field and the overhead field,
+ * respectively.
+ */
+void
+circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
+{
+  if (!circ) return;
+
+  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+
+  circ->n_delivered_read_circ_bw =
+      tor_add_u32_nowrap(circ->n_delivered_read_circ_bw, relay_body_len);
+  circ->n_overhead_read_circ_bw =
+      tor_add_u32_nowrap(circ->n_overhead_read_circ_bw,
+                         RELAY_PAYLOAD_SIZE-relay_body_len);
+}
+
diff --git a/src/or/circuituse.h b/src/or/circuituse.h
index 5973978c45..6458bd6908 100644
--- a/src/or/circuituse.h
+++ b/src/or/circuituse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_CIRCUITUSE_H
 
 void circuit_expire_building(void);
+void circuit_expire_waiting_for_better_guard(void);
 void circuit_remove_handled_ports(smartlist_t *needed_ports);
 int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port,
                                     int min);
@@ -43,6 +44,9 @@ void circuit_build_failed(origin_circuit_t *circ);
 /** Flag to set when the last hop of a circuit doesn't need to be an
  * exit node. */
 #define CIRCLAUNCH_IS_INTERNAL    (1<<3)
+/** Flag to set when we are trying to launch a v3 rendezvous circuit. We need
+ *  to apply some additional filters on the node picked. */
+#define CIRCLAUNCH_IS_V3_RP (1<<4)
 origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose,
                                                 extend_info_t *info,
                                                 int flags);
@@ -59,5 +63,31 @@ int hostname_in_track_host_exits(const or_options_t *options,
                                  const char *address);
 void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ);
 
-#endif
+int circuit_purpose_is_hidden_service(uint8_t);
+int circuit_should_use_vanguards(uint8_t);
+void circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
+void circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
+
+#ifdef TOR_UNIT_TESTS
+/* Used only by circuituse.c and test_circuituse.c */
+
+STATIC int circuit_is_available_for_use(const circuit_t *circ);
+
+STATIC int needs_exit_circuits(time_t now,
+                               int *port_needs_uptime,
+                               int *port_needs_capacity);
+STATIC int needs_hs_server_circuits(time_t now,
+                                    int num_uptime_internal);
+
+STATIC int needs_hs_client_circuits(time_t now,
+                                    int *needs_uptime,
+                                    int *needs_capacity,
+                                    int num_internal,
+                                    int num_uptime_internal);
+
+STATIC int needs_circuits_for_build(int num);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_CIRCUITUSE_H) */
 
diff --git a/src/or/command.c b/src/or/command.c
index 3f5386c5a3..39950f41bf 100644
--- a/src/or/command.c
+++ b/src/or/command.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -46,6 +46,7 @@
 #include "config.h"
 #include "control.h"
 #include "cpuworker.h"
+#include "crypto_util.h"
 #include "dos.h"
 #include "hibernate.h"
 #include "nodelist.h"
@@ -129,7 +130,7 @@ command_time_process_cell(cell_t *cell, channel_t *chan, int *time,
   }
   *time += time_passed;
 }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
 /** Process a <b>cell</b> that was just received on <b>chan</b>. Keep internal
  * statistics about how many of each cell we've processed so far
@@ -166,7 +167,7 @@ command_process_cell(channel_t *chan, cell_t *cell)
     /* remember which second it is, for next time */
     current_second = now;
   }
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
 #ifdef KEEP_TIMING_STATS
 #define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
@@ -174,9 +175,9 @@ command_process_cell(channel_t *chan, cell_t *cell)
     command_time_process_cell(cl, cn, & tp ## time ,            \
                               command_process_ ## tp ## _cell);  \
   } STMT_END
-#else
+#else /* !(defined(KEEP_TIMING_STATS)) */
 #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn)
-#endif
+#endif /* defined(KEEP_TIMING_STATS) */
 
   switch (cell->command) {
     case CELL_CREATE:
@@ -286,7 +287,7 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
              "Received create cell but we're shutting down. Sending back "
              "destroy.");
     channel_send_destroy(cell->circ_id, chan,
-                               END_CIRC_REASON_HIBERNATING);
+                         END_CIRC_REASON_HIBERNATING);
     return;
   }
 
@@ -339,10 +340,15 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
     return;
   }
 
+  if (!channel_is_client(chan)) {
+    /* remember create types we've seen, but don't remember them from
+     * clients, to be extra conservative about client statistics. */
+    rep_hist_note_circuit_handshake_requested(create_cell->handshake_type);
+  }
+
   if (create_cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST) {
     /* hand it off to the cpuworkers, and then return. */
-    if (connection_or_digest_is_known_relay(chan->identity_digest))
-      rep_hist_note_circuit_handshake_requested(create_cell->handshake_type);
+
     if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) {
       log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT);
@@ -375,7 +381,8 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
     created_cell.handshake_len = len;
 
     if (onionskin_answer(circ, &created_cell,
-                         (const char *)keys, rend_circ_nonce)<0) {
+                         (const char *)keys, sizeof(keys),
+                         rend_circ_nonce)<0) {
       log_warn(LD_OR,"Failed to reply to CREATE_FAST cell. Closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
       return;
@@ -489,6 +496,17 @@ command_process_relay_cell(cell_t *cell, channel_t *chan)
     /* if we're a relay and treating connections with recent local
      * traffic better, then this is one of them. */
     channel_timestamp_client(chan);
+
+    /* Count all circuit bytes here for control port accuracy. We want
+     * to count even invalid/dropped relay cells, hence counting
+     * before the recognized check and the connection_edge_process_relay
+     * cell checks.
+     */
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+    /* Count the payload bytes only. We don't care about cell headers */
+    ocirc->n_read_circ_bw = tor_add_u32_nowrap(ocirc->n_read_circ_bw,
+                                               CELL_PAYLOAD_SIZE);
   }
 
   if (!CIRCUIT_IS_ORIGIN(circ) &&
diff --git a/src/or/command.h b/src/or/command.h
index 12cda6a463..c0d1996cbb 100644
--- a/src/or/command.h
+++ b/src/or/command.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,5 +27,5 @@ extern uint64_t stats_n_created_cells_processed;
 extern uint64_t stats_n_relay_cells_processed;
 extern uint64_t stats_n_destroy_cells_processed;
 
-#endif
+#endif /* !defined(TOR_COMMAND_H) */
 
diff --git a/src/or/config.c b/src/or/config.c
index 810f1e9a7a..53c19e3900 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -1,16 +1,67 @@
+
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file config.c
- * \brief Code to parse and interpret configuration files.
+ * \brief Code to interpret the user's configuration of Tor.
+ *
+ * This module handles torrc configuration file, including parsing it,
+ * combining it with torrc.defaults and the command line, allowing
+ * user changes to it (via editing and SIGHUP or via the control port),
+ * writing it back to disk (because of SAVECONF from the control port),
+ * and -- most importantly, acting on it.
+ *
+ * The module additionally has some tools for manipulating and
+ * inspecting values that are calculated as a result of the
+ * configured options.
+ *
+ * <h3>How to add new options</h3>
+ *
+ * To add new items to the torrc, there are a minimum of three places to edit:
+ * <ul>
+ *   <li>The or_options_t structure in or.h, where the options are stored.
+ *   <li>The option_vars_ array below in this module, which configures
+ *       the names of the torrc options, their types, their multiplicities,
+ *       and their mappings to fields in or_options_t.
+ *   <li>The manual in doc/tor.1.txt, to document what the new option
+ *       is, and how it works.
+ * </ul>
+ *
+ * Additionally, you might need to edit these places too:
+ * <ul>
+ *   <li>options_validate() below, in case you want to reject some possible
+ *       values of the new configuration option.
+ *   <li>options_transition_allowed() below, in case you need to
+ *       forbid some or all changes in the option while Tor is
+ *       running.
+ *   <li>options_transition_affects_workers(), in case changes in the option
+ *       might require Tor to relaunch or reconfigure its worker threads.
+ *   <li>options_transition_affects_descriptor(), in case changes in the
+ *       option might require a Tor relay to build and publish a new server
+ *       descriptor.
+ *   <li>options_act() and/or options_act_reversible(), in case there's some
+ *       action that needs to be taken immediately based on the option's
+ *       value.
+ * </ul>
+ *
+ * <h3>Changing the value of an option</h3>
+ *
+ * Because of the SAVECONF command from the control port, it's a bad
+ * idea to change the value of any user-configured option in the
+ * or_options_t.  If you want to sometimes do this anyway, we recommend
+ * that you create a secondary field in or_options_t; that you have the
+ * user option linked only to the secondary field; that you use the
+ * secondary field to initialize the one that Tor actually looks at; and that
+ * you use the one Tor looks as the one that you modify.
  **/
 
 #define CONFIG_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "compat.h"
 #include "addressmap.h"
 #include "channel.h"
@@ -19,18 +70,22 @@
 #include "circuitmux.h"
 #include "circuitmux_ewma.h"
 #include "circuitstats.h"
+#include "compress.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
 #include "connection_or.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "confparse.h"
 #include "cpuworker.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "dns.h"
 #include "dos.h"
 #include "entrynodes.h"
+#include "git_revision.h"
 #include "geoip.h"
 #include "hibernate.h"
 #include "main.h"
@@ -40,6 +95,7 @@
 #include "relay.h"
 #include "rendclient.h"
 #include "rendservice.h"
+#include "hs_config.h"
 #include "rephist.h"
 #include "router.h"
 #include "sandbox.h"
@@ -50,22 +106,25 @@
 #include "statefile.h"
 #include "transports.h"
 #include "ext_orport.h"
-#include "torgzip.h"
+#include "voting_schedule.h"
 #ifdef _WIN32
 #include <shlobj.h>
 #endif
 
 #include "procmon.h"
 
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
+
 #ifdef HAVE_SYSTEMD
 #   if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
 /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-#   endif
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 /* Prefix used to indicate a Unix socket in a FooPort configuration. */
 static const char unix_socket_prefix[] = "unix:";
@@ -73,6 +132,11 @@ static const char unix_socket_prefix[] = "unix:";
  * configuration. */
 static const char unix_q_socket_prefix[] = "unix:\"";
 
+/** macro to help with the bulk rename of *DownloadSchedule to
+ * *DowloadInitialDelay . */
+#define DOWNLOAD_SCHEDULE(name) \
+  { #name "DownloadSchedule", #name "DownloadInitialDelay", 0, 1 }
+
 /** A list of abbreviations and aliases to map command-line options, obsolete
  * option names, or alternative option names, to their current values. */
 static config_abbrev_t option_abbrevs_[] = {
@@ -118,24 +182,55 @@ static config_abbrev_t option_abbrevs_[] = {
   { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0},
   { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0},
   { "VirtualAddrNetwork", "VirtualAddrNetworkIPv4", 0, 0},
+  { "SocksSocketsGroupWritable", "UnixSocksGroupWritable", 0, 1},
+  { "_HSLayer2Nodes", "HSLayer2Nodes", 0, 1 },
+  { "_HSLayer3Nodes", "HSLayer3Nodes", 0, 1 },
+
+  DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthority),
+  DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthorityOnly),
+  DOWNLOAD_SCHEDULE(ClientBootstrapConsensusFallback),
+  DOWNLOAD_SCHEDULE(TestingBridge),
+  DOWNLOAD_SCHEDULE(TestingBridgeBootstrap),
+  DOWNLOAD_SCHEDULE(TestingClient),
+  DOWNLOAD_SCHEDULE(TestingClientConsensus),
+  DOWNLOAD_SCHEDULE(TestingServer),
+  DOWNLOAD_SCHEDULE(TestingServerConsensus),
+
   { NULL, NULL, 0, 0},
 };
 
+/** dummy instance of or_options_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(or_options_t);
+
 /** An entry for config_vars: "The option <b>name</b> has type
  * CONFIG_TYPE_<b>conftype</b>, and corresponds to
  * or_options_t.<b>member</b>"
  */
 #define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(or_options_t, member), \
-      initvalue }
+  { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member),     \
+      initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) }
 /** As VAR, but the option name and member name are the same. */
 #define V(member,conftype,initvalue)                                    \
   VAR(#member, conftype, member, initvalue)
 /** An entry for config_vars: "The option <b>name</b> is obsolete." */
+#ifdef TOR_UNIT_TESTS
+#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} }
+#else
 #define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
+#endif
 
-#define VPORT(member,conftype,initvalue)                                    \
-  VAR(#member, conftype, member ## _lines, initvalue)
+/**
+ * Macro to declare *Port options.  Each one comes in three entries.
+ * For example, most users should use "SocksPort" to configure the
+ * socks port, but TorBrowser wants to use __SocksPort so that it
+ * isn't stored by SAVECONF.  The SocksPortLines virtual option is
+ * used to query both options from the controller.
+ */
+#define VPORT(member)                                           \
+  VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
+  VAR(#member, LINELIST_S, member ## _lines, NULL),             \
+  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
 
 /** Array of configuration options.  Until we disallow nonstandard
  * abbreviations, order is significant, since the first matching option will
@@ -146,11 +241,11 @@ static config_var_t option_vars_[] = {
   VAR("AccountingRule",          STRING,   AccountingRule_option,  "max"),
   V(AccountingStart,             STRING,   NULL),
   V(Address,                     STRING,   NULL),
-  V(AllowDotExit,                BOOL,     "0"),
-  V(AllowInvalidNodes,           CSV,      "middle,rendezvous"),
+  OBSOLETE("AllowDotExit"),
+  OBSOLETE("AllowInvalidNodes"),
   V(AllowNonRFC953Hostnames,     BOOL,     "0"),
-  V(AllowSingleHopCircuits,      BOOL,     "0"),
-  V(AllowSingleHopExits,         BOOL,     "0"),
+  OBSOLETE("AllowSingleHopCircuits"),
+  OBSOLETE("AllowSingleHopExits"),
   V(AlternateBridgeAuthority,    LINELIST, NULL),
   V(AlternateDirAuthority,       LINELIST, NULL),
   OBSOLETE("AlternateHSAuthority"),
@@ -163,14 +258,14 @@ static config_var_t option_vars_[] = {
   V(AuthDirInvalidCCs,           CSV,      ""),
   V(AuthDirFastGuarantee,        MEMUNIT,  "100 KB"),
   V(AuthDirGuardBWGuarantee,     MEMUNIT,  "2 MB"),
-  V(AuthDirPinKeys,              BOOL,     "0"),
+  V(AuthDirPinKeys,              BOOL,     "1"),
   V(AuthDirReject,               LINELIST, NULL),
   V(AuthDirRejectCCs,            CSV,      ""),
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
   V(AuthDirListBadExits,         BOOL,     "0"),
   V(AuthDirMaxServersPerAddr,    UINT,     "2"),
-  V(AuthDirMaxServersPerAuthAddr,UINT,     "5"),
+  OBSOLETE("AuthDirMaxServersPerAuthAddr"),
   V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
   V(AutomapHostsOnResolve,       BOOL,     "0"),
@@ -183,12 +278,17 @@ static config_var_t option_vars_[] = {
   V(BridgePassword,              STRING,   NULL),
   V(BridgeRecordUsageByCountry,  BOOL,     "1"),
   V(BridgeRelay,                 BOOL,     "0"),
+  V(BridgeDistribution,          STRING,   NULL),
+  VAR("CacheDirectory",          FILENAME, CacheDirectory_option, NULL),
+  V(CacheDirectoryGroupReadable, AUTOBOOL,     "auto"),
   V(CellStatistics,              BOOL,     "0"),
+  V(PaddingStatistics,           BOOL,     "1"),
   V(LearnCircuitBuildTimeout,    BOOL,     "1"),
   V(CircuitBuildTimeout,         INTERVAL, "0"),
-  V(CircuitIdleTimeout,          INTERVAL, "1 hour"),
+  OBSOLETE("CircuitIdleTimeout"),
+  V(CircuitsAvailableTimeout,    INTERVAL, "0"),
   V(CircuitStreamTimeout,        INTERVAL, "0"),
-  V(CircuitPriorityHalflife,     DOUBLE,  "-100.0"), /*negative:'Use default'*/
+  V(CircuitPriorityHalflife,     DOUBLE,  "-1.0"), /*negative:'Use default'*/
   V(ClientDNSRejectInternalAddresses, BOOL,"1"),
   V(ClientOnly,                  BOOL,     "0"),
   V(ClientPreferIPv6ORPort,      AUTOBOOL, "auto"),
@@ -203,26 +303,26 @@ static config_var_t option_vars_[] = {
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
   V(ContactInfo,                 STRING,   NULL),
-  V(ControlListenAddress,        LINELIST, NULL),
-  VPORT(ControlPort,                 LINELIST, NULL),
+  OBSOLETE("ControlListenAddress"),
+  VPORT(ControlPort),
   V(ControlPortFileGroupReadable,BOOL,     "0"),
   V(ControlPortWriteToFile,      FILENAME, NULL),
   V(ControlSocket,               LINELIST, NULL),
   V(ControlSocketsGroupWritable, BOOL,     "0"),
-  V(SocksSocketsGroupWritable,   BOOL,     "0"),
+  V(UnixSocksGroupWritable,    BOOL,     "0"),
   V(CookieAuthentication,        BOOL,     "0"),
   V(CookieAuthFileGroupReadable, BOOL,     "0"),
   V(CookieAuthFile,              STRING,   NULL),
   V(CountPrivateBandwidth,       BOOL,     "0"),
-  V(DataDirectory,               FILENAME, NULL),
+  VAR("DataDirectory",           FILENAME, DataDirectory_option, NULL),
   V(DataDirectoryGroupReadable,  BOOL,     "0"),
   V(DisableOOSCheck,             BOOL,     "1"),
   V(DisableNetwork,              BOOL,     "0"),
   V(DirAllowPrivateAddresses,    BOOL,     "0"),
   V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"),
-  V(DirListenAddress,            LINELIST, NULL),
+  OBSOLETE("DirListenAddress"),
   V(DirPolicy,                   LINELIST, NULL),
-  VPORT(DirPort,                     LINELIST, NULL),
+  VPORT(DirPort),
   V(DirPortFrontPage,            FILENAME, NULL),
   VAR("DirReqStatistics",        BOOL,     DirReqStatistics_option, "1"),
   VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
@@ -240,8 +340,8 @@ static config_var_t option_vars_[] = {
   OBSOLETE("DisableIOCP"),
   OBSOLETE("DisableV2DirectoryInfo_"),
   OBSOLETE("DynamicDHGroups"),
-  VPORT(DNSPort,                     LINELIST, NULL),
-  V(DNSListenAddress,            LINELIST, NULL),
+  VPORT(DNSPort),
+  OBSOLETE("DNSListenAddress"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
   V(DoSCircuitCreationMinConnections,      UINT, "0"),
@@ -258,14 +358,14 @@ static config_var_t option_vars_[] = {
   V(DownloadExtraInfo,           BOOL,     "0"),
   V(TestingEnableConnBwEvent,    BOOL,     "0"),
   V(TestingEnableCellStatsEvent, BOOL,     "0"),
-  V(TestingEnableTbEmptyEvent,   BOOL,     "0"),
+  OBSOLETE("TestingEnableTbEmptyEvent"),
   V(EnforceDistinctSubnets,      BOOL,     "1"),
   V(EntryNodes,                  ROUTERSET,   NULL),
   V(EntryStatistics,             BOOL,     "0"),
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
   V(ExcludeNodes,                ROUTERSET, NULL),
   V(ExcludeExitNodes,            ROUTERSET, NULL),
-  V(ExcludeSingleHopRelays,      BOOL,     "1"),
+  OBSOLETE("ExcludeSingleHopRelays"),
   V(ExitNodes,                   ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
@@ -273,17 +373,19 @@ static config_var_t option_vars_[] = {
   V(ExitPortStatistics,          BOOL,     "0"),
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExitRelay,                   AUTOBOOL, "auto"),
-  VPORT(ExtORPort,               LINELIST, NULL),
+  VPORT(ExtORPort),
   V(ExtORPortCookieAuthFile,     STRING,   NULL),
   V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
+  V(ExtendByEd25519ID,           AUTOBOOL, "auto"),
   V(FallbackDir,                 LINELIST, NULL),
+
   V(UseDefaultFallbackDirs,      BOOL,     "1"),
 
   OBSOLETE("FallbackNetworkstatusFile"),
   V(FascistFirewall,             BOOL,     "0"),
   V(FirewallPorts,               CSV,      ""),
-  V(FastFirstHopPK,              AUTOBOOL, "auto"),
+  OBSOLETE("FastFirstHopPK"),
   V(FetchDirInfoEarly,           BOOL,     "0"),
   V(FetchDirInfoExtraEarly,      BOOL,     "0"),
   V(FetchServerDescriptors,      BOOL,     "1"),
@@ -299,11 +401,12 @@ static config_var_t option_vars_[] = {
     SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "geoip"),
   V(GeoIPv6File,                 FILENAME,
     SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "geoip6"),
-#endif
+#endif /* defined(_WIN32) */
   OBSOLETE("Group"),
   V(GuardLifetime,               INTERVAL, "0 minutes"),
   V(HardwareAccel,               BOOL,     "0"),
   V(HeartbeatPeriod,             INTERVAL, "6 hours"),
+  V(MainloopStats,               BOOL,     "0"),
   V(AccelName,                   STRING,   NULL),
   V(AccelDir,                    FILENAME, NULL),
   V(HashedControlPassword,       LINELIST, NULL),
@@ -318,16 +421,17 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreams",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
-  V(HiddenServiceStatistics,     BOOL,     "1"),
+  VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
-  V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"),
-  V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"),
+  OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"),
+  OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"),
   V(HiddenServiceSingleHopMode,  BOOL,     "0"),
   V(HiddenServiceNonAnonymousMode,BOOL,    "0"),
   V(HTTPProxy,                   STRING,   NULL),
   V(HTTPProxyAuthenticator,      STRING,   NULL),
   V(HTTPSProxy,                  STRING,   NULL),
   V(HTTPSProxyAuthenticator,     STRING,   NULL),
+  VPORT(HTTPTunnelPort),
   V(IPv6Exit,                    BOOL,     "0"),
   VAR("ServerTransportPlugin",   LINELIST, ServerTransportPlugin,  NULL),
   V(ServerTransportListenAddr,   LINELIST, NULL),
@@ -337,6 +441,10 @@ static config_var_t option_vars_[] = {
   V(Socks5Proxy,                 STRING,   NULL),
   V(Socks5ProxyUsername,         STRING,   NULL),
   V(Socks5ProxyPassword,         STRING,   NULL),
+  VAR("KeyDirectory",            FILENAME, KeyDirectory_option, NULL),
+  V(KeyDirectoryGroupReadable,   BOOL,     "0"),
+  VAR("HSLayer2Nodes",           ROUTERSET,  HSLayer2Nodes,  NULL),
+  VAR("HSLayer3Nodes",           ROUTERSET,  HSLayer3Nodes,  NULL),
   V(KeepalivePeriod,             INTERVAL, "5 minutes"),
   V(KeepBindCapabilities,            AUTOBOOL, "auto"),
   VAR("Log",                     LINELIST, Logs,             NULL),
@@ -344,33 +452,39 @@ static config_var_t option_vars_[] = {
   V(LogTimeGranularity,          MSEC_INTERVAL, "1 second"),
   V(TruncateLogFile,             BOOL,     "0"),
   V(SyslogIdentityTag,           STRING,   NULL),
+  V(AndroidIdentityTag,          STRING,   NULL),
   V(LongLivedPorts,              CSV,
         "21,22,706,1863,5050,5190,5222,5223,6523,6667,6697,8300"),
   VAR("MapAddress",              LINELIST, AddressMap,           NULL),
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
   V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
   V(MaxOnionQueueDelay,          MSEC_INTERVAL, "1750 msec"),
   V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"),
   V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"),
-  V(MyFamily,                    STRING,   NULL),
+  VAR("MyFamily",                LINELIST, MyFamily_lines,       NULL),
   V(NewCircuitPeriod,            INTERVAL, "30 seconds"),
   OBSOLETE("NamingAuthoritativeDirectory"),
-  V(NATDListenAddress,           LINELIST, NULL),
-  VPORT(NATDPort,                    LINELIST, NULL),
+  OBSOLETE("NATDListenAddress"),
+  VPORT(NATDPort),
   V(Nickname,                    STRING,   NULL),
-  V(PredictedPortsRelevanceTime,  INTERVAL, "1 hour"),
-  V(WarnUnsafeSocks,              BOOL,     "1"),
+  OBSOLETE("PredictedPortsRelevanceTime"),
+  OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
+  V(NoExec,                      BOOL,     "0"),
   V(NumCPUs,                     UINT,     "0"),
   V(NumDirectoryGuards,          UINT,     "0"),
   V(NumEntryGuards,              UINT,     "0"),
+  V(NumPrimaryGuards,            UINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
-  V(ORListenAddress,             LINELIST, NULL),
-  VPORT(ORPort,                      LINELIST, NULL),
+  OBSOLETE("ORListenAddress"),
+  VPORT(ORPort),
   V(OutboundBindAddress,         LINELIST,   NULL),
+  V(OutboundBindAddressOR,       LINELIST,   NULL),
+  V(OutboundBindAddressExit,     LINELIST,   NULL),
 
   OBSOLETE("PathBiasDisableRate"),
   V(PathBiasCircThreshold,       INT,      "-1"),
@@ -403,8 +517,8 @@ static config_var_t option_vars_[] = {
   V(TestingSigningKeySlop,           INTERVAL, "1 day"),
 
   V(OptimisticData,              AUTOBOOL, "auto"),
-  V(PortForwarding,              BOOL,     "0"),
-  V(PortForwardingHelper,        FILENAME, "tor-fw-helper"),
+  OBSOLETE("PortForwarding"),
+  OBSOLETE("PortForwardingHelper"),
   OBSOLETE("PreferTunneledDirConns"),
   V(ProtocolWarnings,            BOOL,     "0"),
   V(PublishServerDescriptor,     CSV,      "1"),
@@ -416,6 +530,8 @@ static config_var_t option_vars_[] = {
   V(RecommendedClientVersions,   LINELIST, NULL),
   V(RecommendedServerVersions,   LINELIST, NULL),
   V(RecommendedPackages,         LINELIST, NULL),
+  V(ReducedConnectionPadding,    BOOL,     "0"),
+  V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
@@ -423,6 +539,7 @@ static config_var_t option_vars_[] = {
   V(RendPostPeriod,              INTERVAL, "1 hour"),
   V(RephistTrackTime,            INTERVAL, "24 hours"),
   V(RunAsDaemon,                 BOOL,     "0"),
+  V(ReducedExitPolicy,           BOOL,     "0"),
   OBSOLETE("RunTesting"), // currently unused
   V(Sandbox,                     BOOL,     "0"),
   V(SafeLogging,                 STRING,   "1"),
@@ -435,13 +552,16 @@ static config_var_t option_vars_[] = {
   V(ServerDNSSearchDomains,      BOOL,     "0"),
   V(ServerDNSTestAddresses,      CSV,
       "www.google.com,www.mit.edu,www.yahoo.com,www.slashdot.org"),
-  V(SchedulerLowWaterMark__,     MEMUNIT,  "100 MB"),
-  V(SchedulerHighWaterMark__,    MEMUNIT,  "101 MB"),
-  V(SchedulerMaxFlushCells__,    UINT,     "1000"),
+  OBSOLETE("SchedulerLowWaterMark__"),
+  OBSOLETE("SchedulerHighWaterMark__"),
+  OBSOLETE("SchedulerMaxFlushCells__"),
+  V(KISTSchedRunInterval,        MSEC_INTERVAL, "0 msec"),
+  V(KISTSockBufSizeFactor,       DOUBLE,   "1.0"),
+  V(Schedulers,                  CSV,      "KIST,KISTLite,Vanilla"),
   V(ShutdownWaitLength,          INTERVAL, "30 seconds"),
-  V(SocksListenAddress,          LINELIST, NULL),
+  OBSOLETE("SocksListenAddress"),
   V(SocksPolicy,                 LINELIST, NULL),
-  VPORT(SocksPort,                   LINELIST, NULL),
+  VPORT(SocksPort),
   V(SocksTimeout,                INTERVAL, "2 minutes"),
   V(SSLKeyLifetime,              INTERVAL, "0"),
   OBSOLETE("StrictEntryNodes"),
@@ -452,23 +572,24 @@ static config_var_t option_vars_[] = {
   V(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
   V(Tor2webMode,                 BOOL,     "0"),
   V(Tor2webRendezvousPoints,      ROUTERSET, NULL),
-  V(TLSECGroup,                  STRING,   NULL),
+  OBSOLETE("TLSECGroup"),
   V(TrackHostExits,              CSV,      NULL),
   V(TrackHostExitsExpire,        INTERVAL, "30 minutes"),
-  V(TransListenAddress,          LINELIST, NULL),
-  VPORT(TransPort,                   LINELIST, NULL),
+  OBSOLETE("TransListenAddress"),
+  VPORT(TransPort),
   V(TransProxyType,              STRING,   "default"),
   OBSOLETE("TunnelDirConns"),
   V(UpdateBridgesFromAuthority,  BOOL,     "0"),
   V(UseBridges,                  BOOL,     "0"),
   VAR("UseEntryGuards",          BOOL,     UseEntryGuards_option, "1"),
-  V(UseEntryGuardsAsDirGuards,   BOOL,     "1"),
+  OBSOLETE("UseEntryGuardsAsDirGuards"),
   V(UseGuardFraction,            AUTOBOOL, "auto"),
   V(UseMicrodescriptors,         AUTOBOOL, "auto"),
   OBSOLETE("UseNTorHandshake"),
   V(User,                        STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
   V(AuthDirSharedRandomness,     BOOL,     "1"),
+  V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
   OBSOLETE("V2AuthoritativeDirectory"),
   VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir,   "0"),
@@ -493,58 +614,58 @@ static config_var_t option_vars_[] = {
   VAR("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
   VAR("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
   VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
+  VAR("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
   VAR("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
   VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword,
       NULL),
   VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
+  VAR("__OwningControllerFD",INT,OwningControllerFD, "-1"),
   V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"),
-  V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, "
-                                 "300, 900, 2147483647"),
-  V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, "
-                                 "2147483647"),
-  V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, "
-                                 "300, 600, 1800, 1800, 1800, 1800, "
-                                 "1800, 3600, 7200"),
-  V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, "
-                                 "300, 600, 1800, 3600, 3600, 3600, "
-                                 "10800, 21600, 43200"),
+  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
   /* With the ClientBootstrapConsensus*Download* below:
    * Clients with only authorities will try:
-   *  - 3 authorities over 10 seconds, then wait 60 minutes.
+   *  - at least 3 authorities over 10 seconds, then exponentially backoff,
+   *    with the next attempt 3-21 seconds later,
    * Clients with authorities and fallbacks will try:
-   *  - 2 authorities and 4 fallbacks over 21 seconds, then wait 60 minutes.
+   *  - at least 2 authorities and 4 fallbacks over 21 seconds, then
+   *    exponentially backoff, with the next attempts 4-33 seconds later,
    * Clients will also retry when an application request arrives.
-   * After a number of failed reqests, clients retry every 3 days + 1 hour.
+   * After a number of failed requests, clients retry every 3 days + 1 hour.
    *
    * Clients used to try 2 authorities over 10 seconds, then wait for
    * 60 minutes or an application request.
    *
    * When clients have authorities and fallbacks available, they use these
    * schedules: (we stagger the times to avoid thundering herds) */
-  V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL,
-    "6, 11, 3600, 10800, 25200, 54000, 111600, 262800" /* 3 days + 1 hour */),
-  V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL,
-    "0, 1, 4, 11, 3600, 10800, 25200, 54000, 111600, 262800"),
+  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "6"),
+  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
   /* When clients only have authorities available, they use this schedule: */
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL,
-    "0, 3, 7, 3600, 10800, 25200, 54000, 111600, 262800"),
+  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
+    "0"),
   /* We don't want to overwhelm slow networks (or mirrors whose replies are
    * blocked), but we also don't want to fail if only some mirrors are
    * blackholed. Clients will try 3 directories simultaneously.
    * (Relays never use simultaneous connections.) */
   V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"),
-  V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "1200, 900, 900, 3600"),
+  /* When a client has any running bridges, check each bridge occasionally,
+    * whether or not that bridge is actually up. */
+  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"),
+  /* When a client is just starting, or has no running bridges, check each
+   * bridge a few times quickly, and then try again later. These schedules
+   * are much longer than the other schedules, because we try each and every
+   * configured bridge with this schedule. */
+  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"),
   V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"),
-  V(TestingConsensusMaxDownloadTries, UINT, "8"),
-  /* Since we try connections rapidly and simultaneously, we can afford
-   * to give up earlier. (This protects against overloading directories.) */
-  V(ClientBootstrapConsensusMaxDownloadTries, UINT, "7"),
-  /* We want to give up much earlier if we're only using authorities. */
-  V(ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries, UINT, "4"),
-  V(TestingDescriptorMaxDownloadTries, UINT, "8"),
-  V(TestingMicrodescMaxDownloadTries, UINT, "8"),
-  V(TestingCertMaxDownloadTries, UINT, "8"),
+  OBSOLETE("TestingConsensusMaxDownloadTries"),
+  OBSOLETE("ClientBootstrapConsensusMaxDownloadTries"),
+  OBSOLETE("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries"),
+  OBSOLETE("TestingDescriptorMaxDownloadTries"),
+  OBSOLETE("TestingMicrodescMaxDownloadTries"),
+  OBSOLETE("TestingCertMaxDownloadTries"),
   V(TestingDirAuthVoteExit, ROUTERSET, NULL),
   V(TestingDirAuthVoteExitIsStrict,  BOOL,     "0"),
   V(TestingDirAuthVoteGuard, ROUTERSET, NULL),
@@ -553,27 +674,21 @@ static config_var_t option_vars_[] = {
   V(TestingDirAuthVoteHSDirIsStrict,  BOOL,     "0"),
   VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"),
 
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+  END_OF_CONFIG_VARS
 };
 
 /** Override default values with these if the user sets the TestingTorNetwork
  * option. */
 static const config_var_t testing_tor_network_defaults[] = {
-  V(ServerDNSAllowBrokenConfig,  BOOL,     "1"),
   V(DirAllowPrivateAddresses,    BOOL,     "1"),
   V(EnforceDistinctSubnets,      BOOL,     "0"),
   V(AssumeReachable,             BOOL,     "1"),
   V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(AuthDirMaxServersPerAuthAddr,UINT,     "0"),
-  V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL,
-    "0, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
-  V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL,
-    "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL,
-    "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
-  V(ClientBootstrapConsensusMaxDownloadTries, UINT, "80"),
-  V(ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries, UINT, "80"),
-  V(ClientDNSRejectInternalAddresses, BOOL,"0"), // deprecated in 0.2.9.2-alpha
+  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
+    "0"),
+  V(ClientDNSRejectInternalAddresses, BOOL,"0"),
   V(ClientRejectInternalAddresses, BOOL,   "0"),
   V(CountPrivateBandwidth,       BOOL,     "1"),
   V(ExitPolicyRejectPrivate,     BOOL,     "0"),
@@ -584,32 +699,23 @@ static const config_var_t testing_tor_network_defaults[] = {
   V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
   V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
   V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
-  V(TestingV3AuthVotingStartOffset, INTERVAL, "0"),
   V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"),
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"),
   V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"),
-  V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, "
-                                 "20, 30, 60"),
-  V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, "
-                                 "30, 60"),
-  V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, "
-                                 "15, 20, 30, 60"),
-  V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, "
-                                 "15, 20, 30, 60"),
-  V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "60, 30, 30, 60"),
+  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
+  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"),
+  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"),
   V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"),
-  V(TestingConsensusMaxDownloadTries, UINT, "80"),
-  V(TestingDescriptorMaxDownloadTries, UINT, "80"),
-  V(TestingMicrodescMaxDownloadTries, UINT, "80"),
-  V(TestingCertMaxDownloadTries, UINT, "80"),
   V(TestingEnableConnBwEvent,    BOOL,     "1"),
   V(TestingEnableCellStatsEvent, BOOL,     "1"),
-  V(TestingEnableTbEmptyEvent,   BOOL,     "1"),
   VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"),
   V(RendPostPeriod,              INTERVAL, "2 minutes"),
 
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+  END_OF_CONFIG_VARS
 };
 
 #undef VAR
@@ -617,39 +723,19 @@ static const config_var_t testing_tor_network_defaults[] = {
 #undef OBSOLETE
 
 static const config_deprecation_t option_deprecation_notes_[] = {
-  /* Deprecated since 0.2.9.2-alpha... */
-  { "AllowDotExit", "Unrestricted use of the .exit notation can be used for "
-    "a wide variety of application-level attacks." },
-  { "AllowInvalidNodes", "There is no reason to enable this option; at best "
-    "it will make you easier to track." },
-  { "AllowSingleHopCircuits", "Almost no relays actually allow single-hop "
-    "exits, making this option pointless." },
-  { "AllowSingleHopExits", "Turning this on will make your relay easier "
-    "to abuse." },
-  { "ClientDNSRejectInternalAddresses", "Turning this on makes your client "
-    "easier to fingerprint, and may open you to esoteric attacks." },
-  { "ExcludeSingleHopRelays", "Turning it on makes your client easier to "
-    "fingerprint." },
-  { "FastFirstHopPK", "Changing this option does not make your client more "
-    "secure, but does make it easier to fingerprint." },
-  { "CloseHSClientCircuitsImmediatelyOnTimeout", "This option makes your "
-    "client easier to fingerprint." },
-  { "CloseHSServiceRendCircuitsImmediatelyOnTimeout", "This option makes "
-    "your hidden services easier to fingerprint." },
-  { "WarnUnsafeSocks", "Changing this option makes it easier for you "
-    "to accidentally lose your anonymity by leaking DNS information" },
-  { "TLSECGroup", "The default is a nice secure choice; the other option "
-    "is less secure." },
-  { "ControlListenAddress", "Use ControlPort instead." },
-  { "DirListenAddress", "Use DirPort instead, possibly with the "
-    "NoAdvertise sub-option" },
-  { "DNSListenAddress", "Use DNSPort instead." },
-  { "SocksListenAddress", "Use SocksPort instead." },
-  { "TransListenAddress", "Use TransPort instead." },
-  { "NATDListenAddress", "Use NATDPort instead." },
-  { "ORListenAddress", "Use ORPort instead, possibly with the "
-    "NoAdvertise sub-option" },
-  /* End of options deprecated since 0.2.9.2-alpha. */
+  /* Deprecated since 0.3.2.0-alpha. */
+  { "HTTPProxy", "It only applies to direct unencrypted HTTP connections "
+    "to your directory server, which your Tor probably wasn't using." },
+  { "HTTPProxyAuthenticator", "HTTPProxy is deprecated in favor of HTTPSProxy "
+    "which should be used with HTTPSProxyAuthenticator." },
+  /* End of options deprecated since 0.3.2.1-alpha */
+
+  /* Options deprecated since 0.3.2.2-alpha */
+  { "ReachableDirAddresses", "It has no effect on relays, and has had no "
+    "effect on clients since 0.2.8." },
+  { "ClientPreferIPv6DirPort", "It has no effect on relays, and has had no "
+    "effect on clients since 0.2.8." },
+  /* End of options deprecated since 0.3.2.2-alpha. */
 
   { NULL, NULL }
 };
@@ -665,7 +751,11 @@ static int options_transition_affects_workers(
       const or_options_t *old_options, const or_options_t *new_options);
 static int options_transition_affects_descriptor(
       const or_options_t *old_options, const or_options_t *new_options);
-static int check_nickname_list(char **lst, const char *name, char **msg);
+static int options_transition_affects_dirauth_timing(
+      const or_options_t *old_options, const or_options_t *new_options);
+static int normalize_nickname_list(config_line_t **normalized_out,
+                                   const config_line_t *lst, const char *name,
+                                   char **msg);
 static char *get_bindaddr_from_transport_listen_line(const char *line,
                                                      const char *transport);
 static int parse_ports(or_options_t *options, int validate_only,
@@ -674,8 +764,7 @@ static int parse_ports(or_options_t *options, int validate_only,
 static int check_server_ports(const smartlist_t *ports,
                               const or_options_t *options,
                               int *num_low_ports_out);
-
-static int validate_data_directory(or_options_t *options);
+static int validate_data_directories(or_options_t *options);
 static int write_configuration_file(const char *fname,
                                     const or_options_t *options);
 static int options_init_logs(const or_options_t *old_options,
@@ -690,8 +779,8 @@ static void config_maybe_load_geoip_files_(const or_options_t *options,
 static int options_validate_cb(void *old_options, void *options,
                                void *default_options,
                                int from_setconf, char **msg);
-static uint64_t compute_real_max_mem_in_queues(const uint64_t val,
-                                               int log_guess);
+static void cleanup_protocol_warning_severity_level(void);
+static void set_protocol_warning_severity_level(int warning_severity);
 
 /** Magic value for or_options_t. */
 #define OR_OPTIONS_MAGIC 9090909
@@ -700,7 +789,7 @@ static uint64_t compute_real_max_mem_in_queues(const uint64_t val,
 STATIC config_format_t options_format = {
   sizeof(or_options_t),
   OR_OPTIONS_MAGIC,
-  STRUCT_OFFSET(or_options_t, magic_),
+  offsetof(or_options_t, magic_),
   option_abbrevs_,
   option_deprecation_notes_,
   option_vars_,
@@ -720,7 +809,7 @@ static or_options_t *global_default_options = NULL;
 /** Name of most recently read torrc file. */
 static char *torrc_fname = NULL;
 /** Name of the most recently read torrc-defaults file.*/
-static char *torrc_defaults_fname;
+static char *torrc_defaults_fname = NULL;
 /** Configuration options set by command line. */
 static config_line_t *global_cmdline_options = NULL;
 /** Non-configuration options set by the command line */
@@ -731,6 +820,11 @@ static int have_parsed_cmdline = 0;
 static char *global_dirfrontpagecontents = NULL;
 /** List of port_cfg_t for all configured ports. */
 static smartlist_t *configured_ports = NULL;
+/** True iff we're currently validating options, and any calls to
+ * get_options() are likely to be bugs. */
+static int in_option_validation = 0;
+/* True iff we've initialized libevent */
+static int libevent_initialized = 0;
 
 /** Return the contents of our frontpage string, or NULL if not configured. */
 MOCK_IMPL(const char*,
@@ -744,6 +838,7 @@ MOCK_IMPL(or_options_t *,
 get_options_mutable, (void))
 {
   tor_assert(global_options);
+  tor_assert_nonfatal(! in_option_validation);
   return global_options;
 }
 
@@ -774,9 +869,13 @@ set_options(or_options_t *new_val, char **msg)
     return -1;
   }
   if (options_act(old_options) < 0) { /* acting on the options failed. die. */
-    log_err(LD_BUG,
-            "Acting on config options left us in a broken state. Dying.");
-    exit(1);
+    if (! tor_event_loop_shutdown_is_pending()) {
+      log_err(LD_BUG,
+              "Acting on config options left us in a broken state. Dying.");
+      tor_shutdown_event_loop_and_exit(1);
+    }
+    global_options = old_options;
+    return -1;
   }
   /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is
    * just starting up then the old_options will be undefined. */
@@ -802,7 +901,7 @@ set_options(or_options_t *new_val, char **msg)
             tor_free(line);
           }
         } else {
-          smartlist_add(elements, tor_strdup(options_format.vars[i].name));
+          smartlist_add_strdup(elements, options_format.vars[i].name);
           smartlist_add(elements, NULL);
         }
       }
@@ -812,14 +911,17 @@ set_options(or_options_t *new_val, char **msg)
     smartlist_free(elements);
   }
 
-  if (old_options != global_options)
+  if (old_options != global_options) {
     or_options_free(old_options);
+    /* If we are here it means we've successfully applied the new options and
+     * that the global options have been changed to the new values. We'll
+     * check if we need to remove or add periodic events. */
+    periodic_events_on_new_options(global_options);
+  }
 
   return 0;
 }
 
-extern const char tor_git_revision[]; /* from tor_main.c */
-
 /** The version of this Tor process, as parsed. */
 static char *the_tor_version = NULL;
 /** A shorter version of this Tor process's version, for export in our router
@@ -859,7 +961,7 @@ get_short_version(void)
 /** Release additional memory allocated in options
  */
 STATIC void
-or_options_free(or_options_t *options)
+or_options_free_(or_options_t *options)
 {
   if (!options)
     return;
@@ -870,9 +972,21 @@ or_options_free(or_options_t *options)
                       rs, routerset_free(rs));
     smartlist_free(options->NodeFamilySets);
   }
+  if (options->SchedulerTypes_) {
+    SMARTLIST_FOREACH(options->SchedulerTypes_, int *, i, tor_free(i));
+    smartlist_free(options->SchedulerTypes_);
+  }
+  if (options->FilesOpenedByIncludes) {
+    SMARTLIST_FOREACH(options->FilesOpenedByIncludes, char *, f, tor_free(f));
+    smartlist_free(options->FilesOpenedByIncludes);
+  }
+  tor_free(options->DataDirectory);
+  tor_free(options->CacheDirectory);
+  tor_free(options->KeyDirectory);
   tor_free(options->BridgePassword_AuthDigest_);
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
+  config_free_lines(options->MyFamily);
   config_free(&options_format, options);
 }
 
@@ -905,6 +1019,11 @@ config_free_all(void)
 
   tor_free(the_short_tor_version);
   tor_free(the_tor_version);
+
+  cleanup_protocol_warning_severity_level();
+
+  have_parsed_cmdline = 0;
+  libevent_initialized = 0;
 }
 
 /** Make <b>address</b> -- a piece of information related to our operation as
@@ -964,6 +1083,52 @@ escaped_safe_str(const char *address)
     return escaped(address);
 }
 
+/**
+ * The severity level that should be used for warnings of severity
+ * LOG_PROTOCOL_WARN.
+ *
+ * We keep this outside the options, and we use an atomic_counter_t, in case
+ * one thread needs to use LOG_PROTOCOL_WARN while an option transition is
+ * happening in the main thread.
+ */
+static atomic_counter_t protocol_warning_severity_level;
+
+/** Return the severity level that should be used for warnings of severity
+ * LOG_PROTOCOL_WARN. */
+int
+get_protocol_warning_severity_level(void)
+{
+  return (int) atomic_counter_get(&protocol_warning_severity_level);
+}
+
+/** Set the protocol warning severity level to <b>severity</b>. */
+static void
+set_protocol_warning_severity_level(int warning_severity)
+{
+  atomic_counter_exchange(&protocol_warning_severity_level,
+                          warning_severity);
+}
+
+/**
+ * Initialize the log warning severity level for protocol warnings. Call
+ * only once at startup.
+ */
+void
+init_protocol_warning_severity_level(void)
+{
+  atomic_counter_init(&protocol_warning_severity_level);
+  set_protocol_warning_severity_level(LOG_WARN);
+}
+
+/**
+ * Tear down protocol_warning_severity_level.
+ */
+static void
+cleanup_protocol_warning_severity_level(void)
+{
+   atomic_counter_destroy(&protocol_warning_severity_level);
+}
+
 /** List of default directory authorities */
 
 static const char *default_authorities[] = {
@@ -1129,6 +1294,69 @@ consider_adding_dir_servers(const or_options_t *options,
   return 0;
 }
 
+/**
+ * Make sure that <b>directory</b> exists, with appropriate ownership and
+ * permissions (as modified by <b>group_readable</b>). If <b>create</b>,
+ * create the directory if it is missing. Return 0 on success.
+ * On failure, return -1 and set *<b>msg_out</b>.
+ */
+static int
+check_and_create_data_directory(int create,
+                                const char *directory,
+                                int group_readable,
+                                const char *owner,
+                                char **msg_out)
+{
+  cpd_check_t cpd_opts = create ? CPD_CREATE : CPD_CHECK;
+  if (group_readable)
+      cpd_opts |= CPD_GROUP_READ;
+  if (check_private_dir(directory,
+                        cpd_opts,
+                        owner) < 0) {
+    tor_asprintf(msg_out,
+                 "Couldn't %s private data directory \"%s\"",
+                 create ? "create" : "access",
+                 directory);
+    return -1;
+  }
+
+#ifndef _WIN32
+  if (group_readable) {
+    /* Only new dirs created get new opts, also enforce group read. */
+    if (chmod(directory, 0750)) {
+      log_warn(LD_FS,"Unable to make %s group-readable: %s",
+               directory, strerror(errno));
+    }
+  }
+#endif /* !defined(_WIN32) */
+
+  return 0;
+}
+
+/**
+ * Ensure that our keys directory exists, with appropriate permissions.
+ * Return 0 on success, -1 on failure.
+ */
+int
+create_keys_directory(const or_options_t *options)
+{
+  /* Make sure DataDirectory exists, and is private. */
+  cpd_check_t cpd_opts = CPD_CREATE;
+  if (options->DataDirectoryGroupReadable)
+    cpd_opts |= CPD_GROUP_READ;
+  if (check_private_dir(options->DataDirectory, cpd_opts, options->User)) {
+    log_err(LD_OR, "Can't create/check datadirectory %s",
+            options->DataDirectory);
+    return -1;
+  }
+
+  /* Check the key directory. */
+  if (check_private_dir(options->KeyDirectory, CPD_CREATE, options->User)) {
+    return -1;
+  }
+  return 0;
+}
+
 /* Helps determine flags to pass to switch_id. */
 static int have_low_ports = -1;
 
@@ -1143,7 +1371,6 @@ options_act_reversible(const or_options_t *old_options, char **msg)
 {
   smartlist_t *new_listeners = smartlist_new();
   smartlist_t *replaced_listeners = smartlist_new();
-  static int libevent_initialized = 0;
   or_options_t *options = get_options_mutable();
   int running_tor = options->command == CMD_RUN_TOR;
   int set_conn_limit = 0;
@@ -1170,13 +1397,13 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                       "on this OS/with this build.");
     goto rollback;
   }
-#else
+#else /* !(!defined(HAVE_SYS_UN_H)) */
   if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
     *msg = tor_strdup("Setting ControlSocketGroupWritable without setting"
                       "a ControlSocket makes no sense.");
     goto rollback;
   }
-#endif
+#endif /* !defined(HAVE_SYS_UN_H) */
 
   if (running_tor) {
     int n_ports=0;
@@ -1223,9 +1450,9 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     consider_hibernation(time(NULL));
 
     /* Launch the listeners.  (We do this before we setuid, so we can bind to
-     * ports under 1024.)  We don't want to rebind if we're hibernating. If
-     * networking is disabled, this will close all but the control listeners,
-     * but disable those. */
+     * ports under 1024.)  We don't want to rebind if we're hibernating or
+     * shutting down. If networking is disabled, this will close all but the
+     * control listeners, but disable those. */
     if (!we_are_hibernating()) {
       if (retry_all_listeners(replaced_listeners, new_listeners,
                               options->DisableNetwork) < 0) {
@@ -1253,7 +1480,7 @@ options_act_reversible(const or_options_t *old_options, char **msg)
       goto rollback;
     }
   }
-#endif
+#endif /* defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) */
 
   /* Attempt to lock all current and future memory with mlockall() only once */
   if (options->DisableAllSwap) {
@@ -1283,29 +1510,47 @@ options_act_reversible(const or_options_t *old_options, char **msg)
   }
 
   /* Ensure data directory is private; create if possible. */
-  cpd_check_t cpd_opts = running_tor ? CPD_CREATE : CPD_CHECK;
-  if (options->DataDirectoryGroupReadable)
-      cpd_opts |= CPD_GROUP_READ;
-  if (check_private_dir(options->DataDirectory,
-                        cpd_opts,
-                        options->User)<0) {
-    tor_asprintf(msg,
-              "Couldn't access/create private data directory \"%s\"",
-              options->DataDirectory);
-
+  /* It's okay to do this in "options_act_reversible()" even though it isn't
+   * actually reversible, since you can't change the DataDirectory while
+   * Tor is running. */
+  if (check_and_create_data_directory(running_tor /* create */,
+                                      options->DataDirectory,
+                                      options->DataDirectoryGroupReadable,
+                                      options->User,
+                                      msg) < 0) {
+    goto done;
+  }
+  if (check_and_create_data_directory(running_tor /* create */,
+                                      options->KeyDirectory,
+                                      options->KeyDirectoryGroupReadable,
+                                      options->User,
+                                      msg) < 0) {
     goto done;
-    /* No need to roll back, since you can't change the value. */
   }
 
-#ifndef _WIN32
-  if (options->DataDirectoryGroupReadable) {
-    /* Only new dirs created get new opts, also enforce group read. */
-    if (chmod(options->DataDirectory, 0750)) {
-      log_warn(LD_FS,"Unable to make %s group-readable: %s",
-               options->DataDirectory, strerror(errno));
-    }
+  /* We need to handle the group-readable flag for the cache directory
+   * specially, since the directory defaults to being the same as the
+   * DataDirectory. */
+  int cache_dir_group_readable;
+  if (options->CacheDirectoryGroupReadable != -1) {
+    /* If the user specified a value, use their setting */
+    cache_dir_group_readable = options->CacheDirectoryGroupReadable;
+  } else if (!strcmp(options->CacheDirectory, options->DataDirectory)) {
+    /* If the user left the value as "auto", and the cache is the same as the
+     * datadirectory, use the datadirectory setting.
+     */
+    cache_dir_group_readable = options->DataDirectoryGroupReadable;
+  } else {
+    /* Otherwise, "auto" means "not group readable". */
+    cache_dir_group_readable = 0;
+  }
+  if (check_and_create_data_directory(running_tor /* create */,
+                                      options->CacheDirectory,
+                                      cache_dir_group_readable,
+                                      options->User,
+                                      msg) < 0) {
+    goto done;
   }
-#endif
 
   /* Bail out at this point if we're not going to be a client or server:
    * we don't run Tor itself. */
@@ -1328,6 +1573,7 @@ options_act_reversible(const or_options_t *old_options, char **msg)
       tor_malloc_zero(sizeof(log_severity_list_t));
     close_temp_logs();
     add_callback_log(severity, control_event_logmsg);
+    logs_set_pending_callback_callback(control_event_logmsg_pending);
     control_adjust_event_log_severity();
     tor_free(severity);
     tor_log_update_sigsafe_err_fds();
@@ -1435,13 +1681,14 @@ options_act_reversible(const or_options_t *old_options, char **msg)
 int
 options_need_geoip_info(const or_options_t *options, const char **reason_out)
 {
-  int bridge_usage =
-    options->BridgeRelay && options->BridgeRecordUsageByCountry;
+  int bridge_usage = should_record_bridge_info(options);
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
+    routerset_needs_geoip(options->HSLayer2Nodes) ||
+    routerset_needs_geoip(options->HSLayer3Nodes) ||
     routerset_needs_geoip(options->Tor2webRendezvousPoints);
 
   if (routerset_usage && reason_out) {
@@ -1482,19 +1729,70 @@ get_effective_bwburst(const or_options_t *options)
   return (uint32_t)bw;
 }
 
-/** Return True if any changes from <b>old_options</b> to
- * <b>new_options</b> needs us to refresh our TLS context. */
+/* Used in the various options_transition_affects* functions. */
+#define YES_IF_CHANGED_BOOL(opt) \
+  if (!CFG_EQ_BOOL(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_INT(opt) \
+  if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_STRING(opt) \
+  if (!CFG_EQ_STRING(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_LINELIST(opt) \
+  if (!CFG_EQ_LINELIST(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_SMARTLIST(opt) \
+  if (!CFG_EQ_SMARTLIST(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_ROUTERSET(opt) \
+  if (!CFG_EQ_ROUTERSET(old_options, new_options, opt)) return 1;
+
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the guard subsystem.
+ */
 static int
-options_transition_requires_fresh_tls_context(const or_options_t *old_options,
-                                              const or_options_t *new_options)
+options_transition_affects_guards(const or_options_t *old_options,
+                                  const or_options_t *new_options)
 {
+  /* NOTE: Make sure this function stays in sync with
+   * node_passes_guard_filter */
+  tor_assert(old_options);
   tor_assert(new_options);
 
-  if (!old_options)
-    return 0;
+  YES_IF_CHANGED_BOOL(UseEntryGuards);
+  YES_IF_CHANGED_BOOL(UseBridges);
+  YES_IF_CHANGED_BOOL(ClientUseIPv4);
+  YES_IF_CHANGED_BOOL(ClientUseIPv6);
+  YES_IF_CHANGED_BOOL(FascistFirewall);
+  YES_IF_CHANGED_ROUTERSET(ExcludeNodes);
+  YES_IF_CHANGED_ROUTERSET(EntryNodes);
+  YES_IF_CHANGED_SMARTLIST(FirewallPorts);
+  YES_IF_CHANGED_LINELIST(Bridges);
+  YES_IF_CHANGED_LINELIST(ReachableORAddresses);
+  YES_IF_CHANGED_LINELIST(ReachableDirAddresses);
+
+  return 0;
+}
 
-  if (!opt_streq(old_options->TLSECGroup, new_options->TLSECGroup))
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the timing of the voting subsystem
+ */
+static int
+options_transition_affects_dirauth_timing(const or_options_t *old_options,
+                                          const or_options_t *new_options)
+{
+  tor_assert(old_options);
+  tor_assert(new_options);
+
+  if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options))
     return 1;
+  if (! authdir_mode_v3(new_options))
+    return 0;
+  YES_IF_CHANGED_INT(V3AuthVotingInterval);
+  YES_IF_CHANGED_INT(V3AuthVoteDelay);
+  YES_IF_CHANGED_INT(V3AuthDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset);
 
   return 0;
 }
@@ -1517,7 +1815,12 @@ options_act(const or_options_t *old_options)
   char *msg=NULL;
   const int transition_affects_workers =
     old_options && options_transition_affects_workers(old_options, options);
-  int old_ewma_enabled;
+  const int transition_affects_guards =
+    old_options && options_transition_affects_guards(old_options, options);
+
+  if (options->NoExec || options->Sandbox) {
+    tor_disable_spawning_background_processes();
+  }
 
   /* disable ptrace and later, other basic debugging techniques */
   {
@@ -1553,8 +1856,16 @@ options_act(const or_options_t *old_options)
       return -1;
   }
 
-  if (consider_adding_dir_servers(options, old_options) < 0)
+  {
+    int warning_severity = options->ProtocolWarnings ? LOG_WARN : LOG_INFO;
+    set_protocol_warning_severity_level(warning_severity);
+  }
+
+  if (consider_adding_dir_servers(options, old_options) < 0) {
+    // XXXX This should get validated earlier, and committed here, to
+    // XXXX lower opportunities for reaching an error case.
     return -1;
+  }
 
   if (rend_non_anonymous_mode_enabled(options)) {
     log_warn(LD_GENERAL, "This copy of Tor was compiled or configured to run "
@@ -1563,6 +1874,7 @@ options_act(const or_options_t *old_options)
 
 #ifdef ENABLE_TOR2WEB_MODE
 /* LCOV_EXCL_START */
+  // XXXX This should move into options_validate()
   if (!options->Tor2webMode) {
     log_err(LD_CONFIG, "This copy of Tor was compiled to run in "
             "'tor2web mode'. It can only be run with the Tor2webMode torrc "
@@ -1570,7 +1882,8 @@ options_act(const or_options_t *old_options)
     return -1;
   }
 /* LCOV_EXCL_STOP */
-#else
+#else /* !(defined(ENABLE_TOR2WEB_MODE)) */
+  // XXXX This should move into options_validate()
   if (options->Tor2webMode) {
     log_err(LD_CONFIG, "This copy of Tor was not compiled to run in "
             "'tor2web mode'. It cannot be run with the Tor2webMode torrc "
@@ -1578,7 +1891,7 @@ options_act(const or_options_t *old_options)
             "--enable-tor2web-mode option.");
     return -1;
   }
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
 
   /* If we are a bridge with a pluggable transport proxy but no
      Extended ORPort, inform the user that they are missing out. */
@@ -1598,25 +1911,49 @@ options_act(const or_options_t *old_options)
     for (cl = options->Bridges; cl; cl = cl->next) {
       bridge_line_t *bridge_line = parse_bridge_line(cl->value);
       if (!bridge_line) {
+        // LCOV_EXCL_START
         log_warn(LD_BUG,
                  "Previously validated Bridge line could not be added!");
         return -1;
+        // LCOV_EXCL_STOP
       }
       bridge_add_from_config(bridge_line);
     }
     sweep_bridge_list();
   }
 
-  if (running_tor && rend_config_services(options, 0)<0) {
+  if (running_tor && hs_config_service_all(options, 0)<0) {
+    // LCOV_EXCL_START
     log_warn(LD_BUG,
        "Previously validated hidden services line could not be added!");
     return -1;
+    // LCOV_EXCL_STOP
   }
 
   if (running_tor && rend_parse_service_authorization(options, 0) < 0) {
+    // LCOV_EXCL_START
     log_warn(LD_BUG, "Previously validated client authorization for "
                      "hidden services could not be added!");
     return -1;
+    // LCOV_EXCL_STOP
+  }
+
+  if (running_tor && !old_options && options->OwningControllerFD != -1) {
+#ifdef _WIN32
+    log_warn(LD_CONFIG, "OwningControllerFD is not supported on Windows. "
+             "If you need it, tell the Tor developers.");
+    return -1;
+#else
+    const unsigned ctrl_flags =
+      CC_LOCAL_FD_IS_OWNER |
+      CC_LOCAL_FD_IS_AUTHENTICATED;
+    tor_socket_t ctrl_sock = (tor_socket_t)options->OwningControllerFD;
+    if (control_connection_add_local_fd(ctrl_sock, ctrl_flags) < 0) {
+      log_warn(LD_CONFIG, "Could not add local controller connection with "
+               "given FD.");
+      return -1;
+    }
+#endif /* defined(_WIN32) */
   }
 
   /* Load state */
@@ -1639,10 +1976,12 @@ options_act(const or_options_t *old_options)
     if (options->ClientTransportPlugin) {
       for (cl = options->ClientTransportPlugin; cl; cl = cl->next) {
         if (parse_transport_line(options, cl->value, 0, 0) < 0) {
+          // LCOV_EXCL_START
           log_warn(LD_BUG,
                    "Previously validated ClientTransportPlugin line "
                    "could not be added!");
           return -1;
+          // LCOV_EXCL_STOP
         }
       }
     }
@@ -1650,10 +1989,12 @@ options_act(const or_options_t *old_options)
     if (options->ServerTransportPlugin && server_mode(options)) {
       for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
         if (parse_transport_line(options, cl->value, 0, 1) < 0) {
+          // LCOV_EXCL_START
           log_warn(LD_BUG,
                    "Previously validated ServerTransportPlugin line "
                    "could not be added!");
           return -1;
+          // LCOV_EXCL_STOP
         }
       }
     }
@@ -1678,6 +2019,9 @@ options_act(const or_options_t *old_options)
     finish_daemon(options->DataDirectory);
   }
 
+  /* See whether we need to enable/disable our once-a-second timer. */
+  reschedule_per_second_timer();
+
   /* We want to reinit keys as needed before we do much of anything else:
      keys are important, and other things can depend on them. */
   if (transition_affects_workers ||
@@ -1687,19 +2031,16 @@ options_act(const or_options_t *old_options)
       log_warn(LD_BUG,"Error initializing keys; exiting");
       return -1;
     }
-  } else if (old_options &&
-             options_transition_requires_fresh_tls_context(old_options,
-                                                           options)) {
-    if (router_initialize_tls_context() < 0) {
-      log_warn(LD_BUG,"Error initializing TLS context.");
-      return -1;
-    }
   }
 
   /* Write our PID to the PID file. If we do not have write permissions we
-   * will log a warning */
+   * will log a warning and exit. */
   if (options->PidFile && !sandbox_is_active()) {
-    write_pidfile(options->PidFile);
+    if (write_pidfile(options->PidFile) < 0) {
+      log_err(LD_CONFIG, "Unable to write PIDFile %s",
+              escaped(options->PidFile));
+      return -1;
+    }
   }
 
   /* Register addressmap directives */
@@ -1714,6 +2055,15 @@ options_act(const or_options_t *old_options)
     return -1;
   }
 
+  if (server_mode(options)) {
+    static int cdm_initialized = 0;
+    if (cdm_initialized == 0) {
+      cdm_initialized = 1;
+      consdiffmgr_configure(NULL);
+      consdiffmgr_validate();
+    }
+  }
+
   if (init_control_cookie_authentication(options->CookieAuthentication) < 0) {
     log_warn(LD_CONFIG,"Error creating control cookie authentication file.");
     return -1;
@@ -1722,35 +2072,27 @@ options_act(const or_options_t *old_options)
   monitor_owning_controller_process(options->OwningControllerProcess);
 
   /* reload keys as needed for rendezvous services. */
-  if (rend_service_load_all_keys(NULL)<0) {
+  if (hs_service_load_all_keys() < 0) {
     log_warn(LD_GENERAL,"Error loading rendezvous service keys");
     return -1;
   }
 
-  /* Set up scheduler thresholds */
-  scheduler_set_watermarks((uint32_t)options->SchedulerLowWaterMark__,
-                           (uint32_t)options->SchedulerHighWaterMark__,
-                           (options->SchedulerMaxFlushCells__ > 0) ?
-                           options->SchedulerMaxFlushCells__ : 1000);
+  /* Inform the scheduler subsystem that a configuration changed happened. It
+   * might be a change of scheduler or parameter. */
+  scheduler_conf_changed();
 
   /* Set up accounting */
   if (accounting_parse_options(options, 0)<0) {
-    log_warn(LD_CONFIG,"Error in accounting options");
+    // LCOV_EXCL_START
+    log_warn(LD_BUG,"Error in previously validated accounting options");
     return -1;
+    // LCOV_EXCL_STOP
   }
   if (accounting_is_enabled(options))
     configure_accounting(time(NULL));
 
-  old_ewma_enabled = cell_ewma_enabled();
   /* Change the cell EWMA settings */
-  cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus());
-  /* If we just enabled ewma, set the cmux policy on all active channels */
-  if (cell_ewma_enabled() && !old_ewma_enabled) {
-    channel_set_cmux_policy_everywhere(&ewma_policy);
-  } else if (!cell_ewma_enabled() && old_ewma_enabled) {
-    /* Turn it off everywhere */
-    channel_set_cmux_policy_everywhere(NULL);
-  }
+  cmux_ewma_set_options(options, networkstatus_get_latest_consensus());
 
   /* Update the BridgePassword's hashed version as needed.  We store this as a
    * digest so that we can do side-channel-proof comparisons on it.
@@ -1759,6 +2101,7 @@ options_act(const or_options_t *old_options)
     char *http_authenticator;
     http_authenticator = alloc_http_authenticator(options->BridgePassword);
     if (!http_authenticator) {
+      // XXXX This should get validated in options_validate().
       log_warn(LD_BUG, "Unable to allocate HTTP authenticator. Not setting "
                "BridgePassword.");
       return -1;
@@ -1771,9 +2114,12 @@ options_act(const or_options_t *old_options)
   }
 
   if (parse_outbound_addresses(options, 0, &msg) < 0) {
-    log_warn(LD_BUG, "Failed parsing outbound bind addresses: %s", msg);
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Failed parsing previously validated outbound "
+             "bind addresses: %s", msg);
     tor_free(msg);
     return -1;
+    // LCOV_EXCL_STOP
   }
 
   config_maybe_load_geoip_files_(options, old_options);
@@ -1794,6 +2140,7 @@ options_act(const or_options_t *old_options)
   if (old_options) {
     int revise_trackexithosts = 0;
     int revise_automap_entries = 0;
+    int abandon_circuits = 0;
     if ((options->UseEntryGuards && !old_options->UseEntryGuards) ||
         options->UseBridges != old_options->UseBridges ||
         (options->UseBridges &&
@@ -1803,6 +2150,10 @@ options_act(const or_options_t *old_options)
                          options->ExcludeExitNodes) ||
         !routerset_equal(old_options->EntryNodes, options->EntryNodes) ||
         !routerset_equal(old_options->ExitNodes, options->ExitNodes) ||
+        !routerset_equal(old_options->HSLayer2Nodes,
+                         options->HSLayer2Nodes) ||
+        !routerset_equal(old_options->HSLayer3Nodes,
+                         options->HSLayer3Nodes) ||
         !routerset_equal(old_options->Tor2webRendezvousPoints,
                          options->Tor2webRendezvousPoints) ||
         options->StrictNodes != old_options->StrictNodes) {
@@ -1810,6 +2161,16 @@ options_act(const or_options_t *old_options)
                "Changed to using entry guards or bridges, or changed "
                "preferred or excluded node lists. "
                "Abandoning previous circuits.");
+      abandon_circuits = 1;
+    }
+
+    if (transition_affects_guards) {
+      if (guards_update_all()) {
+        abandon_circuits = 1;
+      }
+    }
+
+    if (abandon_circuits) {
       circuit_mark_all_unused_circs();
       circuit_mark_all_dirty_circs_as_unusable();
       revise_trackexithosts = 1;
@@ -1840,7 +2201,7 @@ options_act(const or_options_t *old_options)
       addressmap_clear_invalid_automaps(options);
 
 /* How long should we delay counting bridge stats after becoming a bridge?
- * We use this so we don't count people who used our bridge thinking it is
+ * We use this so we don't count clients who used our bridge thinking it is
  * a relay. If you change this, don't forget to change the log message
  * below. It's 4 hours (the time it takes to stop being used by clients)
  * plus some extra time for clock skew. */
@@ -1868,9 +2229,16 @@ options_act(const or_options_t *old_options)
     if (transition_affects_workers) {
       log_info(LD_GENERAL,
                "Worker-related options changed. Rotating workers.");
+      const int server_mode_turned_on =
+        server_mode(options) && !server_mode(old_options);
+      const int dir_server_mode_turned_on =
+        dir_server_mode(options) && !dir_server_mode(old_options);
 
-      if (server_mode(options) && !server_mode(old_options)) {
+      if (server_mode_turned_on || dir_server_mode_turned_on) {
         cpu_init();
+      }
+
+      if (server_mode_turned_on) {
         ip_address_changed(0);
         if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
           inform_testing_reachability();
@@ -1886,11 +2254,23 @@ options_act(const or_options_t *old_options)
     if (options->PerConnBWRate != old_options->PerConnBWRate ||
         options->PerConnBWBurst != old_options->PerConnBWBurst)
       connection_or_update_token_buckets(get_connection_array(), options);
+
+    if (options->BandwidthRate != old_options->BandwidthRate ||
+        options->BandwidthBurst != old_options->BandwidthBurst ||
+        options->RelayBandwidthRate != old_options->RelayBandwidthRate ||
+        options->RelayBandwidthBurst != old_options->RelayBandwidthBurst)
+      connection_bucket_adjust(options);
+
+    if (options->MainloopStats != old_options->MainloopStats) {
+      reset_main_loop_counters();
+    }
   }
 
   /* Only collect directory-request statistics on relays and bridges. */
   options->DirReqStatistics = options->DirReqStatistics_option &&
     server_mode(options);
+  options->HiddenServiceStatistics =
+    options->HiddenServiceStatistics_option && server_mode(options);
 
   if (options->CellStatistics || options->DirReqStatistics ||
       options->EntryStatistics || options->ExitPortStatistics ||
@@ -1905,7 +2285,6 @@ options_act(const or_options_t *old_options)
       options->CellStatistics = 0;
       options->EntryStatistics = 0;
       options->ConnDirectionStatistics = 0;
-      options->HiddenServiceStatistics = 0;
       options->ExitPortStatistics = 0;
     }
 
@@ -1931,6 +2310,11 @@ options_act(const or_options_t *old_options)
     }
     if ((!old_options || !old_options->EntryStatistics) &&
         options->EntryStatistics && !should_record_bridge_info(options)) {
+      /* If we get here, we've started recording bridge info when we didn't
+       * do so before.  Note that "should_record_bridge_info()" will
+       * always be false at this point, because of the earlier block
+       * that cleared EntryStatistics when public_server_mode() was false.
+       * We're leaving it in as defensive programming. */
       if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) {
         geoip_entry_stats_init(now);
         print_notice = 1;
@@ -1991,13 +2375,6 @@ options_act(const or_options_t *old_options)
       !options->BridgeAuthoritativeDir)
     rep_hist_desc_stats_term();
 
-  /* Check if we need to parse and add the EntryNodes config option. */
-  if (options->EntryNodes &&
-      (!old_options ||
-       !routerset_equal(old_options->EntryNodes,options->EntryNodes) ||
-       !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)))
-    entry_nodes_should_be_added();
-
   /* Since our options changed, we might need to regenerate and upload our
    * server descriptor.
    */
@@ -2007,8 +2384,10 @@ options_act(const or_options_t *old_options)
 
   /* We may need to reschedule some directory stuff if our status changed. */
   if (old_options) {
-    if (authdir_mode_v3(options) && !authdir_mode_v3(old_options))
-      dirvote_recalculate_timing(options, time(NULL));
+    if (options_transition_affects_dirauth_timing(old_options, options)) {
+      voting_schedule_recalculate_timing(options, time(NULL));
+      reschedule_dirvote(options);
+    }
     if (!bool_eq(directory_fetches_dir_info_early(options),
                  directory_fetches_dir_info_early(old_options)) ||
         !bool_eq(directory_fetches_dir_info_later(options),
@@ -2065,6 +2444,7 @@ static const struct {
   { "--dump-config",          ARGUMENT_OPTIONAL },
   { "--list-fingerprint",     TAKES_NO_ARGUMENT },
   { "--keygen",               TAKES_NO_ARGUMENT },
+  { "--key-expiration",       ARGUMENT_OPTIONAL },
   { "--newpass",              TAKES_NO_ARGUMENT },
   { "--no-passphrase",        TAKES_NO_ARGUMENT },
   { "--passphrase-fd",        ARGUMENT_NECESSARY },
@@ -2225,24 +2605,36 @@ options_trial_assign(config_line_t *list, unsigned flags, char **msg)
     return r;
   }
 
-  if (options_validate(get_options_mutable(), trial_options,
+  setopt_err_t rv;
+  or_options_t *cur_options = get_options_mutable();
+
+  in_option_validation = 1;
+
+  if (options_validate(cur_options, trial_options,
                        global_default_options, 1, msg) < 0) {
     or_options_free(trial_options);
-    return SETOPT_ERR_PARSE; /*XXX make this a separate return value. */
+    rv = SETOPT_ERR_PARSE; /*XXX make this a separate return value. */
+    goto done;
   }
 
-  if (options_transition_allowed(get_options(), trial_options, msg) < 0) {
+  if (options_transition_allowed(cur_options, trial_options, msg) < 0) {
     or_options_free(trial_options);
-    return SETOPT_ERR_TRANSITION;
+    rv = SETOPT_ERR_TRANSITION;
+    goto done;
   }
+  in_option_validation = 0;
 
   if (set_options(trial_options, msg)<0) {
     or_options_free(trial_options);
-    return SETOPT_ERR_SETTING;
+    rv = SETOPT_ERR_SETTING;
+    goto done;
   }
 
   /* we liked it. put it in place. */
-  return SETOPT_OK;
+  rv = SETOPT_OK;
+ done:
+  in_option_validation = 0;
+  return rv;
 }
 
 /** Print a usage message for tor. */
@@ -2252,7 +2644,7 @@ print_usage(void)
   printf(
 "Copyright (c) 2001-2004, Roger Dingledine\n"
 "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n"
-"Copyright (c) 2007-2016, The Tor Project, Inc.\n\n"
+"Copyright (c) 2007-2017, The Tor Project, Inc.\n\n"
 "tor -f <torrc> [args]\n"
 "See man page for options, or https://www.torproject.org/ for "
 "documentation.\n");
@@ -2333,8 +2725,8 @@ using_default_dir_authorities(const or_options_t *options)
  * Fail if one or more of the following is true:
  *   - DNS name in <b>options-\>Address</b> cannot be resolved.
  *   - <b>options-\>Address</b> is a local host address.
- *   - Attempt to getting local hostname fails.
- *   - Attempt to getting network interface address fails.
+ *   - Attempt at getting local hostname fails.
+ *   - Attempt at getting network interface address fails.
  *
  * Return 0 if all is well, or -1 if we can't find a suitable
  * public IP address.
@@ -2714,13 +3106,13 @@ compute_publishserverdescriptor(or_options_t *options)
 #define MIN_REND_POST_PERIOD (10*60)
 #define MIN_REND_POST_PERIOD_TESTING (5)
 
-/** Higest allowable value for PredictedPortsRelevanceTime; if this is
- * too high, our selection of exits will decrease for an extended
- * period of time to an uncomfortable level .*/
-#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60)
+/** Highest allowable value for CircuitsAvailableTimeout.
+ * If this is too large, client connections will stay open for too long,
+ * incurring extra padding overhead. */
+#define MAX_CIRCS_AVAILABLE_TIME (24*60*60)
 
 /** Highest allowable value for RendPostPeriod. */
-#define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2)
+#define MAX_DIR_PERIOD ((7*24*60*60)/2)
 
 /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor
  * will generate too many circuits and potentially overload the network. */
@@ -2734,10 +3126,6 @@ compute_publishserverdescriptor(or_options_t *options)
  * will generate too many circuits and potentially overload the network. */
 #define MIN_CIRCUIT_STREAM_TIMEOUT 10
 
-/** Lowest allowable value for HeartbeatPeriod; if this is too low, we might
- * expose more information than we're comfortable with. */
-#define MIN_HEARTBEAT_PERIOD (30*60)
-
 /** Lowest recommended value for CircuitBuildTimeout; if it is set too low
  * and LearnCircuitBuildTimeout is off, the failure rate for circuit
  * construction may be very high.  In that case, if it is set below this
@@ -2749,8 +3137,11 @@ static int
 options_validate_cb(void *old_options, void *options, void *default_options,
                     int from_setconf, char **msg)
 {
-  return options_validate(old_options, options, default_options,
+  in_option_validation = 1;
+  int rv = options_validate(old_options, options, default_options,
                           from_setconf, msg);
+  in_option_validation = 0;
+  return rv;
 }
 
 #define REJECT(arg) \
@@ -2761,15 +3152,17 @@ options_validate_cb(void *old_options, void *options, void *default_options,
 #else
 #define COMPLAIN(args, ...)                                     \
   STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END
-#endif
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
 
 /** Log a warning message iff <b>filepath</b> is not absolute.
  * Warning message must contain option name <b>option</b> and
  * an absolute path that <b>filepath</b> will resolve to.
  *
  * In case <b>filepath</b> is absolute, do nothing.
+ *
+ * Return 1 if there were relative paths; 0 otherwise.
  */
-static void
+static int
 warn_if_option_path_is_relative(const char *option,
                                 char *filepath)
 {
@@ -2778,39 +3171,100 @@ warn_if_option_path_is_relative(const char *option,
     COMPLAIN("Path for %s (%s) is relative and will resolve to %s."
              " Is this what you wanted?", option, filepath, abs_path);
     tor_free(abs_path);
+    return 1;
   }
+  return 0;
 }
 
-/** Scan <b>options</b> for occurances of relative file/directory
+/** Scan <b>options</b> for occurrences of relative file/directory
  * path and log a warning whenever it is found.
+ *
+ * Return 1 if there were relative paths; 0 otherwise.
  */
-static void
+static int
 warn_about_relative_paths(or_options_t *options)
 {
   tor_assert(options);
+  int n = 0;
 
-  warn_if_option_path_is_relative("CookieAuthFile",
-                                  options->CookieAuthFile);
-  warn_if_option_path_is_relative("ExtORPortCookieAuthFile",
-                                  options->ExtORPortCookieAuthFile);
-  warn_if_option_path_is_relative("DirPortFrontPage",
-                                  options->DirPortFrontPage);
-  warn_if_option_path_is_relative("V3BandwidthsFile",
-                                  options->V3BandwidthsFile);
-  warn_if_option_path_is_relative("ControlPortWriteToFile",
-                                  options->ControlPortWriteToFile);
-  warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile);
-  warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File);
-  warn_if_option_path_is_relative("Log",options->DebugLogFile);
-  warn_if_option_path_is_relative("AccelDir",options->AccelDir);
-  warn_if_option_path_is_relative("DataDirectory",options->DataDirectory);
-  warn_if_option_path_is_relative("PidFile",options->PidFile);
+  n += warn_if_option_path_is_relative("CookieAuthFile",
+                                       options->CookieAuthFile);
+  n += warn_if_option_path_is_relative("ExtORPortCookieAuthFile",
+                                       options->ExtORPortCookieAuthFile);
+  n += warn_if_option_path_is_relative("DirPortFrontPage",
+                                       options->DirPortFrontPage);
+  n += warn_if_option_path_is_relative("V3BandwidthsFile",
+                                       options->V3BandwidthsFile);
+  n += warn_if_option_path_is_relative("ControlPortWriteToFile",
+                                       options->ControlPortWriteToFile);
+  n += warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile);
+  n += warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File);
+  n += warn_if_option_path_is_relative("Log",options->DebugLogFile);
+  n += warn_if_option_path_is_relative("AccelDir",options->AccelDir);
+  n += warn_if_option_path_is_relative("DataDirectory",options->DataDirectory);
+  n += warn_if_option_path_is_relative("PidFile",options->PidFile);
 
   for (config_line_t *hs_line = options->RendConfigLines; hs_line;
        hs_line = hs_line->next) {
     if (!strcasecmp(hs_line->key, "HiddenServiceDir"))
-      warn_if_option_path_is_relative("HiddenServiceDir",hs_line->value);
+      n += warn_if_option_path_is_relative("HiddenServiceDir",hs_line->value);
   }
+  return n != 0;
+}
+
+/* Validate options related to the scheduler. From the Schedulers list, the
+ * SchedulerTypes_ list is created with int values so once we select the
+ * scheduler, which can happen anytime at runtime, we don't have to parse
+ * strings and thus be quick.
+ *
+ * Return 0 on success else -1 and msg is set with an error message. */
+static int
+options_validate_scheduler(or_options_t *options, char **msg)
+{
+  tor_assert(options);
+  tor_assert(msg);
+
+  if (!options->Schedulers || smartlist_len(options->Schedulers) == 0) {
+    REJECT("Empty Schedulers list. Either remove the option so the defaults "
+           "can be used or set at least one value.");
+  }
+  /* Ok, we do have scheduler types, validate them. */
+  options->SchedulerTypes_ = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(options->Schedulers, const char *, type) {
+    int *sched_type;
+    if (!strcasecmp("KISTLite", type)) {
+      sched_type = tor_malloc_zero(sizeof(int));
+      *sched_type = SCHEDULER_KIST_LITE;
+      smartlist_add(options->SchedulerTypes_, sched_type);
+    } else if (!strcasecmp("KIST", type)) {
+      sched_type = tor_malloc_zero(sizeof(int));
+      *sched_type = SCHEDULER_KIST;
+      smartlist_add(options->SchedulerTypes_, sched_type);
+    } else if (!strcasecmp("Vanilla", type)) {
+      sched_type = tor_malloc_zero(sizeof(int));
+      *sched_type = SCHEDULER_VANILLA;
+      smartlist_add(options->SchedulerTypes_, sched_type);
+    } else {
+      tor_asprintf(msg, "Unknown type %s in option Schedulers. "
+                        "Possible values are KIST, KISTLite and Vanilla.",
+                   escaped(type));
+      return -1;
+    }
+  } SMARTLIST_FOREACH_END(type);
+
+  if (options->KISTSockBufSizeFactor < 0) {
+    REJECT("KISTSockBufSizeFactor must be at least 0");
+  }
+
+  /* Don't need to validate that the Interval is less than anything because
+   * zero is valid and all negative values are valid. */
+  if (options->KISTSchedRunInterval > KIST_SCHED_RUN_INTERVAL_MAX) {
+    tor_asprintf(msg, "KISTSchedRunInterval must not be more than %d (ms)",
+                 KIST_SCHED_RUN_INTERVAL_MAX);
+    return -1;
+  }
+
+  return 0;
 }
 
 /* Validate options related to single onion services.
@@ -2841,13 +3295,13 @@ options_validate_single_onion(or_options_t *options, char **msg)
   const int client_port_set = (options->SocksPort_set ||
                                options->TransPort_set ||
                                options->NATDPort_set ||
-                               options->DNSPort_set);
+                               options->DNSPort_set ||
+                               options->HTTPTunnelPort_set);
   if (rend_service_non_anonymous_mode_enabled(options) && client_port_set &&
       !options->Tor2webMode) {
     REJECT("HiddenServiceNonAnonymousMode is incompatible with using Tor as "
            "an anonymous client. Please set Socks/Trans/NATD/DNSPort to 0, or "
-           "HiddenServiceNonAnonymousMode to 0, or use the non-anonymous "
-           "Tor2webMode.");
+           "revert HiddenServiceNonAnonymousMode to 0.");
   }
 
   /* If you run a hidden service in non-anonymous mode, the hidden service
@@ -2857,12 +3311,12 @@ options_validate_single_onion(or_options_t *options, char **msg)
     REJECT("Non-anonymous (Tor2web) mode is incompatible with using Tor as a "
            "hidden service. Please remove all HiddenServiceDir lines, or use "
            "a version of tor compiled without --enable-tor2web-mode, or use "
-           " HiddenServiceNonAnonymousMode.");
+           "HiddenServiceNonAnonymousMode.");
   }
 
   if (rend_service_allow_non_anonymous_connection(options)
       && options->UseEntryGuards) {
-    /* Single Onion services only use entry guards when uploading descriptors,
+    /* Single Onion services only use entry guards when uploading descriptors;
      * all other connections are one-hop. Further, Single Onions causes the
      * hidden service code to do things which break the path bias
      * detector, and it's far easier to turn off entry guards (and
@@ -2904,13 +3358,21 @@ options_validate(or_options_t *old_options, or_options_t *options,
   tor_assert(msg);
   *msg = NULL;
 
+  if (parse_ports(options, 1, msg, &n_ports,
+                  &world_writable_control_socket) < 0)
+    return -1;
+
   /* Set UseEntryGuards from the configured value, before we check it below.
-   * We change UseEntryGuards whenn it's incompatible with other options,
+   * We change UseEntryGuards when it's incompatible with other options,
    * but leave UseEntryGuards_option with the original value.
    * Always use the value of UseEntryGuards, not UseEntryGuards_option. */
   options->UseEntryGuards = options->UseEntryGuards_option;
 
-  warn_about_relative_paths(options);
+  if (warn_about_relative_paths(options) && options->RunAsDaemon) {
+    REJECT("You have specified at least one relative path (see above) "
+           "with the RunAsDaemon option. RunAsDaemon is not compatible "
+           "with relative paths.");
+  }
 
   if (server_mode(options) &&
       (!strcmpstart(uname, "Windows 95") ||
@@ -2922,19 +3384,15 @@ options_validate(or_options_t *old_options, or_options_t *options,
         "for details.", uname);
   }
 
-  if (parse_ports(options, 1, msg, &n_ports,
-                  &world_writable_control_socket) < 0)
-    return -1;
-
   if (parse_outbound_addresses(options, 1, msg) < 0)
     return -1;
 
-  if (validate_data_directory(options)<0)
+  if (validate_data_directories(options)<0)
     REJECT("Invalid DataDirectory");
 
   if (options->Nickname == NULL) {
     if (server_mode(options)) {
-        options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
+      options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
     }
   } else {
     if (!is_legal_nickname(options->Nickname)) {
@@ -2954,9 +3412,9 @@ options_validate(or_options_t *old_options, or_options_t *options,
   /* Special case on first boot if no Log options are given. */
   if (!options->Logs && !options->RunAsDaemon && !from_setconf) {
     if (quiet_level == 0)
-        config_line_append(&options->Logs, "Log", "notice stdout");
+      config_line_append(&options->Logs, "Log", "notice stdout");
     else if (quiet_level == 1)
-        config_line_append(&options->Logs, "Log", "warn stdout");
+      config_line_append(&options->Logs, "Log", "warn stdout");
   }
 
   /* Validate the tor_log(s) */
@@ -2991,13 +3449,13 @@ options_validate(or_options_t *old_options, or_options_t *options,
     if (!strcasecmp(options->TransProxyType, "default")) {
       options->TransProxyType_parsed = TPT_DEFAULT;
     } else if (!strcasecmp(options->TransProxyType, "pf-divert")) {
-#if !defined(__OpenBSD__) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined( DARWIN )
       /* Later versions of OS X have pf */
       REJECT("pf-divert is a OpenBSD-specific "
              "and OS X/Darwin-specific feature.");
 #else
       options->TransProxyType_parsed = TPT_PF_DIVERT;
-#endif
+#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
     } else if (!strcasecmp(options->TransProxyType, "tproxy")) {
 #if !defined(__linux__)
       REJECT("TPROXY is a Linux-specific feature.");
@@ -3011,22 +3469,20 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "and OS X/Darwin-specific feature.");
 #else
       options->TransProxyType_parsed = TPT_IPFW;
-#endif
+#endif /* !defined(KERNEL_MAY_SUPPORT_IPFW) */
     } else {
       REJECT("Unrecognized value for TransProxyType");
     }
 
     if (strcasecmp(options->TransProxyType, "default") &&
         !options->TransPort_set) {
-      REJECT("Cannot use TransProxyType without any valid TransPort or "
-             "TransListenAddress.");
+      REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
-#else
+#else /* !(defined(USE_TRANSPARENT)) */
   if (options->TransPort_set)
-    REJECT("TransPort and TransListenAddress are disabled "
-           "in this build.");
-#endif
+    REJECT("TransPort is disabled in this build.");
+#endif /* defined(USE_TRANSPARENT) */
 
   if (options->TokenBucketRefillInterval <= 0
       || options->TokenBucketRefillInterval > 1000) {
@@ -3039,17 +3495,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeNodes);
   }
 
-  if (options->SchedulerLowWaterMark__ == 0 ||
-      options->SchedulerLowWaterMark__ > UINT32_MAX) {
-    log_warn(LD_GENERAL, "Bad SchedulerLowWaterMark__ option");
-    return -1;
-  } else if (options->SchedulerHighWaterMark__ <=
-             options->SchedulerLowWaterMark__ ||
-             options->SchedulerHighWaterMark__ > UINT32_MAX) {
-    log_warn(LD_GENERAL, "Bad SchedulerHighWaterMark option");
-    return -1;
-  }
-
   if (options->NodeFamilies) {
     options->NodeFamilySets = smartlist_new();
     for (cl = options->NodeFamilies; cl; cl = cl->next) {
@@ -3062,15 +3507,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
-  if (options->TLSECGroup && (strcasecmp(options->TLSECGroup, "P256") &&
-                              strcasecmp(options->TLSECGroup, "P224"))) {
-    COMPLAIN("Unrecognized TLSECGroup: Falling back to the default.");
-    tor_free(options->TLSECGroup);
-  }
-  if (!evaluate_ecgroup_for_tls(options->TLSECGroup)) {
-    REJECT("Unsupported TLSECGroup.");
-  }
-
   if (options->ExcludeNodes && options->StrictNodes) {
     COMPLAIN("You have asked to exclude certain relays from all positions "
              "in your circuits. Expect hidden services and other Tor "
@@ -3099,19 +3535,21 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("Versioning authoritative dir servers must set "
              "Recommended*Versions.");
 
+#ifdef HAVE_MODULE_DIRAUTH
     char *t;
     /* Call these functions to produce warnings only. */
     t = format_recommended_version_list(options->RecommendedClientVersions, 1);
     tor_free(t);
     t = format_recommended_version_list(options->RecommendedServerVersions, 1);
     tor_free(t);
+#endif
 
     if (options->UseEntryGuards) {
       log_info(LD_CONFIG, "Authoritative directory servers can't set "
                "UseEntryGuards. Disabling.");
       options->UseEntryGuards = 0;
     }
-    if (!options->DownloadExtraInfo && authdir_mode_any_main(options)) {
+    if (!options->DownloadExtraInfo && authdir_mode_v3(options)) {
       log_info(LD_CONFIG, "Authoritative directories always try to download "
                "extra-info documents. Setting DownloadExtraInfo.");
       options->DownloadExtraInfo = 1;
@@ -3265,23 +3703,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "of the Internet, so they must not set Reachable*Addresses "
            "or FascistFirewall or FirewallPorts or ClientUseIPv4 0.");
 
-  /* We check if Reachable*Addresses blocks all addresses in
-   * parse_reachable_addresses(). */
-
-#define WARN_PLEASE_USE_IPV6_LOG_MSG \
-        "ClientPreferIPv6%sPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges."
-
-  if (!fascist_firewall_use_ipv6(options)
-      && options->ClientPreferIPv6ORPort == 1)
-    log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "OR");
-
-  if (!fascist_firewall_use_ipv6(options)
-      && options->ClientPreferIPv6DirPort == 1)
-    log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "Dir");
-
-#undef WARN_PLEASE_USE_IPV6_LOG_MSG
-
   if (options->UseBridges &&
       server_mode(options))
     REJECT("Servers must be able to freely connect to the rest "
@@ -3293,33 +3714,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->UseBridges && options->EntryNodes)
     REJECT("You cannot set both UseBridges and EntryNodes.");
 
+  /* If we have UseBridges as 1 and UseEntryGuards as 0, we end up bypassing
+   * the use of bridges */
+  if (options->UseBridges && !options->UseEntryGuards)
+    REJECT("Setting UseBridges requires also setting UseEntryGuards.");
+
   options->MaxMemInQueues =
     compute_real_max_mem_in_queues(options->MaxMemInQueues_raw,
                                    server_mode(options));
   options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3;
 
-  options->AllowInvalid_ = 0;
-
-  if (options->AllowInvalidNodes) {
-    SMARTLIST_FOREACH_BEGIN(options->AllowInvalidNodes, const char *, cp) {
-        if (!strcasecmp(cp, "entry"))
-          options->AllowInvalid_ |= ALLOW_INVALID_ENTRY;
-        else if (!strcasecmp(cp, "exit"))
-          options->AllowInvalid_ |= ALLOW_INVALID_EXIT;
-        else if (!strcasecmp(cp, "middle"))
-          options->AllowInvalid_ |= ALLOW_INVALID_MIDDLE;
-        else if (!strcasecmp(cp, "introduction"))
-          options->AllowInvalid_ |= ALLOW_INVALID_INTRODUCTION;
-        else if (!strcasecmp(cp, "rendezvous"))
-          options->AllowInvalid_ |= ALLOW_INVALID_RENDEZVOUS;
-        else {
-          tor_asprintf(msg,
-              "Unrecognized value '%s' in AllowInvalidNodes", cp);
-          return -1;
-        }
-    } SMARTLIST_FOREACH_END(cp);
-  }
-
   if (!options->SafeLogging ||
       !strcasecmp(options->SafeLogging, "0")) {
     options->SafeLogging_ = SAFELOG_SCRUB_NONE;
@@ -3355,6 +3759,23 @@ options_validate(or_options_t *old_options, or_options_t *options,
     options->DirPort_set = 0;
   }
 
+  if (server_mode(options) && options->ConnectionPadding != -1) {
+    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
+  }
+
+  if (server_mode(options) && options->ReducedConnectionPadding != 0) {
+    REJECT("Relays cannot set ReducedConnectionPadding. ");
+  }
+
+  if (options->BridgeDistribution) {
+    if (!options->BridgeRelay) {
+      REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
+    }
+    if (check_bridge_distribution_setting(options->BridgeDistribution) < 0) {
+      REJECT("Invalid BridgeDistribution value.");
+    }
+  }
+
   if (options->MinUptimeHidServDirectoryV2 < 0) {
     log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at "
                         "least 0 seconds. Changing to 0.");
@@ -3367,7 +3788,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->RendPostPeriod < min_rendpostperiod) {
     log_warn(LD_CONFIG, "RendPostPeriod option is too short; "
              "raising to %d seconds.", min_rendpostperiod);
-    options->RendPostPeriod = min_rendpostperiod;;
+    options->RendPostPeriod = min_rendpostperiod;
   }
 
   if (options->RendPostPeriod > MAX_DIR_PERIOD) {
@@ -3376,17 +3797,17 @@ options_validate(or_options_t *old_options, or_options_t *options,
     options->RendPostPeriod = MAX_DIR_PERIOD;
   }
 
-  if (options->PredictedPortsRelevanceTime >
-      MAX_PREDICTED_CIRCS_RELEVANCE) {
-    log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; "
-             "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE);
-    options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE;
-  }
-
   /* Check the Single Onion Service options */
   if (options_validate_single_onion(options, msg) < 0)
     return -1;
 
+  if (options->CircuitsAvailableTimeout > MAX_CIRCS_AVAILABLE_TIME) {
+    // options_t is immutable for new code (the above code is older),
+    // so just make the user fix the value themselves rather than
+    // silently keep a shadow value lower than what they asked for.
+    REJECT("CircuitsAvailableTimeout is too large. Max is 24 hours.");
+  }
+
 #ifdef ENABLE_TOR2WEB_MODE
   if (options->Tor2webMode && options->UseEntryGuards) {
     /* tor2web mode clients do not (and should not) use entry guards
@@ -3401,7 +3822,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
                "Tor2WebMode is enabled; disabling UseEntryGuards.");
     options->UseEntryGuards = 0;
   }
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
 
   if (options->Tor2webRendezvousPoints && !options->Tor2webMode) {
     REJECT("Tor2webRendezvousPoints cannot be set without Tor2webMode.");
@@ -3421,6 +3842,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "http://freehaven.net/anonbib/#hs-attack06 for details.");
   }
 
+  if (options->NumPrimaryGuards && options->NumEntryGuards &&
+      options->NumEntryGuards > options->NumPrimaryGuards) {
+    REJECT("NumEntryGuards must not be greater than NumPrimaryGuards.");
+  }
+
   if (options->EntryNodes &&
       routerset_is_list(options->EntryNodes) &&
       (routerset_len(options->EntryNodes) == 1) &&
@@ -3436,6 +3862,20 @@ options_validate(or_options_t *old_options, or_options_t *options,
     return -1;
   }
 
+  /* Inform the hidden service operator that pinning EntryNodes can possibly
+   * be harmful for the service anonymity. */
+  if (options->EntryNodes &&
+      routerset_is_list(options->EntryNodes) &&
+      (options->RendConfigLines != NULL)) {
+    log_warn(LD_CONFIG,
+             "EntryNodes is set with multiple entries and at least one "
+             "hidden service is configured. Pinning entry nodes can possibly "
+             "be harmful to the service anonymity. Because of this, we "
+             "recommend you either don't do that or make sure you know what "
+             "you are doing. For more details, please look at "
+             "https://trac.torproject.org/projects/tor/ticket/21155.");
+  }
+
   /* Single Onion Services: non-anonymous hidden services */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     log_warn(LD_CONFIG,
@@ -3461,7 +3901,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     int severity = LOG_NOTICE;
     /* Be a little quieter if we've deliberately disabled
      * LearnCircuitBuildTimeout. */
-    if (circuit_build_times_disabled()) {
+    if (circuit_build_times_disabled_(options, 1)) {
       severity = LOG_INFO;
     }
     log_fn(severity, LD_CONFIG, "You disabled LearnCircuitBuildTimeout, but "
@@ -3529,11 +3969,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->KeepalivePeriod < 1)
     REJECT("KeepalivePeriod option must be positive.");
 
-  if (options->PortForwarding && options->Sandbox) {
-    REJECT("PortForwarding is not compatible with Sandbox; at most one can "
-           "be set");
-  }
-
   if (ensure_bandwidth_cap(&options->BandwidthRate,
                            "BandwidthRate", msg) < 0)
     return -1;
@@ -3791,13 +4226,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "have it group-readable.");
   }
 
-  if (options->MyFamily && options->BridgeRelay) {
+  if (options->MyFamily_lines && options->BridgeRelay) {
     log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
              "supported: it can reveal bridge fingerprints to censors. "
              "You should also make sure you aren't listing this bridge's "
              "fingerprint in any other MyFamily.");
   }
-  if (check_nickname_list(&options->MyFamily, "MyFamily", msg))
+  if (normalize_nickname_list(&options->MyFamily,
+                              options->MyFamily_lines, "MyFamily", msg))
     return -1;
   for (cl = options->NodeFamilies; cl; cl = cl->next) {
     routerset_t *rs = routerset_new();
@@ -3970,7 +4406,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     COMPLAIN("V3AuthVotingInterval does not divide evenly into 24 hours.");
   }
 
-  if (rend_config_services(options, 1) < 0)
+  if (hs_config_service_all(options, 1) < 0)
     REJECT("Failed to configure rendezvous options. See logs for details.");
 
   /* Parse client-side authorization for hidden services. */
@@ -3994,13 +4430,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "AlternateDirAuthority and AlternateBridgeAuthority configured.");
   }
 
-  if (options->AllowSingleHopExits && !options->DirAuthorities) {
-    COMPLAIN("You have set AllowSingleHopExits; now your relay will allow "
-             "others to make one-hop exits. However, since by default most "
-             "clients avoid relays that set this option, most clients will "
-             "ignore you.");
-  }
-
 #define CHECK_DEFAULT(arg)                                              \
   STMT_BEGIN                                                            \
     if (!options->TestingTorNetwork &&                                  \
@@ -4016,17 +4445,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
   CHECK_DEFAULT(TestingV3AuthVotingStartOffset);
   CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability);
   CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime);
-  CHECK_DEFAULT(TestingServerDownloadSchedule);
-  CHECK_DEFAULT(TestingClientDownloadSchedule);
-  CHECK_DEFAULT(TestingServerConsensusDownloadSchedule);
-  CHECK_DEFAULT(TestingClientConsensusDownloadSchedule);
-  CHECK_DEFAULT(TestingBridgeDownloadSchedule);
+  CHECK_DEFAULT(TestingServerDownloadInitialDelay);
+  CHECK_DEFAULT(TestingClientDownloadInitialDelay);
+  CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay);
+  CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay);
+  CHECK_DEFAULT(TestingBridgeDownloadInitialDelay);
+  CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay);
   CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest);
   CHECK_DEFAULT(TestingDirConnectionMaxStall);
-  CHECK_DEFAULT(TestingConsensusMaxDownloadTries);
-  CHECK_DEFAULT(TestingDescriptorMaxDownloadTries);
-  CHECK_DEFAULT(TestingMicrodescMaxDownloadTries);
-  CHECK_DEFAULT(TestingCertMaxDownloadTries);
   CHECK_DEFAULT(TestingAuthKeyLifetime);
   CHECK_DEFAULT(TestingLinkCertLifetime);
   CHECK_DEFAULT(TestingSigningKeySlop);
@@ -4034,6 +4460,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
   CHECK_DEFAULT(TestingLinkKeySlop);
 #undef CHECK_DEFAULT
 
+  if (!options->ClientDNSRejectInternalAddresses &&
+      !(options->DirAuthorities ||
+        (options->AlternateDirAuthority && options->AlternateBridgeAuthority)))
+    REJECT("ClientDNSRejectInternalAddresses used for default network.");
   if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2)
     REJECT("SigningKeyLifetime is too short.");
   if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2)
@@ -4097,33 +4527,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     COMPLAIN("TestingDirConnectionMaxStall is insanely high.");
   }
 
-  if (options->TestingConsensusMaxDownloadTries < 2) {
-    REJECT("TestingConsensusMaxDownloadTries must be greater than 2.");
-  } else if (options->TestingConsensusMaxDownloadTries > 800) {
-    COMPLAIN("TestingConsensusMaxDownloadTries is insanely high.");
-  }
-
-  if (options->ClientBootstrapConsensusMaxDownloadTries < 2) {
-    REJECT("ClientBootstrapConsensusMaxDownloadTries must be greater "
-           "than 2."
-           );
-  } else if (options->ClientBootstrapConsensusMaxDownloadTries > 800) {
-    COMPLAIN("ClientBootstrapConsensusMaxDownloadTries is insanely "
-             "high.");
-  }
-
-  if (options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries
-      < 2) {
-    REJECT("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries must "
-           "be greater than 2."
-           );
-  } else if (
-        options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries
-        > 800) {
-    COMPLAIN("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries is "
-             "insanely high.");
-  }
-
   if (options->ClientBootstrapConsensusMaxInProgressTries < 1) {
     REJECT("ClientBootstrapConsensusMaxInProgressTries must be greater "
            "than 0.");
@@ -4133,24 +4536,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "high.");
   }
 
-  if (options->TestingDescriptorMaxDownloadTries < 2) {
-    REJECT("TestingDescriptorMaxDownloadTries must be greater than 1.");
-  } else if (options->TestingDescriptorMaxDownloadTries > 800) {
-    COMPLAIN("TestingDescriptorMaxDownloadTries is insanely high.");
-  }
-
-  if (options->TestingMicrodescMaxDownloadTries < 2) {
-    REJECT("TestingMicrodescMaxDownloadTries must be greater than 1.");
-  } else if (options->TestingMicrodescMaxDownloadTries > 800) {
-    COMPLAIN("TestingMicrodescMaxDownloadTries is insanely high.");
-  }
-
-  if (options->TestingCertMaxDownloadTries < 2) {
-    REJECT("TestingCertMaxDownloadTries must be greater than 1.");
-  } else if (options->TestingCertMaxDownloadTries > 800) {
-    COMPLAIN("TestingCertMaxDownloadTries is insanely high.");
-  }
-
   if (options->TestingEnableConnBwEvent &&
       !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) {
     REJECT("TestingEnableConnBwEvent may only be changed in testing "
@@ -4163,12 +4548,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "Tor networks!");
   }
 
-  if (options->TestingEnableTbEmptyEvent &&
-      !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) {
-    REJECT("TestingEnableTbEmptyEvent may only be changed in testing "
-           "Tor networks!");
-  }
-
   if (options->TestingTorNetwork) {
     log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node "
                         "almost unusable in the public Tor network, and is "
@@ -4197,6 +4576,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("BridgeRelay is 1, ORPort is not set. This is an invalid "
              "combination.");
 
+  if (options_validate_scheduler(options, msg) < 0) {
+    return -1;
+  }
+
   return 0;
 }
 
@@ -4206,7 +4589,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
 /* Given the value that the user has set for MaxMemInQueues, compute the
  * actual maximum value.  We clip this value if it's too low, and autodetect
  * it if it's set to 0. */
-static uint64_t
+STATIC uint64_t
 compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 {
   uint64_t result;
@@ -4214,11 +4597,6 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
   if (val == 0) {
 #define ONE_GIGABYTE (U64_LITERAL(1) << 30)
 #define ONE_MEGABYTE (U64_LITERAL(1) << 20)
-#if SIZEOF_VOID_P >= 8
-#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE)
-#else
-#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE)
-#endif
     /* The user didn't pick a memory limit.  Choose a very large one
      * that is still smaller than the system memory */
     static int notice_sent = 0;
@@ -4231,16 +4609,40 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 #else
       /* (presumably) 32-bit system. Let's hope for 1 GB. */
       result = ONE_GIGABYTE;
-#endif
+#endif /* SIZEOF_VOID_P >= 8 */
     } else {
-      /* We detected it, so let's pick 3/4 of the total RAM as our limit. */
-      const uint64_t avail = (ram / 4) * 3;
+      /* We detected the amount of memory available. */
+      uint64_t avail = 0;
+
+#if SIZEOF_SIZE_T > 4
+/* On a 64-bit platform, we consider 8GB "very large". */
+#define RAM_IS_VERY_LARGE(x) ((x) >= (8 * ONE_GIGABYTE))
+#else
+/* On a 32-bit platform, we can't have 8GB of ram. */
+#define RAM_IS_VERY_LARGE(x) (0)
+#endif
 
-      /* Make sure it's in range from 0.25 GB to 8 GB. */
-      if (avail > MAX_DEFAULT_MAXMEM) {
+      if (RAM_IS_VERY_LARGE(ram)) {
+        /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
+         * to 0.4 * RAM. The idea behind this value is that the amount of RAM
+         * is more than enough for a single relay and should allow the relay
+         * operator to run two relays if they have additional bandwidth
+         * available.
+         */
+        avail = (ram / 5) * 2;
+      } else {
+        /* If we have less than 8 GB of RAM available, we use the "old" default
+         * for MaxMemInQueues of 0.75 * RAM.
+         */
+        avail = (ram / 4) * 3;
+      }
+
+      /* Make sure it's in range from 0.25 GB to 8 GB for 64-bit and 0.25 to 2
+       * GB for 32-bit. */
+      if (avail > MAX_DEFAULT_MEMORY_QUEUE_SIZE) {
         /* If you want to use more than this much RAM, you need to configure
            it yourself */
-        result = MAX_DEFAULT_MAXMEM;
+        result = MAX_DEFAULT_MEMORY_QUEUE_SIZE;
       } else if (avail < ONE_GIGABYTE / 4) {
         result = ONE_GIGABYTE / 4;
       } else {
@@ -4266,8 +4668,8 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 }
 
 /* If we have less than 300 MB suggest disabling dircache */
-#define DIRCACHE_MIN_MB_BANDWIDTH 300
-#define DIRCACHE_MIN_BANDWIDTH (DIRCACHE_MIN_MB_BANDWIDTH*ONE_MEGABYTE)
+#define DIRCACHE_MIN_MEM_MB 300
+#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
 #define STRINGIFY(val) #val
 
 /** Create a warning message for emitting if we are a dircache but may not have
@@ -4287,21 +4689,20 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
     }
   }
   if (options->DirCache) {
-    if (total_mem < DIRCACHE_MIN_BANDWIDTH) {
+    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
       if (options->BridgeRelay) {
-        *msg = tor_strdup("Running a Bridge with less than "
-                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
-                      "not recommended.");
+        tor_asprintf(msg, "Running a Bridge with less than %d MB of memory "
+                       "is not recommended.", DIRCACHE_MIN_MEM_MB);
       } else {
-        *msg = tor_strdup("Being a directory cache (default) with less than "
-                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
-                      "not recommended and may consume most of the available "
-                      "resources, consider disabling this functionality by "
-                      "setting the DirCache option to 0.");
+        tor_asprintf(msg, "Being a directory cache (default) with less than "
+                       "%d MB of memory is not recommended and may consume "
+                       "most of the available resources. Consider disabling "
+                       "this functionality by setting the DirCache option "
+                       "to 0.", DIRCACHE_MIN_MEM_MB);
       }
     }
   } else {
-    if (total_mem >= DIRCACHE_MIN_BANDWIDTH) {
+    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
       *msg = tor_strdup("DirCache is disabled and we are configured as a "
                "relay. This may disqualify us from becoming a guard in the "
                "future.");
@@ -4328,120 +4729,61 @@ options_transition_allowed(const or_options_t *old,
   if (!old)
     return 0;
 
-  if (!opt_streq(old->PidFile, new_val->PidFile)) {
-    *msg = tor_strdup("PidFile is not allowed to change.");
-    return -1;
-  }
-
-  if (old->RunAsDaemon != new_val->RunAsDaemon) {
-    *msg = tor_strdup("While Tor is running, changing RunAsDaemon "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (old->Sandbox != new_val->Sandbox) {
-    *msg = tor_strdup("While Tor is running, changing Sandbox "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) {
-    tor_asprintf(msg,
-               "While Tor is running, changing DataDirectory "
-               "(\"%s\"->\"%s\") is not allowed.",
-               old->DataDirectory, new_val->DataDirectory);
-    return -1;
-  }
-
-  if (!opt_streq(old->User, new_val->User)) {
-    *msg = tor_strdup("While Tor is running, changing User is not allowed.");
-    return -1;
-  }
-
-  if (old->KeepBindCapabilities != new_val->KeepBindCapabilities) {
-    *msg = tor_strdup("While Tor is running, changing KeepBindCapabilities is "
-                      "not allowed.");
-    return -1;
-  }
-
-  if (!opt_streq(old->SyslogIdentityTag, new_val->SyslogIdentityTag)) {
-    *msg = tor_strdup("While Tor is running, changing "
-                      "SyslogIdentityTag is not allowed.");
-    return -1;
-  }
-
-  if ((old->HardwareAccel != new_val->HardwareAccel)
-      || !opt_streq(old->AccelName, new_val->AccelName)
-      || !opt_streq(old->AccelDir, new_val->AccelDir)) {
-    *msg = tor_strdup("While Tor is running, changing OpenSSL hardware "
-                      "acceleration engine is not allowed.");
-    return -1;
-  }
-
-  if (old->TestingTorNetwork != new_val->TestingTorNetwork) {
-    *msg = tor_strdup("While Tor is running, changing TestingTorNetwork "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (old->DisableAllSwap != new_val->DisableAllSwap) {
-    *msg = tor_strdup("While Tor is running, changing DisableAllSwap "
-                      "is not allowed.");
-    return -1;
-  }
-
-  if (old->TokenBucketRefillInterval != new_val->TokenBucketRefillInterval) {
-    *msg = tor_strdup("While Tor is running, changing TokenBucketRefill"
-                      "Interval is not allowed");
-    return -1;
-  }
-
-  if (old->HiddenServiceSingleHopMode != new_val->HiddenServiceSingleHopMode) {
-    *msg = tor_strdup("While Tor is running, changing "
-                      "HiddenServiceSingleHopMode is not allowed.");
-    return -1;
-  }
-
-  if (old->HiddenServiceNonAnonymousMode !=
-      new_val->HiddenServiceNonAnonymousMode) {
-    *msg = tor_strdup("While Tor is running, changing "
-                      "HiddenServiceNonAnonymousMode is not allowed.");
-    return -1;
-  }
-
-  if (old->DisableDebuggerAttachment &&
-      !new_val->DisableDebuggerAttachment) {
-    *msg = tor_strdup("While Tor is running, disabling "
-                      "DisableDebuggerAttachment is not allowed.");
-    return -1;
-  }
+#define BAD_CHANGE_TO(opt, how) do {                                    \
+    *msg = tor_strdup("While Tor is running"how", changing " #opt       \
+                      " is not allowed");                               \
+    return -1;                                                          \
+  } while (0)
+
+#define NO_CHANGE_BOOL(opt) \
+  if (! CFG_EQ_BOOL(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
+#define NO_CHANGE_INT(opt) \
+  if (! CFG_EQ_INT(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
+#define NO_CHANGE_STRING(opt) \
+  if (! CFG_EQ_STRING(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
+
+  NO_CHANGE_STRING(PidFile);
+  NO_CHANGE_BOOL(RunAsDaemon);
+  NO_CHANGE_BOOL(Sandbox);
+  NO_CHANGE_STRING(DataDirectory);
+  NO_CHANGE_STRING(KeyDirectory);
+  NO_CHANGE_STRING(CacheDirectory);
+  NO_CHANGE_STRING(User);
+  NO_CHANGE_BOOL(KeepBindCapabilities);
+  NO_CHANGE_STRING(SyslogIdentityTag);
+  NO_CHANGE_STRING(AndroidIdentityTag);
+  NO_CHANGE_BOOL(HardwareAccel);
+  NO_CHANGE_STRING(AccelName);
+  NO_CHANGE_STRING(AccelDir);
+  NO_CHANGE_BOOL(TestingTorNetwork);
+  NO_CHANGE_BOOL(DisableAllSwap);
+  NO_CHANGE_INT(TokenBucketRefillInterval);
+  NO_CHANGE_BOOL(HiddenServiceSingleHopMode);
+  NO_CHANGE_BOOL(HiddenServiceNonAnonymousMode);
+  NO_CHANGE_BOOL(DisableDebuggerAttachment);
+  NO_CHANGE_BOOL(NoExec);
+  NO_CHANGE_INT(OwningControllerFD);
+  NO_CHANGE_BOOL(DisableSignalHandlers);
 
   if (sandbox_is_active()) {
-#define SB_NOCHANGE_STR(opt)                                            \
-    do {                                                                \
-      if (! opt_streq(old->opt, new_val->opt)) {                        \
-        *msg = tor_strdup("Can't change " #opt " while Sandbox is active"); \
-        return -1;                                                      \
-      }                                                                 \
-    } while (0)
+#define SB_NOCHANGE_STR(opt)                      \
+    if (! CFG_EQ_STRING(old, new_val, opt))       \
+      BAD_CHANGE_TO(opt," with Sandbox active")
+#define SB_NOCHANGE_LINELIST(opt)                  \
+    if (! CFG_EQ_LINELIST(old, new_val, opt))      \
+      BAD_CHANGE_TO(opt," with Sandbox active")
+#define SB_NOCHANGE_INT(opt)                       \
+    if (! CFG_EQ_INT(old, new_val, opt))           \
+      BAD_CHANGE_TO(opt," with Sandbox active")
 
     SB_NOCHANGE_STR(Address);
-    SB_NOCHANGE_STR(PidFile);
     SB_NOCHANGE_STR(ServerDNSResolvConfFile);
     SB_NOCHANGE_STR(DirPortFrontPage);
     SB_NOCHANGE_STR(CookieAuthFile);
     SB_NOCHANGE_STR(ExtORPortCookieAuthFile);
+    SB_NOCHANGE_LINELIST(Logs);
+    SB_NOCHANGE_INT(ConnLimit);
 
-#undef SB_NOCHANGE_STR
-
-    if (! config_lines_eq(old->Logs, new_val->Logs)) {
-      *msg = tor_strdup("Can't change Logs while Sandbox is active");
-      return -1;
-    }
-    if (old->ConnLimit != new_val->ConnLimit) {
-      *msg = tor_strdup("Can't change ConnLimit while Sandbox is active");
-      return -1;
-    }
     if (server_mode(old) != server_mode(new_val)) {
       *msg = tor_strdup("Can't start/stop being a server while "
                         "Sandbox is active");
@@ -4449,6 +4791,13 @@ options_transition_allowed(const or_options_t *old,
     }
   }
 
+#undef SB_NOCHANGE_LINELIST
+#undef SB_NOCHANGE_STR
+#undef SB_NOCHANGE_INT
+#undef BAD_CHANGE_TO
+#undef NO_CHANGE_BOOL
+#undef NO_CHANGE_INT
+#undef NO_CHANGE_STRING
   return 0;
 }
 
@@ -4458,21 +4807,20 @@ static int
 options_transition_affects_workers(const or_options_t *old_options,
                                    const or_options_t *new_options)
 {
-  if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) ||
-      old_options->NumCPUs != new_options->NumCPUs ||
-      !config_lines_eq(old_options->ORPort_lines, new_options->ORPort_lines) ||
-      old_options->ServerDNSSearchDomains !=
-                                       new_options->ServerDNSSearchDomains ||
-      old_options->SafeLogging_ != new_options->SafeLogging_ ||
-      old_options->ClientOnly != new_options->ClientOnly ||
-      server_mode(old_options) != server_mode(new_options) ||
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_INT(NumCPUs);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_BOOL(ServerDNSSearchDomains);
+  YES_IF_CHANGED_BOOL(SafeLogging_);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(LogMessageDomains);
+  YES_IF_CHANGED_LINELIST(Logs);
+
+  if (server_mode(old_options) != server_mode(new_options) ||
       public_server_mode(old_options) != public_server_mode(new_options) ||
-      !config_lines_eq(old_options->Logs, new_options->Logs) ||
-      old_options->LogMessageDomains != new_options->LogMessageDomains)
+      dir_server_mode(old_options) != dir_server_mode(new_options))
     return 1;
 
-  /* Check whether log options match. */
-
   /* Nothing that changed matters. */
   return 0;
 }
@@ -4485,35 +4833,34 @@ options_transition_affects_descriptor(const or_options_t *old_options,
 {
   /* XXX We can be smarter here. If your DirPort isn't being
    * published and you just turned it off, no need to republish. Etc. */
-  if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) ||
-      !opt_streq(old_options->Nickname,new_options->Nickname) ||
-      !opt_streq(old_options->Address,new_options->Address) ||
-      !config_lines_eq(old_options->ExitPolicy,new_options->ExitPolicy) ||
-      old_options->ExitRelay != new_options->ExitRelay ||
-      old_options->ExitPolicyRejectPrivate !=
-        new_options->ExitPolicyRejectPrivate ||
-      old_options->ExitPolicyRejectLocalInterfaces !=
-        new_options->ExitPolicyRejectLocalInterfaces ||
-      old_options->IPv6Exit != new_options->IPv6Exit ||
-      !config_lines_eq(old_options->ORPort_lines,
-                       new_options->ORPort_lines) ||
-      !config_lines_eq(old_options->DirPort_lines,
-                       new_options->DirPort_lines) ||
-      old_options->ClientOnly != new_options->ClientOnly ||
-      old_options->DisableNetwork != new_options->DisableNetwork ||
-      old_options->PublishServerDescriptor_ !=
-        new_options->PublishServerDescriptor_ ||
-      get_effective_bwrate(old_options) != get_effective_bwrate(new_options) ||
+
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_STRING(Nickname);
+  YES_IF_CHANGED_STRING(Address);
+  YES_IF_CHANGED_LINELIST(ExitPolicy);
+  YES_IF_CHANGED_BOOL(ExitRelay);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectPrivate);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectLocalInterfaces);
+  YES_IF_CHANGED_BOOL(IPv6Exit);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(DisableNetwork);
+  YES_IF_CHANGED_BOOL(PublishServerDescriptor_);
+  YES_IF_CHANGED_STRING(ContactInfo);
+  YES_IF_CHANGED_STRING(BridgeDistribution);
+  YES_IF_CHANGED_LINELIST(MyFamily);
+  YES_IF_CHANGED_STRING(AccountingStart);
+  YES_IF_CHANGED_INT(AccountingMax);
+  YES_IF_CHANGED_INT(AccountingRule);
+  YES_IF_CHANGED_BOOL(DirCache);
+  YES_IF_CHANGED_BOOL(AssumeReachable);
+
+  if (get_effective_bwrate(old_options) != get_effective_bwrate(new_options) ||
       get_effective_bwburst(old_options) !=
         get_effective_bwburst(new_options) ||
-      !opt_streq(old_options->ContactInfo, new_options->ContactInfo) ||
-      !opt_streq(old_options->MyFamily, new_options->MyFamily) ||
-      !opt_streq(old_options->AccountingStart, new_options->AccountingStart) ||
-      old_options->AccountingMax != new_options->AccountingMax ||
-      old_options->AccountingRule != new_options->AccountingRule ||
-      public_server_mode(old_options) != public_server_mode(new_options) ||
-      old_options->DirCache != new_options->DirCache ||
-      old_options->AssumeReachable != new_options->AssumeReachable)
+      public_server_mode(old_options) != public_server_mode(new_options))
     return 1;
 
   return 0;
@@ -4560,7 +4907,7 @@ get_windows_conf_root(void)
   path[sizeof(path)-1] = '\0';
 #else
   strlcpy(path,tpath,sizeof(path));
-#endif
+#endif /* defined(UNICODE) */
 
   /* Now we need to free the memory that the path-idl was stored in.  In
    * typical Windows fashion, we can't just call 'free()' on it. */
@@ -4576,7 +4923,7 @@ get_windows_conf_root(void)
   is_set = 1;
   return path;
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Return the default location for our torrc file (if <b>defaults_file</b> is
  * false), or for the torrc-defaults file (if <b>defaults_file</b> is true). */
@@ -4600,30 +4947,39 @@ get_default_conf_file(int defaults_file)
   }
 #else
   return defaults_file ? CONFDIR "/torrc-defaults" : CONFDIR "/torrc";
-#endif
+#endif /* defined(DISABLE_SYSTEM_TORRC) || ... */
 }
 
-/** Verify whether lst is a string containing valid-looking comma-separated
- * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname
- * or fingerprint that needs it. Return 0 on success.
+/** Verify whether lst is a list of strings containing valid-looking
+ * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
+ * to any nickname or fingerprint that needs it. Also splits comma-separated
+ * list elements into multiple elements. Return 0 on success.
  * Warn and return -1 on failure.
  */
 static int
-check_nickname_list(char **lst, const char *name, char **msg)
+normalize_nickname_list(config_line_t **normalized_out,
+                        const config_line_t *lst, const char *name,
+                        char **msg)
 {
-  int r = 0;
-  smartlist_t *sl;
-  int changes = 0;
-
-  if (!*lst)
+  if (!lst)
     return 0;
-  sl = smartlist_new();
 
-  smartlist_split_string(sl, *lst, ",",
-    SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+  config_line_t *new_nicknames = NULL;
+  config_line_t **new_nicknames_next = &new_nicknames;
+
+  const config_line_t *cl;
+  for (cl = lst; cl; cl = cl->next) {
+    const char *line = cl->value;
+    if (!line)
+      continue;
 
-  SMARTLIST_FOREACH_BEGIN(sl, char *, s)
+    int valid_line = 1;
+    smartlist_t *sl = smartlist_new();
+    smartlist_split_string(sl, line, ",",
+      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
     {
+      char *normalized = NULL;
       if (!is_legal_nickname_or_hexdigest(s)) {
         // check if first char is dollar
         if (s[0] != '$') {
@@ -4632,36 +4988,45 @@ check_nickname_list(char **lst, const char *name, char **msg)
           tor_asprintf(&prepended, "$%s", s);
 
           if (is_legal_nickname_or_hexdigest(prepended)) {
-            // The nickname is valid when it's prepended, swap the current
-            // version with a prepended one
-            tor_free(s);
-            SMARTLIST_REPLACE_CURRENT(sl, s, prepended);
-            changes = 1;
-            continue;
+            // The nickname is valid when it's prepended, set it as the
+            // normalized version
+            normalized = prepended;
+          } else {
+            // Still not valid, free and fallback to error message
+            tor_free(prepended);
           }
-
-          // Still not valid, free and fallback to error message
-          tor_free(prepended);
         }
 
-        tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
-        r = -1;
-        break;
+        if (!normalized) {
+          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
+          valid_line = 0;
+          break;
+        }
+      } else {
+        normalized = tor_strdup(s);
       }
-    }
-  SMARTLIST_FOREACH_END(s);
 
-  // Replace the caller's nickname list with a fixed one
-  if (changes && r == 0) {
-    char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL);
-    tor_free(*lst);
-    *lst = newNicknames;
+      config_line_t *next = tor_malloc_zero(sizeof(*next));
+      next->key = tor_strdup(cl->key);
+      next->value = normalized;
+      next->next = NULL;
+
+      *new_nicknames_next = next;
+      new_nicknames_next = &next->next;
+    } SMARTLIST_FOREACH_END(s);
+
+    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
+    smartlist_free(sl);
+
+    if (!valid_line) {
+      config_free_lines(new_nicknames);
+      return -1;
+    }
   }
 
-  SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
-  smartlist_free(sl);
+  *normalized_out = new_nicknames;
 
-  return r;
+  return 0;
 }
 
 /** Learn config file name from command line arguments, or use the default.
@@ -4733,9 +5098,9 @@ find_torrc_filename(config_line_t *cmd_arg,
       } else {
         fname = dflt ? tor_strdup(dflt) : NULL;
       }
-#else
+#else /* !(!defined(_WIN32)) */
       fname = dflt ? tor_strdup(dflt) : NULL;
-#endif
+#endif /* !defined(_WIN32) */
     }
   }
   return fname;
@@ -4809,7 +5174,8 @@ load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file)
 /** Read a configuration file into <b>options</b>, finding the configuration
  * file location based on the command line.  After loading the file
  * call options_init_from_string() to load the config.
- * Return 0 if success, -1 if failure. */
+ * Return 0 if success, -1 if failure, and 1 if we succeeded but should exit
+ * anyway. */
 int
 options_init_from_torrc(int argc, char **argv)
 {
@@ -4836,22 +5202,22 @@ options_init_from_torrc(int argc, char **argv)
   if (config_line_find(cmdline_only_options, "-h") ||
       config_line_find(cmdline_only_options, "--help")) {
     print_usage();
-    exit(0);
+    return 1;
   }
   if (config_line_find(cmdline_only_options, "--list-torrc-options")) {
     /* For validating whether we've documented everything. */
     list_torrc_options();
-    exit(0);
+    return 1;
   }
   if (config_line_find(cmdline_only_options, "--list-deprecated-options")) {
     /* For validating whether what we have deprecated really exists. */
     list_deprecated_options();
-    exit(0);
+    return 1;
   }
 
   if (config_line_find(cmdline_only_options, "--version")) {
     printf("Tor version %s.\n",get_version());
-    exit(0);
+    return 1;
   }
 
   if (config_line_find(cmdline_only_options, "--library-versions")) {
@@ -4863,17 +5229,32 @@ options_init_from_torrc(int argc, char **argv)
     printf("OpenSSL \t\t%-15s\t\t%s\n",
                       crypto_openssl_get_header_version_str(),
                       crypto_openssl_get_version_str());
-    printf("Zlib    \t\t%-15s\t\t%s\n",
-                      tor_zlib_get_header_version_str(),
-                      tor_zlib_get_version_str());
+    if (tor_compress_supports_method(ZLIB_METHOD)) {
+      printf("Zlib    \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(ZLIB_METHOD),
+                        tor_compress_header_version_str(ZLIB_METHOD));
+    }
+    if (tor_compress_supports_method(LZMA_METHOD)) {
+      printf("Liblzma \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(LZMA_METHOD),
+                        tor_compress_header_version_str(LZMA_METHOD));
+    }
+    if (tor_compress_supports_method(ZSTD_METHOD)) {
+      printf("Libzstd \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(ZSTD_METHOD),
+                        tor_compress_header_version_str(ZSTD_METHOD));
+    }
     //TODO: Hex versions?
-    exit(0);
+    return 1;
   }
 
   command = CMD_RUN_TOR;
   for (p_index = cmdline_only_options; p_index; p_index = p_index->next) {
     if (!strcmp(p_index->key,"--keygen")) {
       command = CMD_KEYGEN;
+    } else if (!strcmp(p_index->key, "--key-expiration")) {
+      command = CMD_KEY_EXPIRATION;
+      command_arg = p_index->value;
     } else if (!strcmp(p_index->key,"--list-fingerprint")) {
       command = CMD_LIST_FINGERPRINT;
     } else if (!strcmp(p_index->key, "--hash-password")) {
@@ -4925,7 +5306,8 @@ options_init_from_torrc(int argc, char **argv)
       get_options_mutable()->keygen_force_passphrase = FORCE_PASSPHRASE_OFF;
     } else {
       log_err(LD_CONFIG, "--no-passphrase specified without --keygen!");
-      exit(1);
+      retval = -1;
+      goto err;
     }
   }
 
@@ -4934,7 +5316,8 @@ options_init_from_torrc(int argc, char **argv)
       get_options_mutable()->change_key_passphrase = 1;
     } else {
       log_err(LD_CONFIG, "--newpass specified without --keygen!");
-      exit(1);
+      retval = -1;
+      goto err;
     }
   }
 
@@ -4944,17 +5327,20 @@ options_init_from_torrc(int argc, char **argv)
     if (fd_line) {
       if (get_options()->keygen_force_passphrase == FORCE_PASSPHRASE_OFF) {
         log_err(LD_CONFIG, "--no-passphrase specified with --passphrase-fd!");
-        exit(1);
+        retval = -1;
+        goto err;
       } else if (command != CMD_KEYGEN) {
         log_err(LD_CONFIG, "--passphrase-fd specified without --keygen!");
-        exit(1);
+        retval = -1;
+        goto err;
       } else {
         const char *v = fd_line->value;
         int ok = 1;
         long fd = tor_parse_long(v, 10, 0, INT_MAX, &ok, NULL);
         if (fd < 0 || ok == 0) {
           log_err(LD_CONFIG, "Invalid --passphrase-fd value %s", escaped(v));
-          exit(1);
+          retval = -1;
+          goto err;
         }
         get_options_mutable()->keygen_passphrase_fd = (int)fd;
         get_options_mutable()->use_keygen_passphrase_fd = 1;
@@ -4969,7 +5355,8 @@ options_init_from_torrc(int argc, char **argv)
     if (key_line) {
       if (command != CMD_KEYGEN) {
         log_err(LD_CONFIG, "--master-key without --keygen!");
-        exit(1);
+        retval = -1;
+        goto err;
       } else {
         get_options_mutable()->master_key_fname = tor_strdup(key_line->value);
       }
@@ -5005,6 +5392,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   config_line_t *cl;
   int retval;
   setopt_err_t err = SETOPT_ERR_MISC;
+  int cf_has_include = 0;
   tor_assert(msg);
 
   oldoptions = global_options; /* get_options unfortunately asserts if
@@ -5016,12 +5404,16 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   newoptions->command = command;
   newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
 
+  smartlist_t *opened_files = smartlist_new();
   for (int i = 0; i < 2; ++i) {
     const char *body = i==0 ? cf_defaults : cf;
     if (!body)
       continue;
+
     /* get config lines, assign them */
-    retval = config_get_lines(body, &cl, 1);
+    retval = config_get_lines_include(body, &cl, 1,
+                                      body == cf ? &cf_has_include : NULL,
+                                      opened_files);
     if (retval < 0) {
       err = SETOPT_ERR_PARSE;
       goto err;
@@ -5049,6 +5441,9 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     goto err;
   }
 
+  newoptions->IncludeUsed = cf_has_include;
+  newoptions->FilesOpenedByIncludes = opened_files;
+
   /* If this is a testing network configuration, change defaults
    * for a list of dependent config options, re-initialize newoptions
    * with the new defaults, and assign all options to it second time. */
@@ -5087,12 +5482,16 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
 
     /* Assign all options a second time. */
+    opened_files = smartlist_new();
     for (int i = 0; i < 2; ++i) {
       const char *body = i==0 ? cf_defaults : cf;
       if (!body)
         continue;
+
       /* get config lines, assign them */
-      retval = config_get_lines(body, &cl, 1);
+      retval = config_get_lines_include(body, &cl, 1,
+                                        body == cf ? &cf_has_include : NULL,
+                                        opened_files);
       if (retval < 0) {
         err = SETOPT_ERR_PARSE;
         goto err;
@@ -5115,6 +5514,10 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     }
   }
 
+  newoptions->IncludeUsed = cf_has_include;
+  in_option_validation = 1;
+  newoptions->FilesOpenedByIncludes = opened_files;
+
   /* Validate newoptions */
   if (options_validate(oldoptions, newoptions, newdefaultoptions,
                        0, msg) < 0) {
@@ -5126,17 +5529,26 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     err = SETOPT_ERR_TRANSITION;
     goto err;
   }
+  in_option_validation = 0;
 
   if (set_options(newoptions, msg)) {
     err = SETOPT_ERR_SETTING;
     goto err; /* frees and replaces old options */
   }
+
   or_options_free(global_default_options);
   global_default_options = newdefaultoptions;
 
   return SETOPT_OK;
 
  err:
+  in_option_validation = 0;
+  if (opened_files) {
+    SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
+    smartlist_free(opened_files);
+  }
+  // may have been set to opened_files, avoid double free
+  newoptions->FilesOpenedByIncludes = NULL;
   or_options_free(newoptions);
   or_options_free(newdefaultoptions);
   if (*msg) {
@@ -5218,35 +5630,35 @@ addressmap_register_auto(const char *from, const char *to,
   int from_wildcard = 0, to_wildcard = 0;
 
   *msg = "whoops, forgot the error message";
-  if (1) {
-    if (!strcmp(to, "*") || !strcmp(from, "*")) {
-      *msg = "can't remap from or to *";
-      return -1;
-    }
-    /* Detect asterisks in expressions of type: '*.example.com' */
-    if (!strncmp(from,"*.",2)) {
-      from += 2;
-      from_wildcard = 1;
-    }
-    if (!strncmp(to,"*.",2)) {
-      to += 2;
-      to_wildcard = 1;
-    }
 
-    if (to_wildcard && !from_wildcard) {
-      *msg =  "can only use wildcard (i.e. '*.') if 'from' address "
-        "uses wildcard also";
-      return -1;
-    }
+  if (!strcmp(to, "*") || !strcmp(from, "*")) {
+    *msg = "can't remap from or to *";
+    return -1;
+  }
+  /* Detect asterisks in expressions of type: '*.example.com' */
+  if (!strncmp(from,"*.",2)) {
+    from += 2;
+    from_wildcard = 1;
+  }
+  if (!strncmp(to,"*.",2)) {
+    to += 2;
+    to_wildcard = 1;
+  }
 
-    if (address_is_invalid_destination(to, 1)) {
-      *msg = "destination is invalid";
-      return -1;
-    }
+  if (to_wildcard && !from_wildcard) {
+    *msg =  "can only use wildcard (i.e. '*.') if 'from' address "
+      "uses wildcard also";
+    return -1;
+  }
 
-    addressmap_register(from, tor_strdup(to), expires, addrmap_source,
-                        from_wildcard, to_wildcard);
+  if (address_is_invalid_destination(to, 1)) {
+    *msg = "destination is invalid";
+    return -1;
   }
+
+  addressmap_register(from, tor_strdup(to), expires, addrmap_source,
+                      from_wildcard, to_wildcard);
+
   return 0;
 }
 
@@ -5313,7 +5725,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
                            SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
 
     if (smartlist_len(elts) == 0)
-      smartlist_add(elts, tor_strdup("stdout"));
+      smartlist_add_strdup(elts, "stdout");
 
     if (smartlist_len(elts) == 1 &&
         (!strcasecmp(smartlist_get(elts,0), "stdout") ||
@@ -5332,16 +5744,29 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
       }
       goto cleanup;
     }
-    if (smartlist_len(elts) == 1 &&
-        !strcasecmp(smartlist_get(elts,0), "syslog")) {
+    if (smartlist_len(elts) == 1) {
+      if (!strcasecmp(smartlist_get(elts,0), "syslog")) {
 #ifdef HAVE_SYSLOG_H
-      if (!validate_only) {
-        add_syslog_log(severity, options->SyslogIdentityTag);
+        if (!validate_only) {
+          add_syslog_log(severity, options->SyslogIdentityTag);
+        }
+#else
+        log_warn(LD_CONFIG, "Syslog is not supported on this system. Sorry.");
+#endif /* defined(HAVE_SYSLOG_H) */
+        goto cleanup;
       }
+
+      if (!strcasecmp(smartlist_get(elts, 0), "android")) {
+#ifdef HAVE_ANDROID_LOG_H
+        if (!validate_only) {
+          add_android_log(severity, options->AndroidIdentityTag);
+        }
 #else
-      log_warn(LD_CONFIG, "Syslog is not supported on this system. Sorry.");
-#endif
-      goto cleanup;
+        log_warn(LD_CONFIG, "Android logging is not supported"
+                            " on this system. Sorry.");
+#endif // HAVE_ANDROID_LOG_H.
+        goto cleanup;
+      }
     }
 
     if (smartlist_len(elts) == 2 &&
@@ -5427,7 +5852,7 @@ validate_transport_socks_arguments(const smartlist_t *args)
 
 /** Deallocate a bridge_line_t structure. */
 /* private */ void
-bridge_line_free(bridge_line_t *bridge_line)
+bridge_line_free_(bridge_line_t *bridge_line)
 {
   if (!bridge_line)
     return;
@@ -5657,6 +6082,15 @@ parse_transport_line(const or_options_t *options,
     goto err;
   }
 
+  if (is_managed && options->NoExec) {
+    log_warn(LD_CONFIG,
+             "Managed proxies are not compatible with NoExec mode; ignoring."
+             "(%sTransportPlugin line was %s)",
+             server ? "Server" : "Client", escaped(line));
+    r = 0;
+    goto done;
+  }
+
   if (is_managed) {
     /* managed */
 
@@ -5848,7 +6282,7 @@ get_options_from_transport_options_line(const char *line,const char *transport)
     }
 
     /* add it to the options smartlist */
-    smartlist_add(options, tor_strdup(option));
+    smartlist_add_strdup(options, option);
     log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
   } SMARTLIST_FOREACH_END(option);
 
@@ -5927,6 +6361,8 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type,
   dirinfo_type_t type = 0;
   double weight = 1.0;
 
+  memset(v3_digest, 0, sizeof(v3_digest));
+
   items = smartlist_new();
   smartlist_split_string(items, line, NULL,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
@@ -6173,16 +6609,16 @@ port_cfg_new(size_t namelen)
   tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1);
   port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1);
   cfg->entry_cfg.ipv4_traffic = 1;
+  cfg->entry_cfg.ipv6_traffic = 1;
   cfg->entry_cfg.dns_request = 1;
   cfg->entry_cfg.onion_traffic = 1;
-  cfg->entry_cfg.cache_ipv4_answers = 1;
   cfg->entry_cfg.prefer_ipv6_virtaddr = 1;
   return cfg;
 }
 
 /** Free all storage held in <b>port</b> */
 STATIC void
-port_cfg_free(port_cfg_t *port)
+port_cfg_free_(port_cfg_t *port)
 {
   tor_free(port);
 }
@@ -6190,8 +6626,9 @@ port_cfg_free(port_cfg_t *port)
 /** Warn for every port in <b>ports</b> of type <b>listener_type</b> that is
  * on a publicly routable address. */
 static void
-warn_nonlocal_client_ports(const smartlist_t *ports, const char *portname,
-                           int listener_type)
+warn_nonlocal_client_ports(const smartlist_t *ports,
+                           const char *portname,
+                           const int listener_type)
 {
   SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
     if (port->type != listener_type)
@@ -6348,16 +6785,60 @@ warn_client_dns_cache(const char *option, int disabling)
 }
 
 /**
+ * Validate the configured bridge distribution method from a BridgeDistribution
+ * config line.
+ *
+ * The input <b>bd</b>, is a string taken from the BridgeDistribution config
+ * line (if present).  If the option wasn't set, return 0 immediately.  The
+ * BridgeDistribution option is then validated.  Currently valid, recognised
+ * options are:
+ *
+ * - "none"
+ * - "any"
+ * - "https"
+ * - "email"
+ * - "moat"
+ * - "hyphae"
+ *
+ * If the option string is unrecognised, a warning will be logged and 0 is
+ * returned.  If the option string contains an invalid character, -1 is
+ * returned.
+ **/
+STATIC int
+check_bridge_distribution_setting(const char *bd)
+{
+  if (bd == NULL)
+    return 0;
+
+  const char *RECOGNIZED[] = {
+    "none", "any", "https", "email", "moat", "hyphae"
+  };
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(RECOGNIZED); ++i) {
+    if (!strcmp(bd, RECOGNIZED[i]))
+      return 0;
+  }
+
+  const char *cp = bd;
+  //  Method = (KeywordChar | "_") +
+  while (TOR_ISALNUM(*cp) || *cp == '-' || *cp == '_')
+    ++cp;
+
+  if (*cp == 0) {
+    log_warn(LD_CONFIG, "Unrecognized BridgeDistribution value %s. I'll "
+           "assume you know what you are doing...", escaped(bd));
+    return 0; // we reached the end of the string; all is well
+  } else {
+    return -1; // we found a bad character in the string.
+  }
+}
+
+/**
  * Parse port configuration for a single port type.
  *
- * Read entries of the "FooPort" type from the list <b>ports</b>, and
- * entries of the "FooListenAddress" type from the list
- * <b>listenaddrs</b>.  Two syntaxes are supported: a legacy syntax
- * where FooPort is at most a single entry containing a port number and
- * where FooListenAddress has any number of address:port combinations;
- * and a new syntax where there are no FooListenAddress entries and
- * where FooPort can have any number of entries of the format
- * "[Address:][Port] IsolationOptions".
+ * Read entries of the "FooPort" type from the list <b>ports</b>.  Syntax is
+ * that FooPort can have any number of entries of the format
+ *  "[Address:][Port] IsolationOptions".
  *
  * In log messages, describe the port type as <b>portname</b>.
  *
@@ -6371,9 +6852,6 @@ warn_client_dns_cache(const char *option, int disabling)
  * ports are not on a local address.  If CL_PORT_FORBID_NONLOCAL is set,
  * this is a control port with no password set: don't even allow it.
  *
- * Unless CL_PORT_ALLOW_EXTRA_LISTENADDR is set in <b>flags</b>, warn
- * if FooListenAddress is set but FooPort is 0.
- *
  * If CL_PORT_SERVER_OPTIONS is set in <b>flags</b>, do not allow stream
  * isolation options in the FooPort entries; instead allow the
  * server-port option set.
@@ -6388,7 +6866,6 @@ warn_client_dns_cache(const char *option, int disabling)
 STATIC int
 parse_port_config(smartlist_t *out,
                   const config_line_t *ports,
-                  const config_line_t *listenaddrs,
                   const char *portname,
                   int listener_type,
                   const char *defaultaddr,
@@ -6405,90 +6882,12 @@ parse_port_config(smartlist_t *out,
   const unsigned forbid_nonlocal = flags & CL_PORT_FORBID_NONLOCAL;
   const unsigned default_to_group_writable =
     flags & CL_PORT_DFLT_GROUP_WRITABLE;
-  const unsigned allow_spurious_listenaddr =
-    flags & CL_PORT_ALLOW_EXTRA_LISTENADDR;
   const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES;
   const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET;
   int got_zero_port=0, got_nonzero_port=0;
   char *unix_socket_path = NULL;
 
-  /* FooListenAddress is deprecated; let's make it work like it used to work,
-   * though. */
-  if (listenaddrs) {
-    int mainport = defaultport;
-
-    if (ports && ports->next) {
-      log_warn(LD_CONFIG, "%sListenAddress can't be used when there are "
-               "multiple %sPort lines", portname, portname);
-      return -1;
-    } else if (ports) {
-      if (!strcmp(ports->value, "auto")) {
-        mainport = CFG_AUTO_PORT;
-      } else {
-        int ok;
-        mainport = (int)tor_parse_long(ports->value, 10, 0, 65535, &ok, NULL);
-        if (!ok) {
-          log_warn(LD_CONFIG, "%sListenAddress can only be used with a single "
-                   "%sPort with value \"auto\" or 1-65535 and no options set.",
-                   portname, portname);
-          return -1;
-        }
-      }
-    }
-
-    if (mainport == 0) {
-      if (allow_spurious_listenaddr)
-        return 1; /*DOCDOC*/
-      log_warn(LD_CONFIG, "%sPort must be defined if %sListenAddress is used",
-               portname, portname);
-      return -1;
-    }
-
-    if (use_server_options && out) {
-      /* Add a no_listen port. */
-      port_cfg_t *cfg = port_cfg_new(0);
-      cfg->type = listener_type;
-      cfg->port = mainport;
-      tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */
-      cfg->server_cfg.no_listen = 1;
-      cfg->server_cfg.bind_ipv4_only = 1;
-      /* cfg->entry_cfg defaults are already set by port_cfg_new */
-      smartlist_add(out, cfg);
-    }
-
-    for (; listenaddrs; listenaddrs = listenaddrs->next) {
-      tor_addr_t addr;
-      uint16_t port = 0;
-      if (tor_addr_port_lookup(listenaddrs->value, &addr, &port) < 0) {
-        log_warn(LD_CONFIG, "Unable to parse %sListenAddress '%s'",
-                 portname, listenaddrs->value);
-        return -1;
-      }
-      if (out) {
-        port_cfg_t *cfg = port_cfg_new(0);
-        cfg->type = listener_type;
-        cfg->port = port ? port : mainport;
-        tor_addr_copy(&cfg->addr, &addr);
-        cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
-        cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
-        cfg->server_cfg.no_advertise = 1;
-        smartlist_add(out, cfg);
-      }
-    }
-
-    if (warn_nonlocal && out) {
-      if (is_control)
-        warn_nonlocal_controller_ports(out, forbid_nonlocal);
-      else if (is_ext_orport)
-        warn_nonlocal_ext_orports(out, portname);
-      else
-        warn_nonlocal_client_ports(out, portname, listener_type);
-    }
-    return 0;
-  } /* end if (listenaddrs) */
-
-  /* No ListenAddress lines. If there's no FooPort, then maybe make a default
-   * one. */
+  /* If there's no FooPort, then maybe make a default one. */
   if (! ports) {
     if (defaultport && defaultaddr && out) {
       port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
@@ -6526,9 +6925,9 @@ parse_port_config(smartlist_t *out,
     /* This must be kept in sync with port_cfg_new's defaults */
     int no_listen = 0, no_advertise = 0, all_addrs = 0,
       bind_ipv4_only = 0, bind_ipv6_only = 0,
-      ipv4_traffic = 1, ipv6_traffic = 0, prefer_ipv6 = 0, dns_request = 1,
+      ipv4_traffic = 1, ipv6_traffic = 1, prefer_ipv6 = 0, dns_request = 1,
       onion_traffic = 1,
-      cache_ipv4 = 1, use_cached_ipv4 = 0,
+      cache_ipv4 = 0, use_cached_ipv4 = 0,
       cache_ipv6 = 0, use_cached_ipv6 = 0,
       prefer_ipv6_automap = 1, world_writable = 0, group_writable = 0,
       relax_dirmode_check = 0,
@@ -6627,7 +7026,7 @@ parse_port_config(smartlist_t *out,
         } else if (!strcasecmp(elt, "AllAddrs")) {
 
           all_addrs = 1;
-#endif
+#endif /* 0 */
         } else if (!strcasecmp(elt, "IPv4Only")) {
           bind_ipv4_only = 1;
         } else if (!strcasecmp(elt, "IPv6Only")) {
@@ -6889,6 +7288,7 @@ parse_port_config(smartlist_t *out,
     SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
     smartlist_clear(elts);
     tor_free(addrport);
+    tor_free(unix_socket_path);
   }
 
   if (warn_nonlocal && out) {
@@ -6956,39 +7356,48 @@ parse_ports(or_options_t *options, int validate_only,
 
   *n_ports_out = 0;
 
-  const unsigned gw_flag = options->SocksSocketsGroupWritable ?
+  const unsigned gw_flag = options->UnixSocksGroupWritable ?
     CL_PORT_DFLT_GROUP_WRITABLE : 0;
   if (parse_port_config(ports,
-             options->SocksPort_lines, options->SocksListenAddress,
+             options->SocksPort_lines,
              "Socks", CONN_TYPE_AP_LISTENER,
              "127.0.0.1", 9050,
-             CL_PORT_WARN_NONLOCAL|CL_PORT_ALLOW_EXTRA_LISTENADDR|
-             CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) {
-    *msg = tor_strdup("Invalid SocksPort/SocksListenAddress configuration");
+             ((validate_only ? 0 : CL_PORT_WARN_NONLOCAL)
+              | CL_PORT_TAKES_HOSTNAMES | gw_flag)) < 0) {
+    *msg = tor_strdup("Invalid SocksPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->DNSPort_lines, options->DNSListenAddress,
+                        options->DNSPort_lines,
                         "DNS", CONN_TYPE_AP_DNS_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES) < 0) {
-    *msg = tor_strdup("Invalid DNSPort/DNSListenAddress configuration");
+    *msg = tor_strdup("Invalid DNSPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->TransPort_lines, options->TransListenAddress,
+                        options->TransPort_lines,
                         "Trans", CONN_TYPE_AP_TRANS_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL) < 0) {
-    *msg = tor_strdup("Invalid TransPort/TransListenAddress configuration");
+    *msg = tor_strdup("Invalid TransPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->NATDPort_lines, options->NATDListenAddress,
+                        options->NATDPort_lines,
                         "NATD", CONN_TYPE_AP_NATD_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL) < 0) {
-    *msg = tor_strdup("Invalid NatdPort/NatdListenAddress configuration");
+    *msg = tor_strdup("Invalid NatdPort configuration");
+    goto err;
+  }
+  if (parse_port_config(ports,
+                        options->HTTPTunnelPort_lines,
+                        "HTTP Tunnel", CONN_TYPE_AP_HTTP_CONNECT_LISTENER,
+                        "127.0.0.1", 0,
+                        ((validate_only ? 0 : CL_PORT_WARN_NONLOCAL)
+                         | CL_PORT_TAKES_HOSTNAMES | gw_flag)) < 0) {
+    *msg = tor_strdup("Invalid HTTPTunnelPort configuration");
     goto err;
   }
   {
@@ -7004,16 +7413,14 @@ parse_ports(or_options_t *options, int validate_only,
 
     if (parse_port_config(ports,
                           options->ControlPort_lines,
-                          options->ControlListenAddress,
                           "Control", CONN_TYPE_CONTROL_LISTENER,
                           "127.0.0.1", 0,
                           control_port_flags) < 0) {
-      *msg = tor_strdup("Invalid ControlPort/ControlListenAddress "
-                        "configuration");
+      *msg = tor_strdup("Invalid ControlPort configuration");
       goto err;
     }
 
-    if (parse_port_config(ports, options->ControlSocket, NULL,
+    if (parse_port_config(ports, options->ControlSocket,
                           "ControlSocket",
                           CONN_TYPE_CONTROL_LISTENER, NULL, 0,
                           control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) {
@@ -7023,15 +7430,15 @@ parse_ports(or_options_t *options, int validate_only,
   }
   if (! options->ClientOnly) {
     if (parse_port_config(ports,
-                          options->ORPort_lines, options->ORListenAddress,
+                          options->ORPort_lines,
                           "OR", CONN_TYPE_OR_LISTENER,
                           "0.0.0.0", 0,
                           CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid ORPort/ORListenAddress configuration");
+      *msg = tor_strdup("Invalid ORPort configuration");
       goto err;
     }
     if (parse_port_config(ports,
-                          options->ExtORPort_lines, NULL,
+                          options->ExtORPort_lines,
                           "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
                           "127.0.0.1", 0,
                           CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
@@ -7039,11 +7446,11 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
     if (parse_port_config(ports,
-                          options->DirPort_lines, options->DirListenAddress,
+                          options->DirPort_lines,
                           "Dir", CONN_TYPE_DIR_LISTENER,
                           "0.0.0.0", 0,
                           CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid DirPort/DirListenAddress configuration");
+      *msg = tor_strdup("Invalid DirPort configuration");
       goto err;
     }
   }
@@ -7070,6 +7477,8 @@ parse_ports(or_options_t *options, int validate_only,
     !! count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1);
   options->NATDPort_set =
     !! count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1);
+  options->HTTPTunnelPort_set =
+    !! count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1);
   /* Use options->ControlSocket to test if a control socket is set */
   options->ControlPort_set =
     !! count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0);
@@ -7387,60 +7796,81 @@ port_exists_by_type_addr32h_port(int listener_type, uint32_t addr_ipv4h,
                                        check_wildcard);
 }
 
-/** Adjust the value of options->DataDirectory, or fill it in if it's
- * absent. Return 0 on success, -1 on failure. */
-static int
-normalize_data_directory(or_options_t *options)
+/** Allocate and return a good value for the DataDirectory based on
+ *  <b>val</b>, which may be NULL.  Return NULL on failure. */
+static char *
+get_data_directory(const char *val)
 {
 #ifdef _WIN32
-  char *p;
-  if (options->DataDirectory)
-    return 0; /* all set */
-  p = tor_malloc(MAX_PATH);
-  strlcpy(p,get_windows_conf_root(),MAX_PATH);
-  options->DataDirectory = p;
-  return 0;
-#else
-  const char *d = options->DataDirectory;
+  if (val) {
+    return tor_strdup(val);
+  } else {
+    return tor_strdup(get_windows_conf_root());
+  }
+#else /* !(defined(_WIN32)) */
+  const char *d = val;
   if (!d)
     d = "~/.tor";
 
- if (strncmp(d,"~/",2) == 0) {
-   char *fn = expand_filename(d);
-   if (!fn) {
-     log_warn(LD_CONFIG,"Failed to expand filename \"%s\".", d);
-     return -1;
-   }
-   if (!options->DataDirectory && !strcmp(fn,"/.tor")) {
-     /* If our homedir is /, we probably don't want to use it. */
-     /* Default to LOCALSTATEDIR/tor which is probably closer to what we
-      * want. */
-     log_warn(LD_CONFIG,
-              "Default DataDirectory is \"~/.tor\".  This expands to "
-              "\"%s\", which is probably not what you want.  Using "
-              "\"%s"PATH_SEPARATOR"tor\" instead", fn, LOCALSTATEDIR);
-     tor_free(fn);
-     fn = tor_strdup(LOCALSTATEDIR PATH_SEPARATOR "tor");
-   }
-   tor_free(options->DataDirectory);
-   options->DataDirectory = fn;
- }
- return 0;
-#endif
+  if (!strcmpstart(d, "~/")) {
+    char *fn = expand_filename(d);
+    if (!fn) {
+      log_warn(LD_CONFIG,"Failed to expand filename \"%s\".", d);
+      return NULL;
+    }
+    if (!val && !strcmp(fn,"/.tor")) {
+      /* If our homedir is /, we probably don't want to use it. */
+      /* Default to LOCALSTATEDIR/tor which is probably closer to what we
+       * want. */
+      log_warn(LD_CONFIG,
+               "Default DataDirectory is \"~/.tor\".  This expands to "
+               "\"%s\", which is probably not what you want.  Using "
+               "\"%s"PATH_SEPARATOR"tor\" instead", fn, LOCALSTATEDIR);
+      tor_free(fn);
+      fn = tor_strdup(LOCALSTATEDIR PATH_SEPARATOR "tor");
+    }
+    return fn;
+  }
+  return tor_strdup(d);
+#endif /* defined(_WIN32) */
 }
 
-/** Check and normalize the value of options->DataDirectory; return 0 if it
- * is sane, -1 otherwise. */
+/** Check and normalize the values of options->{Key,Data,Cache}Directory;
+ * return 0 if it is sane, -1 otherwise. */
 static int
-validate_data_directory(or_options_t *options)
+validate_data_directories(or_options_t *options)
 {
-  if (normalize_data_directory(options) < 0)
+  tor_free(options->DataDirectory);
+  options->DataDirectory = get_data_directory(options->DataDirectory_option);
+  if (!options->DataDirectory)
     return -1;
-  tor_assert(options->DataDirectory);
   if (strlen(options->DataDirectory) > (512-128)) {
     log_warn(LD_CONFIG, "DataDirectory is too long.");
     return -1;
   }
+
+  tor_free(options->KeyDirectory);
+  if (options->KeyDirectory_option) {
+    options->KeyDirectory = get_data_directory(options->KeyDirectory_option);
+    if (!options->KeyDirectory)
+      return -1;
+  } else {
+    /* Default to the data directory's keys subdir */
+    tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys",
+                 options->DataDirectory);
+  }
+
+  tor_free(options->CacheDirectory);
+  if (options->CacheDirectory_option) {
+    options->CacheDirectory = get_data_directory(
+                                             options->CacheDirectory_option);
+    if (!options->CacheDirectory)
+      return -1;
+  } else {
+    /* Default to the data directory. */
+    options->CacheDirectory = tor_strdup(options->DataDirectory);
+  }
+
   return 0;
 }
 
@@ -7581,53 +8011,56 @@ init_libevent(const or_options_t *options)
   suppress_libevent_log_msg(NULL);
 }
 
-/** Return a newly allocated string holding a filename relative to the data
- * directory.  If <b>sub1</b> is present, it is the first path component after
+/** Return a newly allocated string holding a filename relative to the
+ * directory in <b>options</b> specified by <b>roottype</b>.
+ * If <b>sub1</b> is present, it is the first path component after
  * the data directory.  If <b>sub2</b> is also present, it is the second path
  * component after the data directory.  If <b>suffix</b> is present, it
  * is appended to the filename.
  *
- * Examples:
- *    get_datadir_fname2_suffix("a", NULL, NULL) -> $DATADIR/a
- *    get_datadir_fname2_suffix("a", NULL, ".tmp") -> $DATADIR/a.tmp
- *    get_datadir_fname2_suffix("a", "b", ".tmp") -> $DATADIR/a/b/.tmp
- *    get_datadir_fname2_suffix("a", "b", NULL) -> $DATADIR/a/b
- *
- * Note: Consider using the get_datadir_fname* macros in or.h.
+ * Note: Consider using macros in config.h that wrap this function;
+ * you should probably never need to call it as-is.
  */
 MOCK_IMPL(char *,
-options_get_datadir_fname2_suffix,(const or_options_t *options,
-                                   const char *sub1, const char *sub2,
-                                   const char *suffix))
+options_get_dir_fname2_suffix,(const or_options_t *options,
+                               directory_root_t roottype,
+                               const char *sub1, const char *sub2,
+                               const char *suffix))
 {
-  char *fname = NULL;
-  size_t len;
   tor_assert(options);
-  tor_assert(options->DataDirectory);
-  tor_assert(sub1 || !sub2); /* If sub2 is present, sub1 must be present. */
-  len = strlen(options->DataDirectory);
-  if (sub1) {
-    len += strlen(sub1)+1;
-    if (sub2)
-      len += strlen(sub2)+1;
-  }
-  if (suffix)
-    len += strlen(suffix);
-  len++;
-  fname = tor_malloc(len);
-  if (sub1) {
-    if (sub2) {
-      tor_snprintf(fname, len, "%s"PATH_SEPARATOR"%s"PATH_SEPARATOR"%s",
-                   options->DataDirectory, sub1, sub2);
-    } else {
-      tor_snprintf(fname, len, "%s"PATH_SEPARATOR"%s",
-                   options->DataDirectory, sub1);
-    }
+
+  const char *rootdir = NULL;
+  switch (roottype) {
+    case DIRROOT_DATADIR:
+      rootdir = options->DataDirectory;
+      break;
+    case DIRROOT_CACHEDIR:
+      rootdir = options->CacheDirectory;
+      break;
+    case DIRROOT_KEYDIR:
+      rootdir = options->KeyDirectory;
+      break;
+    default:
+      tor_assert_unreached();
+      break;
+  }
+  tor_assert(rootdir);
+
+  if (!suffix)
+    suffix = "";
+
+  char *fname = NULL;
+
+  if (sub1 == NULL) {
+    tor_asprintf(&fname, "%s%s", rootdir, suffix);
+    tor_assert(!sub2); /* If sub2 is present, sub1 must be present. */
+  } else if (sub2 == NULL) {
+    tor_asprintf(&fname, "%s"PATH_SEPARATOR"%s%s", rootdir, sub1, suffix);
   } else {
-    strlcpy(fname, options->DataDirectory, len);
+    tor_asprintf(&fname, "%s"PATH_SEPARATOR"%s"PATH_SEPARATOR"%s%s",
+                 rootdir, sub1, sub2, suffix);
   }
-  if (suffix)
-    strlcat(fname, suffix, len);
+
   return fname;
 }
 
@@ -7668,28 +8101,6 @@ write_to_data_subdir(const char* subdir, const char* fname,
   return return_val;
 }
 
-/** Given a file name check to see whether the file exists but has not been
- * modified for a very long time.  If so, remove it. */
-void
-remove_file_if_very_old(const char *fname, time_t now)
-{
-#define VERY_OLD_FILE_AGE (28*24*60*60)
-  struct stat st;
-
-  log_debug(LD_FS, "stat()ing %s", fname);
-  if (stat(sandbox_intern_string(fname), &st)==0 &&
-      st.st_mtime < now-VERY_OLD_FILE_AGE) {
-    char buf[ISO_TIME_LEN+1];
-    format_local_iso_time(buf, st.st_mtime);
-    log_notice(LD_GENERAL, "Obsolete file %s hasn't been modified since %s. "
-               "Removing it.", fname, buf);
-    if (unlink(fname) != 0) {
-      log_warn(LD_FS, "Failed to unlink %s: %s",
-               fname, strerror(errno));
-    }
-  }
-}
-
 /** Return a smartlist of ports that must be forwarded by
  *  tor-fw-helper. The smartlist contains the ports in a string format
  *  that is understandable by tor-fw-helper. */
@@ -7762,9 +8173,12 @@ getinfo_helper_config(control_connection_t *conn,
         case CONFIG_TYPE_ISOTIME: type = "Time"; break;
         case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break;
         case CONFIG_TYPE_CSV: type = "CommaList"; break;
-        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break;
+        /* This type accepts more inputs than TimeInterval, but it ignores
+         * everything after the first entry, so we may as well pretend
+         * it's a TimeInterval. */
+        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break;
         case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependant"; break;
+        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
         case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
         default:
         case CONFIG_TYPE_OBSOLETE:
@@ -7846,60 +8260,98 @@ getinfo_helper_config(control_connection_t *conn,
   return 0;
 }
 
-/** Parse outbound bind address option lines. If <b>validate_only</b>
- * is not 0 update OutboundBindAddressIPv4_ and
- * OutboundBindAddressIPv6_ in <b>options</b>. On failure, set
- * <b>msg</b> (if provided) to a newly allocated string containing a
- * description of the problem and return -1. */
+/* Check whether an address has already been set against the options
+ * depending on address family and destination type. Any exsting
+ * value will lead to a fail, even if it is the same value. If not
+ * set and not only validating, copy it into this location too.
+ * Returns 0 on success or -1 if this address is already set.
+ */
 static int
-parse_outbound_addresses(or_options_t *options, int validate_only, char **msg)
+verify_and_store_outbound_address(sa_family_t family, tor_addr_t *addr,
+       outbound_addr_t type, or_options_t *options, int validate_only)
 {
-  const config_line_t *lines = options->OutboundBindAddress;
-  int found_v4 = 0, found_v6 = 0;
-
+  if (type>=OUTBOUND_ADDR_MAX || (family!=AF_INET && family!=AF_INET6)) {
+    return -1;
+  }
+  int fam_index=0;
+  if (family==AF_INET6) {
+    fam_index=1;
+  }
+  tor_addr_t *dest=&options->OutboundBindAddresses[type][fam_index];
+  if (!tor_addr_is_null(dest)) {
+    return -1;
+  }
   if (!validate_only) {
-    memset(&options->OutboundBindAddressIPv4_, 0,
-           sizeof(options->OutboundBindAddressIPv4_));
-    memset(&options->OutboundBindAddressIPv6_, 0,
-           sizeof(options->OutboundBindAddressIPv6_));
+    tor_addr_copy(dest, addr);
   }
+  return 0;
+}
+
+/* Parse a list of address lines for a specific destination type.
+ * Will store them into the options if not validate_only. If a
+ * problem occurs, a suitable error message is store in msg.
+ * Returns 0 on success or -1 if any address is already set.
+ */
+static int
+parse_outbound_address_lines(const config_line_t *lines, outbound_addr_t type,
+           or_options_t *options, int validate_only, char **msg)
+{
+  tor_addr_t addr;
+  sa_family_t family;
   while (lines) {
-    tor_addr_t addr, *dst_addr = NULL;
-    int af = tor_addr_parse(&addr, lines->value);
-    switch (af) {
-    case AF_INET:
-      if (found_v4) {
-        if (msg)
-          tor_asprintf(msg, "Multiple IPv4 outbound bind addresses "
-                       "configured: %s", lines->value);
-        return -1;
-      }
-      found_v4 = 1;
-      dst_addr = &options->OutboundBindAddressIPv4_;
-      break;
-    case AF_INET6:
-      if (found_v6) {
-        if (msg)
-          tor_asprintf(msg, "Multiple IPv6 outbound bind addresses "
-                       "configured: %s", lines->value);
-        return -1;
-      }
-      found_v6 = 1;
-      dst_addr = &options->OutboundBindAddressIPv6_;
-      break;
-    default:
+    family = tor_addr_parse(&addr, lines->value);
+    if (verify_and_store_outbound_address(family, &addr, type,
+                                 options, validate_only)) {
       if (msg)
-        tor_asprintf(msg, "Outbound bind address '%s' didn't parse.",
-                     lines->value);
+        tor_asprintf(msg, "Multiple%s%s outbound bind addresses "
+                     "configured: %s",
+                     family==AF_INET?" IPv4":(family==AF_INET6?" IPv6":""),
+                     type==OUTBOUND_ADDR_OR?" OR":
+                     (type==OUTBOUND_ADDR_EXIT?" exit":""), lines->value);
       return -1;
     }
-    if (!validate_only)
-      tor_addr_copy(dst_addr, &addr);
     lines = lines->next;
   }
   return 0;
 }
 
+/** Parse outbound bind address option lines. If <b>validate_only</b>
+ * is not 0 update OutboundBindAddresses in <b>options</b>.
+ * Only one address can be set for any of these values.
+ * On failure, set <b>msg</b> (if provided) to a newly allocated string
+ * containing a description of the problem and return -1.
+ */
+static int
+parse_outbound_addresses(or_options_t *options, int validate_only, char **msg)
+{
+  if (!validate_only) {
+    memset(&options->OutboundBindAddresses, 0,
+           sizeof(options->OutboundBindAddresses));
+  }
+
+  if (parse_outbound_address_lines(options->OutboundBindAddress,
+                                   OUTBOUND_ADDR_EXIT_AND_OR, options,
+                                   validate_only, msg) < 0) {
+    goto err;
+  }
+
+  if (parse_outbound_address_lines(options->OutboundBindAddressOR,
+                                   OUTBOUND_ADDR_OR, options, validate_only,
+                                   msg) < 0) {
+    goto err;
+  }
+
+  if (parse_outbound_address_lines(options->OutboundBindAddressExit,
+                                   OUTBOUND_ADDR_EXIT, options, validate_only,
+                                   msg)  < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
 /** Load one of the geoip files, <a>family</a> determining which
  * one. <a>default_fname</a> is used if on Windows and
  * <a>fname</a> equals "<default>". */
@@ -7919,10 +8371,10 @@ config_load_geoip_file_(sa_family_t family,
   }
   geoip_load_file(family, fname);
   tor_free(free_fname);
-#else
+#else /* !(defined(_WIN32)) */
   (void)default_fname;
   geoip_load_file(family, fname);
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Load geoip files for IPv4 and IPv6 if <a>options</a> and
@@ -7997,9 +8449,9 @@ init_cookie_authentication(const char *fname, const char *header,
       log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname));
     }
   }
-#else
+#else /* !(!defined(_WIN32)) */
   (void) group_readable;
-#endif
+#endif /* !defined(_WIN32) */
 
   /* Success! */
   log_info(LD_GENERAL, "Generated auth cookie file in '%s'.", escaped(fname));
@@ -8011,3 +8463,17 @@ init_cookie_authentication(const char *fname, const char *header,
   tor_free(cookie_file_str);
   return retval;
 }
+
+/**
+ * Return true if any option is set in <b>options</b> to make us behave
+ * as a client.
+ */
+int
+options_any_client_port_set(const or_options_t *options)
+{
+  return (options->SocksPort_set ||
+          options->TransPort_set ||
+          options->NATDPort_set ||
+          options->DNSPort_set ||
+          options->HTTPTunnelPort_set);
+}
diff --git a/src/or/config.h b/src/or/config.h
index 6645532514..4b41274434 100644
--- a/src/or/config.h
+++ b/src/or/config.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,17 @@
 #define KERNEL_MAY_SUPPORT_IPFW
 #endif
 
+/** Lowest allowable value for HeartbeatPeriod; if this is too low, we might
+ * expose more information than we're comfortable with. */
+#define MIN_HEARTBEAT_PERIOD (30*60)
+
+/** Maximum default value for MaxMemInQueues, in bytes. */
+#if SIZEOF_VOID_P >= 8
+#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (U64_LITERAL(8) << 30)
+#else
+#define MAX_DEFAULT_MEMORY_QUEUE_SIZE (U64_LITERAL(2) << 30)
+#endif
+
 MOCK_DECL(const char*, get_dirportfrontpage, (void));
 MOCK_DECL(const or_options_t *, get_options, (void));
 MOCK_DECL(or_options_t *, get_options_mutable, (void));
@@ -27,6 +38,8 @@ const char *safe_str_client(const char *address);
 const char *safe_str(const char *address);
 const char *escaped_safe_str_client(const char *address);
 const char *escaped_safe_str(const char *address);
+void init_protocol_warning_severity_level(void);
+int get_protocol_warning_severity_level(void);
 const char *get_version(void);
 const char *get_short_version(void);
 setopt_err_t options_trial_assign(config_line_t *list, unsigned flags,
@@ -53,31 +66,77 @@ config_line_t *option_get_assignment(const or_options_t *options,
                                      const char *key);
 int options_save_current(void);
 const char *get_torrc_fname(int defaults_fname);
+typedef enum {
+  DIRROOT_DATADIR,
+  DIRROOT_CACHEDIR,
+  DIRROOT_KEYDIR
+} directory_root_t;
+
 MOCK_DECL(char *,
-          options_get_datadir_fname2_suffix,
+          options_get_dir_fname2_suffix,
           (const or_options_t *options,
+           directory_root_t roottype,
            const char *sub1, const char *sub2,
            const char *suffix));
+
+/* These macros wrap options_get_dir_fname2_suffix to provide a more
+ * convenient API for finding filenames that Tor uses inside its storage
+ * They are named according to a pattern:
+ *    (options_)?get_(cache|key|data)dir_fname(2)?(_suffix)?
+ *
+ * Macros that begin with options_ take an options argument; the others
+ * work with respect to the global options.
+ *
+ * Each macro works relative to the data directory, the key directory,
+ * or the cache directory, as determined by which one is mentioned.
+ *
+ * Macro variants with "2" in their name take two path components; others
+ * take one.
+ *
+ * Macro variants with "_suffix" at the end take an additional suffix
+ * that gets appended to the end of the file
+ */
+#define options_get_datadir_fname2_suffix(options, sub1, sub2, suffix) \
+  options_get_dir_fname2_suffix((options), DIRROOT_DATADIR, \
+                                (sub1), (sub2), (suffix))
+#define options_get_cachedir_fname2_suffix(options, sub1, sub2, suffix) \
+  options_get_dir_fname2_suffix((options), DIRROOT_CACHEDIR, \
+                                (sub1), (sub2), (suffix))
+#define options_get_keydir_fname2_suffix(options, sub1, sub2, suffix) \
+  options_get_dir_fname2_suffix((options), DIRROOT_KEYDIR, \
+                                (sub1), (sub2), (suffix))
+
+#define options_get_datadir_fname(opts,sub1)                    \
+  options_get_datadir_fname2_suffix((opts),(sub1), NULL, NULL)
+#define options_get_datadir_fname2(opts,sub1,sub2)                      \
+  options_get_datadir_fname2_suffix((opts),(sub1), (sub2), NULL)
+
 #define get_datadir_fname2_suffix(sub1, sub2, suffix) \
   options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix))
-/** Return a newly allocated string containing datadir/sub1.  See
- * get_datadir_fname2_suffix.  */
-#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL)
-/** Return a newly allocated string containing datadir/sub1/sub2.  See
- * get_datadir_fname2_suffix.  */
+#define get_datadir_fname(sub1)                 \
+  get_datadir_fname2_suffix((sub1), NULL, NULL)
 #define get_datadir_fname2(sub1,sub2) \
   get_datadir_fname2_suffix((sub1), (sub2), NULL)
-/** Return a newly allocated string containing datadir/sub1/sub2 relative to
- * opts.  See get_datadir_fname2_suffix.  */
-#define options_get_datadir_fname2(opts,sub1,sub2)                      \
-  options_get_datadir_fname2_suffix((opts),(sub1), (sub2), NULL)
-/** Return a newly allocated string containing datadir/sub1suffix.  See
- * get_datadir_fname2_suffix. */
 #define get_datadir_fname_suffix(sub1, suffix) \
   get_datadir_fname2_suffix((sub1), NULL, (suffix))
 
+/** DOCDOC */
+#define options_get_keydir_fname(options, sub1)  \
+  options_get_keydir_fname2_suffix((options), (sub1), NULL, NULL)
+#define get_keydir_fname_suffix(sub1, suffix)   \
+  options_get_keydir_fname2_suffix(get_options(), (sub1), NULL, suffix)
+#define get_keydir_fname(sub1)                  \
+  options_get_keydir_fname2_suffix(get_options(), (sub1), NULL, NULL)
+
+#define get_cachedir_fname(sub1) \
+  options_get_cachedir_fname2_suffix(get_options(), (sub1), NULL, NULL)
+#define get_cachedir_fname_suffix(sub1, suffix) \
+  options_get_cachedir_fname2_suffix(get_options(), (sub1), NULL, (suffix))
+
 int using_default_dir_authorities(const or_options_t *options);
 
+int create_keys_directory(const or_options_t *options);
+
 int check_or_create_data_subdir(const char *subdir);
 int write_to_data_subdir(const char* subdir, const char* fname,
                          const char* str, const char* descr);
@@ -147,17 +206,22 @@ typedef struct bridge_line_t {
                                transport proxy. */
 } bridge_line_t;
 
-void bridge_line_free(bridge_line_t *bridge_line);
+void bridge_line_free_(bridge_line_t *bridge_line);
+#define bridge_line_free(line) \
+  FREE_AND_NULL(bridge_line_t, bridge_line_free_, (line))
 bridge_line_t *parse_bridge_line(const char *line);
 smartlist_t *get_options_from_transport_options_line(const char *line,
                                                      const char *transport);
 smartlist_t *get_options_for_server_transport(const char *transport);
 
+/* Port helper functions. */
+int options_any_client_port_set(const or_options_t *options);
+
 #ifdef CONFIG_PRIVATE
 
 #define CL_PORT_NO_STREAM_OPTIONS (1u<<0)
 #define CL_PORT_WARN_NONLOCAL (1u<<1)
-#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2)
+/* Was CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) */
 #define CL_PORT_SERVER_OPTIONS (1u<<3)
 #define CL_PORT_FORBID_NONLOCAL (1u<<4)
 #define CL_PORT_TAKES_HOSTNAMES (1u<<5)
@@ -170,8 +234,12 @@ extern struct config_format_t options_format;
 #endif
 
 STATIC port_cfg_t *port_cfg_new(size_t namelen);
-STATIC void port_cfg_free(port_cfg_t *port);
-STATIC void or_options_free(or_options_t *options);
+#define port_cfg_free(port) \
+  FREE_AND_NULL(port_cfg_t, port_cfg_free_, (port))
+STATIC void port_cfg_free_(port_cfg_t *port);
+#define or_options_free(opt) \
+  FREE_AND_NULL(or_options_t, or_options_free_, (opt))
+STATIC void or_options_free_(or_options_t *options);
 STATIC int options_validate_single_onion(or_options_t *options,
                                          char **msg);
 STATIC int options_validate(or_options_t *old_options,
@@ -193,13 +261,18 @@ STATIC int have_enough_mem_for_dircache(const or_options_t *options,
                                         size_t total_mem, char **msg);
 STATIC int parse_port_config(smartlist_t *out,
                   const config_line_t *ports,
-                  const config_line_t *listenaddrs,
                   const char *portname,
                   int listener_type,
                   const char *defaultaddr,
                   int defaultport,
                   const unsigned flags);
-#endif
 
-#endif
+STATIC int check_bridge_distribution_setting(const char *bd);
+
+STATIC uint64_t compute_real_max_mem_in_queues(const uint64_t val,
+                                               int log_guess);
+
+#endif /* defined(CONFIG_PRIVATE) */
+
+#endif /* !defined(TOR_CONFIG_H) */
 
diff --git a/src/or/confparse.c b/src/or/confparse.c
index efcf4f981e..6bab790945 100644
--- a/src/or/confparse.c
+++ b/src/or/confparse.c
@@ -1,7 +1,8 @@
+
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +10,16 @@
  *
  * \brief Back-end for parsing and generating key-value files, used to
  *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
  */
 
 #include "or.h"
@@ -67,120 +78,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option,
   return option;
 }
 
-/** Helper: allocate a new configuration option mapping 'key' to 'val',
- * append it to *<b>lst</b>. */
-void
-config_line_append(config_line_t **lst,
-                   const char *key,
-                   const char *val)
-{
-  config_line_t *newline;
-
-  newline = tor_malloc_zero(sizeof(config_line_t));
-  newline->key = tor_strdup(key);
-  newline->value = tor_strdup(val);
-  newline->next = NULL;
-  while (*lst)
-    lst = &((*lst)->next);
-
-  (*lst) = newline;
-}
-
-/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL
- * if no such key exists. For handling commandline-only options only; other
- * options should be looked up in the appropriate data structure. */
-const config_line_t *
-config_line_find(const config_line_t *lines,
-                 const char *key)
-{
-  const config_line_t *cl;
-  for (cl = lines; cl; cl = cl->next) {
-    if (!strcmp(cl->key, key))
-      return cl;
-  }
-  return NULL;
-}
-
-/** Helper: parse the config string and strdup into key/value
- * strings. Set *result to the list, or NULL if parsing the string
- * failed.  Return 0 on success, -1 on failure. Warn and ignore any
- * misformatted lines.
- *
- * If <b>extended</b> is set, then treat keys beginning with / and with + as
- * indicating "clear" and "append" respectively. */
-int
-config_get_lines(const char *string, config_line_t **result, int extended)
-{
-  config_line_t *list = NULL, **next;
-  char *k, *v;
-  const char *parse_err;
-
-  next = &list;
-  do {
-    k = v = NULL;
-    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
-    if (!string) {
-      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
-               parse_err?parse_err:"<unknown>");
-      config_free_lines(list);
-      tor_free(k);
-      tor_free(v);
-      return -1;
-    }
-    if (k && v) {
-      unsigned command = CONFIG_LINE_NORMAL;
-      if (extended) {
-        if (k[0] == '+') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          command = CONFIG_LINE_APPEND;
-        } else if (k[0] == '/') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          tor_free(v);
-          v = tor_strdup("");
-          command = CONFIG_LINE_CLEAR;
-        }
-      }
-      /* This list can get long, so we keep a pointer to the end of it
-       * rather than using config_line_append over and over and getting
-       * n^2 performance. */
-      *next = tor_malloc_zero(sizeof(config_line_t));
-      (*next)->key = k;
-      (*next)->value = v;
-      (*next)->next = NULL;
-      (*next)->command = command;
-      next = &((*next)->next);
-    } else {
-      tor_free(k);
-      tor_free(v);
-    }
-  } while (*string);
-
-  *result = list;
-  return 0;
-}
-
-/**
- * Free all the configuration lines on the linked list <b>front</b>.
- */
-void
-config_free_lines(config_line_t *front)
-{
-  config_line_t *tmp;
-
-  while (front) {
-    tmp = front;
-    front = tmp->next;
-
-    tor_free(tmp->key);
-    tor_free(tmp->value);
-    tor_free(tmp);
-  }
-}
-
 /** If <b>key</b> is a deprecated configuration option, return the message
  * explaining why it is deprecated (which may be an empty string). Return NULL
  * if it is not deprecated. The <b>key</b> field must be fully expanded. */
@@ -265,8 +162,6 @@ config_assign_value(const config_format_t *fmt, void *options,
   int i, ok;
   const config_var_t *var;
   void *lvalue;
-  int *csv_int;
-  smartlist_t *csv_str;
 
   CONFIG_CHECK(fmt, options);
 
@@ -298,6 +193,30 @@ config_assign_value(const config_format_t *fmt, void *options,
     *(int *)lvalue = i;
     break;
 
+  case CONFIG_TYPE_CSV_INTERVAL: {
+    /* We used to have entire smartlists here.  But now that all of our
+     * download schedules use exponential backoff, only the first part
+     * matters. */
+    const char *comma = strchr(c->value, ',');
+    const char *val = c->value;
+    char *tmp = NULL;
+    if (comma) {
+      tmp = tor_strndup(c->value, comma - c->value);
+      val = tmp;
+    }
+
+    i = config_parse_interval(val, &ok);
+    if (!ok) {
+      tor_asprintf(msg,
+          "Interval '%s %s' is malformed or out of bounds.",
+          c->key, c->value);
+      return -1;
+    }
+    *(int *)lvalue = i;
+    tor_free(tmp);
+    break;
+  }
+
   case CONFIG_TYPE_INTERVAL: {
     i = config_parse_interval(c->value, &ok);
     if (!ok) {
@@ -401,36 +320,6 @@ config_assign_value(const config_format_t *fmt, void *options,
                            SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
     break;
 
-  case CONFIG_TYPE_CSV_INTERVAL:
-    if (*(smartlist_t**)lvalue) {
-      SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp));
-      smartlist_clear(*(smartlist_t**)lvalue);
-    } else {
-      *(smartlist_t**)lvalue = smartlist_new();
-    }
-    csv_str = smartlist_new();
-    smartlist_split_string(csv_str, c->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    SMARTLIST_FOREACH_BEGIN(csv_str, char *, str)
-      {
-        i = config_parse_interval(str, &ok);
-        if (!ok) {
-          tor_asprintf(msg,
-              "Interval in '%s %s' is malformed or out of bounds.",
-              c->key, c->value);
-          SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp));
-          smartlist_free(csv_str);
-          return -1;
-        }
-        csv_int = tor_malloc_zero(sizeof(int));
-        *csv_int = i;
-        smartlist_add(*(smartlist_t**)lvalue, csv_int);
-      }
-    SMARTLIST_FOREACH_END(str);
-    SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp));
-    smartlist_free(csv_str);
-    break;
-
   case CONFIG_TYPE_LINELIST:
   case CONFIG_TYPE_LINELIST_S:
     {
@@ -620,23 +509,6 @@ config_value_needs_escape(const char *value)
   return 0;
 }
 
-/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
-config_line_t *
-config_lines_dup(const config_line_t *inp)
-{
-  config_line_t *result = NULL;
-  config_line_t **next_out = &result;
-  while (inp) {
-    *next_out = tor_malloc_zero(sizeof(config_line_t));
-    (*next_out)->key = tor_strdup(inp->key);
-    (*next_out)->value = tor_strdup(inp->value);
-    inp = inp->next;
-    next_out = &((*next_out)->next);
-  }
-  (*next_out) = NULL;
-  return result;
-}
-
 /** Return newly allocated line or lines corresponding to <b>key</b> in the
  * configuration <b>options</b>.  If <b>escape_val</b> is true and a
  * value needs to be quoted before it's put in a config file, quote and
@@ -648,7 +520,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
   const config_var_t *var;
   const void *value;
   config_line_t *result;
-  smartlist_t *csv_str;
   tor_assert(options && key);
 
   CONFIG_CHECK(fmt, options);
@@ -691,6 +562,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
         break;
       }
       /* fall through */
+    case CONFIG_TYPE_CSV_INTERVAL:
     case CONFIG_TYPE_INTERVAL:
     case CONFIG_TYPE_MSEC_INTERVAL:
     case CONFIG_TYPE_UINT:
@@ -731,20 +603,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
       else
         result->value = tor_strdup("");
       break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-      if (*(smartlist_t**)value) {
-        csv_str = smartlist_new();
-        SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i)
-          {
-            smartlist_add_asprintf(csv_str, "%d", *i);
-          }
-        SMARTLIST_FOREACH_END(i);
-        result->value = smartlist_join_strings(csv_str, ",", 0, NULL);
-        SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp));
-        smartlist_free(csv_str);
-      } else
-        result->value = tor_strdup("");
-      break;
     case CONFIG_TYPE_OBSOLETE:
       log_fn(LOG_INFO, LD_CONFIG,
              "You asked me for the value of an obsolete config option '%s'.",
@@ -753,11 +611,11 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
       tor_free(result);
       return NULL;
     case CONFIG_TYPE_LINELIST_S:
-      log_warn(LD_CONFIG,
-               "Can't return context-sensitive '%s' on its own", key);
       tor_free(result->key);
       tor_free(result);
-      return NULL;
+      result = config_lines_dup_and_filter(*(const config_line_t **)value,
+                                           key);
+      break;
     case CONFIG_TYPE_LINELIST:
     case CONFIG_TYPE_LINELIST_V:
       tor_free(result->key);
@@ -909,6 +767,7 @@ config_clear(const config_format_t *fmt, void *options,
     case CONFIG_TYPE_ISOTIME:
       *(time_t*)lvalue = 0;
       break;
+    case CONFIG_TYPE_CSV_INTERVAL:
     case CONFIG_TYPE_INTERVAL:
     case CONFIG_TYPE_MSEC_INTERVAL:
     case CONFIG_TYPE_UINT:
@@ -936,13 +795,6 @@ config_clear(const config_format_t *fmt, void *options,
         *(smartlist_t **)lvalue = NULL;
       }
       break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-      if (*(smartlist_t**)lvalue) {
-        SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp));
-        smartlist_free(*(smartlist_t **)lvalue);
-        *(smartlist_t **)lvalue = NULL;
-      }
-      break;
     case CONFIG_TYPE_LINELIST:
     case CONFIG_TYPE_LINELIST_S:
       config_free_lines(*(config_line_t **)lvalue);
@@ -983,7 +835,7 @@ config_reset(const config_format_t *fmt, void *options,
 
 /** Release storage held by <b>options</b>. */
 void
-config_free(const config_format_t *fmt, void *options)
+config_free_(const config_format_t *fmt, void *options)
 {
   int i;
 
@@ -1002,36 +854,6 @@ config_free(const config_format_t *fmt, void *options)
   tor_free(options);
 }
 
-/** Return true iff a and b contain identical keys and values in identical
- * order. */
-int
-config_lines_eq(config_line_t *a, config_line_t *b)
-{
-  while (a && b) {
-    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
-      return 0;
-    a = a->next;
-    b = b->next;
-  }
-  if (a || b)
-    return 0;
-  return 1;
-}
-
-/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
-int
-config_count_key(const config_line_t *a, const char *key)
-{
-  int n = 0;
-  while (a) {
-    if (!strcasecmp(a->key, key)) {
-      ++n;
-    }
-    a = a->next;
-  }
-  return n;
-}
-
 /** Return true iff the option <b>name</b> has the same value in <b>o1</b>
  * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
  */
@@ -1148,6 +970,11 @@ config_dump(const config_format_t *fmt, const void *default_options,
       config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
 
     for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
       smartlist_add_asprintf(elements, "%s%s %s\n",
                    comment_option ? "# " : "",
                    line->key, line->value);
@@ -1213,6 +1040,8 @@ static struct unit_table_t memory_units[] = {
   { "gbits",     1<<27 },
   { "gbit",      1<<27 },
   { "tb",        U64_LITERAL(1)<<40 },
+  { "tbyte",     U64_LITERAL(1)<<40 },
+  { "tbytes",    U64_LITERAL(1)<<40 },
   { "terabyte",  U64_LITERAL(1)<<40 },
   { "terabytes", U64_LITERAL(1)<<40 },
   { "terabits",  U64_LITERAL(1)<<37 },
@@ -1333,8 +1162,6 @@ config_parse_msec_interval(const char *s, int *ok)
 {
   uint64_t r;
   r = config_parse_units(s, time_msec_units, ok);
-  if (!ok)
-    return -1;
   if (r > INT_MAX) {
     log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
     *ok = 0;
@@ -1352,8 +1179,6 @@ config_parse_interval(const char *s, int *ok)
 {
   uint64_t r;
   r = config_parse_units(s, time_units, ok);
-  if (!ok)
-    return -1;
   if (r > INT_MAX) {
     log_warn(LD_CONFIG, "Interval '%s' is too long", s);
     *ok = 0;
diff --git a/src/or/confparse.h b/src/or/confparse.h
index 8d915d266b..4b4bf0adb4 100644
--- a/src/or/confparse.h
+++ b/src/or/confparse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CONFPARSE_H
@@ -28,7 +28,9 @@ typedef enum config_type_t {
                               * optional whitespace. */
   CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
                               * optional whitespace, representing intervals in
-                              * seconds, with optional units */
+                              * seconds, with optional units.  We allow
+                              * multiple values here for legacy reasons, but
+                              * ignore every value after the first. */
   CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
   CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
                              * mixed with other keywords. */
@@ -40,6 +42,36 @@ typedef enum config_type_t {
   CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
 } config_type_t;
 
+#ifdef TOR_UNIT_TESTS
+/**
+ * Union used when building in test mode typechecking the members of a type
+ * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
+ * it is used. */
+typedef union {
+  char **STRING;
+  char **FILENAME;
+  int *UINT; /* yes, really: Even though the confparse type is called
+              * "UINT", it still uses the C int type -- it just enforces that
+              * the values are in range [0,INT_MAX].
+              */
+  int *INT;
+  int *PORT;
+  int *INTERVAL;
+  int *MSEC_INTERVAL;
+  uint64_t *MEMUNIT;
+  double *DOUBLE;
+  int *BOOL;
+  int *AUTOBOOL;
+  time_t *ISOTIME;
+  smartlist_t **CSV;
+  int *CSV_INTERVAL;
+  config_line_t **LINELIST;
+  config_line_t **LINELIST_S;
+  config_line_t **LINELIST_V;
+  routerset_t **ROUTERSET;
+} confparse_dummy_values_t;
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** An abbreviation for a configuration option allowed on the command line. */
 typedef struct config_abbrev_t {
   const char *abbreviated;
@@ -64,8 +96,52 @@ typedef struct config_var_t {
                        * value. */
   off_t var_offset; /**< Offset of the corresponding member of or_options_t. */
   const char *initvalue; /**< String (or null) describing initial value. */
+
+#ifdef TOR_UNIT_TESTS
+  /** Used for compiler-magic to typecheck the corresponding field in the
+   * corresponding struct. Only used in unit test mode, at compile-time. */
+  confparse_dummy_values_t var_ptr_dummy;
+#endif
 } config_var_t;
 
+/* Macros to define extra members inside config_var_t fields, and at the
+ * end of a list of them.
+ */
+#ifdef TOR_UNIT_TESTS
+/* This is a somewhat magic type-checking macro for users of confparse.c.
+ * It initializes a union member "confparse_dummy_values_t.conftype" with
+ * the address of a static member "tp_dummy.member".   This
+ * will give a compiler warning unless the member field is of the correct
+ * type.
+ *
+ * (This warning is mandatory, because a type mismatch here violates the type
+ * compatibility constraint for simple assignment, and requires a diagnostic,
+ * according to the C spec.)
+ *
+ * For example, suppose you say:
+ *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
+ * Then this macro will evaluate to:
+ *     { .STRING = &or_options_t_dummy.Address }
+ * And since confparse_dummy_values_t.STRING has type "char **", that
+ * expression will create a warning unless or_options_t.Address also
+ * has type "char *".
+ */
+#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
+  { . conftype = &tp ## _dummy . member }
+#define CONF_TEST_MEMBERS(tp, conftype, member) \
+  , CONF_CHECK_VAR_TYPE(tp, conftype, member)
+#define END_OF_CONFIG_VARS                                      \
+  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } }
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  static tp tp ## _dummy
+#else /* !(defined(TOR_UNIT_TESTS)) */
+#define CONF_TEST_MEMBERS(tp, conftype, member)
+#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  struct tor_semicolon_eater
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Type of a callback to validate whether a given configuration is
  * well-formed and consistent. See options_trial_assign() for documentation
  * of arguments. */
@@ -103,14 +179,12 @@ typedef struct config_format_t {
 #define CAL_WARN_DEPRECATIONS (1u<<2)
 
 void *config_new(const config_format_t *fmt);
-void config_line_append(config_line_t **lst,
-                        const char *key, const char *val);
-config_line_t *config_lines_dup(const config_line_t *inp);
-const config_line_t *config_line_find(const config_line_t *lines,
-                                      const char *key);
-void config_free(const config_format_t *fmt, void *options);
-int config_lines_eq(config_line_t *a, config_line_t *b);
-int config_count_key(const config_line_t *a, const char *key);
+void config_free_(const config_format_t *fmt, void *options);
+#define config_free(fmt, options) do {                \
+    config_free_((fmt), (options));                   \
+    (options) = NULL;                                 \
+  } while (0)
+
 config_line_t *config_get_assigned_option(const config_format_t *fmt,
                                           const void *options, const char *key,
                                           int escape_val);
@@ -131,13 +205,18 @@ const char *config_find_deprecation(const config_format_t *fmt,
                                      const char *key);
 const config_var_t *config_find_option(const config_format_t *fmt,
                                        const char *key);
-
-int config_get_lines(const char *string, config_line_t **result, int extended);
-void config_free_lines(config_line_t *front);
 const char *config_expand_abbrev(const config_format_t *fmt,
                                  const char *option,
                                  int command_line, int warn_obsolete);
 void warn_deprecated_option(const char *what, const char *why);
 
-#endif
+/* Helper macros to compare an option across two configuration objects */
+#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
+#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
+#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
+#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
+
+#endif /* !defined(TOR_CONFPARSE_H) */
 
diff --git a/src/or/connection.c b/src/or/connection.c
index 791fd95c27..0a2a635096 100644
--- a/src/or/connection.c
+++ b/src/or/connection.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,10 +34,10 @@
  * they become able to read or write register the fact with the event main
  * loop by calling connection_watch_events(), connection_start_reading(), or
  * connection_start_writing().  When they no longer want to read or write,
- * they call connection_stop_reading() or connection_start_writing().
+ * they call connection_stop_reading() or connection_stop_writing().
  *
  * To queue data to be written on a connection, call
- * connection_write_to_buf().  When data arrives, the
+ * connection_buf_add().  When data arrives, the
  * connection_process_inbuf() callback is invoked, which dispatches to a
  * type-specific function (such as connection_edge_process_inbuf() for
  * example). Connection types that need notice of when data has been written
@@ -56,7 +56,9 @@
 
 #define CONNECTION_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "buffers.h"
+#include "buffers_tls.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
@@ -74,6 +76,7 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "crypto_util.h"
 #include "directory.h"
 #include "dirserv.h"
 #include "dns.h"
@@ -83,7 +86,12 @@
 #include "ext_orport.h"
 #include "geoip.h"
 #include "main.h"
+#include "hibernate.h"
+#include "hs_common.h"
+#include "hs_ident.h"
 #include "nodelist.h"
+#include "proto_http.h"
+#include "proto_socks.h"
 #include "policies.h"
 #include "reasons.h"
 #include "relay.h"
@@ -95,7 +103,6 @@
 #include "transports.h"
 #include "routerparse.h"
 #include "sandbox.h"
-#include "transports.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -113,17 +120,14 @@ static connection_t *connection_listener_new(
                                const port_cfg_t *portcfg);
 static void connection_init(time_t now, connection_t *conn, int type,
                             int socket_family);
-static int connection_init_accepted_conn(connection_t *conn,
-                          const listener_connection_t *listener);
 static int connection_handle_listener_read(connection_t *conn, int new_type);
-static int connection_bucket_should_increase(int bucket,
-                                             or_connection_t *conn);
 static int connection_finished_flushing(connection_t *conn);
 static int connection_flushed_some(connection_t *conn);
 static int connection_finished_connecting(connection_t *conn);
 static int connection_reached_eof(connection_t *conn);
-static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
-                                  int *socket_error);
+static int connection_buf_read_from_socket(connection_t *conn,
+                                           ssize_t *max_to_read,
+                                           int *socket_error);
 static int connection_process_inbuf(connection_t *conn, int package_partial);
 static void client_check_address_changed(tor_socket_t sock);
 static void set_constrained_socket_buffers(tor_socket_t sock, int size);
@@ -133,6 +137,10 @@ static int connection_read_https_proxy_response(connection_t *conn);
 static void connection_send_socks5_connect(connection_t *conn);
 static const char *proxy_type_to_string(int proxy_type);
 static int get_proxy_type(void);
+const tor_addr_t *conn_get_outbound_address(sa_family_t family,
+                  const or_options_t *options, unsigned int conn_type);
+static void reenable_blocked_connection_init(const or_options_t *options);
+static void reenable_blocked_connection_schedule(void);
 
 /** The last addresses that our network interface seemed to have been
  * binding to.  We use this as one way to detect when our IP changes.
@@ -154,7 +162,8 @@ static smartlist_t *outgoing_addrs = NULL;
     case CONN_TYPE_CONTROL_LISTENER: \
     case CONN_TYPE_AP_TRANS_LISTENER: \
     case CONN_TYPE_AP_NATD_LISTENER: \
-    case CONN_TYPE_AP_DNS_LISTENER
+    case CONN_TYPE_AP_DNS_LISTENER: \
+    case CONN_TYPE_AP_HTTP_CONNECT_LISTENER
 
 /**************************************************************/
 
@@ -181,6 +190,7 @@ conn_type_to_string(int type)
     case CONN_TYPE_CONTROL: return "Control";
     case CONN_TYPE_EXT_OR: return "Extended OR";
     case CONN_TYPE_EXT_OR_LISTENER: return "Extended OR listener";
+    case CONN_TYPE_AP_HTTP_CONNECT_LISTENER: return "HTTP tunnel listener";
     default:
       log_warn(LD_BUG, "unknown connection type %d", type);
       tor_snprintf(buf, sizeof(buf), "unknown [%d]", type);
@@ -327,8 +337,6 @@ entry_connection_new(int type, int socket_family)
     entry_conn->entry_cfg.ipv4_traffic = 1;
   else if (socket_family == AF_INET6)
     entry_conn->entry_cfg.ipv6_traffic = 1;
-  else if (socket_family == AF_UNIX)
-    entry_conn->is_socks_socket = 1;
   return entry_conn;
 }
 
@@ -453,8 +461,8 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family)
   }
 
   conn->timestamp_created = now;
-  conn->timestamp_lastread = now;
-  conn->timestamp_lastwritten = now;
+  conn->timestamp_last_read_allowed = now;
+  conn->timestamp_last_write_allowed = now;
 }
 
 /** Create a link between <b>conn_a</b> and <b>conn_b</b>. */
@@ -491,7 +499,7 @@ conn_listener_type_supports_af_unix(int type)
  * if <b>conn</b> is an OR or OP connection.
  */
 STATIC void
-connection_free_(connection_t *conn)
+connection_free_minimal(connection_t *conn)
 {
   void *mem;
   size_t memlen;
@@ -602,6 +610,7 @@ connection_free_(connection_t *conn)
   }
   if (CONN_IS_EDGE(conn)) {
     rend_data_free(TO_EDGE_CONN(conn)->rend_data);
+    hs_ident_edge_conn_free(TO_EDGE_CONN(conn)->hs_ident);
   }
   if (conn->type == CONN_TYPE_CONTROL) {
     control_connection_t *control_conn = TO_CONTROL_CONN(conn);
@@ -625,14 +634,20 @@ connection_free_(connection_t *conn)
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     tor_free(dir_conn->requested_resource);
 
-    tor_zlib_free(dir_conn->zlib_state);
-    if (dir_conn->fingerprint_stack) {
-      SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp));
-      smartlist_free(dir_conn->fingerprint_stack);
+    tor_compress_free(dir_conn->compress_state);
+    if (dir_conn->spool) {
+      SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled,
+                        spooled_resource_free(spooled));
+      smartlist_free(dir_conn->spool);
     }
 
-    cached_dir_decref(dir_conn->cached_dir);
     rend_data_free(dir_conn->rend_data);
+    hs_ident_dir_conn_free(dir_conn->hs_ident);
+    if (dir_conn->guard_state) {
+      /* Cancel before freeing, if it's still there. */
+      entry_guard_cancel(&dir_conn->guard_state);
+    }
+    circuit_guard_state_free(dir_conn->guard_state);
   }
 
   if (SOCKET_OK(conn->s)) {
@@ -644,7 +659,7 @@ connection_free_(connection_t *conn)
   if (conn->type == CONN_TYPE_OR &&
       !tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest)) {
     log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest");
-    connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+    connection_or_clear_identity(TO_OR_CONN(conn));
   }
   if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) {
     connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn));
@@ -660,7 +675,7 @@ connection_free_(connection_t *conn)
 /** Make sure <b>conn</b> isn't in any of the global conn lists; then free it.
  */
 MOCK_IMPL(void,
-connection_free,(connection_t *conn))
+connection_free_,(connection_t *conn))
 {
   if (!conn)
     return;
@@ -675,7 +690,7 @@ connection_free,(connection_t *conn))
   }
   if (connection_speaks_cells(conn)) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest)) {
-      connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+      connection_or_clear_identity(TO_OR_CONN(conn));
     }
   }
   if (conn->type == CONN_TYPE_CONTROL) {
@@ -687,7 +702,7 @@ connection_free,(connection_t *conn))
     connection_ap_warn_and_unmark_if_pending_circ(TO_ENTRY_CONN(conn),
                                                   "connection_free");
   }
-#endif
+#endif /* 1 */
 
   /* Notify the circuit creation DoS mitigation subsystem that an OR client
    * connection has been closed. And only do that if we track it. */
@@ -696,7 +711,7 @@ connection_free,(connection_t *conn))
   }
 
   connection_unregister_events(conn);
-  connection_free_(conn);
+  connection_free_minimal(conn);
 }
 
 /**
@@ -761,8 +776,8 @@ connection_close_immediate(connection_t *conn)
   connection_unregister_events(conn);
 
   /* Prevent the event from getting unblocked. */
-  conn->read_blocked_on_bw =
-    conn->write_blocked_on_bw = 0;
+  conn->read_blocked_on_bw = 0;
+  conn->write_blocked_on_bw = 0;
 
   if (SOCKET_OK(conn->s))
     tor_close_socket(conn->s);
@@ -806,7 +821,7 @@ connection_mark_for_close_(connection_t *conn, int line, const char *file)
  * CONN_TYPE_OR checks; this should be called when you either are sure that
  * if this is an or_connection_t the controlling channel has been notified
  * (e.g. with connection_or_notify_error()), or you actually are the
- * connection_or_close_for_error() or connection_or_close_normally function.
+ * connection_or_close_for_error() or connection_or_close_normally() function.
  * For all other cases, use connection_mark_and_flush() instead, which
  * checks for or_connection_t properly, instead.  See below.
  */
@@ -845,7 +860,7 @@ connection_mark_for_close_internal_, (connection_t *conn,
   /* in case we're going to be held-open-til-flushed, reset
    * the number of seconds since last successful write, so
    * we get our whole 15 seconds */
-  conn->timestamp_lastwritten = time(NULL);
+  conn->timestamp_last_write_allowed = time(NULL);
 }
 
 /** Find each connection that has hold_open_until_flushed set to
@@ -867,7 +882,7 @@ connection_expire_held_open(void)
      */
     if (conn->hold_open_until_flushed) {
       tor_assert(conn->marked_for_close);
-      if (now - conn->timestamp_lastwritten >= 15) {
+      if (now - conn->timestamp_last_write_allowed >= 15) {
         int severity;
         if (conn->type == CONN_TYPE_EXIT ||
             (conn->type == CONN_TYPE_DIR &&
@@ -922,7 +937,7 @@ create_unix_sockaddr(const char *listenaddress, char **readable_address,
   *len_out = sizeof(struct sockaddr_un);
   return sockaddr;
 }
-#else
+#else /* !(defined(HAVE_SYS_UN_H) || defined(RUNNING_DOXYGEN)) */
 static struct sockaddr *
 create_unix_sockaddr(const char *listenaddress, char **readable_address,
                      socklen_t *len_out)
@@ -935,7 +950,7 @@ create_unix_sockaddr(const char *listenaddress, char **readable_address,
   tor_fragile_assert();
   return NULL;
 }
-#endif /* HAVE_SYS_UN_H */
+#endif /* defined(HAVE_SYS_UN_H) || defined(RUNNING_DOXYGEN) */
 
 /** Warn that an accept or a connect has failed because we're running out of
  * TCP sockets we can use on current system.  Rate-limit these warnings so
@@ -1050,7 +1065,7 @@ check_location_for_unix_socket(const or_options_t *options, const char *path,
   tor_free(p);
   return r;
 }
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
 
 /** Tell the TCP stack that it shouldn't wait for a long time after
  * <b>sock</b> has closed before reusing its port. Return 0 on success,
@@ -1073,7 +1088,7 @@ make_socket_reuseable(tor_socket_t sock)
     return -1;
   }
   return 0;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 #ifdef _WIN32
@@ -1094,12 +1109,12 @@ make_win32_socket_exclusive(tor_socket_t sock)
     return -1;
   }
   return 0;
-#else
+#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */
   (void) sock;
   return 0;
-#endif
+#endif /* defined(SO_EXCLUSIVEADDRUSE) */
 }
-#endif
+#endif /* defined(_WIN32) */
 
 /** Max backlog to pass to listen.  We start at */
 static int listen_limit = INT_MAX;
@@ -1189,7 +1204,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                conn_type_to_string(type),
                tor_socket_strerror(errno));
     }
-#endif
+#endif /* defined(_WIN32) */
 
 #if defined(USE_TRANSPARENT) && defined(IP_TRANSPARENT)
     if (options->TransProxyType_parsed == TPT_TPROXY &&
@@ -1206,7 +1221,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                  tor_socket_strerror(e), extra);
       }
     }
-#endif
+#endif /* defined(USE_TRANSPARENT) && defined(IP_TRANSPARENT) */
 
 #ifdef IPV6_V6ONLY
     if (listensockaddr->sa_family == AF_INET6) {
@@ -1221,7 +1236,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
         /* Keep going; probably not harmful. */
       }
     }
-#endif
+#endif /* defined(IPV6_V6ONLY) */
 
     if (bind(s,listensockaddr,socklen) < 0) {
       const char *helpfulhint = "";
@@ -1324,7 +1339,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
         goto err;
       }
     }
-#endif
+#endif /* defined(HAVE_PWD_H) */
 
     {
       unsigned mode;
@@ -1355,7 +1370,7 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                tor_socket_strerror(tor_socket_errno(s)));
       goto err;
     }
-#endif /* HAVE_SYS_UN_H */
+#endif /* defined(HAVE_SYS_UN_H) */
   } else {
     log_err(LD_BUG, "Got unexpected address family %d.",
             listensockaddr->sa_family);
@@ -1664,11 +1679,15 @@ connection_handle_listener_read(connection_t *conn, int new_type)
 }
 
 /** Initialize states for newly accepted connection <b>conn</b>.
+ *
  * If conn is an OR, start the TLS handshake.
+ *
  * If conn is a transparent AP, get its original destination
  * and place it in circuit_wait.
+ *
+ * The <b>listener</b> parameter is only used for AP connections.
  */
-static int
+int
 connection_init_accepted_conn(connection_t *conn,
                               const listener_connection_t *listener)
 {
@@ -1710,6 +1729,8 @@ connection_init_accepted_conn(connection_t *conn,
           TO_ENTRY_CONN(conn)->is_transparent_ap = 1;
           conn->state = AP_CONN_STATE_NATD_WAIT;
           break;
+        case CONN_TYPE_AP_HTTP_CONNECT_LISTENER:
+          conn->state = AP_CONN_STATE_HTTP_CONNECT_WAIT;
       }
       break;
     case CONN_TYPE_DIR:
@@ -1744,9 +1765,13 @@ connection_connect_sockaddr,(connection_t *conn,
   tor_assert(sa);
   tor_assert(socket_error);
 
-  if (get_options()->DisableNetwork) {
-    /* We should never even try to connect anyplace if DisableNetwork is set.
-     * Warn if we do, and refuse to make the connection. */
+  if (net_is_completely_disabled()) {
+    /* We should never even try to connect anyplace if the network is
+     * completely shut off.
+     *
+     * (We don't check net_is_disabled() here, since we still sometimes
+     * want to open connections when we're in soft hibernation.)
+     */
     static ratelim_t disablenet_violated = RATELIM_INIT(30*60);
     *socket_error = SOCK_ERRNO(ENETUNREACH);
     log_fn_ratelim(&disablenet_violated, LOG_WARN, LD_BUG,
@@ -1784,7 +1809,7 @@ connection_connect_sockaddr,(connection_t *conn,
 
   /*
    * We've got the socket open; give the OOS handler a chance to check
-   * against configuured maximum socket number, but tell it no exhaustion
+   * against configured maximum socket number, but tell it no exhaustion
    * failure.
    */
   connection_check_oos(get_n_open_sockets(), 0);
@@ -1903,6 +1928,55 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
   }
 }
 
+/** Retrieve the outbound address depending on the protocol (IPv4 or IPv6)
+ * and the connection type (relay, exit, ...)
+ * Return a socket address or NULL in case nothing is configured.
+ **/
+const tor_addr_t *
+conn_get_outbound_address(sa_family_t family,
+             const or_options_t *options, unsigned int conn_type)
+{
+  const tor_addr_t *ext_addr = NULL;
+
+  int fam_index;
+  switch (family) {
+    case AF_INET:
+      fam_index = 0;
+      break;
+    case AF_INET6:
+      fam_index = 1;
+      break;
+    default:
+      return NULL;
+  }
+
+  // If an exit connection, use the exit address (if present)
+  if (conn_type == CONN_TYPE_EXIT) {
+    if (!tor_addr_is_null(
+        &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT][fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT]
+                 [fam_index];
+    } else if (!tor_addr_is_null(
+                 &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index];
+    }
+  } else { // All non-exit connections
+    if (!tor_addr_is_null(
+           &options->OutboundBindAddresses[OUTBOUND_ADDR_OR][fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_OR]
+                 [fam_index];
+    } else if (!tor_addr_is_null(
+                 &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index];
+    }
+  }
+  return ext_addr;
+}
+
 /** Take conn, make a nonblocking socket; try to connect to
  * addr:port (port arrives in *host order*). If fail, return -1 and if
  * applicable put your best guess about errno into *<b>socket_error</b>.
@@ -1924,26 +1998,15 @@ connection_connect(connection_t *conn, const char *address,
   struct sockaddr *bind_addr = NULL;
   struct sockaddr *dest_addr;
   int dest_addr_len, bind_addr_len = 0;
-  const or_options_t *options = get_options();
-  int protocol_family;
 
   /* Log if we didn't stick to ClientUseIPv4/6 or ClientPreferIPv6OR/DirPort
    */
   connection_connect_log_client_use_ip_version(conn);
 
-  if (tor_addr_family(addr) == AF_INET6)
-    protocol_family = PF_INET6;
-  else
-    protocol_family = PF_INET;
-
   if (!tor_addr_is_loopback(addr)) {
     const tor_addr_t *ext_addr = NULL;
-    if (protocol_family == AF_INET &&
-        !tor_addr_is_null(&options->OutboundBindAddressIPv4_))
-      ext_addr = &options->OutboundBindAddressIPv4_;
-    else if (protocol_family == AF_INET6 &&
-             !tor_addr_is_null(&options->OutboundBindAddressIPv6_))
-      ext_addr = &options->OutboundBindAddressIPv6_;
+    ext_addr = conn_get_outbound_address(tor_addr_family(addr), get_options(),
+                                         conn->type);
     if (ext_addr) {
       memset(&bind_addr_ss, 0, sizeof(bind_addr_ss));
       bind_addr_len = tor_addr_to_sockaddr(ext_addr, 0,
@@ -2110,7 +2173,7 @@ connection_proxy_connect(connection_t *conn, int type)
                      fmt_addrport(&conn->addr, conn->port));
       }
 
-      connection_write_to_buf(buf, strlen(buf), conn);
+      connection_buf_add(buf, strlen(buf), conn);
       conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
       break;
     }
@@ -2176,7 +2239,7 @@ connection_proxy_connect(connection_t *conn, int type)
         buf[8] = 0; /* no userid */
       }
 
-      connection_write_to_buf((char *)buf, buf_size, conn);
+      connection_buf_add((char *)buf, buf_size, conn);
       tor_free(buf);
 
       conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
@@ -2207,7 +2270,7 @@ connection_proxy_connect(connection_t *conn, int type)
         conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
       }
 
-      connection_write_to_buf((char *)buf, 2 + buf[1], conn);
+      connection_buf_add((char *)buf, 2 + buf[1], conn);
       break;
     }
 
@@ -2313,7 +2376,7 @@ connection_send_socks5_connect(connection_t *conn)
     memcpy(buf + 20, &port, 2);
   }
 
-  connection_write_to_buf((char *)buf, reqsize, conn);
+  connection_buf_add((char *)buf, reqsize, conn);
 
   conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK;
 }
@@ -2439,7 +2502,7 @@ connection_read_proxy_handshake(connection_t *conn)
         if (socks_args_string)
           tor_free(socks_args_string);
 
-        connection_write_to_buf((char *)buf, reqsize, conn);
+        connection_buf_add((char *)buf, reqsize, conn);
 
         conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK;
         ret = 0;
@@ -2588,7 +2651,7 @@ retry_listener_ports(smartlist_t *old_conns,
     if (port->is_unix_addr && !geteuid() && (options->User) &&
         strcmp(options->User, "root"))
       continue;
-#endif
+#endif /* !defined(_WIN32) */
 
     if (port->is_unix_addr) {
       listensockaddr = (struct sockaddr *)
@@ -2758,10 +2821,10 @@ connection_is_rate_limited(connection_t *conn)
     return 1;
 }
 
-/** Did either global write bucket run dry last second? If so,
- * we are likely to run dry again this second, so be stingy with the
- * tokens we just put in. */
-static int write_buckets_empty_last_second = 0;
+/** When was either global write bucket last empty? If this was recent, then
+ * we're probably low on bandwidth, and we should be stingy with our bandwidth
+ * usage. */
+static time_t write_buckets_last_empty_at = -100;
 
 /** How many seconds of no active local circuits will make the
  * connection revert to the "relayed" bandwidth class? */
@@ -2789,25 +2852,25 @@ connection_counts_as_relayed_traffic(connection_t *conn, time_t now)
  * write many of them or just a few; and <b>conn_bucket</b> (if
  * non-negative) provides an upper limit for our answer. */
 static ssize_t
-connection_bucket_round_robin(int base, int priority,
-                              ssize_t global_bucket, ssize_t conn_bucket)
+connection_bucket_get_share(int base, int priority,
+                            ssize_t global_bucket_val, ssize_t conn_bucket)
 {
   ssize_t at_most;
   ssize_t num_bytes_high = (priority ? 32 : 16) * base;
   ssize_t num_bytes_low = (priority ? 4 : 2) * base;
 
-  /* Do a rudimentary round-robin so one circuit can't hog a connection.
+  /* Do a rudimentary limiting so one circuit can't hog a connection.
    * Pick at most 32 cells, at least 4 cells if possible, and if we're in
    * the middle pick 1/8 of the available bandwidth. */
-  at_most = global_bucket / 8;
+  at_most = global_bucket_val / 8;
   at_most -= (at_most % base); /* round down */
   if (at_most > num_bytes_high) /* 16 KB, or 8 KB for low-priority */
     at_most = num_bytes_high;
   else if (at_most < num_bytes_low) /* 2 KB, or 1 KB for low-priority */
     at_most = num_bytes_low;
 
-  if (at_most > global_bucket)
-    at_most = global_bucket;
+  if (at_most > global_bucket_val)
+    at_most = global_bucket_val;
 
   if (conn_bucket >= 0 && at_most > conn_bucket)
     at_most = conn_bucket;
@@ -2823,13 +2886,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now)
 {
   int base = RELAY_PAYLOAD_SIZE;
   int priority = conn->type != CONN_TYPE_DIR;
-  int conn_bucket = -1;
-  int global_bucket = global_read_bucket;
+  ssize_t conn_bucket = -1;
+  size_t global_bucket_val = token_bucket_rw_get_read(&global_bucket);
 
   if (connection_speaks_cells(conn)) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
     if (conn->state == OR_CONN_STATE_OPEN)
-      conn_bucket = or_conn->read_bucket;
+      conn_bucket = token_bucket_rw_get_read(&or_conn->bucket);
     base = get_cell_network_size(or_conn->wide_circ_ids);
   }
 
@@ -2838,12 +2901,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now)
     return conn_bucket>=0 ? conn_bucket : 1<<14;
   }
 
-  if (connection_counts_as_relayed_traffic(conn, now) &&
-      global_relayed_read_bucket <= global_read_bucket)
-    global_bucket = global_relayed_read_bucket;
+  if (connection_counts_as_relayed_traffic(conn, now)) {
+    size_t relayed = token_bucket_rw_get_read(&global_relayed_bucket);
+    global_bucket_val = MIN(global_bucket_val, relayed);
+  }
 
-  return connection_bucket_round_robin(base, priority,
-                                       global_bucket, conn_bucket);
+  return connection_bucket_get_share(base, priority,
+                                     global_bucket_val, conn_bucket);
 }
 
 /** How many bytes at most can we write onto this connection? */
@@ -2852,8 +2916,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
 {
   int base = RELAY_PAYLOAD_SIZE;
   int priority = conn->type != CONN_TYPE_DIR;
-  int conn_bucket = (int)conn->outbuf_flushlen;
-  int global_bucket = global_write_bucket;
+  size_t conn_bucket = conn->outbuf_flushlen;
+  size_t global_bucket_val = token_bucket_rw_get_write(&global_bucket);
 
   if (!connection_is_rate_limited(conn)) {
     /* be willing to write to local conns even if our buckets are empty */
@@ -2861,22 +2925,21 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
   }
 
   if (connection_speaks_cells(conn)) {
-    /* use the per-conn write limit if it's lower, but if it's less
-     * than zero just use zero */
+    /* use the per-conn write limit if it's lower */
     or_connection_t *or_conn = TO_OR_CONN(conn);
     if (conn->state == OR_CONN_STATE_OPEN)
-      if (or_conn->write_bucket < conn_bucket)
-        conn_bucket = or_conn->write_bucket >= 0 ?
-                        or_conn->write_bucket : 0;
+      conn_bucket = MIN(conn_bucket,
+                        token_bucket_rw_get_write(&or_conn->bucket));
     base = get_cell_network_size(or_conn->wide_circ_ids);
   }
 
-  if (connection_counts_as_relayed_traffic(conn, now) &&
-      global_relayed_write_bucket <= global_write_bucket)
-    global_bucket = global_relayed_write_bucket;
+  if (connection_counts_as_relayed_traffic(conn, now)) {
+    size_t relayed = token_bucket_rw_get_write(&global_relayed_bucket);
+    global_bucket_val = MIN(global_bucket_val, relayed);
+  }
 
-  return connection_bucket_round_robin(base, priority,
-                                       global_bucket, conn_bucket);
+  return connection_bucket_get_share(base, priority,
+                                     global_bucket_val, conn_bucket);
 }
 
 /** Return 1 if the global write buckets are low enough that we
@@ -2901,27 +2964,31 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
 int
 global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
 {
-  int smaller_bucket = global_write_bucket < global_relayed_write_bucket ?
-                       global_write_bucket : global_relayed_write_bucket;
+  size_t smaller_bucket =
+    MIN(token_bucket_rw_get_write(&global_bucket),
+        token_bucket_rw_get_write(&global_relayed_bucket));
   if (authdir_mode(get_options()) && priority>1)
     return 0; /* there's always room to answer v2 if we're an auth dir */
 
   if (!connection_is_rate_limited(conn))
     return 0; /* local conns don't get limited */
 
-  if (smaller_bucket < (int)attempt)
+  if (smaller_bucket < attempt)
     return 1; /* not enough space no matter the priority */
 
-  if (write_buckets_empty_last_second)
-    return 1; /* we're already hitting our limits, no more please */
+  {
+    const time_t diff = approx_time() - write_buckets_last_empty_at;
+    if (diff <= 1)
+      return 1; /* we're already hitting our limits, no more please */
+  }
 
   if (priority == 1) { /* old-style v1 query */
     /* Could we handle *two* of these requests within the next two seconds? */
     const or_options_t *options = get_options();
-    int64_t can_write = (int64_t)smaller_bucket
+    size_t can_write = (size_t) (smaller_bucket
       + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate :
-                                         options->BandwidthRate);
-    if (can_write < 2*(int64_t)attempt)
+           options->BandwidthRate));
+    if (can_write < 2*attempt)
       return 1;
   } else { /* v2 query */
     /* no further constraints yet */
@@ -2929,6 +2996,10 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
   return 0;
 }
 
+/** When did we last tell the accounting subsystem about transmitted
+ * bandwidth? */
+static time_t last_recorded_accounting_at = 0;
+
 /** Helper: adjusts our bandwidth history and informs the controller as
  * appropriate, given that we have just read <b>num_read</b> bytes and written
  * <b>num_written</b> bytes on <b>conn</b>. */
@@ -2959,59 +3030,22 @@ record_num_bytes_transferred_impl(connection_t *conn,
   }
   if (conn->type == CONN_TYPE_EXIT)
     rep_hist_note_exit_bytes(conn->port, num_written, num_read);
-}
 
-/** Helper: convert given <b>tvnow</b> time value to milliseconds since
- * midnight. */
-static uint32_t
-msec_since_midnight(const struct timeval *tvnow)
-{
-  return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) +
-         ((uint32_t)tvnow->tv_usec / (uint32_t)1000L));
-}
+  /* Remember these bytes towards statistics. */
+  stats_increment_bytes_read_and_written(num_read, num_written);
 
-/** Helper: return the time in milliseconds since <b>last_empty_time</b>
- * when a bucket ran empty that previously had <b>tokens_before</b> tokens
- * now has <b>tokens_after</b> tokens after refilling at timestamp
- * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since
- * last refilling that bucket.  Return 0 if the bucket has not been empty
- * since the last refill or has not been refilled. */
-uint32_t
-bucket_millis_empty(int tokens_before, uint32_t last_empty_time,
-                    int tokens_after, int milliseconds_elapsed,
-                    const struct timeval *tvnow)
-{
-  uint32_t result = 0, refilled;
-  if (tokens_before <= 0 && tokens_after > tokens_before) {
-    refilled = msec_since_midnight(tvnow);
-    result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) %
-             (86400L * 1000L));
-    if (result > (uint32_t)milliseconds_elapsed)
-      result = (uint32_t)milliseconds_elapsed;
-  }
-  return result;
-}
-
-/** Check if a bucket which had <b>tokens_before</b> tokens and which got
- * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run
- * out of tokens, and if so, note the milliseconds since midnight in
- * <b>timestamp_var</b> for the next TB_EMPTY event. */
-void
-connection_buckets_note_empty_ts(uint32_t *timestamp_var,
-                                 int tokens_before, size_t tokens_removed,
-                                 const struct timeval *tvnow)
-{
-  if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed)
-    *timestamp_var = msec_since_midnight(tvnow);
+  /* Remember these bytes towards accounting. */
+  if (accounting_is_enabled(get_options())) {
+    if (now > last_recorded_accounting_at && last_recorded_accounting_at) {
+      accounting_add_bytes(num_read, num_written,
+                           (int)(now - last_recorded_accounting_at));
+    } else {
+      accounting_add_bytes(num_read, num_written, 0);
+    }
+    last_recorded_accounting_at = now;
+  }
 }
 
-/** Last time at which the global or relay buckets were emptied in msec
- * since midnight. */
-static uint32_t global_relayed_read_emptied = 0,
-                global_relayed_write_emptied = 0,
-                global_read_emptied = 0,
-                global_write_emptied = 0;
-
 /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes
  * onto <b>conn</b>. Decrement buckets appropriately. */
 static void
@@ -3024,9 +3058,11 @@ connection_buckets_decrement(connection_t *conn, time_t now,
              (unsigned long)num_read, (unsigned long)num_written,
              conn_type_to_string(conn->type),
              conn_state_to_string(conn->type, conn->state));
-    if (num_written >= INT_MAX) num_written = 1;
-    if (num_read >= INT_MAX) num_read = 1;
-    tor_fragile_assert();
+    tor_assert_nonfatal_unreached();
+    if (num_written >= INT_MAX)
+      num_written = 1;
+    if (num_read >= INT_MAX)
+      num_read = 1;
   }
 
   record_num_bytes_transferred_impl(conn, now, num_read, num_written);
@@ -3034,45 +3070,54 @@ connection_buckets_decrement(connection_t *conn, time_t now,
   if (!connection_is_rate_limited(conn))
     return; /* local IPs are free */
 
-  /* If one or more of our token buckets ran dry just now, note the
-   * timestamp for TB_EMPTY events. */
-  if (get_options()->TestingEnableTbEmptyEvent) {
-    struct timeval tvnow;
-    tor_gettimeofday_cached(&tvnow);
-    if (connection_counts_as_relayed_traffic(conn, now)) {
-      connection_buckets_note_empty_ts(&global_relayed_read_emptied,
-                         global_relayed_read_bucket, num_read, &tvnow);
-      connection_buckets_note_empty_ts(&global_relayed_write_emptied,
-                         global_relayed_write_bucket, num_written, &tvnow);
-    }
-    connection_buckets_note_empty_ts(&global_read_emptied,
-                       global_read_bucket, num_read, &tvnow);
-    connection_buckets_note_empty_ts(&global_write_emptied,
-                       global_write_bucket, num_written, &tvnow);
-    if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      connection_buckets_note_empty_ts(&or_conn->read_emptied_time,
-                         or_conn->read_bucket, num_read, &tvnow);
-      connection_buckets_note_empty_ts(&or_conn->write_emptied_time,
-                         or_conn->write_bucket, num_written, &tvnow);
-    }
+  unsigned flags = 0;
+  if (connection_counts_as_relayed_traffic(conn, now)) {
+    flags = token_bucket_rw_dec(&global_relayed_bucket, num_read, num_written);
   }
+  flags |= token_bucket_rw_dec(&global_bucket, num_read, num_written);
 
-  if (connection_counts_as_relayed_traffic(conn, now)) {
-    global_relayed_read_bucket -= (int)num_read;
-    global_relayed_write_bucket -= (int)num_written;
+  if (flags & TB_WRITE) {
+    write_buckets_last_empty_at = now;
   }
-  global_read_bucket -= (int)num_read;
-  global_write_bucket -= (int)num_written;
   if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
-    TO_OR_CONN(conn)->read_bucket -= (int)num_read;
-    TO_OR_CONN(conn)->write_bucket -= (int)num_written;
+    or_connection_t *or_conn = TO_OR_CONN(conn);
+    token_bucket_rw_dec(&or_conn->bucket, num_read, num_written);
   }
 }
 
+/**
+ * Mark <b>conn</b> as needing to stop reading because bandwidth has been
+ * exhausted.  If <b>is_global_bw</b>, it is closing because global bandwidth
+ * limit has been exhausted.  Otherwise, it is closing because its own
+ * bandwidth limit has been exhausted.
+ */
+void
+connection_read_bw_exhausted(connection_t *conn, bool is_global_bw)
+{
+  (void)is_global_bw;
+  conn->read_blocked_on_bw = 1;
+  connection_stop_reading(conn);
+  reenable_blocked_connection_schedule();
+}
+
+/**
+ * Mark <b>conn</b> as needing to stop reading because write bandwidth has
+ * been exhausted.  If <b>is_global_bw</b>, it is closing because global
+ * bandwidth limit has been exhausted.  Otherwise, it is closing because its
+ * own bandwidth limit has been exhausted.
+*/
+void
+connection_write_bw_exhausted(connection_t *conn, bool is_global_bw)
+{
+  (void)is_global_bw;
+  conn->write_blocked_on_bw = 1;
+  connection_stop_writing(conn);
+  reenable_blocked_connection_schedule();
+}
+
 /** If we have exhausted our global buckets, or the buckets for conn,
  * stop reading. */
-static void
+void
 connection_consider_empty_read_buckets(connection_t *conn)
 {
   const char *reason;
@@ -3080,26 +3125,28 @@ connection_consider_empty_read_buckets(connection_t *conn)
   if (!connection_is_rate_limited(conn))
     return; /* Always okay. */
 
-  if (global_read_bucket <= 0) {
+  int is_global = 1;
+
+  if (token_bucket_rw_get_read(&global_bucket) <= 0) {
     reason = "global read bucket exhausted. Pausing.";
   } else if (connection_counts_as_relayed_traffic(conn, approx_time()) &&
-             global_relayed_read_bucket <= 0) {
+             token_bucket_rw_get_read(&global_relayed_bucket) <= 0) {
     reason = "global relayed read bucket exhausted. Pausing.";
   } else if (connection_speaks_cells(conn) &&
              conn->state == OR_CONN_STATE_OPEN &&
-             TO_OR_CONN(conn)->read_bucket <= 0) {
+             token_bucket_rw_get_read(&TO_OR_CONN(conn)->bucket) <= 0) {
     reason = "connection read bucket exhausted. Pausing.";
+    is_global = false;
   } else
     return; /* all good, no need to stop it */
 
   LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason));
-  conn->read_blocked_on_bw = 1;
-  connection_stop_reading(conn);
+  connection_read_bw_exhausted(conn, is_global);
 }
 
 /** If we have exhausted our global buckets, or the buckets for conn,
  * stop writing. */
-static void
+void
 connection_consider_empty_write_buckets(connection_t *conn)
 {
   const char *reason;
@@ -3107,238 +3154,171 @@ connection_consider_empty_write_buckets(connection_t *conn)
   if (!connection_is_rate_limited(conn))
     return; /* Always okay. */
 
-  if (global_write_bucket <= 0) {
+  bool is_global = true;
+  if (token_bucket_rw_get_write(&global_bucket) <= 0) {
     reason = "global write bucket exhausted. Pausing.";
   } else if (connection_counts_as_relayed_traffic(conn, approx_time()) &&
-             global_relayed_write_bucket <= 0) {
+             token_bucket_rw_get_write(&global_relayed_bucket) <= 0) {
     reason = "global relayed write bucket exhausted. Pausing.";
   } else if (connection_speaks_cells(conn) &&
              conn->state == OR_CONN_STATE_OPEN &&
-             TO_OR_CONN(conn)->write_bucket <= 0) {
+             token_bucket_rw_get_write(&TO_OR_CONN(conn)->bucket) <= 0) {
     reason = "connection write bucket exhausted. Pausing.";
+    is_global = false;
   } else
     return; /* all good, no need to stop it */
 
   LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason));
-  conn->write_blocked_on_bw = 1;
-  connection_stop_writing(conn);
+  connection_write_bw_exhausted(conn, is_global);
 }
 
-/** Initialize the global read bucket to options-\>BandwidthBurst. */
+/** Initialize the global buckets to the values configured in the
+ * options */
 void
 connection_bucket_init(void)
 {
   const or_options_t *options = get_options();
-  /* start it at max traffic */
-  global_read_bucket = (int)options->BandwidthBurst;
-  global_write_bucket = (int)options->BandwidthBurst;
+  const uint32_t now_ts = monotime_coarse_get_stamp();
+  token_bucket_rw_init(&global_bucket,
+                    (int32_t)options->BandwidthRate,
+                    (int32_t)options->BandwidthBurst,
+                    now_ts);
   if (options->RelayBandwidthRate) {
-    global_relayed_read_bucket = (int)options->RelayBandwidthBurst;
-    global_relayed_write_bucket = (int)options->RelayBandwidthBurst;
+    token_bucket_rw_init(&global_relayed_bucket,
+                      (int32_t)options->RelayBandwidthRate,
+                      (int32_t)options->RelayBandwidthBurst,
+                      now_ts);
   } else {
-    global_relayed_read_bucket = (int)options->BandwidthBurst;
-    global_relayed_write_bucket = (int)options->BandwidthBurst;
+    token_bucket_rw_init(&global_relayed_bucket,
+                      (int32_t)options->BandwidthRate,
+                      (int32_t)options->BandwidthBurst,
+                      now_ts);
   }
+
+  reenable_blocked_connection_init(options);
 }
 
-/** Refill a single <b>bucket</b> called <b>name</b> with bandwidth rate per
- * second <b>rate</b> and bandwidth burst <b>burst</b>, assuming that
- * <b>milliseconds_elapsed</b> milliseconds have passed since the last
- * call. */
-static void
-connection_bucket_refill_helper(int *bucket, int rate, int burst,
-                                int milliseconds_elapsed,
-                                const char *name)
+/** Update the global connection bucket settings to a new value. */
+void
+connection_bucket_adjust(const or_options_t *options)
 {
-  int starting_bucket = *bucket;
-  if (starting_bucket < burst && milliseconds_elapsed > 0) {
-    int64_t incr = (((int64_t)rate) * milliseconds_elapsed) / 1000;
-    if ((burst - starting_bucket) < incr) {
-      *bucket = burst;  /* We would overflow the bucket; just set it to
-                         * the maximum. */
-    } else {
-      *bucket += (int)incr;
-      if (*bucket > burst || *bucket < starting_bucket) {
-        /* If we overflow the burst, or underflow our starting bucket,
-         * cap the bucket value to burst. */
-        /* XXXX this might be redundant now, but it doesn't show up
-         * in profiles.  Remove it after analysis. */
-        *bucket = burst;
-      }
-    }
-    log_debug(LD_NET,"%s now %d.", name, *bucket);
+  token_bucket_rw_adjust(&global_bucket,
+                      (int32_t)options->BandwidthRate,
+                      (int32_t)options->BandwidthBurst);
+  if (options->RelayBandwidthRate) {
+    token_bucket_rw_adjust(&global_relayed_bucket,
+                        (int32_t)options->RelayBandwidthRate,
+                        (int32_t)options->RelayBandwidthBurst);
+  } else {
+    token_bucket_rw_adjust(&global_relayed_bucket,
+                        (int32_t)options->BandwidthRate,
+                        (int32_t)options->BandwidthBurst);
   }
 }
 
-/** Time has passed; increment buckets appropriately. */
-void
-connection_bucket_refill(int milliseconds_elapsed, time_t now)
+/**
+ * Cached value of the last coarse-timestamp when we refilled the
+ * global buckets.
+ */
+static uint32_t last_refilled_global_buckets_ts=0;
+/**
+ * Refill the token buckets for a single connection <b>conn</b>, and the
+ * global token buckets as appropriate.  Requires that <b>now_ts</b> is
+ * the time in coarse timestamp units.
+ */
+static void
+connection_bucket_refill_single(connection_t *conn, uint32_t now_ts)
 {
-  const or_options_t *options = get_options();
-  smartlist_t *conns = get_connection_array();
-  int bandwidthrate, bandwidthburst, relayrate, relayburst;
+  /* Note that we only check for equality here: the underlying
+   * token bucket functions can handle moving backwards in time if they
+   * need to. */
+  if (now_ts != last_refilled_global_buckets_ts) {
+    token_bucket_rw_refill(&global_bucket, now_ts);
+    token_bucket_rw_refill(&global_relayed_bucket, now_ts);
+    last_refilled_global_buckets_ts = now_ts;
+  }
 
-  int prev_global_read = global_read_bucket;
-  int prev_global_write = global_write_bucket;
-  int prev_relay_read = global_relayed_read_bucket;
-  int prev_relay_write = global_relayed_write_bucket;
-  struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */
+  if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
+    or_connection_t *or_conn = TO_OR_CONN(conn);
+    token_bucket_rw_refill(&or_conn->bucket, now_ts);
+  }
+}
 
-  bandwidthrate = (int)options->BandwidthRate;
-  bandwidthburst = (int)options->BandwidthBurst;
+/**
+ * Event to re-enable all connections that were previously blocked on read or
+ * write.
+ */
+static mainloop_event_t *reenable_blocked_connections_ev = NULL;
 
-  if (options->RelayBandwidthRate) {
-    relayrate = (int)options->RelayBandwidthRate;
-    relayburst = (int)options->RelayBandwidthBurst;
-  } else {
-    relayrate = bandwidthrate;
-    relayburst = bandwidthburst;
-  }
-
-  tor_assert(milliseconds_elapsed >= 0);
-
-  write_buckets_empty_last_second =
-    global_relayed_write_bucket <= 0 || global_write_bucket <= 0;
-
-  /* refill the global buckets */
-  connection_bucket_refill_helper(&global_read_bucket,
-                                  bandwidthrate, bandwidthburst,
-                                  milliseconds_elapsed,
-                                  "global_read_bucket");
-  connection_bucket_refill_helper(&global_write_bucket,
-                                  bandwidthrate, bandwidthburst,
-                                  milliseconds_elapsed,
-                                  "global_write_bucket");
-  connection_bucket_refill_helper(&global_relayed_read_bucket,
-                                  relayrate, relayburst,
-                                  milliseconds_elapsed,
-                                  "global_relayed_read_bucket");
-  connection_bucket_refill_helper(&global_relayed_write_bucket,
-                                  relayrate, relayburst,
-                                  milliseconds_elapsed,
-                                  "global_relayed_write_bucket");
-
-  /* If buckets were empty before and have now been refilled, tell any
-   * interested controllers. */
-  if (get_options()->TestingEnableTbEmptyEvent) {
-    uint32_t global_read_empty_time, global_write_empty_time,
-             relay_read_empty_time, relay_write_empty_time;
-    tor_gettimeofday_cached(&tvnow);
-    global_read_empty_time = bucket_millis_empty(prev_global_read,
-                             global_read_emptied, global_read_bucket,
-                             milliseconds_elapsed, &tvnow);
-    global_write_empty_time = bucket_millis_empty(prev_global_write,
-                              global_write_emptied, global_write_bucket,
-                              milliseconds_elapsed, &tvnow);
-    control_event_tb_empty("GLOBAL", global_read_empty_time,
-                           global_write_empty_time, milliseconds_elapsed);
-    relay_read_empty_time = bucket_millis_empty(prev_relay_read,
-                            global_relayed_read_emptied,
-                            global_relayed_read_bucket,
-                            milliseconds_elapsed, &tvnow);
-    relay_write_empty_time = bucket_millis_empty(prev_relay_write,
-                             global_relayed_write_emptied,
-                             global_relayed_write_bucket,
-                             milliseconds_elapsed, &tvnow);
-    control_event_tb_empty("RELAY", relay_read_empty_time,
-                           relay_write_empty_time, milliseconds_elapsed);
-  }
-
-  /* refill the per-connection buckets */
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
-    if (connection_speaks_cells(conn)) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      int orbandwidthrate = or_conn->bandwidthrate;
-      int orbandwidthburst = or_conn->bandwidthburst;
-
-      int prev_conn_read = or_conn->read_bucket;
-      int prev_conn_write = or_conn->write_bucket;
-
-      if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) {
-        connection_bucket_refill_helper(&or_conn->read_bucket,
-                                        orbandwidthrate,
-                                        orbandwidthburst,
-                                        milliseconds_elapsed,
-                                        "or_conn->read_bucket");
-      }
-      if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) {
-        connection_bucket_refill_helper(&or_conn->write_bucket,
-                                        orbandwidthrate,
-                                        orbandwidthburst,
-                                        milliseconds_elapsed,
-                                        "or_conn->write_bucket");
-      }
+/** True iff reenable_blocked_connections_ev is currently scheduled. */
+static int reenable_blocked_connections_is_scheduled = 0;
 
-      /* If buckets were empty before and have now been refilled, tell any
-       * interested controllers. */
-      if (get_options()->TestingEnableTbEmptyEvent) {
-        char *bucket;
-        uint32_t conn_read_empty_time, conn_write_empty_time;
-        tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT,
-                     U64_PRINTF_ARG(or_conn->base_.global_identifier));
-        conn_read_empty_time = bucket_millis_empty(prev_conn_read,
-                               or_conn->read_emptied_time,
-                               or_conn->read_bucket,
-                               milliseconds_elapsed, &tvnow);
-        conn_write_empty_time = bucket_millis_empty(prev_conn_write,
-                                or_conn->write_emptied_time,
-                                or_conn->write_bucket,
-                                milliseconds_elapsed, &tvnow);
-        control_event_tb_empty(bucket, conn_read_empty_time,
-                               conn_write_empty_time,
-                               milliseconds_elapsed);
-        tor_free(bucket);
-      }
-    }
+/** Delay after which to run reenable_blocked_connections_ev. */
+static struct timeval reenable_blocked_connections_delay;
 
-    if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */
-        && global_read_bucket > 0 /* and we're allowed to read */
-        && (!connection_counts_as_relayed_traffic(conn, now) ||
-            global_relayed_read_bucket > 0) /* even if we're relayed traffic */
-        && (!connection_speaks_cells(conn) ||
-            conn->state != OR_CONN_STATE_OPEN ||
-            TO_OR_CONN(conn)->read_bucket > 0)) {
-        /* and either a non-cell conn or a cell conn with non-empty bucket */
-      LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET,
-                         "waking up conn (fd %d) for read", (int)conn->s));
-      conn->read_blocked_on_bw = 0;
+/**
+ * Re-enable all connections that were previously blocked on read or write.
+ * This event is scheduled after enough time has elapsed to be sure
+ * that the buckets will refill when the connections have something to do.
+ */
+static void
+reenable_blocked_connections_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    if (conn->read_blocked_on_bw == 1) {
       connection_start_reading(conn);
+      conn->read_blocked_on_bw = 0;
     }
-
-    if (conn->write_blocked_on_bw == 1
-        && global_write_bucket > 0 /* and we're allowed to write */
-        && (!connection_counts_as_relayed_traffic(conn, now) ||
-            global_relayed_write_bucket > 0) /* even if it's relayed traffic */
-        && (!connection_speaks_cells(conn) ||
-            conn->state != OR_CONN_STATE_OPEN ||
-            TO_OR_CONN(conn)->write_bucket > 0)) {
-      LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET,
-                         "waking up conn (fd %d) for write", (int)conn->s));
-      conn->write_blocked_on_bw = 0;
+    if (conn->write_blocked_on_bw == 1) {
       connection_start_writing(conn);
+      conn->write_blocked_on_bw = 0;
     }
   } SMARTLIST_FOREACH_END(conn);
+
+  reenable_blocked_connections_is_scheduled = 0;
 }
 
-/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we
- * should add another pile of tokens to it?
+/**
+ * Initialize the mainloop event that we use to wake up connections that
+ * find themselves blocked on bandwidth.
  */
-static int
-connection_bucket_should_increase(int bucket, or_connection_t *conn)
+static void
+reenable_blocked_connection_init(const or_options_t *options)
 {
-  tor_assert(conn);
-
-  if (conn->base_.state != OR_CONN_STATE_OPEN)
-    return 0; /* only open connections play the rate limiting game */
-  if (bucket >= conn->bandwidthburst)
-    return 0;
+  if (! reenable_blocked_connections_ev) {
+    reenable_blocked_connections_ev =
+      mainloop_event_new(reenable_blocked_connections_cb, NULL);
+    reenable_blocked_connections_is_scheduled = 0;
+  }
+  time_t sec = options->TokenBucketRefillInterval / 1000;
+  int msec = (options->TokenBucketRefillInterval % 1000);
+  reenable_blocked_connections_delay.tv_sec = sec;
+  reenable_blocked_connections_delay.tv_usec = msec * 1000;
+}
 
-  return 1;
+/**
+ * Called when we have blocked a connection for being low on bandwidth:
+ * schedule an event to reenable such connections, if it is not already
+ * scheduled.
+ */
+static void
+reenable_blocked_connection_schedule(void)
+{
+  if (reenable_blocked_connections_is_scheduled)
+    return;
+  if (BUG(reenable_blocked_connections_ev == NULL)) {
+    reenable_blocked_connection_init(get_options());
+  }
+  mainloop_event_schedule(reenable_blocked_connections_ev,
+                          &reenable_blocked_connections_delay);
+  reenable_blocked_connections_is_scheduled = 1;
 }
 
 /** Read bytes from conn-\>s and process them.
  *
- * It calls connection_read_to_buf() to bring in any new bytes,
+ * It calls connection_buf_read_from_socket() to bring in any new bytes,
  * and then calls connection_process_inbuf() to process them.
  *
  * Mark the connection and return -1 if you want to close it, else
@@ -3354,7 +3334,9 @@ connection_handle_read_impl(connection_t *conn)
   if (conn->marked_for_close)
     return 0; /* do nothing */
 
-  conn->timestamp_lastread = approx_time();
+  conn->timestamp_last_read_allowed = approx_time();
+
+  connection_bucket_refill_single(conn, monotime_coarse_get_stamp());
 
   switch (conn->type) {
     case CONN_TYPE_OR_LISTENER:
@@ -3364,6 +3346,7 @@ connection_handle_read_impl(connection_t *conn)
     case CONN_TYPE_AP_LISTENER:
     case CONN_TYPE_AP_TRANS_LISTENER:
     case CONN_TYPE_AP_NATD_LISTENER:
+    case CONN_TYPE_AP_HTTP_CONNECT_LISTENER:
       return connection_handle_listener_read(conn, CONN_TYPE_AP);
     case CONN_TYPE_DIR_LISTENER:
       return connection_handle_listener_read(conn, CONN_TYPE_DIR);
@@ -3380,7 +3363,7 @@ connection_handle_read_impl(connection_t *conn)
   tor_assert(!conn->marked_for_close);
 
   before = buf_datalen(conn->inbuf);
-  if (connection_read_to_buf(conn, &max_to_read, &socket_error) < 0) {
+  if (connection_buf_read_from_socket(conn, &max_to_read, &socket_error) < 0) {
     /* There's a read error; kill the connection.*/
     if (conn->type == CONN_TYPE_OR) {
       connection_or_notify_error(TO_OR_CONN(conn),
@@ -3460,8 +3443,7 @@ int
 connection_handle_read(connection_t *conn)
 {
   int res;
-
-  tor_gettimeofday_cache_clear();
+  update_current_time(time(NULL));
   res = connection_handle_read_impl(conn);
   return res;
 }
@@ -3477,7 +3459,7 @@ connection_handle_read(connection_t *conn)
  * Return -1 if we want to break conn, else return 0.
  */
 static int
-connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
+connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read,
                        int *socket_error)
 {
   int result;
@@ -3518,7 +3500,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
 
     initial_size = buf_datalen(conn->inbuf);
     /* else open, or closing */
-    result = read_to_buf_tls(or_conn->tls, at_most, conn->inbuf);
+    result = buf_read_from_tls(conn->inbuf, or_conn->tls, at_most);
     if (TOR_TLS_IS_ERROR(result) || result == TOR_TLS_CLOSE)
       or_conn->tls_error = result;
     else
@@ -3554,10 +3536,8 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
             connection_start_reading(conn);
         }
         /* we're already reading, one hopes */
-        result = 0;
         break;
       case TOR_TLS_DONE: /* no data read, so nothing to process */
-        result = 0;
         break; /* so we call bucket_decrement below */
       default:
         break;
@@ -3567,7 +3547,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
       /* If we have any pending bytes, we read them now.  This *can*
        * take us over our read allotment, but really we shouldn't be
        * believing that SSL bytes are the same as TCP bytes anyway. */
-      int r2 = read_to_buf_tls(or_conn->tls, pending, conn->inbuf);
+      int r2 = buf_read_from_tls(conn->inbuf, or_conn->tls, pending);
       if (BUG(r2<0)) {
         log_warn(LD_BUG, "apparently, reading pending bytes can fail.");
         return -1;
@@ -3579,7 +3559,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
               result, (long)n_read, (long)n_written);
   } else if (conn->linked) {
     if (conn->linked_conn) {
-      result = move_buf_to_buf(conn->inbuf, conn->linked_conn->outbuf,
+      result = buf_move_to_buf(conn->inbuf, conn->linked_conn->outbuf,
                                &conn->linked_conn->outbuf_flushlen);
     } else {
       result = 0;
@@ -3597,8 +3577,10 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
     /* !connection_speaks_cells, !conn->linked_conn. */
     int reached_eof = 0;
     CONN_LOG_PROTECT(conn,
-        result = read_to_buf(conn->s, at_most, conn->inbuf, &reached_eof,
-                             socket_error));
+                     result = buf_read_from_socket(conn->inbuf, conn->s,
+                                                   at_most,
+                                                   &reached_eof,
+                                                   socket_error));
     if (reached_eof)
       conn->inbuf_reached_eof = 1;
 
@@ -3613,25 +3595,15 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
      /* change *max_to_read */
     *max_to_read = at_most - n_read;
 
-    /* Update edge_conn->n_read and ocirc->n_read_circ_bw */
+    /* Update edge_conn->n_read */
     if (conn->type == CONN_TYPE_AP) {
       edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-      circuit_t *circ = circuit_get_by_edge_conn(edge_conn);
-      origin_circuit_t *ocirc;
 
       /* Check for overflow: */
       if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read))
         edge_conn->n_read += (int)n_read;
       else
         edge_conn->n_read = UINT32_MAX;
-
-      if (circ && CIRCUIT_IS_ORIGIN(circ)) {
-        ocirc = TO_ORIGIN_CIRCUIT(circ);
-        if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read))
-          ocirc->n_read_circ_bw += (int)n_read;
-        else
-          ocirc->n_read_circ_bw = UINT32_MAX;
-      }
     }
 
     /* If CONN_BW events are enabled, update conn->n_read_conn_bw for
@@ -3667,17 +3639,17 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read,
 
 /** A pass-through to fetch_from_buf. */
 int
-connection_fetch_from_buf(char *string, size_t len, connection_t *conn)
+connection_buf_get_bytes(char *string, size_t len, connection_t *conn)
 {
-  return fetch_from_buf(string, len, conn->inbuf);
+  return buf_get_bytes(conn->inbuf, string, len);
 }
 
-/** As fetch_from_buf_line(), but read from a connection's input buffer. */
+/** As buf_get_line(), but read from a connection's input buffer. */
 int
-connection_fetch_from_buf_line(connection_t *conn, char *data,
+connection_buf_get_line(connection_t *conn, char *data,
                                size_t *data_len)
 {
-  return fetch_from_buf_line(conn->inbuf, data, data_len);
+  return buf_get_line(conn->inbuf, data, data_len);
 }
 
 /** As fetch_from_buf_http, but fetches from a connection's input buffer_t as
@@ -3710,13 +3682,50 @@ connection_outbuf_too_full(connection_t *conn)
   return (conn->outbuf_flushlen > 10*CELL_PAYLOAD_SIZE);
 }
 
+/**
+ * On Windows Vista and Windows 7, tune the send buffer size according to a
+ * hint from the OS.
+ *
+ * This should help fix slow upload rates.
+ */
+static void
+update_send_buffer_size(tor_socket_t sock)
+{
+#ifdef _WIN32
+  /* We only do this on Vista and 7, because earlier versions of Windows
+   * don't have the SIO_IDEAL_SEND_BACKLOG_QUERY functionality, and on
+   * later versions it isn't necessary. */
+  static int isVistaOr7 = -1;
+  if (isVistaOr7 == -1) {
+    isVistaOr7 = 0;
+    OSVERSIONINFO osvi = { 0 };
+    osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    GetVersionEx(&osvi);
+    if (osvi.dwMajorVersion == 6 && osvi.dwMinorVersion < 2)
+      isVistaOr7 = 1;
+  }
+  if (!isVistaOr7)
+    return;
+  if (get_options()->ConstrainedSockets)
+    return;
+  ULONG isb = 0;
+  DWORD bytesReturned = 0;
+  if (!WSAIoctl(sock, SIO_IDEAL_SEND_BACKLOG_QUERY, NULL, 0,
+      &isb, sizeof(isb), &bytesReturned, NULL, NULL)) {
+    setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb));
+  }
+#else
+  (void) sock;
+#endif
+}
+
 /** Try to flush more bytes onto <b>conn</b>-\>s.
  *
  * This function gets called either from conn_write_callback() in main.c
  * when libevent tells us that conn wants to write, or below
- * from connection_write_to_buf() when an entire TLS record is ready.
+ * from connection_buf_add() when an entire TLS record is ready.
  *
- * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf
+ * Update <b>conn</b>-\>timestamp_last_write_allowed to now, and call flush_buf
  * or flush_buf_tls appropriately. If it succeeds and there are no more
  * more bytes on <b>conn</b>-\>outbuf, then call connection_finished_flushing
  * on it too.
@@ -3749,7 +3758,9 @@ connection_handle_write_impl(connection_t *conn, int force)
     return 0;
   }
 
-  conn->timestamp_lastwritten = now;
+  conn->timestamp_last_write_allowed = now;
+
+  connection_bucket_refill_single(conn, monotime_coarse_get_stamp());
 
   /* Sometimes, "writable" means "connected". */
   if (connection_state_is_connecting(conn)) {
@@ -3825,9 +3836,12 @@ connection_handle_write_impl(connection_t *conn, int force)
 
     /* else open, or closing */
     initial_size = buf_datalen(conn->outbuf);
-    result = flush_buf_tls(or_conn->tls, conn->outbuf,
+    result = buf_flush_to_tls(conn->outbuf, or_conn->tls,
                            max_to_write, &conn->outbuf_flushlen);
 
+    if (result >= 0)
+      update_send_buffer_size(conn->s);
+
     /* If we just flushed the last bytes, tell the channel on the
      * or_conn to check if it needs to geoip_change_dirreq_state() */
     /* XXXX move this to flushed_some or finished_flushing -NM */
@@ -3861,8 +3875,7 @@ connection_handle_write_impl(connection_t *conn, int force)
         /* Make sure to avoid a loop if the receive buckets are empty. */
         log_debug(LD_NET,"wanted read.");
         if (!connection_is_reading(conn)) {
-          connection_stop_writing(conn);
-          conn->write_blocked_on_bw = 1;
+          connection_write_bw_exhausted(conn, true);
           /* we'll start reading again when we get more tokens in our
            * read bucket; then we'll start writing again too.
            */
@@ -3888,8 +3901,8 @@ connection_handle_write_impl(connection_t *conn, int force)
     result = (int)(initial_size-buf_datalen(conn->outbuf));
   } else {
     CONN_LOG_PROTECT(conn,
-             result = flush_buf(conn->s, conn->outbuf,
-                                max_to_write, &conn->outbuf_flushlen));
+                     result = buf_flush_to_socket(conn->outbuf, conn->s,
+                                        max_to_write, &conn->outbuf_flushlen));
     if (result < 0) {
       if (CONN_IS_EDGE(conn))
         connection_edge_end_errno(TO_EDGE_CONN(conn));
@@ -3902,27 +3915,18 @@ connection_handle_write_impl(connection_t *conn, int force)
       connection_mark_for_close(conn);
       return -1;
     }
+    update_send_buffer_size(conn->s);
     n_written = (size_t) result;
   }
 
   if (n_written && conn->type == CONN_TYPE_AP) {
     edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-    circuit_t *circ = circuit_get_by_edge_conn(edge_conn);
-    origin_circuit_t *ocirc;
 
     /* Check for overflow: */
     if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written))
       edge_conn->n_written += (int)n_written;
     else
       edge_conn->n_written = UINT32_MAX;
-
-    if (circ && CIRCUIT_IS_ORIGIN(circ)) {
-      ocirc = TO_ORIGIN_CIRCUIT(circ);
-      if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written))
-        ocirc->n_written_circ_bw += (int)n_written;
-      else
-        ocirc->n_written_circ_bw = UINT32_MAX;
-    }
   }
 
   /* If CONN_BW events are enabled, update conn->n_written_conn_bw for
@@ -3982,7 +3986,7 @@ int
 connection_handle_write(connection_t *conn, int force)
 {
     int res;
-    tor_gettimeofday_cache_clear();
+    update_current_time(time(NULL));
     conn->in_connection_handle_write = 1;
     res = connection_handle_write_impl(conn, force);
     conn->in_connection_handle_write = 0;
@@ -4004,6 +4008,68 @@ connection_flush(connection_t *conn)
   return connection_handle_write(conn, 1);
 }
 
+/** Helper for connection_write_to_buf_impl and connection_write_buf_to_buf:
+ *
+ * Return true iff it is okay to queue bytes on <b>conn</b>'s outbuf for
+ * writing.
+ */
+static int
+connection_may_write_to_buf(connection_t *conn)
+{
+  /* if it's marked for close, only allow write if we mean to flush it */
+  if (conn->marked_for_close && !conn->hold_open_until_flushed)
+    return 0;
+
+  return 1;
+}
+
+/** Helper for connection_write_to_buf_impl and connection_write_buf_to_buf:
+ *
+ * Called when an attempt to add bytes on <b>conn</b>'s outbuf has failed;
+ * mark the connection and warn as appropriate.
+ */
+static void
+connection_write_to_buf_failed(connection_t *conn)
+{
+  if (CONN_IS_EDGE(conn)) {
+    /* if it failed, it means we have our package/delivery windows set
+       wrong compared to our max outbuf size. close the whole circuit. */
+    log_warn(LD_NET,
+             "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s);
+    circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)),
+                           END_CIRC_REASON_INTERNAL);
+  } else if (conn->type == CONN_TYPE_OR) {
+    or_connection_t *orconn = TO_OR_CONN(conn);
+    log_warn(LD_NET,
+             "write_to_buf failed on an orconn; notifying of error "
+             "(fd %d)", (int)(conn->s));
+    connection_or_close_for_error(orconn, 0);
+  } else {
+    log_warn(LD_NET,
+             "write_to_buf failed. Closing connection (fd %d).",
+             (int)conn->s);
+    connection_mark_for_close(conn);
+  }
+}
+
+/** Helper for connection_write_to_buf_impl and connection_write_buf_to_buf:
+ *
+ * Called when an attempt to add bytes on <b>conn</b>'s outbuf has succeeded:
+ * record the number of bytes added.
+ */
+static void
+connection_write_to_buf_commit(connection_t *conn, size_t len)
+{
+  /* If we receive optimistic data in the EXIT_CONN_STATE_RESOLVING
+   * state, we don't want to try to write it right away, since
+   * conn->write_event won't be set yet.  Otherwise, write data from
+   * this conn as the socket is available. */
+  if (conn->write_event) {
+    connection_start_writing(conn);
+  }
+  conn->outbuf_flushlen += len;
+}
+
 /** Append <b>len</b> bytes of <b>string</b> onto <b>conn</b>'s
  * outbuf, and ask it to start writing.
  *
@@ -4011,10 +4077,6 @@ connection_flush(connection_t *conn)
  * its contents compressed or decompressed as they're written.  If zlib is
  * negative, this is the last data to be compressed, and the connection's zlib
  * state should be flushed.
- *
- * If it's a local control connection and a 64k chunk is ready, try to flush
- * it all, so we don't end up with many megabytes of controller info queued at
- * once.
  */
 MOCK_IMPL(void,
 connection_write_to_buf_impl_,(const char *string, size_t len,
@@ -4022,58 +4084,84 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
 {
   /* XXXX This function really needs to return -1 on failure. */
   int r;
-  size_t old_datalen;
   if (!len && !(zlib<0))
     return;
-  /* if it's marked for close, only allow write if we mean to flush it */
-  if (conn->marked_for_close && !conn->hold_open_until_flushed)
+
+  if (!connection_may_write_to_buf(conn))
     return;
 
-  old_datalen = buf_datalen(conn->outbuf);
+  size_t written;
+
   if (zlib) {
+    size_t old_datalen = buf_datalen(conn->outbuf);
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     int done = zlib < 0;
-    CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf,
-                                                 dir_conn->zlib_state,
-                                                 string, len, done));
+    CONN_LOG_PROTECT(conn, r = buf_add_compress(conn->outbuf,
+                                                dir_conn->compress_state,
+                                                string, len, done));
+    written = buf_datalen(conn->outbuf) - old_datalen;
   } else {
-    CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf));
+    CONN_LOG_PROTECT(conn, r = buf_add(conn->outbuf, string, len));
+    written = len;
   }
   if (r < 0) {
-    if (CONN_IS_EDGE(conn)) {
-      /* if it failed, it means we have our package/delivery windows set
-         wrong compared to our max outbuf size. close the whole circuit. */
-      log_warn(LD_NET,
-               "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s);
-      circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)),
-                             END_CIRC_REASON_INTERNAL);
-    } else if (conn->type == CONN_TYPE_OR) {
-      or_connection_t *orconn = TO_OR_CONN(conn);
-      log_warn(LD_NET,
-               "write_to_buf failed on an orconn; notifying of error "
-               "(fd %d)", (int)(conn->s));
-      connection_or_close_for_error(orconn, 0);
-    } else {
-      log_warn(LD_NET,
-               "write_to_buf failed. Closing connection (fd %d).",
-               (int)conn->s);
-      connection_mark_for_close(conn);
-    }
+    connection_write_to_buf_failed(conn);
     return;
   }
+  connection_write_to_buf_commit(conn, written);
+}
 
-  /* If we receive optimistic data in the EXIT_CONN_STATE_RESOLVING
-   * state, we don't want to try to write it right away, since
-   * conn->write_event won't be set yet.  Otherwise, write data from
-   * this conn as the socket is available. */
-  if (conn->write_event) {
-    connection_start_writing(conn);
-  }
-  if (zlib) {
-    conn->outbuf_flushlen += buf_datalen(conn->outbuf) - old_datalen;
-  } else {
-    conn->outbuf_flushlen += len;
-  }
+/**
+ * Add all bytes from <b>buf</b> to <b>conn</b>'s outbuf, draining them
+ * from <b>buf</b>. (If the connection is marked and will soon be closed,
+ * nothing is drained.)
+ */
+void
+connection_buf_add_buf(connection_t *conn, buf_t *buf)
+{
+  tor_assert(conn);
+  tor_assert(buf);
+  size_t len = buf_datalen(buf);
+  if (len == 0)
+    return;
+
+  if (!connection_may_write_to_buf(conn))
+    return;
+
+  buf_move_all(conn->outbuf, buf);
+  connection_write_to_buf_commit(conn, len);
+}
+
+#define CONN_GET_ALL_TEMPLATE(var, test) \
+  STMT_BEGIN \
+    smartlist_t *conns = get_connection_array();   \
+    smartlist_t *ret_conns = smartlist_new();     \
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, var) { \
+      if (var && (test) && !var->marked_for_close) \
+        smartlist_add(ret_conns, var); \
+    } SMARTLIST_FOREACH_END(var);                                            \
+    return ret_conns; \
+  STMT_END
+
+/* Return a list of connections that aren't close and matches the given type
+ * and state. The returned list can be empty and must be freed using
+ * smartlist_free(). The caller does NOT have ownership of the objects in the
+ * list so it must not free them nor reference them as they can disappear. */
+smartlist_t *
+connection_list_by_type_state(int type, int state)
+{
+  CONN_GET_ALL_TEMPLATE(conn, (conn->type == type && conn->state == state));
+}
+
+/* Return a list of connections that aren't close and matches the given type
+ * and purpose. The returned list can be empty and must be freed using
+ * smartlist_free(). The caller does NOT have ownership of the objects in the
+ * list so it must not free them nor reference them as they can disappear. */
+smartlist_t *
+connection_list_by_type_purpose(int type, int purpose)
+{
+  CONN_GET_ALL_TEMPLATE(conn,
+                        (conn->type == type && conn->purpose == purpose));
 }
 
 /** Return a connection_t * from get_connection_array() that satisfies test on
@@ -4149,12 +4237,12 @@ connection_get_by_type_state_rendquery(int type, int state,
          (type == CONN_TYPE_DIR &&
           TO_DIR_CONN(conn)->rend_data &&
           !rend_cmp_service_ids(rendquery,
-                                TO_DIR_CONN(conn)->rend_data->onion_address))
+                    rend_data_get_address(TO_DIR_CONN(conn)->rend_data)))
          ||
               (CONN_IS_EDGE(conn) &&
                TO_EDGE_CONN(conn)->rend_data &&
                !rend_cmp_service_ids(rendquery,
-                            TO_EDGE_CONN(conn)->rend_data->onion_address))
+                    rend_data_get_address(TO_EDGE_CONN(conn)->rend_data)))
          ));
 }
 
@@ -4260,6 +4348,7 @@ connection_is_listener(connection_t *conn)
       conn->type == CONN_TYPE_AP_TRANS_LISTENER ||
       conn->type == CONN_TYPE_AP_DNS_LISTENER ||
       conn->type == CONN_TYPE_AP_NATD_LISTENER ||
+      conn->type == CONN_TYPE_AP_HTTP_CONNECT_LISTENER ||
       conn->type == CONN_TYPE_DIR_LISTENER ||
       conn->type == CONN_TYPE_CONTROL_LISTENER)
     return 1;
@@ -4336,8 +4425,6 @@ alloc_http_authenticator(const char *authenticator)
 static void
 client_check_address_changed(tor_socket_t sock)
 {
-  struct sockaddr_storage out_sockaddr;
-  socklen_t out_addr_len = (socklen_t) sizeof(out_sockaddr);
   tor_addr_t out_addr, iface_addr;
   tor_addr_t **last_interface_ip_ptr;
   sa_family_t family;
@@ -4345,13 +4432,12 @@ client_check_address_changed(tor_socket_t sock)
   if (!outgoing_addrs)
     outgoing_addrs = smartlist_new();
 
-  if (getsockname(sock, (struct sockaddr*)&out_sockaddr, &out_addr_len)<0) {
+  if (tor_addr_from_getsockname(&out_addr, sock) < 0) {
     int e = tor_socket_errno(sock);
     log_warn(LD_NET, "getsockname() to check for address change failed: %s",
              tor_socket_strerror(e));
     return;
   }
-  tor_addr_from_sockaddr(&out_addr, (struct sockaddr*)&out_sockaddr, NULL);
   family = tor_addr_family(&out_addr);
 
   if (family == AF_INET)
@@ -4927,9 +5013,9 @@ assert_connection_ok(connection_t *conn, time_t now)
 
   /* buffers */
   if (conn->inbuf)
-    assert_buf_ok(conn->inbuf);
+    buf_assert_ok(conn->inbuf);
   if (conn->outbuf)
-    assert_buf_ok(conn->outbuf);
+    buf_assert_ok(conn->outbuf);
 
   if (conn->type == CONN_TYPE_OR) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
@@ -5103,8 +5189,8 @@ proxy_type_to_string(int proxy_type)
   return NULL; /*Unreached*/
 }
 
-/** Call connection_free_() on every connection in our array, and release all
- * storage held by connection.c.
+/** Call connection_free_minimal() on every connection in our array, and
+ * release all storage held by connection.c.
  *
  * Don't do the checks in connection_free(), because they will
  * fail.
@@ -5128,7 +5214,8 @@ connection_free_all(void)
   /* Clear out our list of broken connections */
   clear_broken_connection_map(0);
 
-  SMARTLIST_FOREACH(conns, connection_t *, conn, connection_free_(conn));
+  SMARTLIST_FOREACH(conns, connection_t *, conn,
+                    connection_free_minimal(conn));
 
   if (outgoing_addrs) {
     SMARTLIST_FOREACH(outgoing_addrs, tor_addr_t *, addr, tor_free(addr));
@@ -5138,6 +5225,11 @@ connection_free_all(void)
 
   tor_free(last_interface_ipv4);
   tor_free(last_interface_ipv6);
+  last_recorded_accounting_at = 0;
+
+  mainloop_event_free(reenable_blocked_connections_ev);
+  reenable_blocked_connections_is_scheduled = 0;
+  memset(&reenable_blocked_connections_delay, 0, sizeof(struct timeval));
 }
 
 /** Log a warning, and possibly emit a control event, that <b>received</b> came
@@ -5145,13 +5237,13 @@ connection_free_all(void)
  * that we had more faith in and therefore the warning level should have higher
  * severity.
  */
-void
-clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted,
-                   log_domain_mask_t domain, const char *received,
-                   const char *source)
+MOCK_IMPL(void,
+clock_skew_warning, (const connection_t *conn, long apparent_skew, int trusted,
+                     log_domain_mask_t domain, const char *received,
+                     const char *source))
 {
   char dbuf[64];
-  char *ext_source = NULL;
+  char *ext_source = NULL, *warn = NULL;
   format_time_interval(dbuf, sizeof(dbuf), apparent_skew);
   if (conn)
     tor_asprintf(&ext_source, "%s:%s:%d", source, conn->address, conn->port);
@@ -5165,9 +5257,14 @@ clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted,
          apparent_skew > 0 ? "ahead" : "behind", dbuf,
          apparent_skew > 0 ? "behind" : "ahead",
          (!conn || trusted) ? "" : ", or they are sending us the wrong time");
-  if (trusted)
+  if (trusted) {
     control_event_general_status(LOG_WARN, "CLOCK_SKEW SKEW=%ld SOURCE=%s",
                                  apparent_skew, ext_source);
+    tor_asprintf(&warn, "Clock skew %ld in %s from %s", apparent_skew,
+                 received, source);
+    control_event_bootstrap_problem(warn, "CLOCK_SKEW", conn, 1);
+  }
+  tor_free(warn);
   tor_free(ext_source);
 }
 
diff --git a/src/or/connection.h b/src/or/connection.h
index d25e002fa4..ad3129c9d8 100644
--- a/src/or/connection.h
+++ b/src/or/connection.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,9 +26,12 @@ entry_connection_t *entry_connection_new(int type, int socket_family);
 control_connection_t *control_connection_new(int socket_family);
 listener_connection_t *listener_connection_new(int type, int socket_family);
 connection_t *connection_new(int type, int socket_family);
-
+int connection_init_accepted_conn(connection_t *conn,
+                                  const listener_connection_t *listener);
 void connection_link_connections(connection_t *conn_a, connection_t *conn_b);
-MOCK_DECL(void,connection_free,(connection_t *conn));
+MOCK_DECL(void,connection_free_,(connection_t *conn));
+#define connection_free(conn) \
+  FREE_AND_NULL(connection_t, connection_free_, (conn))
 void connection_free_all(void);
 void connection_about_to_close_connection(connection_t *conn);
 void connection_close_immediate(connection_t *conn);
@@ -119,12 +122,18 @@ void connection_mark_all_noncontrol_connections(void);
 ssize_t connection_bucket_write_limit(connection_t *conn, time_t now);
 int global_write_bucket_low(connection_t *conn, size_t attempt, int priority);
 void connection_bucket_init(void);
-void connection_bucket_refill(int seconds_elapsed, time_t now);
+void connection_bucket_adjust(const or_options_t *options);
+void connection_bucket_refill_all(time_t now,
+                                  uint32_t now_ts);
+void connection_read_bw_exhausted(connection_t *conn, bool is_global_bw);
+void connection_write_bw_exhausted(connection_t *conn, bool is_global_bw);
+void connection_consider_empty_read_buckets(connection_t *conn);
+void connection_consider_empty_write_buckets(connection_t *conn);
 
 int connection_handle_read(connection_t *conn);
 
-int connection_fetch_from_buf(char *string, size_t len, connection_t *conn);
-int connection_fetch_from_buf_line(connection_t *conn, char *data,
+int connection_buf_get_bytes(char *string, size_t len, connection_t *conn);
+int connection_buf_get_line(connection_t *conn, char *data,
                                    size_t *data_len);
 int connection_fetch_from_buf_http(connection_t *conn,
                                char **headers_out, size_t max_headerlen,
@@ -139,22 +148,23 @@ int connection_flush(connection_t *conn);
 MOCK_DECL(void, connection_write_to_buf_impl_,
           (const char *string, size_t len, connection_t *conn, int zlib));
 /* DOCDOC connection_write_to_buf */
-static void connection_write_to_buf(const char *string, size_t len,
+static void connection_buf_add(const char *string, size_t len,
                                     connection_t *conn);
-/* DOCDOC connection_write_to_buf_zlib */
-static void connection_write_to_buf_zlib(const char *string, size_t len,
-                                         dir_connection_t *conn, int done);
+/* DOCDOC connection_write_to_buf_compress */
+static void connection_buf_add_compress(const char *string, size_t len,
+                                             dir_connection_t *conn, int done);
 static inline void
-connection_write_to_buf(const char *string, size_t len, connection_t *conn)
+connection_buf_add(const char *string, size_t len, connection_t *conn)
 {
   connection_write_to_buf_impl_(string, len, conn, 0);
 }
 static inline void
-connection_write_to_buf_zlib(const char *string, size_t len,
-                             dir_connection_t *conn, int done)
+connection_buf_add_compress(const char *string, size_t len,
+                                 dir_connection_t *conn, int done)
 {
   connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1);
 }
+void connection_buf_add_buf(connection_t *conn, buf_t *buf);
 
 /* DOCDOC connection_get_inbuf_len */
 static size_t connection_get_inbuf_len(connection_t *conn);
@@ -182,6 +192,8 @@ MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type,
 connection_t *connection_get_by_type_state(int type, int state);
 connection_t *connection_get_by_type_state_rendquery(int type, int state,
                                                      const char *rendquery);
+smartlist_t *connection_list_by_type_state(int type, int state);
+smartlist_t *connection_list_by_type_purpose(int type, int purpose);
 smartlist_t *connection_dir_list_by_purpose_and_resource(
                                                   int purpose,
                                                   const char *resource);
@@ -241,11 +253,11 @@ char *alloc_http_authenticator(const char *authenticator);
 void assert_connection_ok(connection_t *conn, time_t now);
 int connection_or_nonopen_was_started_here(or_connection_t *conn);
 void connection_dump_buffer_mem_stats(int severity);
-void remove_file_if_very_old(const char *fname, time_t now);
 
-void clock_skew_warning(const connection_t *conn, long apparent_skew,
-                        int trusted, log_domain_mask_t domain,
-                        const char *received, const char *source);
+MOCK_DECL(void, clock_skew_warning,
+          (const connection_t *conn, long apparent_skew, int trusted,
+           log_domain_mask_t domain, const char *received,
+           const char *source));
 
 /** Check if a connection is on the way out so the OOS handler doesn't try
  * to kill more than it needs. */
@@ -264,16 +276,9 @@ connection_is_moribund(connection_t *conn)
 void connection_check_oos(int n_socks, int failed);
 
 #ifdef CONNECTION_PRIVATE
-STATIC void connection_free_(connection_t *conn);
+STATIC void connection_free_minimal(connection_t *conn);
 
 /* Used only by connection.c and test*.c */
-uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time,
-                             int tokens_after, int milliseconds_elapsed,
-                             const struct timeval *tvnow);
-void connection_buckets_note_empty_ts(uint32_t *timestamp_var,
-                                      int tokens_before,
-                                      size_t tokens_removed,
-                                      const struct timeval *tvnow);
 MOCK_DECL(STATIC int,connection_connect_sockaddr,
                                             (connection_t *conn,
                                              const struct sockaddr *sa,
@@ -284,7 +289,7 @@ MOCK_DECL(STATIC int,connection_connect_sockaddr,
 MOCK_DECL(STATIC void, kill_conn_list_for_oos, (smartlist_t *conns));
 MOCK_DECL(STATIC smartlist_t *, pick_oos_victims, (int n));
 
-#endif
+#endif /* defined(CONNECTION_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CONNECTION_H) */
 
diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c
index 7a97c632d1..88e346e9b8 100644
--- a/src/or/connection_edge.c
+++ b/src/or/connection_edge.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,7 @@
  *   <li>DNS lookup streams, created on the exit side in response to
  *     a RELAY_RESOLVE cell from a client.
  *   <li>Tunneled directory streams, created on the directory cache side
- *     in response to a RELAY_BEGINDIR cell.  These streams attach directly
+ *     in response to a RELAY_BEGIN_DIR cell.  These streams attach directly
  *     to a dir_connection_t object without ever using TCP.
  *   </ul>
  *
@@ -70,14 +70,22 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "crypto_util.h"
 #include "dns.h"
 #include "dnsserv.h"
 #include "directory.h"
 #include "dirserv.h"
 #include "hibernate.h"
+#include "hs_common.h"
+#include "hs_cache.h"
+#include "hs_client.h"
+#include "hs_circuit.h"
 #include "main.h"
+#include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "proto_http.h"
+#include "proto_socks.h"
 #include "reasons.h"
 #include "relay.h"
 #include "rendclient.h"
@@ -108,7 +116,7 @@
 #define TRANS_NETFILTER
 #define TRANS_NETFILTER_IPV6
 #endif
-#endif
+#endif /* defined(HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H) */
 
 #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
 #include <net/if.h>
@@ -151,7 +159,9 @@ connection_mark_unattached_ap_,(entry_connection_t *conn, int endreason,
    * but we should fix it someday anyway. */
   if ((edge_conn->on_circuit != NULL || edge_conn->edge_has_sent_end) &&
       connection_edge_is_rendezvous_stream(edge_conn)) {
-    rend_client_note_connection_attempt_ended(edge_conn->rend_data);
+    if (edge_conn->rend_data) {
+      rend_client_note_connection_attempt_ended(edge_conn->rend_data);
+    }
   }
 
   if (base_conn->marked_for_close) {
@@ -235,6 +245,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial)
         return -1;
       }
       return 0;
+    case AP_CONN_STATE_HTTP_CONNECT_WAIT:
+      if (connection_ap_process_http_connect(EDGE_TO_ENTRY_CONN(conn)) < 0) {
+        return -1;
+      }
+      return 0;
     case AP_CONN_STATE_OPEN:
     case EXIT_CONN_STATE_OPEN:
       if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) {
@@ -325,10 +340,41 @@ relay_send_end_cell_from_edge(streamid_t stream_id, circuit_t *circ,
 
   payload[0] = (char) reason;
 
+  /* Note: we have to use relay_send_command_from_edge here, not
+   * connection_edge_end or connection_edge_send_command, since those require
+   * that we have a stream connected to a circuit, and we don't connect to a
+   * circuit until we have a pending/successful resolve. */
   return relay_send_command_from_edge(stream_id, circ, RELAY_COMMAND_END,
                                       payload, 1, cpath_layer);
 }
 
+/* If the connection <b>conn</b> is attempting to connect to an external
+ * destination that is an hidden service and the reason is a connection
+ * refused or timeout, log it so the operator can take appropriate actions.
+ * The log statement is a rate limited warning. */
+static void
+warn_if_hs_unreachable(const edge_connection_t *conn, uint8_t reason)
+{
+  tor_assert(conn);
+
+  if (conn->base_.type == CONN_TYPE_EXIT &&
+      connection_edge_is_rendezvous_stream(conn) &&
+      (reason == END_STREAM_REASON_CONNECTREFUSED ||
+       reason == END_STREAM_REASON_TIMEOUT)) {
+#define WARN_FAILED_HS_CONNECTION 300
+    static ratelim_t warn_limit = RATELIM_INIT(WARN_FAILED_HS_CONNECTION);
+    char *m;
+    if ((m = rate_limit_log(&warn_limit, approx_time()))) {
+      log_warn(LD_EDGE, "Onion service connection to %s failed (%s)",
+               (conn->base_.socket_family == AF_UNIX) ?
+               safe_str(conn->base_.address) :
+               safe_str(fmt_addrport(&conn->base_.addr, conn->base_.port)),
+               stream_end_reason_to_string(reason));
+      tor_free(m);
+    }
+  }
+}
+
 /** Send a relay end cell from stream <b>conn</b> down conn's circuit, and
  * remember that we've done so.  If this is not a client connection, set the
  * relay end cell's reason for closing as <b>reason</b>.
@@ -384,8 +430,17 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
   if (circ && !circ->marked_for_close) {
     log_debug(LD_EDGE,"Sending end on conn (fd "TOR_SOCKET_T_FORMAT").",
               conn->base_.s);
+
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      connection_half_edge_add(conn, origin_circ);
+    }
+
     connection_edge_send_command(conn, RELAY_COMMAND_END,
                                  payload, payload_len);
+    /* We'll log warn if the connection was an hidden service and couldn't be
+     * made because the service wasn't available. */
+    warn_if_hs_unreachable(conn, control_reason);
   } else {
     log_debug(LD_EDGE,"No circ to send end on conn "
               "(fd "TOR_SOCKET_T_FORMAT").",
@@ -397,6 +452,236 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
   return 0;
 }
 
+/**
+ * Helper function for bsearch.
+ *
+ * As per smartlist_bsearch, return < 0 if key preceeds member,
+ * > 0 if member preceeds key, and 0 if they are equal.
+ *
+ * This is equivalent to subtraction of the values of key - member
+ * (why does no one ever say that explicitly?).
+ */
+static int
+connection_half_edge_compare_bsearch(const void *key, const void **member)
+{
+  const half_edge_t *e2;
+  tor_assert(key);
+  tor_assert(member && *(half_edge_t**)member);
+  e2 = *(const half_edge_t **)member;
+
+  return *(const streamid_t*)key - e2->stream_id;
+}
+
+/** Total number of half_edge_t objects allocated */
+static size_t n_half_conns_allocated = 0;
+
+/**
+ * Add a half-closed connection to the list, to watch for activity.
+ *
+ * These connections are removed from the list upon receiving an end
+ * cell.
+ */
+STATIC void
+connection_half_edge_add(const edge_connection_t *conn,
+                         origin_circuit_t *circ)
+{
+  half_edge_t *half_conn = NULL;
+  int insert_at = 0;
+  int ignored;
+
+  /* Double-check for re-insertion. This should not happen,
+   * but this check is cheap compared to the sort anyway */
+  if (connection_half_edge_find_stream_id(circ->half_streams,
+                                          conn->stream_id)) {
+    log_warn(LD_BUG, "Duplicate stream close for stream %d on circuit %d",
+             conn->stream_id, circ->global_identifier);
+    return;
+  }
+
+  half_conn = tor_malloc_zero(sizeof(half_edge_t));
+  ++n_half_conns_allocated;
+
+  if (!circ->half_streams) {
+    circ->half_streams = smartlist_new();
+  }
+
+  half_conn->stream_id = conn->stream_id;
+
+  // How many sendme's should I expect?
+  half_conn->sendmes_pending =
+   (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT;
+
+   // Is there a connected cell pending?
+  half_conn->connected_pending = conn->base_.state ==
+      AP_CONN_STATE_CONNECT_WAIT;
+
+  /* Data should only arrive if we're not waiting on a resolved cell.
+   * It can arrive after waiting on connected, because of optimistic
+   * data. */
+  if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) {
+    // How many more data cells can arrive on this id?
+    half_conn->data_pending = conn->deliver_window;
+  }
+
+  insert_at = smartlist_bsearch_idx(circ->half_streams, &half_conn->stream_id,
+                                    connection_half_edge_compare_bsearch,
+                                    &ignored);
+  smartlist_insert(circ->half_streams, insert_at, half_conn);
+}
+
+/** Release space held by <b>he</b> */
+void
+half_edge_free_(half_edge_t *he)
+{
+  if (!he)
+    return;
+  --n_half_conns_allocated;
+  tor_free(he);
+}
+
+/** Return the number of bytes devoted to storing info on half-open streams. */
+size_t
+half_streams_get_total_allocation(void)
+{
+  return n_half_conns_allocated * sizeof(half_edge_t);
+}
+
+/**
+ * Find a stream_id_t in the list in O(lg(n)).
+ *
+ * Returns NULL if the list is empty or element is not found.
+ * Returns a pointer to the element if found.
+ */
+STATIC half_edge_t *
+connection_half_edge_find_stream_id(const smartlist_t *half_conns,
+                                    streamid_t stream_id)
+{
+  if (!half_conns)
+    return NULL;
+
+  return smartlist_bsearch(half_conns, &stream_id,
+                           connection_half_edge_compare_bsearch);
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so,
+ * check if it still has data cells pending, and decrement that
+ * window if so.
+ *
+ * Return 1 if the data window was not empty.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_data(const smartlist_t *half_conns,
+                                   streamid_t stream_id)
+{
+  half_edge_t *half = connection_half_edge_find_stream_id(half_conns,
+                                                          stream_id);
+
+  if (!half)
+    return 0;
+
+  if (half->data_pending > 0) {
+    half->data_pending--;
+    return 1;
+  }
+
+  return 0;
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so,
+ * check if it still has a connected cell pending, and decrement
+ * that window if so.
+ *
+ * Return 1 if the connected window was not empty.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_connected(const smartlist_t *half_conns,
+                                        streamid_t stream_id)
+{
+  half_edge_t *half = connection_half_edge_find_stream_id(half_conns,
+                                                          stream_id);
+
+  if (!half)
+    return 0;
+
+  if (half->connected_pending) {
+    half->connected_pending = 0;
+    return 1;
+  }
+
+  return 0;
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so,
+ * check if it still has sendme cells pending, and decrement that
+ * window if so.
+ *
+ * Return 1 if the sendme window was not empty.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_sendme(const smartlist_t *half_conns,
+                                     streamid_t stream_id)
+{
+  half_edge_t *half = connection_half_edge_find_stream_id(half_conns,
+                                                          stream_id);
+
+  if (!half)
+    return 0;
+
+  if (half->sendmes_pending > 0) {
+    half->sendmes_pending--;
+    return 1;
+  }
+
+  return 0;
+}
+
+/**
+ * Check if this stream_id is in a half-closed state. If so, remove
+ * it from the list. No other data should come after the END cell.
+ *
+ * Return 1 if stream_id was in half-closed state.
+ * Return 0 otherwise.
+ */
+int
+connection_half_edge_is_valid_end(smartlist_t *half_conns,
+                                  streamid_t stream_id)
+{
+  half_edge_t *half;
+  int found, remove_idx;
+
+  if (!half_conns)
+    return 0;
+
+  remove_idx = smartlist_bsearch_idx(half_conns, &stream_id,
+                                    connection_half_edge_compare_bsearch,
+                                    &found);
+  if (!found)
+    return 0;
+
+  half = smartlist_get(half_conns, remove_idx);
+  smartlist_del_keeporder(half_conns, remove_idx);
+  half_edge_free(half);
+  return 1;
+}
+
+/**
+ * Streams that were used to send a RESOLVE cell are closed
+ * when they get the RESOLVED, without an end. So treat
+ * a RESOLVED just like an end, and remove from the list.
+ */
+int
+connection_half_edge_is_valid_resolved(smartlist_t *half_conns,
+                                       streamid_t stream_id)
+{
+  return connection_half_edge_is_valid_end(half_conns, stream_id);
+}
+
 /** An error has just occurred on an operation on an edge connection
  * <b>conn</b>.  Extract the errno; convert it to an end reason, and send an
  * appropriate relay end cell to the other end of the connection's circuit.
@@ -454,6 +739,7 @@ connection_edge_finished_flushing(edge_connection_t *conn)
     case AP_CONN_STATE_CONNECT_WAIT:
     case AP_CONN_STATE_CONTROLLER_WAIT:
     case AP_CONN_STATE_RESOLVE_WAIT:
+    case AP_CONN_STATE_HTTP_CONNECT_WAIT:
       return 0;
     default:
       log_warn(LD_BUG, "Called in unexpected state %d.",conn->base_.state);
@@ -562,6 +848,12 @@ static smartlist_t *pending_entry_connections = NULL;
 
 static int untried_pending_connections = 0;
 
+/**
+ * Mainloop event to tell us to scan for pending connections that can
+ * be attached.
+ */
+static mainloop_event_t *attach_pending_entry_connections_ev = NULL;
+
 /** Common code to connection_(ap|exit)_about_to_close. */
 static void
 connection_edge_about_to_close(edge_connection_t *edge_conn)
@@ -616,7 +908,7 @@ connection_ap_about_to_close(entry_connection_t *entry_conn)
     connection_ap_warn_and_unmark_if_pending_circ(entry_conn,
                                                   "about_to_close");
   }
-#endif
+#endif /* 1 */
 
   control_event_stream_bandwidth(edge_conn);
   control_event_stream_status(entry_conn, STREAM_EVENT_CLOSED,
@@ -690,7 +982,7 @@ connection_ap_expire_beginning(void)
     /* if it's an internal linked connection, don't yell its status. */
     severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port)
       ? LOG_INFO : LOG_NOTICE;
-    seconds_idle = (int)( now - base_conn->timestamp_lastread );
+    seconds_idle = (int)( now - base_conn->timestamp_last_read_allowed );
     seconds_since_born = (int)( now - base_conn->timestamp_created );
 
     if (base_conn->state == AP_CONN_STATE_OPEN)
@@ -743,7 +1035,10 @@ connection_ap_expire_beginning(void)
       }
       continue;
     }
+
     if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL &&
+        circ->purpose != CIRCUIT_PURPOSE_C_HSDIR_GET &&
+        circ->purpose != CIRCUIT_PURPOSE_S_HSDIR_POST &&
         circ->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT &&
         circ->purpose != CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
       log_warn(LD_BUG, "circuit->purpose == CIRCUIT_PURPOSE_C_GENERAL failed. "
@@ -773,7 +1068,7 @@ connection_ap_expire_beginning(void)
     mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ));
 
     /* give our stream another 'cutoff' seconds to try */
-    conn->base_.timestamp_lastread += cutoff;
+    conn->base_.timestamp_last_read_allowed += cutoff;
     if (entry_conn->num_socks_retries < 250) /* avoid overflow */
       entry_conn->num_socks_retries++;
     /* move it back into 'pending' state, and try to attach. */
@@ -826,12 +1121,13 @@ connection_ap_rescan_and_attach_pending(void)
     entry_conn->marked_pending_circ_line = 0;   \
     entry_conn->marked_pending_circ_file = 0;   \
   } while (0)
-#else
+#else /* !(defined(DEBUGGING_17659)) */
 #define UNMARK() do { } while (0)
-#endif
+#endif /* defined(DEBUGGING_17659) */
 
 /** Tell any AP streams that are listed as waiting for a new circuit to try
- * again, either attaching to an available circ or launching a new one.
+ * again.  If there is an available circuit for a stream, attach it. Otherwise,
+ * launch a new circuit.
  *
  * If <b>retry</b> is false, only check the list if it contains at least one
  * streams that we have not yet tried to attach to a circuit.
@@ -846,8 +1142,9 @@ connection_ap_attach_pending(int retry)
   if (untried_pending_connections == 0 && !retry)
     return;
 
-  /* Don't allow modifications to pending_entry_connections while we are
-   * iterating over it. */
+  /* Don't allow any modifications to list while we are iterating over
+   * it.  We'll put streams back on this list if we can't attach them
+   * immediately. */
   smartlist_t *pending = pending_entry_connections;
   pending_entry_connections = smartlist_new();
 
@@ -866,9 +1163,7 @@ connection_ap_attach_pending(int retry)
       continue;
     }
     if (conn->state != AP_CONN_STATE_CIRCUIT_WAIT) {
-      // XXXX 030 -- this is downgraded in 0.2.9, since we apparently
-      // XXXX are running into it in practice.  It's harmless.
-      log_info(LD_BUG, "%p is no longer in circuit_wait. Its current state "
+      log_warn(LD_BUG, "%p is no longer in circuit_wait. Its current state "
                "is %s. Why is it on pending_entry_connections?",
                entry_conn,
                conn_state_to_string(conn->type, conn->state));
@@ -876,6 +1171,7 @@ connection_ap_attach_pending(int retry)
       continue;
     }
 
+    /* Okay, we're through the sanity checks. Try to handle this stream. */
     if (connection_ap_handshake_attach_circuit(entry_conn) < 0) {
       if (!conn->marked_for_close)
         connection_mark_unattached_ap(entry_conn,
@@ -885,12 +1181,17 @@ connection_ap_attach_pending(int retry)
     if (! conn->marked_for_close &&
         conn->type == CONN_TYPE_AP &&
         conn->state == AP_CONN_STATE_CIRCUIT_WAIT) {
+      /* Is it still waiting for a circuit? If so, we didn't attach it,
+       * so it's still pending.  Put it back on the list.
+       */
       if (!smartlist_contains(pending_entry_connections, entry_conn)) {
         smartlist_add(pending_entry_connections, entry_conn);
         continue;
       }
     }
 
+    /* If we got here, then we either closed the connection, or
+     * we attached it. */
     UNMARK();
   } SMARTLIST_FOREACH_END(entry_conn);
 
@@ -898,6 +1199,14 @@ connection_ap_attach_pending(int retry)
   untried_pending_connections = 0;
 }
 
+static void
+attach_pending_entry_connections_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  connection_ap_attach_pending(0);
+}
+
 /** Mark <b>entry_conn</b> as needing to get attached to a circuit.
  *
  * And <b>entry_conn</b> must be in AP_CONN_STATE_CIRCUIT_WAIT,
@@ -915,9 +1224,13 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
   if (conn->marked_for_close)
     return;
 
-  if (PREDICT_UNLIKELY(NULL == pending_entry_connections))
+  if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) {
     pending_entry_connections = smartlist_new();
-
+  }
+  if (PREDICT_UNLIKELY(NULL == attach_pending_entry_connections_ev)) {
+    attach_pending_entry_connections_ev = mainloop_event_postloop_new(
+                                  attach_pending_entry_connections_cb, NULL);
+  }
   if (PREDICT_UNLIKELY(smartlist_contains(pending_entry_connections,
                                           entry_conn))) {
     log_warn(LD_BUG, "What?? pending_entry_connections already contains %p! "
@@ -928,7 +1241,7 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
     log_warn(LD_BUG, "(Previously called from %s:%d.)\n",
              f2 ? f2 : "<NULL>",
              entry_conn->marked_pending_circ_line);
-#endif
+#endif /* defined(DEBUGGING_17659) */
     log_backtrace(LOG_WARN, LD_BUG, "To debug, this may help");
     return;
   }
@@ -941,14 +1254,7 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn,
   untried_pending_connections = 1;
   smartlist_add(pending_entry_connections, entry_conn);
 
-  /* Work-around for bug 19969: we handle pending_entry_connections at
-   * the end of run_main_loop_once(), but in many cases that function will
-   * take a very long time, if ever, to finish its call to event_base_loop().
-   *
-   * So the fix is to tell it right now that it ought to finish its loop at
-   * its next available opportunity.
-   */
-  tell_event_loop_to_finish();
+  mainloop_event_activate(attach_pending_entry_connections_ev);
 }
 
 /** Mark <b>entry_conn</b> as no longer waiting for a circuit. */
@@ -1036,7 +1342,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info)
     if (!entry_conn->chosen_exit_optional &&
         !entry_conn->chosen_exit_retries)
       continue;
-    r1 = node_get_by_nickname(entry_conn->chosen_exit_name, 0);
+    r1 = node_get_by_nickname(entry_conn->chosen_exit_name,
+                              NNF_NO_WARN_UNNAMED);
     r2 = node_get_by_id(info->identity_digest);
     if (!r1 || !r2 || r1 != r2)
       continue;
@@ -1076,7 +1383,7 @@ connection_ap_detach_retriable(entry_connection_t *conn,
                                int reason)
 {
   control_event_stream_status(conn, STREAM_EVENT_FAILED_RETRIABLE, reason);
-  ENTRY_TO_CONN(conn)->timestamp_lastread = time(NULL);
+  ENTRY_TO_CONN(conn)->timestamp_last_read_allowed = time(NULL);
 
   /* Roll back path bias use state so that we probe the circuit
    * if nothing else succeeds on it */
@@ -1139,10 +1446,10 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port)
  *  See connection_ap_handshake_rewrite_and_attach()'s
  *  documentation for arguments and return value.
  */
-int
-connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn,
-                                            origin_circuit_t *circ,
-                                            crypt_path_t *cpath)
+MOCK_IMPL(int,
+connection_ap_rewrite_and_attach_if_allowed,(entry_connection_t *conn,
+                                             origin_circuit_t *circ,
+                                             crypt_path_t *cpath))
 {
   const or_options_t *options = get_options();
 
@@ -1185,10 +1492,9 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   /* Check for whether this is a .exit address.  By default, those are
    * disallowed when they're coming straight from the client, but you're
    * allowed to have them in MapAddress commands and so forth. */
-  if (!strcmpend(socks->address, ".exit") && !options->AllowDotExit) {
+  if (!strcmpend(socks->address, ".exit")) {
     log_warn(LD_APP, "The  \".exit\" notation is disabled in Tor due to "
-             "security risks. Set AllowDotExit in your torrc to enable "
-             "it (at your own risk).");
+             "security risks.");
     control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                 escaped(socks->address));
     out->end_reason = END_STREAM_REASON_TORPROTOCOL;
@@ -1198,6 +1504,8 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
 
   /* Remember the original address so we can tell the user about what
    * they actually said, not just what it turned into. */
+  /* XXX yes, this is the same as out->orig_address above. One is
+   * in the output, and one is in the connection. */
   if (! conn->original_dest_address) {
     /* Is the 'if' necessary here? XXXX */
     conn->original_dest_address = tor_strdup(conn->socks_request->address);
@@ -1205,7 +1513,7 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
 
   /* First, apply MapAddress and MAPADDRESS mappings. We need to do
    * these only for non-reverse lookups, since they don't exist for those.
-   * We need to do this before we consider automapping, since we might
+   * We also need to do this before we consider automapping, since we might
    * e.g. resolve irc.oftc.net into irconionaddress.onion, at which point
    * we'd need to automap it. */
   if (socks->command != SOCKS_COMMAND_RESOLVE_PTR) {
@@ -1217,9 +1525,12 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
     }
   }
 
-  /* Now, handle automapping.  Automapping happens when we're asked to
-   * resolve a hostname, and AutomapHostsOnResolve is set, and
-   * the hostname has a suffix listed in AutomapHostsSuffixes.
+  /* Now see if we need to create or return an existing Hostname->IP
+   * automapping.  Automapping happens when we're asked to resolve a
+   * hostname, and AutomapHostsOnResolve is set, and the hostname has a
+   * suffix listed in AutomapHostsSuffixes.  It's a handy feature
+   * that lets you have Tor assign e.g. IPv6 addresses for .onion
+   * names, and return them safely from DNSPort.
    */
   if (socks->command == SOCKS_COMMAND_RESOLVE &&
       tor_addr_parse(&addr_tmp, socks->address)<0 &&
@@ -1259,7 +1570,8 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   }
 
   /* Now handle reverse lookups, if they're in the cache.  This doesn't
-   * happen too often, since client-side DNS caching is off by default. */
+   * happen too often, since client-side DNS caching is off by default,
+   * and very deprecated. */
   if (socks->command == SOCKS_COMMAND_RESOLVE_PTR) {
     unsigned rewrite_flags = 0;
     if (conn->entry_cfg.use_cached_ipv4_answers)
@@ -1288,7 +1600,7 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
      * an internal address?  If so, we should reject it if we're configured to
      * do so. */
     if (options->ClientDNSRejectInternalAddresses) {
-      /* Don't let people try to do a reverse lookup on 10.0.0.1. */
+      /* Don't let clients try to do a reverse lookup on 10.0.0.1. */
       tor_addr_t addr;
       int ok;
       ok = tor_addr_parse_PTR_name(
@@ -1304,11 +1616,12 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
     }
   }
 
-  /* If we didn't automap it before, then this is still the address
-   * that came straight from the user, mapped according to any
-   * MapAddress/MAPADDRESS commands.  Now other mappings, including
-   * previously registered Automap entries, TrackHostExits entries,
-   * and client-side DNS cache entries (not recommended).
+  /* If we didn't automap it before, then this is still the address that
+   * came straight from the user, mapped according to any
+   * MapAddress/MAPADDRESS commands.  Now apply other mappings,
+   * including previously registered Automap entries (IP back to
+   * hostname), TrackHostExits entries, and client-side DNS cache
+   * entries (if they're turned on).
    */
   if (socks->command != SOCKS_COMMAND_RESOLVE_PTR &&
       !out->automap) {
@@ -1347,6 +1660,199 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   }
 }
 
+/** We just received a SOCKS request in <b>conn</b> to an onion address of type
+ *  <b>addresstype</b>. Start connecting to the onion service. */
+static int
+connection_ap_handle_onion(entry_connection_t *conn,
+                           socks_request_t *socks,
+                           origin_circuit_t *circ,
+                           hostname_type_t addresstype)
+{
+  time_t now = approx_time();
+  connection_t *base_conn = ENTRY_TO_CONN(conn);
+
+  /* If .onion address requests are disabled, refuse the request */
+  if (!conn->entry_cfg.onion_traffic) {
+    log_warn(LD_APP, "Onion address %s requested from a port with .onion "
+             "disabled", safe_str_client(socks->address));
+    connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+    return -1;
+  }
+
+  /* Check whether it's RESOLVE or RESOLVE_PTR.  We don't handle those
+   * for hidden service addresses. */
+  if (SOCKS_COMMAND_IS_RESOLVE(socks->command)) {
+    /* if it's a resolve request, fail it right now, rather than
+     * building all the circuits and then realizing it won't work. */
+    log_warn(LD_APP,
+             "Resolve requests to hidden services not allowed. Failing.");
+    connection_ap_handshake_socks_resolved(conn,RESOLVED_TYPE_ERROR,
+                                           0,NULL,-1,TIME_MAX);
+    connection_mark_unattached_ap(conn,
+                               END_STREAM_REASON_SOCKSPROTOCOL |
+                               END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED);
+    return -1;
+  }
+
+  /* If we were passed a circuit, then we need to fail.  .onion addresses
+   * only work when we launch our own circuits for now. */
+  if (circ) {
+    log_warn(LD_CONTROL, "Attachstream to a circuit is not "
+             "supported for .onion addresses currently. Failing.");
+    connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
+    return -1;
+  }
+
+  /* Interface: Regardless of HS version after the block below we should have
+     set onion_address, rend_cache_lookup_result, and descriptor_is_usable. */
+  const char *onion_address = NULL;
+  int rend_cache_lookup_result = -ENOENT;
+  int descriptor_is_usable = 0;
+
+  if (addresstype == ONION_V2_HOSTNAME) { /* it's a v2 hidden service */
+    rend_cache_entry_t *entry = NULL;
+    /* Look up if we have client authorization configured for this hidden
+     * service.  If we do, associate it with the rend_data. */
+    rend_service_authorization_t *client_auth =
+      rend_client_lookup_service_authorization(socks->address);
+
+    const uint8_t *cookie = NULL;
+    rend_auth_type_t auth_type = REND_NO_AUTH;
+    if (client_auth) {
+      log_info(LD_REND, "Using previously configured client authorization "
+               "for hidden service request.");
+      auth_type = client_auth->auth_type;
+      cookie = client_auth->descriptor_cookie;
+    }
+
+    /* Fill in the rend_data field so we can start doing a connection to
+     * a hidden service. */
+    rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data =
+      rend_data_client_create(socks->address, NULL, (char *) cookie,
+                              auth_type);
+    if (rend_data == NULL) {
+      return -1;
+    }
+    onion_address = rend_data_get_address(rend_data);
+    log_info(LD_REND,"Got a hidden service request for ID '%s'",
+             safe_str_client(onion_address));
+
+    rend_cache_lookup_result = rend_cache_lookup_entry(onion_address,-1,
+                                                       &entry);
+    if (!rend_cache_lookup_result && entry) {
+      descriptor_is_usable = rend_client_any_intro_points_usable(entry);
+    }
+  } else { /* it's a v3 hidden service */
+    tor_assert(addresstype == ONION_V3_HOSTNAME);
+    const hs_descriptor_t *cached_desc = NULL;
+    int retval;
+    /* Create HS conn identifier with HS pubkey */
+    hs_ident_edge_conn_t *hs_conn_ident =
+      tor_malloc_zero(sizeof(hs_ident_edge_conn_t));
+
+    retval = hs_parse_address(socks->address, &hs_conn_ident->identity_pk,
+                              NULL, NULL);
+    if (retval < 0) {
+      log_warn(LD_GENERAL, "failed to parse hs address");
+      tor_free(hs_conn_ident);
+      return -1;
+    }
+    ENTRY_TO_EDGE_CONN(conn)->hs_ident = hs_conn_ident;
+
+    onion_address = socks->address;
+
+    /* Check the v3 desc cache */
+    cached_desc = hs_cache_lookup_as_client(&hs_conn_ident->identity_pk);
+    if (cached_desc) {
+      rend_cache_lookup_result = 0;
+      descriptor_is_usable =
+        hs_client_any_intro_points_usable(&hs_conn_ident->identity_pk,
+                                          cached_desc);
+      log_info(LD_GENERAL, "Found %s descriptor in cache for %s. %s.",
+               (descriptor_is_usable) ? "usable" : "unusable",
+               safe_str_client(onion_address),
+               (descriptor_is_usable) ? "Not fetching." : "Refecting.");
+    } else {
+      rend_cache_lookup_result = -ENOENT;
+    }
+  }
+
+  /* Lookup the given onion address. If invalid, stop right now.
+   * Otherwise, we might have it in the cache or not. */
+  unsigned int refetch_desc = 0;
+  if (rend_cache_lookup_result < 0) {
+    switch (-rend_cache_lookup_result) {
+    case EINVAL:
+      /* We should already have rejected this address! */
+      log_warn(LD_BUG,"Invalid service name '%s'",
+               safe_str_client(onion_address));
+      connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
+      return -1;
+    case ENOENT:
+      /* We didn't have this; we should look it up. */
+      log_info(LD_REND, "No descriptor found in our cache for %s. Fetching.",
+               safe_str_client(onion_address));
+      refetch_desc = 1;
+      break;
+    default:
+      log_warn(LD_BUG, "Unknown cache lookup error %d",
+               rend_cache_lookup_result);
+      return -1;
+    }
+  }
+
+  /* Help predict that we'll want to do hidden service circuits in the
+   * future. We're not sure if it will need a stable circuit yet, but
+   * we know we'll need *something*. */
+  rep_hist_note_used_internal(now, 0, 1);
+
+  /* Now we have a descriptor but is it usable or not? If not, refetch.
+   * Also, a fetch could have been requested if the onion address was not
+   * found in the cache previously. */
+  if (refetch_desc || !descriptor_is_usable) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn);
+    connection_ap_mark_as_non_pending_circuit(conn);
+    base_conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+    if (addresstype == ONION_V2_HOSTNAME) {
+      tor_assert(edge_conn->rend_data);
+      rend_client_refetch_v2_renddesc(edge_conn->rend_data);
+      /* Whatever the result of the refetch, we don't go further. */
+      return 0;
+    } else {
+      tor_assert(addresstype == ONION_V3_HOSTNAME);
+      tor_assert(edge_conn->hs_ident);
+      /* Attempt to fetch the hsv3 descriptor. Check the retval to see how it
+       * went and act accordingly. */
+      int ret = hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+      switch (ret) {
+      case HS_CLIENT_FETCH_MISSING_INFO:
+        /* Keeping the connection in descriptor wait state is fine because
+         * once we get enough dirinfo or a new live consensus, the HS client
+         * subsystem is notified and every connection in that state will
+         * trigger a fetch for the service key. */
+      case HS_CLIENT_FETCH_LAUNCHED:
+      case HS_CLIENT_FETCH_PENDING:
+      case HS_CLIENT_FETCH_HAVE_DESC:
+        return 0;
+      case HS_CLIENT_FETCH_ERROR:
+      case HS_CLIENT_FETCH_NO_HSDIRS:
+      case HS_CLIENT_FETCH_NOT_ALLOWED:
+        /* Can't proceed further and better close the SOCKS request. */
+        return -1;
+      }
+    }
+  }
+
+  /* We have the descriptor!  So launch a connection to the HS. */
+  log_info(LD_REND, "Descriptor is here. Great.");
+
+  base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
+  /* We'll try to attach it at the next event loop, or whenever
+   * we call connection_ap_attach_pending() */
+  connection_ap_mark_as_pending_circuit(conn);
+  return 0;
+}
+
 /** Connection <b>conn</b> just finished its socks handshake, or the
  * controller asked us to take care of it. If <b>circ</b> is defined,
  * then that's where we'll want to attach it. Otherwise we have to
@@ -1373,11 +1879,14 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   time_t now = time(NULL);
   rewrite_result_t rr;
 
+  /* First we'll do the rewrite part.  Let's see if we get a reasonable
+   * answer.
+   */
   memset(&rr, 0, sizeof(rr));
   connection_ap_handshake_rewrite(conn,&rr);
 
   if (rr.should_close) {
-    /* connection_ap_handshake_rewrite told us to close the connection,
+    /* connection_ap_handshake_rewrite told us to close the connection:
      * either because it sent back an answer, or because it sent back an
      * error */
     connection_mark_unattached_ap(conn, rr.end_reason);
@@ -1391,8 +1900,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   const int automap = rr.automap;
   const addressmap_entry_source_t exit_source = rr.exit_source;
 
-  /* Parse the address provided by SOCKS.  Modify it in-place if it
-   * specifies a hidden-service (.onion) or particular exit node (.exit).
+  /* Now, we parse the address to see if it's an .onion or .exit or
+   * other special address.
    */
   const hostname_type_t addresstype = parse_extended_hostname(socks->address);
 
@@ -1406,8 +1915,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   }
 
   /* If this is a .exit hostname, strip off the .name.exit part, and
-   * see whether we're going to connect there, and otherwise handle it.
-   * (The ".exit" part got stripped off by "parse_extended_hostname").
+   * see whether we're willing to connect there, and and otherwise handle the
+   * .exit address.
    *
    * We'll set chosen_exit_name and/or close the connection as appropriate.
    */
@@ -1419,23 +1928,23 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     const node_t *node = NULL;
 
     /* If this .exit was added by an AUTOMAP, then it came straight from
-     * a user.  Make sure that options->AllowDotExit permits that. */
-    if (exit_source == ADDRMAPSRC_AUTOMAP && !options->AllowDotExit) {
-      /* Whoops; this one is stale.  It must have gotten added earlier,
-       * when AllowDotExit was on. */
-      log_warn(LD_APP,"Stale automapped address for '%s.exit', with "
-               "AllowDotExit disabled. Refusing.",
+     * a user.  That's not safe. */
+    if (exit_source == ADDRMAPSRC_AUTOMAP) {
+      /* Whoops; this one is stale.  It must have gotten added earlier?
+       * (Probably this is not possible, since AllowDotExit no longer
+       * exists.) */
+      log_warn(LD_APP,"Stale automapped address for '%s.exit'. Refusing.",
                safe_str_client(socks->address));
       control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                   escaped(socks->address));
       connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
+      tor_assert_nonfatal_unreached();
       return -1;
     }
 
     /* Double-check to make sure there are no .exits coming from
      * impossible/weird sources. */
-    if (exit_source == ADDRMAPSRC_DNS ||
-        (exit_source == ADDRMAPSRC_NONE && !options->AllowDotExit)) {
+    if (exit_source == ADDRMAPSRC_DNS || exit_source == ADDRMAPSRC_NONE) {
       /* It shouldn't be possible to get a .exit address from any of these
        * sources. */
       log_warn(LD_BUG,"Address '%s.exit', with impossible source for the "
@@ -1448,14 +1957,19 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     }
 
     tor_assert(!automap);
-    /* Now, find the character before the .(name) part. */
+
+    /* Now, find the character before the .(name) part.
+     * (The ".exit" part got stripped off by "parse_extended_hostname").
+     *
+     * We're going to put the exit name into conn->chosen_exit_name, and
+     * look up a node correspondingly. */
     char *s = strrchr(socks->address,'.');
     if (s) {
       /* The address was of the form "(stuff).(name).exit */
       if (s[1] != '\0') {
         /* Looks like a real .exit one. */
         conn->chosen_exit_name = tor_strdup(s+1);
-        node = node_get_by_nickname(conn->chosen_exit_name, 1);
+        node = node_get_by_nickname(conn->chosen_exit_name, 0);
 
         if (exit_source == ADDRMAPSRC_TRACKEXIT) {
           /* We 5 tries before it expires the addressmap */
@@ -1476,7 +1990,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
        * form that means (foo's address).foo.exit. */
 
       conn->chosen_exit_name = tor_strdup(socks->address);
-      node = node_get_by_nickname(conn->chosen_exit_name, 1);
+      node = node_get_by_nickname(conn->chosen_exit_name, 0);
       if (node) {
         *socks->address = 0;
         node_get_address_string(node, socks->address, sizeof(socks->address));
@@ -1504,10 +2018,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
        implies no. */
   }
 
-  /* Now, handle everything that isn't a .onion address. */
-  if (addresstype != ONION_HOSTNAME) {
+  /* Now, we handle everything that isn't a .onion address. */
+  if (addresstype != ONION_V2_HOSTNAME && addresstype != ONION_V3_HOSTNAME) {
     /* Not a hidden-service request.  It's either a hostname or an IP,
-     * possibly with a .exit that we stripped off. */
+     * possibly with a .exit that we stripped off.  We're going to check
+     * if we're allowed to connect/resolve there, and then launch the
+     * appropriate request. */
 
     /* Check for funny characters in the address. */
     if (address_is_invalid_destination(socks->address, 1)) {
@@ -1530,7 +2046,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
       return -1;
     }
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
 
     /* socks->address is a non-onion hostname or IP address.
      * If we can't do any non-onion requests, refuse the connection.
@@ -1554,30 +2070,37 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     }
 
     /* Then check if we have a hostname or IP address, and whether DNS or
-     * the IP address family are permitted */
+     * the IP address family are permitted.  Reject if not. */
     tor_addr_t dummy_addr;
     int socks_family = tor_addr_parse(&dummy_addr, socks->address);
     /* family will be -1 for a non-onion hostname that's not an IP */
-    if (socks_family == -1 && !conn->entry_cfg.dns_request) {
-      log_warn(LD_APP, "Refusing to connect to hostname %s "
-               "because Port has NoDNSRequest set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    } else if (socks_family == AF_INET && !conn->entry_cfg.ipv4_traffic) {
-      log_warn(LD_APP, "Refusing to connect to IPv4 address %s because "
-               "Port has NoIPv4Traffic set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    } else if (socks_family == AF_INET6 && !conn->entry_cfg.ipv6_traffic) {
-      log_warn(LD_APP, "Refusing to connect to IPv6 address %s because "
-               "Port has NoIPv6Traffic set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
+    if (socks_family == -1) {
+      if (!conn->entry_cfg.dns_request) {
+        log_warn(LD_APP, "Refusing to connect to hostname %s "
+                 "because Port has NoDNSRequest set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else if (socks_family == AF_INET) {
+      if (!conn->entry_cfg.ipv4_traffic) {
+        log_warn(LD_APP, "Refusing to connect to IPv4 address %s because "
+                 "Port has NoIPv4Traffic set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else if (socks_family == AF_INET6) {
+      if (!conn->entry_cfg.ipv6_traffic) {
+        log_warn(LD_APP, "Refusing to connect to IPv6 address %s because "
+                 "Port has NoIPv6Traffic set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else {
+      tor_assert_nonfatal_unreached_once();
     }
-    /* No else, we've covered all possible returned value. */
 
     /* See if this is a hostname lookup that we can answer immediately.
      * (For example, an attempt to look up the IP address for an IP address.)
@@ -1598,7 +2121,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       tor_assert(!automap);
       rep_hist_note_used_resolve(now); /* help predict this next time */
     } else if (socks->command == SOCKS_COMMAND_CONNECT) {
-      /* Special handling for attempts to connect */
+      /* Now see if this is a connect request that we can reject immediately */
+
       tor_assert(!automap);
       /* Don't allow connections to port 0. */
       if (socks->port == 0) {
@@ -1607,7 +2131,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         return -1;
       }
       /* You can't make connections to internal addresses, by default.
-       * Exceptions are begindir requests (where the address is meaningless,
+       * Exceptions are begindir requests (where the address is meaningless),
        * or cases where you've hand-configured a particular exit, thereby
        * making the local address meaningful. */
       if (options->ClientRejectInternalAddresses &&
@@ -1651,7 +2175,9 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       } /* end "if we should check for internal addresses" */
 
       /* Okay.  We're still doing a CONNECT, and it wasn't a private
-       * address.  Do special handling for literal IP addresses */
+       * address.  Here we do special handling for literal IP addresses,
+       * to see if we should reject this preemptively, and to set up
+       * fields in conn->entry_cfg to tell the exit what AF we want. */
       {
         tor_addr_t addr;
         /* XXX Duplicate call to tor_addr_parse. */
@@ -1694,11 +2220,15 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         }
       }
 
+      /* we never allow IPv6 answers on socks4. (TODO: Is this smart?) */
       if (socks->socks_version == 4)
         conn->entry_cfg.ipv6_traffic = 0;
 
       /* Still handling CONNECT. Now, check for exit enclaves.  (Which we
-       * don't do on BEGINDIR, or there is a chosen exit.)
+       * don't do on BEGIN_DIR, or when there is a chosen exit.)
+       *
+       * TODO: Should we remove this?  Exit enclaves are nutty and don't
+       * work very well
        */
       if (!conn->use_begindir && !conn->chosen_exit_name && !circ) {
         /* see if we can find a suitable enclave exit */
@@ -1722,7 +2252,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         if (consider_plaintext_ports(conn, socks->port) < 0)
           return -1;
 
-      /* Remember the port so that we do predicted requests there. */
+      /* Remember the port so that we will predict that more requests
+         there will happen in the future. */
       if (!conn->use_begindir) {
         /* help predict this next time */
         rep_hist_note_used_port(now, socks->port);
@@ -1731,7 +2262,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       rep_hist_note_used_resolve(now); /* help predict this next time */
       /* no extra processing needed */
     } else {
-      /* We should only be doing CONNECT or RESOLVE! */
+      /* We should only be doing CONNECT, RESOLVE, or RESOLVE_PTR! */
       tor_fragile_assert();
     }
 
@@ -1747,6 +2278,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     if (circ) {
       rv = connection_ap_handshake_attach_chosen_circuit(conn, circ, cpath);
     } else {
+      /* We'll try to attach it at the next event loop, or whenever
+       * we call connection_ap_attach_pending() */
       connection_ap_mark_as_pending_circuit(conn);
       rv = 0;
     }
@@ -1764,110 +2297,10 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     return 0;
   } else {
     /* If we get here, it's a request for a .onion address! */
+    tor_assert(addresstype == ONION_V2_HOSTNAME ||
+               addresstype == ONION_V3_HOSTNAME);
     tor_assert(!automap);
-
-    /* If .onion address requests are disabled, refuse the request */
-    if (!conn->entry_cfg.onion_traffic) {
-      log_warn(LD_APP, "Onion address %s requested from a port with .onion "
-                       "disabled", safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    }
-
-    /* Check whether it's RESOLVE or RESOLVE_PTR.  We don't handle those
-     * for hidden service addresses. */
-    if (SOCKS_COMMAND_IS_RESOLVE(socks->command)) {
-      /* if it's a resolve request, fail it right now, rather than
-       * building all the circuits and then realizing it won't work. */
-      log_warn(LD_APP,
-               "Resolve requests to hidden services not allowed. Failing.");
-      connection_ap_handshake_socks_resolved(conn,RESOLVED_TYPE_ERROR,
-                                             0,NULL,-1,TIME_MAX);
-      connection_mark_unattached_ap(conn,
-                                END_STREAM_REASON_SOCKSPROTOCOL |
-                                END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED);
-      return -1;
-    }
-
-    /* If we were passed a circuit, then we need to fail.  .onion addresses
-     * only work when we launch our own circuits for now. */
-    if (circ) {
-      log_warn(LD_CONTROL, "Attachstream to a circuit is not "
-               "supported for .onion addresses currently. Failing.");
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
-      return -1;
-    }
-
-    /* Look up if we have client authorization configured for this hidden
-     * service.  If we do, associate it with the rend_data. */
-    rend_service_authorization_t *client_auth =
-      rend_client_lookup_service_authorization(socks->address);
-
-    const uint8_t *cookie = NULL;
-    rend_auth_type_t auth_type = REND_NO_AUTH;
-    if (client_auth) {
-      log_info(LD_REND, "Using previously configured client authorization "
-               "for hidden service request.");
-      auth_type = client_auth->auth_type;
-      cookie = client_auth->descriptor_cookie;
-    }
-
-    /* Fill in the rend_data field so we can start doing a connection to
-     * a hidden service. */
-    rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data =
-      rend_data_client_create(socks->address, NULL, (char *) cookie,
-                              auth_type);
-    if (rend_data == NULL) {
-      return -1;
-    }
-    log_info(LD_REND,"Got a hidden service request for ID '%s'",
-             safe_str_client(rend_data->onion_address));
-
-    /* Lookup the given onion address. If invalid, stop right now else we
-     * might have it in the cache or not, it will be tested later on. */
-    unsigned int refetch_desc = 0;
-    rend_cache_entry_t *entry = NULL;
-    const int rend_cache_lookup_result =
-      rend_cache_lookup_entry(rend_data->onion_address, -1, &entry);
-    if (rend_cache_lookup_result < 0) {
-      switch (-rend_cache_lookup_result) {
-      case EINVAL:
-        /* We should already have rejected this address! */
-        log_warn(LD_BUG,"Invalid service name '%s'",
-            safe_str_client(rend_data->onion_address));
-        connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
-        return -1;
-      case ENOENT:
-        refetch_desc = 1;
-        break;
-      default:
-        log_warn(LD_BUG, "Unknown cache lookup error %d",
-            rend_cache_lookup_result);
-        return -1;
-      }
-    }
-
-    /* Help predict this next time. We're not sure if it will need
-     * a stable circuit yet, but we know we'll need *something*. */
-    rep_hist_note_used_internal(now, 0, 1);
-
-    /* Now we have a descriptor but is it usable or not? If not, refetch.
-     * Also, a fetch could have been requested if the onion address was not
-     * found in the cache previously. */
-    if (refetch_desc || !rend_client_any_intro_points_usable(entry)) {
-      connection_ap_mark_as_non_pending_circuit(conn);
-      base_conn->state = AP_CONN_STATE_RENDDESC_WAIT;
-      log_info(LD_REND, "Unknown descriptor %s. Fetching.",
-          safe_str_client(rend_data->onion_address));
-      rend_client_refetch_v2_renddesc(rend_data);
-      return 0;
-    }
-
-    /* We have the descriptor so launch a connection to the HS. */
-    base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
-    log_info(LD_REND, "Descriptor is here. Great.");
-    connection_ap_mark_as_pending_circuit(conn);
-    return 0;
+    return connection_ap_handle_onion(conn, socks, circ, addresstype);
   }
 
   return 0; /* unreached but keeps the compiler happy */
@@ -1883,13 +2316,13 @@ get_pf_socket(void)
   if (pf_socket >= 0)
     return pf_socket;
 
-#ifdef OPENBSD
+#if defined(OpenBSD)
   /* only works on OpenBSD */
   pf = tor_open_cloexec("/dev/pf", O_RDONLY, 0);
 #else
   /* works on NetBSD and FreeBSD */
   pf = tor_open_cloexec("/dev/pf", O_RDWR, 0);
-#endif
+#endif /* defined(OpenBSD) */
 
   if (pf < 0) {
     log_warn(LD_NET, "open(\"/dev/pf\") failed: %s", strerror(errno));
@@ -1899,9 +2332,10 @@ get_pf_socket(void)
   pf_socket = pf;
   return pf_socket;
 }
-#endif
+#endif /* defined(TRANS_PF) */
 
-#if defined(TRANS_NETFILTER) || defined(TRANS_PF) || defined(TRANS_TPROXY)
+#if defined(TRANS_NETFILTER) || defined(TRANS_PF) || \
+  defined(TRANS_TPROXY)
 /** Try fill in the address of <b>req</b> from the socket configured
  * with <b>conn</b>. */
 static int
@@ -1921,7 +2355,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
     }
     goto done;
   }
-#endif
+#endif /* defined(TRANS_TPROXY) */
 
 #ifdef TRANS_NETFILTER
   int rv = -1;
@@ -1931,13 +2365,13 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
       rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST,
                   (struct sockaddr*)&orig_dst, &orig_dst_len);
       break;
-#endif
+#endif /* defined(TRANS_NETFILTER_IPV4) */
 #ifdef TRANS_NETFILTER_IPV6
     case AF_INET6:
       rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IPV6, IP6T_SO_ORIGINAL_DST,
                   (struct sockaddr*)&orig_dst, &orig_dst_len);
       break;
-#endif
+#endif /* defined(TRANS_NETFILTER_IPV6) */
     default:
       log_warn(LD_BUG,
                "Received transparent data from an unsuported socket family %d",
@@ -1963,7 +2397,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
   (void)req;
   log_warn(LD_BUG, "Unable to determine destination from socket.");
   return -1;
-#endif
+#endif /* defined(TRANS_NETFILTER) || ... */
 
  done:
   tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port);
@@ -1971,7 +2405,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req)
 
   return 0;
 }
-#endif
+#endif /* defined(TRANS_NETFILTER) || defined(TRANS_PF) || ... */
 
 #ifdef TRANS_PF
 static int
@@ -2005,7 +2439,7 @@ destination_from_pf(entry_connection_t *conn, socks_request_t *req)
 
     return 0;
   }
-#endif
+#endif /* defined(__FreeBSD__) */
 
   memset(&pnl, 0, sizeof(pnl));
   pnl.proto           = IPPROTO_TCP;
@@ -2054,7 +2488,7 @@ destination_from_pf(entry_connection_t *conn, socks_request_t *req)
 
   return 0;
 }
-#endif
+#endif /* defined(TRANS_PF) */
 
 /** Fetch the original destination address and port from a
  * system-specific interface and put them into a
@@ -2090,7 +2524,7 @@ connection_ap_get_original_destination(entry_connection_t *conn,
   log_warn(LD_BUG, "Called connection_ap_get_original_destination, but no "
            "transparent proxy method was configured.");
   return -1;
-#endif
+#endif /* defined(TRANS_NETFILTER) || ... */
 }
 
 /** connection_edge_process_inbuf() found a conn in state
@@ -2125,7 +2559,7 @@ connection_ap_handshake_process_socks(entry_connection_t *conn)
 
   if (socks->replylen) {
     had_reply = 1;
-    connection_write_to_buf((const char*)socks->reply, socks->replylen,
+    connection_buf_add((const char*)socks->reply, socks->replylen,
                             base_conn);
     socks->replylen = 0;
     if (sockshere == -1) {
@@ -2222,7 +2656,7 @@ connection_ap_process_natd(entry_connection_t *conn)
 
   /* look for LF-terminated "[DEST ip_addr port]"
    * where ip_addr is a dotted-quad and port is in string form */
-  err = connection_fetch_from_buf_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen);
+  err = connection_buf_get_line(ENTRY_TO_CONN(conn), tmp_buf, &tlen);
   if (err == 0)
     return 0;
   if (err < 0) {
@@ -2271,6 +2705,111 @@ connection_ap_process_natd(entry_connection_t *conn)
   return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
 }
 
+/** Called on an HTTP CONNECT entry connection when some bytes have arrived,
+ * but we have not yet received a full HTTP CONNECT request.  Try to parse an
+ * HTTP CONNECT request from the connection's inbuf.  On success, set up the
+ * connection's socks_request field and try to attach the connection.  On
+ * failure, send an HTTP reply, and mark the connection.
+ */
+STATIC int
+connection_ap_process_http_connect(entry_connection_t *conn)
+{
+  if (BUG(ENTRY_TO_CONN(conn)->state != AP_CONN_STATE_HTTP_CONNECT_WAIT))
+    return -1;
+
+  char *headers = NULL, *body = NULL;
+  char *command = NULL, *addrport = NULL;
+  char *addr = NULL;
+  size_t bodylen = 0;
+
+  const char *errmsg = NULL;
+  int rv = 0;
+
+  const int http_status =
+    fetch_from_buf_http(ENTRY_TO_CONN(conn)->inbuf, &headers, 8192,
+                        &body, &bodylen, 1024, 0);
+  if (http_status < 0) {
+    /* Bad http status */
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    goto err;
+  } else if (http_status == 0) {
+    /* no HTTP request yet. */
+    goto done;
+  }
+
+  const int cmd_status = parse_http_command(headers, &command, &addrport);
+  if (cmd_status < 0) {
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    goto err;
+  }
+  tor_assert(command);
+  tor_assert(addrport);
+  if (strcasecmp(command, "connect")) {
+    errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n";
+    goto err;
+  }
+
+  tor_assert(conn->socks_request);
+  socks_request_t *socks = conn->socks_request;
+  uint16_t port;
+  if (tor_addr_port_split(LOG_WARN, addrport, &addr, &port) < 0) {
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    goto err;
+  }
+  if (strlen(addr) >= MAX_SOCKS_ADDR_LEN) {
+    errmsg = "HTTP/1.0 414 Request-URI Too Long\r\n\r\n";
+    goto err;
+  }
+
+  /* Abuse the 'username' and 'password' fields here. They are already an
+  * abuse. */
+  {
+    char *authorization = http_get_header(headers, "Proxy-Authorization: ");
+    if (authorization) {
+      socks->username = authorization; // steal reference
+      socks->usernamelen = strlen(authorization);
+    }
+    char *isolation = http_get_header(headers, "X-Tor-Stream-Isolation: ");
+    if (isolation) {
+      socks->password = isolation; // steal reference
+      socks->passwordlen = strlen(isolation);
+    }
+  }
+
+  socks->command = SOCKS_COMMAND_CONNECT;
+  socks->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  strlcpy(socks->address, addr, sizeof(socks->address));
+  socks->port = port;
+
+  control_event_stream_status(conn, STREAM_EVENT_NEW, 0);
+
+  rv = connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
+
+  // XXXX send a "100 Continue" message?
+
+  goto done;
+
+ err:
+  if (BUG(errmsg == NULL))
+    errmsg = "HTTP/1.0 400 Bad Request\r\n\r\n";
+  log_info(LD_EDGE, "HTTP tunnel error: saying %s", escaped(errmsg));
+  connection_buf_add(errmsg, strlen(errmsg), ENTRY_TO_CONN(conn));
+  /* Mark it as "has_finished" so that we don't try to send an extra socks
+   * reply. */
+  conn->socks_request->has_finished = 1;
+  connection_mark_unattached_ap(conn,
+                                END_STREAM_REASON_HTTPPROTOCOL|
+                                END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED);
+
+ done:
+  tor_free(headers);
+  tor_free(body);
+  tor_free(command);
+  tor_free(addrport);
+  tor_free(addr);
+  return rv;
+}
+
 /** Iterate over the two bytes of stream_id until we get one that is not
  * already in use; return it. Return 0 if can't get a unique stream_id.
  */
@@ -2293,6 +2832,11 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ)
   for (tmpconn = circ->p_streams; tmpconn; tmpconn=tmpconn->next_stream)
     if (tmpconn->stream_id == test_stream_id)
       goto again;
+
+  if (connection_half_edge_find_stream_id(circ->half_streams,
+                                           test_stream_id))
+    goto again;
+
   return test_stream_id;
 }
 
@@ -2307,6 +2851,8 @@ connection_ap_supports_optimistic_data(const entry_connection_t *conn)
   if (edge_conn->on_circuit == NULL ||
       edge_conn->on_circuit->state != CIRCUIT_STATE_OPEN ||
       (edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL &&
+       edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_HSDIR_GET &&
+       edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_S_HSDIR_POST &&
        edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_REND_JOINED))
     return 0;
 
@@ -2378,8 +2924,8 @@ connection_ap_get_begincell_flags(entry_connection_t *ap_conn)
  *
  * If ap_conn is broken, mark it for close and return -1. Else return 0.
  */
-int
-connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
+MOCK_IMPL(int,
+connection_ap_handshake_send_begin,(entry_connection_t *ap_conn))
 {
   char payload[CELL_PAYLOAD_SIZE];
   int payload_len;
@@ -2451,8 +2997,10 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
     /* Sensitive directory connections must have an anonymous path length.
      * Otherwise, directory connections are typically one-hop.
      * This matches the earlier check for directory connection path anonymity
-     * in directory_initiate_command_rend(). */
-    if (is_sensitive_dir_purpose(linked_dir_conn_base->purpose)) {
+     * in directory_initiate_request(). */
+    if (purpose_needs_anonymity(linked_dir_conn_base->purpose,
+                    TO_DIR_CONN(linked_dir_conn_base)->router_purpose,
+                    TO_DIR_CONN(linked_dir_conn_base)->requested_resource)) {
       assert_circ_anonymity_ok(circ, options);
     }
   } else {
@@ -2888,15 +3436,22 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
     return;
   }
   if (replylen) { /* we already have a reply in mind */
-    connection_write_to_buf(reply, replylen, ENTRY_TO_CONN(conn));
+    connection_buf_add(reply, replylen, ENTRY_TO_CONN(conn));
     conn->socks_request->has_finished = 1;
     return;
   }
-  if (conn->socks_request->socks_version == 4) {
+  if (conn->socks_request->listener_type ==
+       CONN_TYPE_AP_HTTP_CONNECT_LISTENER) {
+    const char *response = end_reason_to_http_connect_response_line(endreason);
+    if (!response) {
+      response = "HTTP/1.0 400 Bad Request\r\n\r\n";
+    }
+    connection_buf_add(response, strlen(response), ENTRY_TO_CONN(conn));
+  } else if (conn->socks_request->socks_version == 4) {
     memset(buf,0,SOCKS4_NETWORK_LEN);
     buf[1] = (status==SOCKS5_SUCCEEDED ? SOCKS4_GRANTED : SOCKS4_REJECT);
     /* leave version, destport, destip zero */
-    connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn));
+    connection_buf_add(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn));
   } else if (conn->socks_request->socks_version == 5) {
     size_t buf_len;
     memset(buf,0,sizeof(buf));
@@ -2915,7 +3470,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
       /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */
       buf_len = 22;
     }
-    connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn));
+    connection_buf_add(buf,buf_len,ENTRY_TO_CONN(conn));
   }
   /* If socks_version isn't 4 or 5, don't send anything.
    * This can happen in the case of AP bridges. */
@@ -2923,12 +3478,12 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
   return;
 }
 
-/** Read a RELAY_BEGIN or RELAY_BEGINDIR cell from <b>cell</b>, decode it, and
+/** Read a RELAY_BEGIN or RELAY_BEGIN_DIR cell from <b>cell</b>, decode it, and
  * place the result in <b>bcell</b>.  On success return 0; on failure return
  * <0 and set *<b>end_reason_out</b> to the end reason we should send back to
  * the client.
  *
- * Return -1 in the case where want to send a RELAY_END cell, and < -1 when
+ * Return -1 in the case where we want to send a RELAY_END cell, and < -1 when
  * we don't.
  **/
 STATIC int
@@ -2987,6 +3542,88 @@ begin_cell_parse(const cell_t *cell, begin_cell_t *bcell,
   return 0;
 }
 
+/** For the given <b>circ</b> and the edge connection <b>conn</b>, setup the
+ * connection, attach it to the circ and connect it. Return 0 on success
+ * or END_CIRC_AT_ORIGIN if we can't find the requested hidden service port
+ * where the caller should close the circuit. */
+static int
+handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn)
+{
+  int ret;
+  origin_circuit_t *origin_circ;
+
+  assert_circuit_ok(circ);
+  tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+  tor_assert(conn);
+
+  log_debug(LD_REND, "Connecting the hidden service rendezvous circuit "
+                     "to the service destination.");
+
+  origin_circ = TO_ORIGIN_CIRCUIT(circ);
+  conn->base_.address = tor_strdup("(rendezvous)");
+  conn->base_.state = EXIT_CONN_STATE_CONNECTING;
+
+  /* The circuit either has an hs identifier for v3+ or a rend_data for legacy
+   * service. */
+  if (origin_circ->rend_data) {
+    conn->rend_data = rend_data_dup(origin_circ->rend_data);
+    tor_assert(connection_edge_is_rendezvous_stream(conn));
+    ret = rend_service_set_connection_addr_port(conn, origin_circ);
+  } else if (origin_circ->hs_ident) {
+    /* Setup the identifier to be the one for the circuit service. */
+    conn->hs_ident =
+      hs_ident_edge_conn_new(&origin_circ->hs_ident->identity_pk);
+    tor_assert(connection_edge_is_rendezvous_stream(conn));
+    ret = hs_service_set_conn_addr_port(origin_circ, conn);
+  } else {
+    /* We should never get here if the circuit's purpose is rendezvous. */
+    tor_assert_nonfatal_unreached();
+    return -1;
+  }
+  if (ret < 0) {
+    log_info(LD_REND, "Didn't find rendezvous service (addr%s, port %d)",
+             fmt_addr(&TO_CONN(conn)->addr), TO_CONN(conn)->port);
+    /* Send back reason DONE because we want to make hidden service port
+     * scanning harder thus instead of returning that the exit policy
+     * didn't match, which makes it obvious that the port is closed,
+     * return DONE and kill the circuit. That way, a user (malicious or
+     * not) needs one circuit per bad port unless it matches the policy of
+     * the hidden service. */
+    relay_send_end_cell_from_edge(conn->stream_id, circ,
+                                  END_STREAM_REASON_DONE,
+                                  origin_circ->cpath->prev);
+    connection_free_(TO_CONN(conn));
+
+    /* Drop the circuit here since it might be someone deliberately
+     * scanning the hidden service ports. Note that this mitigates port
+     * scanning by adding more work on the attacker side to successfully
+     * scan but does not fully solve it. */
+    if (ret < -1) {
+      return END_CIRC_AT_ORIGIN;
+    } else {
+      return 0;
+    }
+  }
+
+  /* Link the circuit and the connection crypt path. */
+  conn->cpath_layer = origin_circ->cpath->prev;
+
+  /* Add it into the linked list of p_streams on this circuit */
+  conn->next_stream = origin_circ->p_streams;
+  origin_circ->p_streams = conn;
+  conn->on_circuit = circ;
+  assert_circuit_ok(circ);
+
+  hs_inc_rdv_stream_counter(origin_circ);
+
+  /* Connect tor to the hidden service destination. */
+  connection_exit_connect(conn);
+
+  /* For path bias: This circuit was used successfully */
+  pathbias_mark_use_success(origin_circ);
+  return 0;
+}
+
 /** A relay 'begin' or 'begin_dir' cell has arrived, and either we are
  * an exit hop for the circuit, or we are the origin and it is a
  * rendezvous begin.
@@ -3013,24 +3650,26 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   char *address = NULL;
   uint16_t port = 0;
   or_circuit_t *or_circ = NULL;
+  origin_circuit_t *origin_circ = NULL;
+  crypt_path_t *layer_hint = NULL;
   const or_options_t *options = get_options();
   begin_cell_t bcell;
   int rv;
   uint8_t end_reason=0;
 
   assert_circuit_ok(circ);
-  if (!CIRCUIT_IS_ORIGIN(circ))
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
     or_circ = TO_OR_CIRCUIT(circ);
+  } else {
+    tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    layer_hint = origin_circ->cpath->prev;
+  }
 
   relay_header_unpack(&rh, cell->payload);
   if (rh.length > RELAY_PAYLOAD_SIZE)
     return -END_CIRC_REASON_TORPROTOCOL;
 
-  /* Note: we have to use relay_send_command_from_edge here, not
-   * connection_edge_end or connection_edge_send_command, since those require
-   * that we have a stream connected to a circuit, and we don't connect to a
-   * circuit until we have a pending/successful resolve. */
-
   if (!server_mode(options) &&
       circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
@@ -3045,7 +3684,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     return -END_CIRC_REASON_TORPROTOCOL;
   } else if (rv == -1) {
     tor_free(bcell.address);
-    relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, NULL);
+    relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, layer_hint);
     return 0;
   }
 
@@ -3055,22 +3694,21 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     port = bcell.port;
 
     if (or_circ && or_circ->p_chan) {
-      if (!options->AllowSingleHopExits &&
-           (or_circ->is_first_hop ||
-            (!connection_or_digest_is_known_relay(
+      const int client_chan = channel_is_client(or_circ->p_chan);
+      if ((client_chan ||
+           (!connection_or_digest_is_known_relay(
                 or_circ->p_chan->identity_digest) &&
           should_refuse_unknown_exits(options)))) {
-        /* Don't let clients use us as a single-hop proxy, unless the user
-         * has explicitly allowed that in the config. It attracts attackers
-         * and users who'd be better off with, well, single-hop proxies.
-         */
+        /* Don't let clients use us as a single-hop proxy. It attracts
+         * attackers and users who'd be better off with, well, single-hop
+         * proxies. */
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "Attempt by %s to open a stream %s. Closing.",
                safe_str(channel_get_canonical_remote_descr(or_circ->p_chan)),
-               or_circ->is_first_hop ? "on first hop of circuit" :
-                                       "from unknown relay");
+               client_chan ? "on first hop of circuit" :
+                             "from unknown relay");
         relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                      or_circ->is_first_hop ?
+                                      client_chan ?
                                         END_STREAM_REASON_TORPROTOCOL :
                                         END_STREAM_REASON_MISC,
                                       NULL);
@@ -3082,7 +3720,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     if (!directory_permits_begindir_requests(options) ||
         circ->purpose != CIRCUIT_PURPOSE_OR) {
       relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_NOTDIRECTORY, NULL);
+                                  END_STREAM_REASON_NOTDIRECTORY, layer_hint);
       return 0;
     }
     /* Make sure to get the 'real' address of the previous hop: the
@@ -3099,7 +3737,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   } else {
     log_warn(LD_BUG, "Got an unexpected command %d", (int)rh.command);
     relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                  END_STREAM_REASON_INTERNAL, NULL);
+                                  END_STREAM_REASON_INTERNAL, layer_hint);
     return 0;
   }
 
@@ -3110,7 +3748,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     if (bcell.flags & BEGIN_FLAG_IPV4_NOT_OK) {
       tor_free(address);
       relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_EXITPOLICY, NULL);
+                                    END_STREAM_REASON_EXITPOLICY, layer_hint);
       return 0;
     }
   }
@@ -3133,68 +3771,28 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   n_stream->deliver_window = STREAMWINDOW_START;
 
   if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
-    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
-    log_info(LD_REND,"begin is for rendezvous. configuring stream.");
-    n_stream->base_.address = tor_strdup("(rendezvous)");
-    n_stream->base_.state = EXIT_CONN_STATE_CONNECTING;
-    n_stream->rend_data = rend_data_dup(origin_circ->rend_data);
-    tor_assert(connection_edge_is_rendezvous_stream(n_stream));
-    assert_circuit_ok(circ);
-
-    const int r = rend_service_set_connection_addr_port(n_stream, origin_circ);
-    if (r < 0) {
-      log_info(LD_REND,"Didn't find rendezvous service (port %d)",
-               n_stream->base_.port);
-      /* Send back reason DONE because we want to make hidden service port
-       * scanning harder thus instead of returning that the exit policy
-       * didn't match, which makes it obvious that the port is closed,
-       * return DONE and kill the circuit. That way, a user (malicious or
-       * not) needs one circuit per bad port unless it matches the policy of
-       * the hidden service. */
-      relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_DONE,
-                                    origin_circ->cpath->prev);
-      connection_free(TO_CONN(n_stream));
-      tor_free(address);
+    int ret;
+    tor_free(address);
+    /* We handle this circuit and stream in this function for all supported
+     * hidden service version. */
+    ret = handle_hs_exit_conn(circ, n_stream);
 
-      /* Drop the circuit here since it might be someone deliberately
-       * scanning the hidden service ports. Note that this mitigates port
-       * scanning by adding more work on the attacker side to successfully
-       * scan but does not fully solve it. */
-      if (r < -1)
-        return END_CIRC_AT_ORIGIN;
-      else
-        return 0;
+    if (ret == 0) {
+      /* This was a valid cell. Count it as delivered + overhead. */
+      circuit_read_valid_data(origin_circ, rh.length);
     }
-    assert_circuit_ok(circ);
-    log_debug(LD_REND,"Finished assigning addr/port");
-    n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */
-
-    /* add it into the linked list of p_streams on this circuit */
-    n_stream->next_stream = origin_circ->p_streams;
-    n_stream->on_circuit = circ;
-    origin_circ->p_streams = n_stream;
-    assert_circuit_ok(circ);
-
-    origin_circ->rend_data->nr_streams++;
-
-    connection_exit_connect(n_stream);
-
-    /* For path bias: This circuit was used successfully */
-    pathbias_mark_use_success(origin_circ);
-
-    tor_free(address);
-    return 0;
+    return ret;
   }
   tor_strlower(address);
   n_stream->base_.address = address;
   n_stream->base_.state = EXIT_CONN_STATE_RESOLVEFAILED;
   /* default to failed, change in dns_resolve if it turns out not to fail */
 
+  /* If we're hibernating or shutting down, we refuse to open new streams. */
   if (we_are_hibernating()) {
     relay_send_end_cell_from_edge(rh.stream_id, circ,
                                   END_STREAM_REASON_HIBERNATING, NULL);
-    connection_free(TO_CONN(n_stream));
+    connection_free_(TO_CONN(n_stream));
     return 0;
   }
 
@@ -3272,7 +3870,7 @@ connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ)
       return 0;
     case 1: /* The result was cached; a resolved cell was sent. */
       if (!dummy_conn->base_.marked_for_close)
-        connection_free(TO_CONN(dummy_conn));
+        connection_free_(TO_CONN(dummy_conn));
       return 0;
     case 0: /* resolve added to pending list */
       assert_circuit_ok(TO_CIRCUIT(circ));
@@ -3445,8 +4043,8 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
 
   if (connection_add(TO_CONN(exitconn))<0) {
     connection_edge_end(exitconn, END_STREAM_REASON_RESOURCELIMIT);
-    connection_free(TO_CONN(exitconn));
-    connection_free(TO_CONN(dirconn));
+    connection_free_(TO_CONN(exitconn));
+    connection_free_(TO_CONN(dirconn));
     return 0;
   }
 
@@ -3458,7 +4056,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
     connection_edge_end(exitconn, END_STREAM_REASON_RESOURCELIMIT);
     connection_close_immediate(TO_CONN(exitconn));
     connection_mark_for_close(TO_CONN(exitconn));
-    connection_free(TO_CONN(dirconn));
+    connection_free_(TO_CONN(dirconn));
     return 0;
   }
 
@@ -3479,11 +4077,15 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
  * it is a general stream.
  */
 int
-connection_edge_is_rendezvous_stream(edge_connection_t *conn)
+connection_edge_is_rendezvous_stream(const edge_connection_t *conn)
 {
   tor_assert(conn);
-  if (conn->rend_data)
+  /* It should not be possible to set both of these structs */
+  tor_assert_nonfatal(!(conn->rend_data && conn->hs_ident));
+
+  if (conn->rend_data || conn->hs_ident) {
     return 1;
+  }
   return 0;
 }
 
@@ -3507,7 +4109,7 @@ connection_ap_can_use_exit(const entry_connection_t *conn,
    */
   if (conn->chosen_exit_name) {
     const node_t *chosen_exit =
-      node_get_by_nickname(conn->chosen_exit_name, 1);
+      node_get_by_nickname(conn->chosen_exit_name, 0);
     if (!chosen_exit || tor_memneq(chosen_exit->identity,
                                exit_node->identity, DIGEST_LEN)) {
       /* doesn't match */
@@ -3556,10 +4158,12 @@ connection_ap_can_use_exit(const entry_connection_t *conn,
 }
 
 /** If address is of the form "y.onion" with a well-formed handle y:
- *     Put a NUL after y, lower-case it, and return ONION_HOSTNAME.
+ *     Put a NUL after y, lower-case it, and return ONION_V2_HOSTNAME or
+ *     ONION_V3_HOSTNAME depending on the HS version.
  *
  *  If address is of the form "x.y.onion" with a well-formed handle x:
- *     Drop "x.", put a NUL after y, lower-case it, and return ONION_HOSTNAME.
+ *     Drop "x.", put a NUL after y, lower-case it, and return
+ *     ONION_V2_HOSTNAME or ONION_V3_HOSTNAME depending on the HS version.
  *
  * If address is of the form "y.onion" with a badly-formed handle y:
  *     Return BAD_HOSTNAME and log a message.
@@ -3575,7 +4179,7 @@ parse_extended_hostname(char *address)
 {
     char *s;
     char *q;
-    char query[REND_SERVICE_ID_LEN_BASE32+1];
+    char query[HS_SERVICE_ADDR_LEN_BASE32+1];
 
     s = strrchr(address,'.');
     if (!s)
@@ -3595,14 +4199,17 @@ parse_extended_hostname(char *address)
       goto failed; /* reject sub-domain, as DNS does */
     }
     q = (NULL == q) ? address : q + 1;
-    if (strlcpy(query, q, REND_SERVICE_ID_LEN_BASE32+1) >=
-        REND_SERVICE_ID_LEN_BASE32+1)
+    if (strlcpy(query, q, HS_SERVICE_ADDR_LEN_BASE32+1) >=
+        HS_SERVICE_ADDR_LEN_BASE32+1)
       goto failed;
     if (q != address) {
       memmove(address, q, strlen(q) + 1 /* also get \0 */);
     }
-    if (rend_valid_service_id(query)) {
-      return ONION_HOSTNAME; /* success */
+    if (rend_valid_v2_service_id(query)) {
+      return ONION_V2_HOSTNAME; /* success */
+    }
+    if (hs_address_is_valid(query)) {
+      return ONION_V3_HOSTNAME;
     }
  failed:
     /* otherwise, return to previous state and return 0 */
@@ -3822,4 +4429,5 @@ connection_edge_free_all(void)
   untried_pending_connections = 0;
   smartlist_free(pending_entry_connections);
   pending_entry_connections = NULL;
+  mainloop_event_free(attach_pending_entry_connections_ev);
 }
diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h
index 5dfc8af901..8bef97dc41 100644
--- a/src/or/connection_edge.h
+++ b/src/or/connection_edge.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,7 +33,8 @@ int connection_edge_finished_connecting(edge_connection_t *conn);
 void connection_ap_about_to_close(entry_connection_t *edge_conn);
 void connection_exit_about_to_close(edge_connection_t *edge_conn);
 
-int connection_ap_handshake_send_begin(entry_connection_t *ap_conn);
+MOCK_DECL(int,
+          connection_ap_handshake_send_begin,(entry_connection_t *ap_conn));
 int connection_ap_handshake_send_resolve(entry_connection_t *ap_conn);
 
 entry_connection_t  *connection_ap_make_link(connection_t *partner,
@@ -60,7 +61,7 @@ void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn,
 int connection_exit_begin_conn(cell_t *cell, circuit_t *circ);
 int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ);
 void connection_exit_connect(edge_connection_t *conn);
-int connection_edge_is_rendezvous_stream(edge_connection_t *conn);
+int connection_edge_is_rendezvous_stream(const edge_connection_t *conn);
 int connection_ap_can_use_exit(const entry_connection_t *conn,
                                const node_t *exit);
 void connection_ap_expire_beginning(void);
@@ -88,16 +89,18 @@ int connection_ap_process_transparent(entry_connection_t *conn);
 
 int address_is_invalid_destination(const char *address, int client);
 
-int connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn,
-                                                origin_circuit_t *circ,
-                                                crypt_path_t *cpath);
+MOCK_DECL(int, connection_ap_rewrite_and_attach_if_allowed,
+                                                (entry_connection_t *conn,
+                                                 origin_circuit_t *circ,
+                                                 crypt_path_t *cpath));
 int connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
                                                origin_circuit_t *circ,
                                                crypt_path_t *cpath);
 
 /** Possible return values for parse_extended_hostname. */
 typedef enum hostname_type_t {
-  NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME
+  NORMAL_HOSTNAME, ONION_V2_HOSTNAME, ONION_V3_HOSTNAME,
+  EXIT_HOSTNAME, BAD_HOSTNAME
 } hostname_type_t;
 hostname_type_t parse_extended_hostname(char *address);
 
@@ -119,6 +122,22 @@ void connection_ap_warn_and_unmark_if_pending_circ(
                                              entry_connection_t *entry_conn,
                                              const char *where);
 
+int connection_half_edge_is_valid_data(const smartlist_t *half_conns,
+                                       streamid_t stream_id);
+int connection_half_edge_is_valid_sendme(const smartlist_t *half_conns,
+                                         streamid_t stream_id);
+int connection_half_edge_is_valid_connected(const smartlist_t *half_conns,
+                                            streamid_t stream_id);
+int connection_half_edge_is_valid_end(smartlist_t *half_conns,
+                                      streamid_t stream_id);
+int connection_half_edge_is_valid_resolved(smartlist_t *half_conns,
+                                           streamid_t stream_id);
+
+size_t half_streams_get_total_allocation(void);
+void half_edge_free_(half_edge_t *he);
+#define half_edge_free(he) \
+  FREE_AND_NULL(half_edge_t, half_edge_free_, (he))
+
 /** @name Begin-cell flags
  *
  * These flags are used in RELAY_BEGIN cells to change the default behavior
@@ -186,7 +205,14 @@ typedef struct {
 
 STATIC void connection_ap_handshake_rewrite(entry_connection_t *conn,
                                             rewrite_result_t *out);
-#endif
 
-#endif
+STATIC int connection_ap_process_http_connect(entry_connection_t *conn);
+struct half_edge_t;
+STATIC void connection_half_edge_add(const edge_connection_t *conn,
+                                     origin_circuit_t *circ);
+STATIC struct half_edge_t *connection_half_edge_find_stream_id(
+                                     const smartlist_t *half_conns,
+                                     streamid_t stream_id);
+#endif /* defined(CONNECTION_EDGE_PRIVATE) */
 
+#endif /* !defined(TOR_CONNECTION_EDGE_H) */
diff --git a/src/or/connection_or.c b/src/or/connection_or.c
index a01d086279..99c0ac077f 100644
--- a/src/or/connection_or.c
+++ b/src/or/connection_or.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,12 +21,14 @@
  * This module also implements the client side of the v3 Tor link handshake,
  **/
 #include "or.h"
+#include "bridges.h"
 #include "buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
 #define TOR_CHANNEL_INTERNAL_
+#define CONNECTION_OR_PRIVATE
 #include "channel.h"
 #include "channeltls.h"
 #include "circuitbuild.h"
@@ -37,6 +39,8 @@
 #include "connection.h"
 #include "connection_or.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "dirserv.h"
 #include "entrynodes.h"
 #include "geoip.h"
@@ -45,14 +49,18 @@
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
+#include "proto_cell.h"
 #include "reasons.h"
 #include "relay.h"
 #include "rendcommon.h"
 #include "rephist.h"
 #include "router.h"
+#include "routerkeys.h"
 #include "routerlist.h"
 #include "ext_orport.h"
 #include "scheduler.h"
+#include "torcert.h"
+#include "channelpadding.h"
 
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
@@ -74,56 +82,25 @@ static void connection_or_mark_bad_for_new_circs(or_connection_t *or_conn);
 
 static void connection_or_change_state(or_connection_t *conn, uint8_t state);
 
-/**************************************************************/
+static void connection_or_check_canonicity(or_connection_t *conn,
+                                           int started_here);
 
-/** Map from identity digest of connected OR or desired OR to a connection_t
- * with that identity digest.  If there is more than one such connection_t,
- * they form a linked list, with next_with_same_id as the next pointer. */
-static digestmap_t *orconn_identity_map = NULL;
+/**************************************************************/
 
 /** Global map between Extended ORPort identifiers and OR
  *  connections. */
 static digestmap_t *orconn_ext_or_id_map = NULL;
 
-/** If conn is listed in orconn_identity_map, remove it, and clear
- * conn->identity_digest.  Otherwise do nothing. */
+/** Clear clear conn->identity_digest and update other data
+ * structures as appropriate.*/
 void
-connection_or_remove_from_identity_map(or_connection_t *conn)
+connection_or_clear_identity(or_connection_t *conn)
 {
-  or_connection_t *tmp;
   tor_assert(conn);
-  if (!orconn_identity_map)
-    return;
-  tmp = digestmap_get(orconn_identity_map, conn->identity_digest);
-  if (!tmp) {
-    if (!tor_digest_is_zero(conn->identity_digest)) {
-      log_warn(LD_BUG, "Didn't find connection '%s' on identity map when "
-               "trying to remove it.",
-               conn->nickname ? conn->nickname : "NULL");
-    }
-    return;
-  }
-  if (conn == tmp) {
-    if (conn->next_with_same_id)
-      digestmap_set(orconn_identity_map, conn->identity_digest,
-                    conn->next_with_same_id);
-    else
-      digestmap_remove(orconn_identity_map, conn->identity_digest);
-  } else {
-    while (tmp->next_with_same_id) {
-      if (tmp->next_with_same_id == conn) {
-        tmp->next_with_same_id = conn->next_with_same_id;
-        break;
-      }
-      tmp = tmp->next_with_same_id;
-    }
-  }
   memset(conn->identity_digest, 0, DIGEST_LEN);
-  conn->next_with_same_id = NULL;
 }
 
-/** Remove all entries from the identity-to-orconn map, and clear
- * all identities in OR conns.*/
+/** Clear all identities in OR conns.*/
 void
 connection_or_clear_identity_map(void)
 {
@@ -131,57 +108,72 @@ connection_or_clear_identity_map(void)
   SMARTLIST_FOREACH(conns, connection_t *, conn,
   {
     if (conn->type == CONN_TYPE_OR) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      memset(or_conn->identity_digest, 0, DIGEST_LEN);
-      or_conn->next_with_same_id = NULL;
+      connection_or_clear_identity(TO_OR_CONN(conn));
     }
   });
-
-  digestmap_free(orconn_identity_map, NULL);
-  orconn_identity_map = NULL;
 }
 
 /** Change conn->identity_digest to digest, and add conn into
- * orconn_digest_map. */
+ * the appropriate digest maps.
+ *
+ * NOTE that this function only allows two kinds of transitions: from
+ * unset identity to set identity, and from idempotent re-settings
+ * of the same identity.  It's not allowed to clear an identity or to
+ * change an identity.  Return 0 on success, and -1 if the transition
+ * is not allowed.
+ **/
 static void
-connection_or_set_identity_digest(or_connection_t *conn, const char *digest)
+connection_or_set_identity_digest(or_connection_t *conn,
+                                  const char *rsa_digest,
+                                  const ed25519_public_key_t *ed_id)
 {
-  or_connection_t *tmp;
+  channel_t *chan = NULL;
   tor_assert(conn);
-  tor_assert(digest);
+  tor_assert(rsa_digest);
 
-  if (!orconn_identity_map)
-    orconn_identity_map = digestmap_new();
-  if (tor_memeq(conn->identity_digest, digest, DIGEST_LEN))
+  if (conn->chan)
+    chan = TLS_CHAN_TO_BASE(conn->chan);
+
+  log_info(LD_HANDSHAKE, "Set identity digest for %p (%s): %s %s.",
+           conn,
+           escaped_safe_str(conn->base_.address),
+           hex_str(rsa_digest, DIGEST_LEN),
+           ed25519_fmt(ed_id));
+  log_info(LD_HANDSHAKE, "   (Previously: %s %s)",
+           hex_str(conn->identity_digest, DIGEST_LEN),
+           chan ? ed25519_fmt(&chan->ed25519_identity) : "<null>");
+
+  const int rsa_id_was_set = ! tor_digest_is_zero(conn->identity_digest);
+  const int ed_id_was_set =
+    chan && !ed25519_public_key_is_zero(&chan->ed25519_identity);
+  const int rsa_changed =
+    tor_memneq(conn->identity_digest, rsa_digest, DIGEST_LEN);
+  const int ed_changed = ed_id_was_set &&
+    (!ed_id || !ed25519_pubkey_eq(ed_id, &chan->ed25519_identity));
+
+  tor_assert(!rsa_changed || !rsa_id_was_set);
+  tor_assert(!ed_changed || !ed_id_was_set);
+
+  if (!rsa_changed && !ed_changed)
     return;
 
   /* If the identity was set previously, remove the old mapping. */
-  if (! tor_digest_is_zero(conn->identity_digest)) {
-    connection_or_remove_from_identity_map(conn);
-    if (conn->chan)
-      channel_clear_identity_digest(TLS_CHAN_TO_BASE(conn->chan));
+  if (rsa_id_was_set) {
+    connection_or_clear_identity(conn);
+    if (chan)
+      channel_clear_identity_digest(chan);
   }
 
-  memcpy(conn->identity_digest, digest, DIGEST_LEN);
+  memcpy(conn->identity_digest, rsa_digest, DIGEST_LEN);
 
-  /* If we're setting the ID to zero, don't add a mapping. */
-  if (tor_digest_is_zero(digest))
+  /* If we're initializing the IDs to zero, don't add a mapping yet. */
+  if (tor_digest_is_zero(rsa_digest) &&
+      (!ed_id || ed25519_public_key_is_zero(ed_id)))
     return;
 
-  tmp = digestmap_set(orconn_identity_map, digest, conn);
-  conn->next_with_same_id = tmp;
-
   /* Deal with channels */
-  if (conn->chan)
-    channel_set_identity_digest(TLS_CHAN_TO_BASE(conn->chan), digest);
-
-#if 1
-  /* Testing code to check for bugs in representation. */
-  for (; tmp; tmp = tmp->next_with_same_id) {
-    tor_assert(tor_memeq(tmp->identity_digest, digest, DIGEST_LEN));
-    tor_assert(tmp != conn);
-  }
-#endif
+  if (chan)
+    channel_set_identity_digest(chan, rsa_digest, ed_id);
 }
 
 /** Remove the Extended ORPort identifier of <b>conn</b> from the
@@ -485,7 +477,7 @@ var_cell_pack_header(const var_cell_t *cell, char *hdr_out, int wide_circ_ids)
 var_cell_t *
 var_cell_new(uint16_t payload_len)
 {
-  size_t size = STRUCT_OFFSET(var_cell_t, payload) + payload_len;
+  size_t size = offsetof(var_cell_t, payload) + payload_len;
   var_cell_t *cell = tor_malloc_zero(size);
   cell->payload_len = payload_len;
   cell->command = 0;
@@ -504,7 +496,7 @@ var_cell_copy(const var_cell_t *src)
   size_t size = 0;
 
   if (src != NULL) {
-    size = STRUCT_OFFSET(var_cell_t, payload) + src->payload_len;
+    size = offsetof(var_cell_t, payload) + src->payload_len;
     copy = tor_malloc_zero(size);
     copy->payload_len = src->payload_len;
     copy->command = src->command;
@@ -517,7 +509,7 @@ var_cell_copy(const var_cell_t *src)
 
 /** Release all space held by <b>cell</b>. */
 void
-var_cell_free(var_cell_t *cell)
+var_cell_free_(var_cell_t *cell)
 {
   tor_free(cell);
 }
@@ -604,8 +596,9 @@ connection_or_flushed_some(or_connection_t *conn)
 {
   size_t datalen;
 
-  /* The channel will want to update its estimated queue size */
-  channel_update_xmit_queue_size(TLS_CHAN_TO_BASE(conn->chan));
+  /* Update the channel's active timestamp if there is one */
+  if (conn->chan)
+    channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
   /* If we're under the low water mark, add cells until we're just over the
    * high water mark. */
@@ -667,6 +660,11 @@ connection_or_finished_flushing(or_connection_t *conn)
       tor_fragile_assert();
       return -1;
   }
+
+  /* Update the channel's active timestamp if there is one */
+  if (conn->chan)
+    channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
+
   return 0;
 }
 
@@ -711,7 +709,6 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 void
 connection_or_about_to_close(or_connection_t *or_conn)
 {
-  time_t now = time(NULL);
   connection_t *conn = TO_CONN(or_conn);
 
   /* Tell the controlling channel we're closed */
@@ -731,15 +728,14 @@ connection_or_about_to_close(or_connection_t *or_conn)
     if (connection_or_nonopen_was_started_here(or_conn)) {
       const or_options_t *options = get_options();
       connection_or_note_state_when_broken(or_conn);
-      rep_hist_note_connect_failed(or_conn->identity_digest, now);
-      entry_guard_register_connect_status(or_conn->identity_digest,0,
-                                          !options->HTTPSProxy, now);
+      /* Tell the new guard API about the channel failure */
+      entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
         control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
                                      reason);
         if (!authdir_mode_tests_reachability(options))
-          control_event_bootstrap_problem(
+          control_event_bootstrap_prob_or(
                 orconn_end_reason_to_control_string(reason),
                 reason, or_conn);
       }
@@ -747,11 +743,9 @@ connection_or_about_to_close(or_connection_t *or_conn)
   } else if (conn->hold_open_until_flushed) {
     /* We only set hold_open_until_flushed when we're intentionally
      * closing a connection. */
-    rep_hist_note_disconnect(or_conn->identity_digest, now);
     control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   } else if (!tor_digest_is_zero(or_conn->identity_digest)) {
-    rep_hist_note_connection_died(or_conn->identity_digest, now);
     control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
@@ -802,18 +796,10 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset,
                                 (int)options->BandwidthBurst, 1, INT32_MAX);
   }
 
-  conn->bandwidthrate = rate;
-  conn->bandwidthburst = burst;
-  if (reset) { /* set up the token buckets to be full */
-    conn->read_bucket = conn->write_bucket = burst;
-    return;
+  token_bucket_rw_adjust(&conn->bucket, rate, burst);
+  if (reset) {
+    token_bucket_rw_reset(&conn->bucket, monotime_coarse_get_stamp());
   }
-  /* If the new token bucket is smaller, take out the extra tokens.
-   * (If it's larger, don't -- the buckets can grow to reach the cap.) */
-  if (conn->read_bucket > burst)
-    conn->read_bucket = burst;
-  if (conn->write_bucket > burst)
-    conn->write_bucket = burst;
 }
 
 /** Either our set of relays or our per-conn rate limits have changed.
@@ -830,24 +816,6 @@ connection_or_update_token_buckets(smartlist_t *conns,
   });
 }
 
-/** How long do we wait before killing non-canonical OR connections with no
- * circuits?  In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15
- * minutes before cancelling these connections, which caused fast relays to
- * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough
- * that it kills most idle connections, without being so low that we cause
- * clients to bounce on and off.
- *
- * For canonical connections, the limit is higher, at 15-22.5 minutes.
- *
- * For each OR connection, we randomly add up to 50% extra to its idle_timeout
- * field, to avoid exposing when exactly the last circuit closed.  Since we're
- * storing idle_timeout in a uint16_t, don't let these values get higher than
- * 12 hours or so without revising connection_or_set_canonical and/or expanding
- * idle_timeout.
- */
-#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180
-#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900
-
 /* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and
  * non-canonical otherwise. Adjust idle_timeout accordingly.
  */
@@ -855,9 +823,6 @@ void
 connection_or_set_canonical(or_connection_t *or_conn,
                             int is_canonical)
 {
-  const unsigned int timeout_base = is_canonical ?
-    IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL;
-
   if (bool_eq(is_canonical, or_conn->is_canonical) &&
       or_conn->idle_timeout != 0) {
     /* Don't recalculate an existing idle_timeout unless the canonical
@@ -866,7 +831,14 @@ connection_or_set_canonical(or_connection_t *or_conn,
   }
 
   or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */
-  or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2);
+  or_conn->idle_timeout = channelpadding_get_channel_idle_timeout(
+          TLS_CHAN_TO_BASE(or_conn->chan), is_canonical);
+
+  log_info(LD_CIRC,
+          "Channel " U64_FORMAT " chose an idle timeout of %d.",
+          or_conn->chan ?
+          U64_PRINTF_ARG(TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier):0,
+          or_conn->idle_timeout);
 }
 
 /** If we don't necessarily know the router we're connecting to, but we
@@ -878,15 +850,47 @@ void
 connection_or_init_conn_from_address(or_connection_t *conn,
                                      const tor_addr_t *addr, uint16_t port,
                                      const char *id_digest,
+                                     const ed25519_public_key_t *ed_id,
                                      int started_here)
 {
-  const node_t *r = node_get_by_id(id_digest);
-  connection_or_set_identity_digest(conn, id_digest);
+  log_debug(LD_HANDSHAKE, "init conn from address %s: %s, %s (%d)",
+            fmt_addr(addr),
+            hex_str((const char*)id_digest, DIGEST_LEN),
+            ed25519_fmt(ed_id),
+            started_here);
+
+  connection_or_set_identity_digest(conn, id_digest, ed_id);
   connection_or_update_token_buckets_helper(conn, 1, get_options());
 
   conn->base_.port = port;
   tor_addr_copy(&conn->base_.addr, addr);
   tor_addr_copy(&conn->real_addr, addr);
+
+  connection_or_check_canonicity(conn, started_here);
+}
+
+/** Check whether the identity of <b>conn</b> matches a known node.  If it
+ * does, check whether the address of conn matches the expected address, and
+ * update the connection's is_canonical flag, nickname, and address fields as
+ * appropriate. */
+static void
+connection_or_check_canonicity(or_connection_t *conn, int started_here)
+{
+  const char *id_digest = conn->identity_digest;
+  const ed25519_public_key_t *ed_id = NULL;
+  const tor_addr_t *addr = &conn->real_addr;
+  if (conn->chan)
+    ed_id = & TLS_CHAN_TO_BASE(conn->chan)->ed25519_identity;
+
+  const node_t *r = node_get_by_id(id_digest);
+  if (r &&
+      node_supports_ed25519_link_authentication(r, 1) &&
+      ! node_ed25519_id_matches(r, ed_id)) {
+    /* If this node is capable of proving an ed25519 ID,
+     * we can't call this a canonical connection unless both IDs match. */
+     r = NULL;
+  }
+
   if (r) {
     tor_addr_port_t node_ap;
     node_get_pref_orport(r, &node_ap);
@@ -908,10 +912,12 @@ connection_or_init_conn_from_address(or_connection_t *conn,
       tor_addr_copy(&conn->base_.addr, &node_ap.addr);
       conn->base_.port = node_ap.port;
     }
+    tor_free(conn->nickname);
     conn->nickname = tor_strdup(node_get_nickname(r));
     tor_free(conn->base_.address);
     conn->base_.address = tor_addr_to_str_dup(&node_ap.addr);
   } else {
+    tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
     conn->nickname[0] = '$';
     base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1,
@@ -957,7 +963,37 @@ connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
  * too old for new circuits? */
 #define TIME_BEFORE_OR_CONN_IS_TOO_OLD (60*60*24*7)
 
-/** Given the head of the linked list for all the or_connections with a given
+/** Expire an or_connection if it is too old. Helper for
+ * connection_or_group_set_badness_ and fast path for
+ * channel_rsa_id_group_set_badness.
+ *
+ * Returns 1 if the connection was already expired, else 0.
+ */
+int
+connection_or_single_set_badness_(time_t now,
+                                  or_connection_t *or_conn,
+                                  int force)
+{
+  /* XXXX this function should also be about channels? */
+  if (or_conn->base_.marked_for_close ||
+      connection_or_is_bad_for_new_circs(or_conn))
+    return 1;
+
+  if (force ||
+      or_conn->base_.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD
+        < now) {
+    log_info(LD_OR,
+             "Marking OR conn to %s:%d as too old for new circuits "
+             "(fd "TOR_SOCKET_T_FORMAT", %d secs old).",
+             or_conn->base_.address, or_conn->base_.port, or_conn->base_.s,
+             (int)(now - or_conn->base_.timestamp_created));
+    connection_or_mark_bad_for_new_circs(or_conn);
+  }
+
+  return 0;
+}
+
+/** Given a list of all the or_connections with a given
  * identity, set elements of that list as is_bad_for_new_circs as
  * appropriate. Helper for connection_or_set_bad_connections().
  *
@@ -974,29 +1010,21 @@ connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
  * See channel_is_better() in channel.c for our idea of what makes one OR
  * connection better than another.
  */
-static void
-connection_or_group_set_badness(or_connection_t *head, int force)
+void
+connection_or_group_set_badness_(smartlist_t *group, int force)
 {
-  or_connection_t *or_conn = NULL, *best = NULL;
+  /* XXXX this function should be entirely about channels, not OR
+   * XXXX connections. */
+
+  or_connection_t *best = NULL;
   int n_old = 0, n_inprogress = 0, n_canonical = 0, n_other = 0;
   time_t now = time(NULL);
 
   /* Pass 1: expire everything that's old, and see what the status of
    * everything else is. */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
-    if (or_conn->base_.marked_for_close ||
-        connection_or_is_bad_for_new_circs(or_conn))
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
+    if (connection_or_single_set_badness_(now, or_conn, force))
       continue;
-    if (force ||
-        or_conn->base_.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD
-          < now) {
-      log_info(LD_OR,
-               "Marking OR conn to %s:%d as too old for new circuits "
-               "(fd "TOR_SOCKET_T_FORMAT", %d secs old).",
-               or_conn->base_.address, or_conn->base_.port, or_conn->base_.s,
-               (int)(now - or_conn->base_.timestamp_created));
-      connection_or_mark_bad_for_new_circs(or_conn);
-    }
 
     if (connection_or_is_bad_for_new_circs(or_conn)) {
       ++n_old;
@@ -1007,11 +1035,11 @@ connection_or_group_set_badness(or_connection_t *head, int force)
     } else {
       ++n_other;
     }
-  }
+  } SMARTLIST_FOREACH_END(or_conn);
 
   /* Pass 2: We know how about how good the best connection is.
    * expire everything that's worse, and find the very best if we can. */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn))
       continue; /* This one doesn't need to be marked bad. */
@@ -1032,13 +1060,11 @@ connection_or_group_set_badness(or_connection_t *head, int force)
     }
 
     if (!best ||
-        channel_is_better(now,
-                          TLS_CHAN_TO_BASE(or_conn->chan),
-                          TLS_CHAN_TO_BASE(best->chan),
-                          0)) {
+        channel_is_better(TLS_CHAN_TO_BASE(or_conn->chan),
+                          TLS_CHAN_TO_BASE(best->chan))) {
       best = or_conn;
     }
-  }
+  } SMARTLIST_FOREACH_END(or_conn);
 
   if (!best)
     return;
@@ -1057,17 +1083,15 @@ connection_or_group_set_badness(or_connection_t *head, int force)
    *   0.1.2.x dies out, the first case will go away, and the second one is
    *   "mostly harmless", so a fix can wait until somebody is bored.
    */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn) ||
         or_conn->base_.state != OR_CONN_STATE_OPEN)
       continue;
     if (or_conn != best &&
-        channel_is_better(now,
-                          TLS_CHAN_TO_BASE(best->chan),
-                          TLS_CHAN_TO_BASE(or_conn->chan), 1)) {
-      /* This isn't the best conn, _and_ the best conn is better than it,
-         even when we're being forgiving. */
+        channel_is_better(TLS_CHAN_TO_BASE(best->chan),
+                          TLS_CHAN_TO_BASE(or_conn->chan))) {
+      /* This isn't the best conn, _and_ the best conn is better than it */
       if (best->is_canonical) {
         log_info(LD_OR,
                  "Marking OR conn to %s:%d as unsuitable for new circuits: "
@@ -1091,24 +1115,217 @@ connection_or_group_set_badness(or_connection_t *head, int force)
         connection_or_mark_bad_for_new_circs(or_conn);
       }
     }
+  } SMARTLIST_FOREACH_END(or_conn);
+}
+
+/* Lifetime of a connection failure. After that, we'll retry. This is in
+ * seconds. */
+#define OR_CONNECT_FAILURE_LIFETIME 60
+/* The interval to use with when to clean up the failure cache. */
+#define OR_CONNECT_FAILURE_CLEANUP_INTERVAL 60
+
+/* When is the next time we have to cleanup the failure map. We keep this
+ * because we clean it opportunistically. */
+static time_t or_connect_failure_map_next_cleanup_ts = 0;
+
+/* OR connection failure entry data structure. It is kept in the connection
+ * failure map defined below and indexed by OR identity digest, address and
+ * port.
+ *
+ * We need to identify a connection failure with these three values because we
+ * want to avoid to wrongfully blacklist a relay if someone is trying to
+ * extend to a known identity digest but with the wrong IP/port. For instance,
+ * it can happen if a relay changed its port but the client still has an old
+ * descriptor with the old port. We want to stop connecting to that
+ * IP/port/identity all together, not only the relay identity. */
+typedef struct or_connect_failure_entry_t {
+  HT_ENTRY(or_connect_failure_entry_t) node;
+  /* Identity digest of the connection where it is connecting to. */
+  uint8_t identity_digest[DIGEST_LEN];
+  /* This is the connection address from the base connection_t. After the
+   * connection is checked for canonicity, the base address should represent
+   * what we know instead of where we are connecting to. This is what we need
+   * so we can correlate known relays within the consensus. */
+  tor_addr_t addr;
+  uint16_t port;
+  /* Last time we were unable to connect. */
+  time_t last_failed_connect_ts;
+} or_connect_failure_entry_t;
+
+/* Map where we keep connection failure entries. They are indexed by addr,
+ * port and identity digest. */
+static HT_HEAD(or_connect_failure_ht, or_connect_failure_entry_t)
+       or_connect_failures_map = HT_INITIALIZER();
+
+/* Helper: Hashtable equal function. Return 1 if equal else 0. */
+static int
+or_connect_failure_ht_eq(const or_connect_failure_entry_t *a,
+                         const or_connect_failure_entry_t *b)
+{
+  return fast_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN) &&
+         tor_addr_eq(&a->addr, &b->addr) &&
+         a->port == b->port;
+}
+
+/* Helper: Return the hash for the hashtable of the given entry. For this
+ * table, it is a combination of address, port and identity digest. */
+static unsigned int
+or_connect_failure_ht_hash(const or_connect_failure_entry_t *entry)
+{
+  size_t offset = 0, addr_size;
+  const void *addr_ptr;
+  /* Largest size is IPv6 and IPv4 is smaller so it is fine. */
+  uint8_t data[16 + sizeof(uint16_t) + DIGEST_LEN];
+
+  /* Get the right address bytes depending on the family. */
+  switch (tor_addr_family(&entry->addr)) {
+  case AF_INET:
+    addr_size = 4;
+    addr_ptr = &entry->addr.addr.in_addr.s_addr;
+    break;
+  case AF_INET6:
+    addr_size = 16;
+    addr_ptr = &entry->addr.addr.in6_addr.s6_addr;
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+    return 0;
+  }
+
+  memcpy(data, addr_ptr, addr_size);
+  offset += addr_size;
+  memcpy(data + offset, entry->identity_digest, DIGEST_LEN);
+  offset += DIGEST_LEN;
+  set_uint16(data + offset, entry->port);
+  offset += sizeof(uint16_t);
+
+  return (unsigned int) siphash24g(data, offset);
+}
+
+HT_PROTOTYPE(or_connect_failure_ht, or_connect_failure_entry_t, node,
+             or_connect_failure_ht_hash, or_connect_failure_ht_eq)
+
+HT_GENERATE2(or_connect_failure_ht, or_connect_failure_entry_t, node,
+             or_connect_failure_ht_hash, or_connect_failure_ht_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+/* Initialize a given connect failure entry with the given identity_digest,
+ * addr and port. All field are optional except ocf. */
+static void
+or_connect_failure_init(const char *identity_digest, const tor_addr_t *addr,
+                        uint16_t port, or_connect_failure_entry_t *ocf)
+{
+  tor_assert(ocf);
+  if (identity_digest) {
+    memcpy(ocf->identity_digest, identity_digest,
+           sizeof(ocf->identity_digest));
+  }
+  if (addr) {
+    tor_addr_copy(&ocf->addr, addr);
   }
+  ocf->port = port;
 }
 
-/** Go through all the OR connections (or if <b>digest</b> is non-NULL, just
- * the OR connections with that digest), and set the is_bad_for_new_circs
- * flag based on the rules in connection_or_group_set_badness() (or just
- * always set it if <b>force</b> is true).
- */
-void
-connection_or_set_bad_connections(const char *digest, int force)
+/* Return a newly allocated connection failure entry. It is initialized with
+ * the given or_conn data. This can't fail. */
+static or_connect_failure_entry_t *
+or_connect_failure_new(const or_connection_t *or_conn)
 {
-  if (!orconn_identity_map)
-    return;
+  or_connect_failure_entry_t *ocf = tor_malloc_zero(sizeof(*ocf));
+  or_connect_failure_init(or_conn->identity_digest, &or_conn->real_addr,
+                          TO_CONN(or_conn)->port, ocf);
+  return ocf;
+}
+
+/* Return a connection failure entry matching the given or_conn. NULL is
+ * returned if not found. */
+static or_connect_failure_entry_t *
+or_connect_failure_find(const or_connection_t *or_conn)
+{
+  or_connect_failure_entry_t lookup;
+  tor_assert(or_conn);
+  or_connect_failure_init(or_conn->identity_digest, &TO_CONN(or_conn)->addr,
+                          TO_CONN(or_conn)->port, &lookup);
+  return HT_FIND(or_connect_failure_ht, &or_connect_failures_map, &lookup);
+}
+
+/* Note down in the connection failure cache that a failure occurred on the
+ * given or_conn. */
+STATIC void
+note_or_connect_failed(const or_connection_t *or_conn)
+{
+  or_connect_failure_entry_t *ocf = NULL;
+
+  tor_assert(or_conn);
+
+  ocf = or_connect_failure_find(or_conn);
+  if (ocf == NULL) {
+    ocf = or_connect_failure_new(or_conn);
+    HT_INSERT(or_connect_failure_ht, &or_connect_failures_map, ocf);
+  }
+  ocf->last_failed_connect_ts = approx_time();
+}
+
+/* Cleanup the connection failure cache and remove all entries below the
+ * given cutoff. */
+static void
+or_connect_failure_map_cleanup(time_t cutoff)
+{
+  or_connect_failure_entry_t **ptr, **next, *entry;
+
+  for (ptr = HT_START(or_connect_failure_ht, &or_connect_failures_map);
+       ptr != NULL; ptr = next) {
+    entry = *ptr;
+    if (entry->last_failed_connect_ts <= cutoff) {
+      next = HT_NEXT_RMV(or_connect_failure_ht, &or_connect_failures_map, ptr);
+      tor_free(entry);
+    } else {
+      next = HT_NEXT(or_connect_failure_ht, &or_connect_failures_map, ptr);
+    }
+  }
+}
+
+/* Return true iff the given OR connection can connect to its destination that
+ * is the triplet identity_digest, address and port.
+ *
+ * The or_conn MUST have gone through connection_or_check_canonicity() so the
+ * base address is properly set to what we know or doesn't know. */
+STATIC int
+should_connect_to_relay(const or_connection_t *or_conn)
+{
+  time_t now, cutoff;
+  time_t connect_failed_since_ts = 0;
+  or_connect_failure_entry_t *ocf;
+
+  tor_assert(or_conn);
+
+  now = approx_time();
+  cutoff = now - OR_CONNECT_FAILURE_LIFETIME;
+
+  /* Opportunistically try to cleanup the failure cache. We do that at regular
+   * interval so it doesn't grow too big. */
+  if (or_connect_failure_map_next_cleanup_ts <= now) {
+    or_connect_failure_map_cleanup(cutoff);
+    or_connect_failure_map_next_cleanup_ts =
+      now + OR_CONNECT_FAILURE_CLEANUP_INTERVAL;
+  }
+
+  /* Look if we have failed previously to the same destination as this
+   * OR connection. */
+  ocf = or_connect_failure_find(or_conn);
+  if (ocf) {
+    connect_failed_since_ts = ocf->last_failed_connect_ts;
+  }
+  /* If we do have an unable to connect timestamp and it is below cutoff, we
+   * can connect. Or we have never failed before so let it connect. */
+  if (connect_failed_since_ts > cutoff) {
+    goto no_connect;
+  }
 
-  DIGESTMAP_FOREACH(orconn_identity_map, identity, or_connection_t *, conn) {
-    if (!digest || tor_memeq(digest, conn->identity_digest, DIGEST_LEN))
-      connection_or_group_set_badness(conn, force);
-  } DIGESTMAP_FOREACH_END;
+  /* Ok we can connect! */
+  return 1;
+ no_connect:
+  return 0;
 }
 
 /** <b>conn</b> is in the 'connecting' state, and it failed to complete
@@ -1123,7 +1340,8 @@ connection_or_connect_failed(or_connection_t *conn,
 {
   control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason);
   if (!authdir_mode_tests_reachability(get_options()))
-    control_event_bootstrap_problem(msg, reason, conn);
+    control_event_bootstrap_prob_or(msg, reason, conn);
+  note_or_connect_failed(conn);
 }
 
 /** <b>conn</b> got an error in connection_handle_read_impl() or
@@ -1174,7 +1392,9 @@ connection_or_notify_error(or_connection_t *conn,
 
 MOCK_IMPL(or_connection_t *,
 connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
-                        const char *id_digest, channel_tls_t *chan))
+                        const char *id_digest,
+                        const ed25519_public_key_t *ed_id,
+                        channel_tls_t *chan))
 {
   or_connection_t *conn;
   const or_options_t *options = get_options();
@@ -1194,6 +1414,11 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     log_info(LD_PROTOCOL,"Client asked me to connect to myself. Refusing.");
     return NULL;
   }
+  if (server_mode(options) && router_ed25519_id_is_me(ed_id)) {
+    log_info(LD_PROTOCOL,"Client asked me to connect to myself by Ed25519 "
+             "identity. Refusing.");
+    return NULL;
+  }
 
   conn = or_connection_new(CONN_TYPE_OR, tor_addr_family(&addr));
 
@@ -1206,7 +1431,20 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
    */
   conn->chan = chan;
   chan->conn = conn;
-  connection_or_init_conn_from_address(conn, &addr, port, id_digest, 1);
+  connection_or_init_conn_from_address(conn, &addr, port, id_digest, ed_id, 1);
+
+  /* We have a proper OR connection setup, now check if we can connect to it
+   * that is we haven't had a failure earlier. This is to avoid to try to
+   * constantly connect to relays that we think are not reachable. */
+  if (!should_connect_to_relay(conn)) {
+    log_info(LD_GENERAL, "Can't connect to identity %s at %s:%u because we "
+                         "failed earlier. Refusing.",
+             hex_str(id_digest, DIGEST_LEN), fmt_addr(&TO_CONN(conn)->addr),
+             TO_CONN(conn)->port);
+    connection_free_(TO_CONN(conn));
+    return NULL;
+  }
+
   connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
   control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
 
@@ -1239,7 +1477,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
                fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port),
                transport_name, transport_name);
 
-      control_event_bootstrap_problem(
+      control_event_bootstrap_prob_or(
                                 "Can't connect to bridge",
                                 END_OR_CONN_REASON_PT_MISSING,
                                 conn);
@@ -1250,7 +1488,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
                fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port));
     }
 
-    connection_free(TO_CONN(conn));
+    connection_free_(TO_CONN(conn));
     return NULL;
   }
 
@@ -1263,7 +1501,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
       connection_or_connect_failed(conn,
                                    errno_to_orconn_end_reason(socket_error),
                                    tor_socket_strerror(socket_error));
-      connection_free(TO_CONN(conn));
+      connection_free_(TO_CONN(conn));
       return NULL;
     case 0:
       connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
@@ -1373,7 +1611,6 @@ connection_tls_start_handshake,(or_connection_t *conn, int receiving))
   connection_start_reading(TO_CONN(conn));
   log_debug(LD_HANDSHAKE,"starting TLS handshake on fd "TOR_SOCKET_T_FORMAT,
             conn->base_.s);
-  note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C);
 
   if (connection_tls_continue_handshake(conn) < 0)
     return -1;
@@ -1553,7 +1790,10 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
   }
 
   if (identity_rcvd) {
-    crypto_pk_get_digest(identity_rcvd, digest_rcvd_out);
+    if (crypto_pk_get_digest(identity_rcvd, digest_rcvd_out) < 0) {
+      crypto_pk_free(identity_rcvd);
+      return -1;
+    }
   } else {
     memset(digest_rcvd_out, 0, DIGEST_LEN);
   }
@@ -1563,18 +1803,25 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
 
   crypto_pk_free(identity_rcvd);
 
-  if (started_here)
+  if (started_here) {
+    /* A TLS handshake can't teach us an Ed25519 ID, so we set it to NULL
+     * here. */
+    log_debug(LD_HANDSHAKE, "Calling client_learned_peer_id from "
+              "check_valid_tls_handshake");
     return connection_or_client_learned_peer_id(conn,
-                                     (const uint8_t*)digest_rcvd_out);
+                                        (const uint8_t*)digest_rcvd_out,
+                                        NULL);
+  }
 
   return 0;
 }
 
 /** Called when we (as a connection initiator) have definitively,
  * authenticatedly, learned that ID of the Tor instance on the other
- * side of <b>conn</b> is <b>peer_id</b>.  For v1 and v2 handshakes,
+ * side of <b>conn</b> is <b>rsa_peer_id</b> and optionally <b>ed_peer_id</b>.
+ * For v1 and v2 handshakes,
  * this is right after we get a certificate chain in a TLS handshake
- * or renegotiation.  For v3 handshakes, this is right after we get a
+ * or renegotiation.  For v3+ handshakes, this is right after we get a
  * certificate chain in a CERTS cell.
  *
  * If we did not know the ID before, record the one we got.
@@ -1595,12 +1842,31 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
  */
 int
 connection_or_client_learned_peer_id(or_connection_t *conn,
-                                     const uint8_t *peer_id)
+                                     const uint8_t *rsa_peer_id,
+                                     const ed25519_public_key_t *ed_peer_id)
 {
   const or_options_t *options = get_options();
-
-  if (tor_digest_is_zero(conn->identity_digest)) {
-    connection_or_set_identity_digest(conn, (const char*)peer_id);
+  channel_tls_t *chan_tls = conn->chan;
+  channel_t *chan = channel_tls_to_base(chan_tls);
+  int changed_identity = 0;
+  tor_assert(chan);
+
+  const int expected_rsa_key =
+    ! tor_digest_is_zero(conn->identity_digest);
+  const int expected_ed_key =
+    ! ed25519_public_key_is_zero(&chan->ed25519_identity);
+
+  log_info(LD_HANDSHAKE, "learned peer id for %p (%s): %s, %s",
+           conn,
+           safe_str_client(conn->base_.address),
+           hex_str((const char*)rsa_peer_id, DIGEST_LEN),
+           ed25519_fmt(ed_peer_id));
+
+  if (! expected_rsa_key && ! expected_ed_key) {
+    log_info(LD_HANDSHAKE, "(we had no ID in mind when we made this "
+             "connection.");
+    connection_or_set_identity_digest(conn,
+                                      (const char*)rsa_peer_id, ed_peer_id);
     tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
     conn->nickname[0] = '$';
@@ -1612,16 +1878,39 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     /* if it's a bridge and we didn't know its identity fingerprint, now
      * we do -- remember it for future attempts. */
     learned_router_identity(&conn->base_.addr, conn->base_.port,
-                            (const char*)peer_id);
+                            (const char*)rsa_peer_id, ed_peer_id);
+    changed_identity = 1;
   }
 
-  if (tor_memneq(peer_id, conn->identity_digest, DIGEST_LEN)) {
+  const int rsa_mismatch = expected_rsa_key &&
+    tor_memneq(rsa_peer_id, conn->identity_digest, DIGEST_LEN);
+  /* It only counts as an ed25519 mismatch if we wanted an ed25519 identity
+   * and didn't get it. It's okay if we get one that we didn't ask for. */
+  const int ed25519_mismatch =
+    expected_ed_key &&
+    (ed_peer_id == NULL ||
+     ! ed25519_pubkey_eq(&chan->ed25519_identity, ed_peer_id));
+
+  if (rsa_mismatch || ed25519_mismatch) {
     /* I was aiming for a particular digest. I didn't get it! */
-    char seen[HEX_DIGEST_LEN+1];
-    char expected[HEX_DIGEST_LEN+1];
-    base16_encode(seen, sizeof(seen), (const char*)peer_id, DIGEST_LEN);
-    base16_encode(expected, sizeof(expected), conn->identity_digest,
+    char seen_rsa[HEX_DIGEST_LEN+1];
+    char expected_rsa[HEX_DIGEST_LEN+1];
+    char seen_ed[ED25519_BASE64_LEN+1];
+    char expected_ed[ED25519_BASE64_LEN+1];
+    base16_encode(seen_rsa, sizeof(seen_rsa),
+                  (const char*)rsa_peer_id, DIGEST_LEN);
+    base16_encode(expected_rsa, sizeof(expected_rsa), conn->identity_digest,
                   DIGEST_LEN);
+    if (ed_peer_id) {
+      ed25519_public_to_base64(seen_ed, ed_peer_id);
+    } else {
+      strlcpy(seen_ed, "no ed25519 key", sizeof(seen_ed));
+    }
+    if (! ed25519_public_key_is_zero(&chan->ed25519_identity)) {
+      ed25519_public_to_base64(expected_ed, &chan->ed25519_identity);
+    } else {
+      strlcpy(expected_ed, "no ed25519 key", sizeof(expected_ed));
+    }
     const int using_hardcoded_fingerprints =
       !networkstatus_get_reasonably_live_consensus(time(NULL),
                                                    usable_consensus_flavor());
@@ -1659,30 +1948,48 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     }
 
     log_fn(severity, LD_HANDSHAKE,
-           "Tried connecting to router at %s:%d, but identity key was not "
-           "as expected: wanted %s but got %s.%s",
-           conn->base_.address, conn->base_.port, expected, seen, extra_log);
-    entry_guard_register_connect_status(conn->identity_digest, 0, 1,
-                                        time(NULL));
+           "Tried connecting to router at %s:%d, but RSA + ed25519 identity "
+           "keys were not as expected: wanted %s + %s but got %s + %s.%s",
+           conn->base_.address, conn->base_.port,
+           expected_rsa, expected_ed, seen_rsa, seen_ed, extra_log);
+
+    /* Tell the new guard API about the channel failure */
+    entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
     control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
                                  END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
-      control_event_bootstrap_problem(
+      control_event_bootstrap_prob_or(
                                 "Unexpected identity in router certificate",
                                 END_OR_CONN_REASON_OR_IDENTITY,
                                 conn);
     return -1;
   }
+
+  if (!expected_ed_key && ed_peer_id) {
+    log_info(LD_HANDSHAKE, "(We had no Ed25519 ID in mind when we made this "
+             "connection.)");
+    connection_or_set_identity_digest(conn,
+                                      (const char*)rsa_peer_id, ed_peer_id);
+    changed_identity = 1;
+  }
+
+  if (changed_identity) {
+    /* If we learned an identity for this connection, then we might have
+     * just discovered it to be canonical. */
+    connection_or_check_canonicity(conn, conn->handshake_state->started_here);
+  }
+
   if (authdir_mode_tests_reachability(options)) {
     dirserv_orconn_tls_done(&conn->base_.addr, conn->base_.port,
-                            (const char*)peer_id);
+                            (const char*)rsa_peer_id, ed_peer_id);
   }
 
   return 0;
 }
 
-/** Return when a client used this, for connection.c, since client_used
- * is now one of the timestamps of channel_t */
+/** Return when we last used this channel for client activity (origin
+ * circuits). This is called from connection.c, since client_used is now one
+ * of the timestamps in channel_t */
 
 time_t
 connection_or_client_used(or_connection_t *conn)
@@ -1696,7 +2003,7 @@ connection_or_client_used(or_connection_t *conn)
 
 /** The v1/v2 TLS handshake is finished.
  *
- * Make sure we are happy with the person we just handshaked with.
+ * Make sure we are happy with the peer we just handshaked with.
  *
  * If they initiated the connection, make sure they're not already connected,
  * then initialize conn from the information in router.
@@ -1731,7 +2038,8 @@ connection_tls_finish_handshake(or_connection_t *conn)
   if (tor_tls_used_v1_handshake(conn->tls)) {
     conn->link_proto = 1;
     connection_or_init_conn_from_address(conn, &conn->base_.addr,
-                                         conn->base_.port, digest_rcvd, 0);
+                                         conn->base_.port, digest_rcvd,
+                                         NULL, 0);
     tor_tls_block_renegotiation(conn->tls);
     rep_hist_note_negotiated_link_proto(1, started_here);
     return connection_or_set_state_open(conn);
@@ -1740,7 +2048,8 @@ connection_tls_finish_handshake(or_connection_t *conn)
     if (connection_init_or_handshake_state(conn, started_here) < 0)
       return -1;
     connection_or_init_conn_from_address(conn, &conn->base_.addr,
-                                         conn->base_.port, digest_rcvd, 0);
+                                         conn->base_.port, digest_rcvd,
+                                         NULL, 0);
     return connection_or_send_versions(conn, 0);
   }
 }
@@ -1779,19 +2088,24 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here)
   s->started_here = started_here ? 1 : 0;
   s->digest_sent_data = 1;
   s->digest_received_data = 1;
+  if (! started_here && get_current_link_cert_cert()) {
+    s->own_link_cert = tor_cert_dup(get_current_link_cert_cert());
+  }
+  s->certs = or_handshake_certs_new();
+  s->certs->started_here = s->started_here;
   return 0;
 }
 
 /** Free all storage held by <b>state</b>. */
 void
-or_handshake_state_free(or_handshake_state_t *state)
+or_handshake_state_free_(or_handshake_state_t *state)
 {
   if (!state)
     return;
   crypto_digest_free(state->digest_sent);
   crypto_digest_free(state->digest_received);
-  tor_x509_cert_free(state->auth_cert);
-  tor_x509_cert_free(state->id_cert);
+  or_handshake_certs_free(state->certs);
+  tor_cert_free(state->own_link_cert);
   memwipe(state, 0xBE, sizeof(or_handshake_state_t));
   tor_free(state);
 }
@@ -1912,12 +2226,23 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
-  connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn));
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (cell->command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+
+  connection_buf_add(networkcell.body, cell_network_size, TO_CONN(conn));
 
   /* Touch the channel's active timestamp if there is one */
-  if (conn->chan)
+  if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
+    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+      if (cell->command == CELL_PADDING)
+        rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+    }
+  }
+
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0);
 }
@@ -1935,8 +2260,8 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   tor_assert(cell);
   tor_assert(conn);
   n = var_cell_pack_header(cell, hdr, conn->wide_circ_ids);
-  connection_write_to_buf(hdr, n, TO_CONN(conn));
-  connection_write_to_buf((char*)cell->payload,
+  connection_buf_add(hdr, n, TO_CONN(conn));
+  connection_buf_add((char*)cell->payload,
                           cell->payload_len, TO_CONN(conn));
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
@@ -2011,7 +2336,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
         channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
       circuit_build_times_network_is_live(get_circuit_build_times_mutable());
-      connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn));
+      connection_buf_get_bytes(buf, cell_network_size, TO_CONN(conn));
 
       /* retrieve cell info from buf (create the host-order struct from the
        * network-order string) */
@@ -2023,7 +2348,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
 }
 
 /** Array of recognized link protocol versions. */
-static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4 };
+static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4, 5 };
 /** Number of versions in <b>or_protocol_versions</b>. */
 static const int n_or_protocol_versions =
   (int)( sizeof(or_protocol_versions)/sizeof(uint16_t) );
@@ -2144,66 +2469,187 @@ connection_or_send_netinfo,(or_connection_t *conn))
   return 0;
 }
 
+/** Helper used to add an encoded certs to a cert cell */
+static void
+add_certs_cell_cert_helper(certs_cell_t *certs_cell,
+                           uint8_t cert_type,
+                           const uint8_t *cert_encoded,
+                           size_t cert_len)
+{
+  tor_assert(cert_len <= UINT16_MAX);
+  certs_cell_cert_t *ccc = certs_cell_cert_new();
+  ccc->cert_type = cert_type;
+  ccc->cert_len = cert_len;
+  certs_cell_cert_setlen_body(ccc, cert_len);
+  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
+
+  certs_cell_add_certs(certs_cell, ccc);
+}
+
+/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
+ * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
+ * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
+ * (If <b>cert</b> is NULL, take no action.) */
+static void
+add_x509_cert(certs_cell_t *certs_cell,
+              uint8_t cert_type,
+              const tor_x509_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  const uint8_t *cert_encoded = NULL;
+  size_t cert_len;
+  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+
+  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
+}
+
+/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
+ * that we are building in <b>certs_cell</b>.  Set its type field to
+ * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
+static void
+add_ed25519_cert(certs_cell_t *certs_cell,
+                 uint8_t cert_type,
+                 const tor_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  add_certs_cell_cert_helper(certs_cell, cert_type,
+                             cert->encoded, cert->encoded_len);
+}
+
+#ifdef TOR_UNIT_TESTS
+int certs_cell_ed25519_disabled_for_testing = 0;
+#else
+#define certs_cell_ed25519_disabled_for_testing 0
+#endif
+
 /** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
  * on failure. */
 int
 connection_or_send_certs_cell(or_connection_t *conn)
 {
-  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL,
-    *using_link_cert = NULL;
+  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
   tor_x509_cert_t *own_link_cert = NULL;
-  const uint8_t *link_encoded = NULL, *id_encoded = NULL;
-  size_t link_len, id_len;
   var_cell_t *cell;
-  size_t cell_len;
-  ssize_t pos;
+
+  certs_cell_t *certs_cell = NULL;
 
   tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
 
   if (! conn->handshake_state)
     return -1;
+
   const int conn_in_server_mode = ! conn->handshake_state->started_here;
+
+  /* Get the encoded values of the X509 certificates */
   if (tor_tls_get_my_certs(conn_in_server_mode,
                            &global_link_cert, &id_cert) < 0)
     return -1;
+
+  if (conn_in_server_mode) {
+    own_link_cert = tor_tls_get_own_cert(conn->tls);
+  }
+  tor_assert(id_cert);
+
+  certs_cell = certs_cell_new();
+
+  /* Start adding certs.  First the link cert or auth1024 cert. */
   if (conn_in_server_mode) {
-    using_link_cert = own_link_cert = tor_tls_get_own_cert(conn->tls);
+    tor_assert_nonfatal(own_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
   } else {
-    using_link_cert = global_link_cert;
+    tor_assert(global_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
   }
-  tor_x509_cert_get_der(using_link_cert, &link_encoded, &link_len);
-  tor_x509_cert_get_der(id_cert, &id_encoded, &id_len);
 
-  cell_len = 1 /* 1 byte: num certs in cell */ +
-             2 * ( 1 + 2 ) /* For each cert: 1 byte for type, 2 for length */ +
-             link_len + id_len;
-  cell = var_cell_new(cell_len);
-  cell->command = CELL_CERTS;
-  cell->payload[0] = 2;
-  pos = 1;
+  /* Next the RSA->RSA ID cert */
+  add_x509_cert(certs_cell,
+                OR_CERT_TYPE_ID_1024, id_cert);
 
-  if (conn_in_server_mode)
-    cell->payload[pos] = OR_CERT_TYPE_TLS_LINK; /* Link cert  */
-  else
-    cell->payload[pos] = OR_CERT_TYPE_AUTH_1024; /* client authentication */
-  set_uint16(&cell->payload[pos+1], htons(link_len));
-  memcpy(&cell->payload[pos+3], link_encoded, link_len);
-  pos += 3 + link_len;
+  /* Next the Ed25519 certs */
+  add_ed25519_cert(certs_cell,
+                   CERTTYPE_ED_ID_SIGN,
+                   get_master_signing_key_cert());
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
+                        certs_cell_ed25519_disabled_for_testing);
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_LINK,
+                     conn->handshake_state->own_link_cert);
+  } else {
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_AUTH,
+                     get_current_auth_key_cert());
+  }
+
+  /* And finally the crosscert. */
+  {
+    const uint8_t *crosscert=NULL;
+    size_t crosscert_len;
+    get_master_rsa_crosscert(&crosscert, &crosscert_len);
+    if (crosscert) {
+      add_certs_cell_cert_helper(certs_cell,
+                               CERTTYPE_RSA1024_ID_EDID,
+                               crosscert, crosscert_len);
+    }
+  }
 
-  cell->payload[pos] = OR_CERT_TYPE_ID_1024; /* ID cert */
-  set_uint16(&cell->payload[pos+1], htons(id_len));
-  memcpy(&cell->payload[pos+3], id_encoded, id_len);
-  pos += 3 + id_len;
+  /* We've added all the certs; make the cell. */
+  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
 
-  tor_assert(pos == (int)cell_len); /* Otherwise we just smashed the heap */
+  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
+  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
+  cell = var_cell_new(alloc_len);
+  cell->command = CELL_CERTS;
+  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
+  tor_assert(enc_len > 0 && enc_len <= alloc_len);
+  cell->payload_len = enc_len;
 
   connection_or_write_var_cell_to_buf(cell, conn);
   var_cell_free(cell);
+  certs_cell_free(certs_cell);
   tor_x509_cert_free(own_link_cert);
 
   return 0;
 }
 
+/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
+ * we can send and receive. */
+int
+authchallenge_type_is_supported(uint16_t challenge_type)
+{
+  switch (challenge_type) {
+     case AUTHTYPE_RSA_SHA256_TLSSECRET:
+     case AUTHTYPE_ED25519_SHA256_RFC5705:
+       return 1;
+     case AUTHTYPE_RSA_SHA256_RFC5705:
+     default:
+       return 0;
+  }
+}
+
+/** Return true iff <b>challenge_type_a</b> is one that we would rather
+ * use than <b>challenge_type_b</b>. */
+int
+authchallenge_type_is_better(uint16_t challenge_type_a,
+                             uint16_t challenge_type_b)
+{
+  /* Any supported type is better than an unsupported one;
+   * all unsupported types are equally bad. */
+  if (!authchallenge_type_is_supported(challenge_type_a))
+    return 0;
+  if (!authchallenge_type_is_supported(challenge_type_b))
+    return 1;
+  /* It happens that types are superior in numerically ascending order.
+   * If that ever changes, this must change too. */
+  return (challenge_type_a > challenge_type_b);
+}
+
 /** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
  * on success, -1 on failure. */
 int
@@ -2218,17 +2664,26 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
 
   auth_challenge_cell_t *ac = auth_challenge_cell_new();
 
+  tor_assert(sizeof(ac->challenge) == 32);
   crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
 
   auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
+  /* Disabled, because everything that supports this method also supports
+   * the much-superior ED25519_SHA256_RFC5705 */
+  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
+  auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
   auth_challenge_cell_set_n_methods(ac,
                                     auth_challenge_cell_getlen_methods(ac));
 
   cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
   ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
                                            ac);
-  if (len != cell->payload_len)
+  if (len != cell->payload_len) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
     goto done;
+    /* LCOV_EXCL_STOP */
+  }
   cell->command = CELL_AUTH_CHALLENGE;
 
   connection_or_write_var_cell_to_buf(cell, conn);
@@ -2242,8 +2697,8 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
 }
 
 /** Compute the main body of an AUTHENTICATE cell that a client can use
- * to authenticate itself on a v3 handshake for <b>conn</b>.  Write it to the
- * <b>outlen</b>-byte buffer at <b>out</b>.
+ * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
+ * in a var_cell_t.
  *
  * If <b>server</b> is true, only calculate the first
  * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
@@ -2259,24 +2714,44 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
  *
  * Return the length of the cell body on success, and -1 on failure.
  */
-int
+var_cell_t *
 connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                             uint8_t *out, size_t outlen,
+                                             const int authtype,
                                              crypto_pk_t *signing_key,
-                                             int server)
+                                      const ed25519_keypair_t *ed_signing_key,
+                                      int server)
 {
   auth1_t *auth = NULL;
   auth_ctx_t *ctx = auth_ctx_new();
-  int result;
+  var_cell_t *result = NULL;
+  int old_tlssecrets_algorithm = 0;
+  const char *authtype_str = NULL;
 
-  /* assert state is reasonable XXXX */
+  int is_ed = 0;
 
-  ctx->is_ed = 0;
+  /* assert state is reasonable XXXX */
+  switch (authtype) {
+  case AUTHTYPE_RSA_SHA256_TLSSECRET:
+    authtype_str = "AUTH0001";
+    old_tlssecrets_algorithm = 1;
+    break;
+  case AUTHTYPE_RSA_SHA256_RFC5705:
+    authtype_str = "AUTH0002";
+    break;
+  case AUTHTYPE_ED25519_SHA256_RFC5705:
+    authtype_str = "AUTH0003";
+    is_ed = 1;
+    break;
+  default:
+    tor_assert(0);
+    break;
+  }
 
   auth = auth1_new();
+  ctx->is_ed = is_ed;
 
   /* Type: 8 bytes. */
-  memcpy(auth1_getarray_type(auth), "AUTH0001", 8);
+  memcpy(auth1_getarray_type(auth), authtype_str, 8);
 
   {
     const tor_x509_cert_t *id_cert=NULL;
@@ -2286,7 +2761,7 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
       goto err;
     my_digests = tor_x509_cert_get_id_digests(id_cert);
     their_digests =
-      tor_x509_cert_get_id_digests(conn->handshake_state->id_cert);
+      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
     tor_assert(my_digests);
     tor_assert(their_digests);
     my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
@@ -2302,6 +2777,22 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
     memcpy(auth->sid, server_id, 32);
   }
 
+  if (is_ed) {
+    const ed25519_public_key_t *my_ed_id, *their_ed_id;
+    if (!conn->handshake_state->certs->ed_id_sign) {
+      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
+      goto err;
+    }
+    my_ed_id = get_master_identity_key();
+    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
+
+    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
+    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
+
+    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
+    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
+  }
+
   {
     crypto_digest_t *server_d, *client_d;
     if (server) {
@@ -2328,7 +2819,8 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
       cert = tor_tls_get_peer_cert(conn->tls);
     }
     if (!cert) {
-      log_warn(LD_OR, "Unable to find cert when making AUTH1 data.");
+      log_warn(LD_OR, "Unable to find cert when making %s data.",
+               authtype_str);
       goto err;
     }
 
@@ -2339,36 +2831,85 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
   }
 
   /* HMAC of clientrandom and serverrandom using master key : 32 octets */
-  tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets);
+  if (old_tlssecrets_algorithm) {
+    tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets);
+  } else {
+    char label[128];
+    tor_snprintf(label, sizeof(label),
+                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
+    int r = tor_tls_export_key_material(conn->tls, auth->tlssecrets,
+                                        auth->cid, sizeof(auth->cid),
+                                        label);
+    if (r < 0) {
+      if (r != -2)
+        log_warn(LD_BUG, "TLS key export failed for unknown reason.");
+      // If r == -2, this was openssl bug 7712.
+      goto err;
+    }
+  }
 
   /* 8 octets were reserved for the current time, but we're trying to get out
    * of the habit of sending time around willynilly.  Fortunately, nothing
    * checks it.  That's followed by 16 bytes of nonce. */
   crypto_rand((char*)auth->rand, 24);
 
+  ssize_t maxlen = auth1_encoded_len(auth, ctx);
+  if (ed_signing_key && is_ed) {
+    maxlen += ED25519_SIG_LEN;
+  } else if (signing_key && !is_ed) {
+    maxlen += crypto_pk_keysize(signing_key);
+  }
+
+  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
+  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
+  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
+  const size_t outlen = maxlen;
   ssize_t len;
+
+  result->command = CELL_AUTHENTICATE;
+  set_uint16(result->payload, htons(authtype));
+
   if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
-    log_warn(LD_OR, "Unable to encode signed part of AUTH1 data.");
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
     goto err;
+    /* LCOV_EXCL_STOP */
   }
 
   if (server) {
     auth1_t *tmp = NULL;
     ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
     if (!tmp) {
-      log_warn(LD_OR, "Unable to parse signed part of AUTH1 data.");
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
+               "we just encoded");
       goto err;
+      /* LCOV_EXCL_STOP */
     }
-    result = (int) (tmp->end_of_fixed_part - out);
+    result->payload_len = (tmp->end_of_signed - result->payload);
+
     auth1_free(tmp);
     if (len2 != len) {
-      log_warn(LD_OR, "Mismatched length when re-parsing AUTH1 data.");
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
       goto err;
+      /* LCOV_EXCL_STOP */
     }
     goto done;
   }
 
-  if (signing_key) {
+  if (ed_signing_key && is_ed) {
+    ed25519_signature_t sig;
+    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    auth1_setlen_sig(auth, ED25519_SIG_LEN);
+    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
+
+  } else if (signing_key && !is_ed) {
     auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
 
     char d[32];
@@ -2383,18 +2924,24 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
     }
 
     auth1_setlen_sig(auth, siglen);
+  }
 
-    len = auth1_encode(out, outlen, auth, ctx);
-    if (len < 0) {
-      log_warn(LD_OR, "Unable to encode signed AUTH1 data.");
-      goto err;
-    }
+  len = auth1_encode(out, outlen, auth, ctx);
+  if (len < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
   }
-  result = (int) len;
+  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
+  result->payload_len = len + AUTH_CELL_HEADER_LEN;
+  set_uint16(result->payload+2, htons(len));
+
   goto done;
 
  err:
-  result = -1;
+  var_cell_free(result);
+  result = NULL;
  done:
   auth1_free(auth);
   auth_ctx_free(ctx);
@@ -2408,47 +2955,29 @@ connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
 {
   var_cell_t *cell;
   crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
-  int authlen;
-  size_t cell_maxlen;
   /* XXXX make sure we're actually supposed to send this! */
 
   if (!pk) {
     log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
     return -1;
   }
-  if (authtype != AUTHTYPE_RSA_SHA256_TLSSECRET) {
+  if (! authchallenge_type_is_supported(authtype)) {
     log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
              "authentication type %d", authtype);
     return -1;
   }
 
-  cell_maxlen = 4 + /* overhead */
-    V3_AUTH_BODY_LEN + /* Authentication body */
-    crypto_pk_keysize(pk) + /* Max signature length */
-    16 /* add a few extra bytes just in case. */;
-
-  cell = var_cell_new(cell_maxlen);
-  cell->command = CELL_AUTHENTICATE;
-  set_uint16(cell->payload, htons(AUTHTYPE_RSA_SHA256_TLSSECRET));
-  /* skip over length ; we don't know that yet. */
-
-  authlen = connection_or_compute_authenticate_cell_body(conn,
-                                                         cell->payload+4,
-                                                         cell_maxlen-4,
-                                                         pk,
-                                                         0 /* not server */);
-  if (authlen < 0) {
-    log_warn(LD_BUG, "Unable to compute authenticate cell!");
-    var_cell_free(cell);
+  cell = connection_or_compute_authenticate_cell_body(conn,
+                                                 authtype,
+                                                 pk,
+                                                 get_current_auth_keypair(),
+                                                 0 /* not server */);
+  if (! cell) {
+    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unable to compute authenticate cell!");
     return -1;
   }
-  tor_assert(authlen + 4 <= cell->payload_len);
-  set_uint16(cell->payload+2, htons(authlen));
-  cell->payload_len = authlen + 4;
-
   connection_or_write_var_cell_to_buf(cell, conn);
   var_cell_free(cell);
 
   return 0;
 }
-
diff --git a/src/or/connection_or.h b/src/or/connection_or.h
index 2e8c6066cc..158eb1fdad 100644
--- a/src/or/connection_or.h
+++ b/src/or/connection_or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,14 +12,13 @@
 #ifndef TOR_CONNECTION_OR_H
 #define TOR_CONNECTION_OR_H
 
-void connection_or_remove_from_identity_map(or_connection_t *conn);
+void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
 void clear_broken_connection_map(int disable);
 or_connection_t *connection_or_get_for_extend(const char *digest,
                                               const tor_addr_t *target_addr,
                                               const char **msg_out,
                                               int *launch_out);
-void connection_or_set_bad_connections(const char *digest, int force);
 
 void connection_or_block_renegotiation(or_connection_t *conn);
 int connection_or_reached_eof(or_connection_t *conn);
@@ -40,7 +39,9 @@ void connection_or_notify_error(or_connection_t *conn,
 MOCK_DECL(or_connection_t *,
           connection_or_connect,
           (const tor_addr_t *addr, uint16_t port,
-           const char *id_digest, channel_tls_t *chan));
+           const char *id_digest,
+           const ed25519_public_key_t *ed_id,
+           channel_tls_t *chan));
 
 void connection_or_close_normally(or_connection_t *orconn, int flush);
 MOCK_DECL(void,connection_or_close_for_error,
@@ -59,13 +60,17 @@ int connection_init_or_handshake_state(or_connection_t *conn,
 void connection_or_init_conn_from_address(or_connection_t *conn,
                                           const tor_addr_t *addr,
                                           uint16_t port,
-                                          const char *id_digest,
+                                          const char *rsa_id_digest,
+                                          const ed25519_public_key_t *ed_id,
                                           int started_here);
 int connection_or_client_learned_peer_id(or_connection_t *conn,
-                                         const uint8_t *peer_id);
+                              const uint8_t *rsa_peer_id,
+                              const ed25519_public_key_t *ed_peer_id);
 time_t connection_or_client_used(or_connection_t *conn);
 MOCK_DECL(int, connection_or_get_num_circuits, (or_connection_t *conn));
-void or_handshake_state_free(or_handshake_state_t *state);
+void or_handshake_state_free_(or_handshake_state_t *state);
+#define or_handshake_state_free(state) \
+  FREE_AND_NULL(or_handshake_state_t, or_handshake_state_free_, (state))
 void or_handshake_state_record_cell(or_connection_t *conn,
                                     or_handshake_state_t *state,
                                     const cell_t *cell,
@@ -84,10 +89,14 @@ int connection_or_send_versions(or_connection_t *conn, int v3_plus);
 MOCK_DECL(int,connection_or_send_netinfo,(or_connection_t *conn));
 int connection_or_send_certs_cell(or_connection_t *conn);
 int connection_or_send_auth_challenge_cell(or_connection_t *conn);
-int connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                                 uint8_t *out, size_t outlen,
-                                                 crypto_pk_t *signing_key,
-                                                 int server);
+int authchallenge_type_is_supported(uint16_t challenge_type);
+int authchallenge_type_is_better(uint16_t challenge_type_a,
+                                 uint16_t challenge_type_b);
+var_cell_t *connection_or_compute_authenticate_cell_body(or_connection_t *conn,
+                                       const int authtype,
+                                       crypto_pk_t *signing_key,
+                                       const ed25519_keypair_t *ed_signing_key,
+                                       int server);
 MOCK_DECL(int,connection_or_send_authenticate_cell,
           (or_connection_t *conn, int type));
 
@@ -98,10 +107,27 @@ int var_cell_pack_header(const var_cell_t *cell, char *hdr_out,
                          int wide_circ_ids);
 var_cell_t *var_cell_new(uint16_t payload_len);
 var_cell_t *var_cell_copy(const var_cell_t *src);
-void var_cell_free(var_cell_t *cell);
+void var_cell_free_(var_cell_t *cell);
+#define var_cell_free(cell) FREE_AND_NULL(var_cell_t, var_cell_free_, (cell))
 
 /* DOCDOC */
 #define MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS 4
+#define MIN_LINK_PROTO_FOR_CHANNEL_PADDING 5
+#define MAX_LINK_PROTO MIN_LINK_PROTO_FOR_CHANNEL_PADDING
 
+int connection_or_single_set_badness_(time_t now,
+                                      or_connection_t *or_conn,
+                                      int force);
+void connection_or_group_set_badness_(smartlist_t *group, int force);
+
+#ifdef CONNECTION_OR_PRIVATE
+STATIC int should_connect_to_relay(const or_connection_t *or_conn);
+STATIC void note_or_connect_failed(const or_connection_t *or_conn);
+#endif
+
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
 #endif
 
+#endif /* !defined(TOR_CONNECTION_OR_H) */
+
diff --git a/src/or/conscache.c b/src/or/conscache.c
new file mode 100644
index 0000000000..51dc9d621f
--- /dev/null
+++ b/src/or/conscache.c
@@ -0,0 +1,627 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+#include "config.h"
+#include "conscache.h"
+#include "crypto_util.h"
+#include "storagedir.h"
+
+#define CCE_MAGIC 0x17162253
+
+#ifdef _WIN32
+/* On Windows, unlink won't work on a file if the file is actively mmap()ed.
+ * That forces us to be less aggressive about unlinking files, and causes other
+ * changes throughout our logic.
+ */
+#define MUST_UNMAP_TO_UNLINK
+#endif /* defined(_WIN32) */
+
+/**
+ * A consensus_cache_entry_t is a reference-counted handle to an
+ * item in a consensus_cache_t.  It can be mmapped into RAM, or not,
+ * depending whether it's currently in use.
+ */
+struct consensus_cache_entry_t {
+  uint32_t magic; /**< Must be set to CCE_MAGIC */
+  HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t);
+  int32_t refcnt; /**< Reference count. */
+  unsigned can_remove : 1; /**< If true, we want to delete this file. */
+  /** If true, we intend to unmap this file as soon as we're done with it. */
+  unsigned release_aggressively : 1;
+
+  /** Filename for this object within the storage_dir_t */
+  char *fname;
+  /** Labels associated with this object. Immutable once the object
+   * is created. */
+  config_line_t *labels;
+  /** Pointer to the cache that includes this entry (if any). */
+  consensus_cache_t *in_cache;
+
+  /** Since what time has this object been mapped into RAM, but with the cache
+   * being the only having a reference to it? */
+  time_t unused_since;
+  /** mmaped contents of the underlying file.  May be NULL */
+  tor_mmap_t *map;
+  /** Length of the body within <b>map</b>. */
+  size_t bodylen;
+  /** Pointer to the body within <b>map</b>. */
+  const uint8_t *body;
+};
+
+/**
+ * A consensus_cache_t holds a directory full of labeled items.
+ */
+struct consensus_cache_t {
+  /** Underling storage_dir_t to handle persistence */
+  storage_dir_t *dir;
+  /** List of all the entries in the directory. */
+  smartlist_t *entries;
+
+  /** The maximum number of entries that we'd like to allow in this cache.
+   * This is the same as the storagedir limit when MUST_UNMAP_TO_UNLINK is
+   * not defined. */
+  unsigned max_entries;
+};
+
+static void consensus_cache_clear(consensus_cache_t *cache);
+static void consensus_cache_rescan(consensus_cache_t *);
+static void consensus_cache_entry_map(consensus_cache_t *,
+                                      consensus_cache_entry_t *);
+static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
+
+/**
+ * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
+ * data directory, to hold up to <b>max_entries</b> of data.
+ */
+consensus_cache_t *
+consensus_cache_open(const char *subdir, int max_entries)
+{
+  int storagedir_max_entries;
+  consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
+  char *directory = get_cachedir_fname(subdir);
+  cache->max_entries = max_entries;
+
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* If we can't unlink the files that we're still using, then we need to
+   * tell the storagedir backend to allow far more files than this consensus
+   * cache actually wants, so that it can hold files which, from this cache's
+   * perspective, have become useless.
+   */
+#define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
+  storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
+#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+  /* Otherwise, we can just tell the storagedir to use the same limits
+   * as this cache. */
+  storagedir_max_entries = max_entries;
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+
+  cache->dir = storage_dir_new(directory, storagedir_max_entries);
+  tor_free(directory);
+  if (!cache->dir) {
+    tor_free(cache);
+    return NULL;
+  }
+
+  consensus_cache_rescan(cache);
+  return cache;
+}
+
+/** Return true if it's okay to put more entries in this cache than
+ * its official file limit.
+ *
+ * (We need this method on Windows, where we can't unlink files that are still
+ * in use, and therefore might need to temporarily exceed the file limit until
+ * the no-longer-wanted files are deletable.)
+ */
+int
+consensus_cache_may_overallocate(consensus_cache_t *cache)
+{
+  (void) cache;
+#ifdef MUST_UNMAP_TO_UNLINK
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>cache</b> will need.
+ */
+int
+consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                      struct sandbox_cfg_elem **cfg)
+{
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* Our Linux sandbox doesn't support huge file lists like the one that would
+   * be generated by using VERY_LARGE_STORAGEDIR_LIMIT above in
+   * consensus_cache_open().  Since the Linux sandbox is the only one we have
+   * right now, we just assert that we never reach this point when we've had
+   * to use VERY_LARGE_STORAGEDIR_LIMIT.
+   *
+   * If at some point in the future we have a different sandbox mechanism that
+   * can handle huge file lists, we can remove this assertion or make it
+   * conditional.
+   */
+  tor_assert_nonfatal_unreached();
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+  return storage_dir_register_with_sandbox(cache->dir, cfg);
+}
+
+/**
+ * Helper: clear all entries from <b>cache</b> (but do not delete
+ * any that aren't marked for removal
+ */
+static void
+consensus_cache_clear(consensus_cache_t *cache)
+{
+  consensus_cache_delete_pending(cache, 0);
+
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    ent->in_cache = NULL;
+    consensus_cache_entry_decref(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(cache->entries);
+  cache->entries = NULL;
+}
+
+/**
+ * Drop all storage held by <b>cache</b>.
+ */
+void
+consensus_cache_free_(consensus_cache_t *cache)
+{
+  if (! cache)
+    return;
+
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+  storage_dir_free(cache->dir);
+  tor_free(cache);
+}
+
+/**
+ * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
+ * labeling that data with <b>labels</b>.  On failure, return NULL. On
+ * success, return a newly created consensus_cache_entry_t.
+ *
+ * The returned value will be owned by the cache, and you will have a
+ * reference to it.  Call consensus_cache_entry_decref() when you are
+ * done with it.
+ *
+ * The provided <b>labels</b> MUST have distinct keys: if they don't,
+ * this API does not specify which values (if any) for the duplicate keys
+ * will be considered.
+ */
+consensus_cache_entry_t *
+consensus_cache_add(consensus_cache_t *cache,
+                    const config_line_t *labels,
+                    const uint8_t *data,
+                    size_t datalen)
+{
+  char *fname = NULL;
+  int r = storage_dir_save_labeled_to_file(cache->dir,
+                                            labels, data, datalen, &fname);
+  if (r < 0 || fname == NULL) {
+    return NULL;
+  }
+  consensus_cache_entry_t *ent =
+    tor_malloc_zero(sizeof(consensus_cache_entry_t));
+  ent->magic = CCE_MAGIC;
+  ent->fname = fname;
+  ent->labels = config_lines_dup(labels);
+  ent->in_cache = cache;
+  ent->unused_since = TIME_MAX;
+  smartlist_add(cache->entries, ent);
+  /* Start the reference count at 2: the caller owns one copy, and the
+   * cache owns another.
+   */
+  ent->refcnt = 2;
+
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
+ * Return NULL if no such entry exists.
+ *
+ * Does not adjust reference counts.
+ */
+consensus_cache_entry_t *
+consensus_cache_find_first(consensus_cache_t *cache,
+                           const char *key,
+                           const char *value)
+{
+  smartlist_t *tmp = smartlist_new();
+  consensus_cache_find_all(tmp, cache, key, value);
+  consensus_cache_entry_t *ent = NULL;
+  if (smartlist_len(tmp))
+    ent = smartlist_get(tmp, 0);
+  smartlist_free(tmp);
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
+ *
+ * Do not add any entry that has been marked for removal.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_find_all(smartlist_t *out,
+                         consensus_cache_t *cache,
+                         const char *key,
+                         const char *value)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    if (ent->can_remove == 1) {
+      /* We want to delete this; pretend it isn't there. */
+      continue;
+    }
+    if (! key) {
+      smartlist_add(out, ent);
+      continue;
+    }
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (found_val && !strcmp(value, found_val)) {
+      smartlist_add(out, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Given a list of consensus_cache_entry_t, remove all those entries
+ * that do not have <b>key</b>=<b>value</b> in their labels.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_filter_list(smartlist_t *lst,
+                            const char *key,
+                            const char *value)
+{
+  if (BUG(lst == NULL))
+    return; // LCOV_EXCL_LINE
+  if (key == NULL)
+    return;
+  SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (! found_val || strcmp(value, found_val)) {
+      SMARTLIST_DEL_CURRENT(lst, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * If <b>ent</b> has a label with the given <b>key</b>, return its
+ * value.  Otherwise return NULL.
+ *
+ * The return value is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const char *
+consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                const char *key)
+{
+  const config_line_t *match = config_line_find(ent->labels, key);
+  if (match)
+    return match->value;
+  else
+    return NULL;
+}
+
+/**
+ * Return a pointer to the labels in <b>ent</b>.
+ *
+ * This pointer is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const config_line_t *
+consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
+{
+  return ent->labels;
+}
+
+/**
+ * Increase the reference count of <b>ent</b>.
+ */
+void
+consensus_cache_entry_incref(consensus_cache_entry_t *ent)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+  ++ent->refcnt;
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Release a reference held to <b>ent</b>.
+ *
+ * If it was the last reference, ent will be freed. Therefore, you must not
+ * use <b>ent</b> after calling this function.
+ */
+void
+consensus_cache_entry_decref(consensus_cache_entry_t *ent)
+{
+  if (! ent)
+    return;
+  if (BUG(ent->refcnt <= 0))
+    return; // LCOV_EXCL_LINE
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+
+  --ent->refcnt;
+
+  if (ent->refcnt == 1 && ent->in_cache) {
+    /* Only the cache has a reference: we don't need to keep the file
+     * mapped */
+    if (ent->map) {
+      if (ent->release_aggressively) {
+        consensus_cache_entry_unmap(ent);
+      } else {
+        ent->unused_since = approx_time();
+      }
+    }
+    return;
+  }
+
+  if (ent->refcnt > 0)
+    return;
+
+  /* Refcount is zero; we can free it. */
+  if (ent->map) {
+    consensus_cache_entry_unmap(ent);
+  }
+  tor_free(ent->fname);
+  config_free_lines(ent->labels);
+  consensus_cache_entry_handles_clear(ent);
+  memwipe(ent, 0, sizeof(consensus_cache_entry_t));
+  tor_free(ent);
+}
+
+/**
+ * Mark <b>ent</b> for deletion from the cache.  Deletion will not occur
+ * until the cache is the only place that holds a reference to <b>ent</b>.
+ */
+void
+consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
+{
+  ent->can_remove = 1;
+}
+
+/**
+ * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
+ * any longer than we're actually using it.
+ */
+void
+consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
+{
+  ent->release_aggressively = 1;
+}
+
+/**
+ * Try to read the body of <b>ent</b> into memory if it isn't already
+ * loaded.  On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
+ * to its size, and return 0.  On failure return -1.
+ *
+ * The resulting body pointer will only be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+int
+consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                               const uint8_t **body_out,
+                               size_t *sz_out)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return -1; // LCOV_EXCL_LINE
+
+  if (! ent->map) {
+    if (! ent->in_cache)
+      return -1;
+
+    consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
+                              (consensus_cache_entry_t *)ent);
+    if (! ent->map) {
+      return -1;
+    }
+  }
+
+  *body_out = ent->body;
+  *sz_out = ent->bodylen;
+  return 0;
+}
+
+/**
+ * Unmap every mmap'd element of <b>cache</b> that has been unused
+ * since <b>cutoff</b>.
+ */
+void
+consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+      /* Somebody is using this entry right now */
+      continue;
+    }
+    if (ent->unused_since > cutoff) {
+      /* Has been unused only for a little while */
+      continue;
+    }
+    if (ent->map == NULL) {
+      /* Not actually mapped. */
+      continue;
+    }
+    consensus_cache_entry_unmap(ent);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Return the number of currently unused filenames available in this cache.
+ */
+int
+consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
+{
+  tor_assert(cache);
+  int max = cache->max_entries;
+  int used = smartlist_len(storage_dir_list(cache->dir));
+#ifdef MUST_UNMAP_TO_UNLINK
+  if (used > max)
+    return 0;
+#else
+  tor_assert_nonfatal(max >= used);
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+  return max - used;
+}
+
+/**
+ * Delete every element of <b>cache</b> has been marked with
+ * consensus_cache_entry_mark_for_removal. If <b>force</b> is false,
+ * retain those entries which are in use by something other than the cache.
+ */
+void
+consensus_cache_delete_pending(consensus_cache_t *cache, int force)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    int force_ent = force;
+#ifdef MUST_UNMAP_TO_UNLINK
+    /* We cannot delete anything with an active mmap on win32, so no
+     * force-deletion. */
+    if (ent->map) {
+      force_ent = 0;
+    }
+#endif /* defined(MUST_UNMAP_TO_UNLINK) */
+    if (! force_ent) {
+      if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+        /* Somebody is using this entry right now */
+        continue;
+      }
+    }
+    if (ent->can_remove == 0) {
+      /* Don't want to delete this. */
+      continue;
+    }
+    if (BUG(ent->refcnt <= 0)) {
+      continue; // LCOV_EXCL_LINE
+    }
+
+    SMARTLIST_DEL_CURRENT(cache->entries, ent);
+    ent->in_cache = NULL;
+    char *fname = tor_strdup(ent->fname); /* save a copy */
+    consensus_cache_entry_decref(ent);
+    storage_dir_remove_file(cache->dir, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Internal helper: rescan <b>cache</b> and rebuild its list of entries.
+ */
+static void
+consensus_cache_rescan(consensus_cache_t *cache)
+{
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+
+  cache->entries = smartlist_new();
+  const smartlist_t *fnames = storage_dir_list(cache->dir);
+  SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
+    tor_mmap_t *map = NULL;
+    config_line_t *labels = NULL;
+    const uint8_t *body;
+    size_t bodylen;
+    map = storage_dir_map_labeled(cache->dir, fname,
+                                  &labels, &body, &bodylen);
+    if (! map) {
+      /* The ERANGE error might come from tor_mmap_file() -- it means the file
+       * was empty. EINVAL might come from ..map_labeled() -- it means the
+       * file was misformatted. In both cases, we should just delete it.
+       */
+      if (errno == ERANGE || errno == EINVAL) {
+        log_warn(LD_FS, "Found %s file %s in consensus cache; removing it.",
+                 errno == ERANGE ? "empty" : "misformatted",
+                 escaped(fname));
+        storage_dir_remove_file(cache->dir, fname);
+      } else {
+        /* Can't load this; continue */
+        log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
+                 escaped(fname), strerror(errno));
+      }
+      continue;
+    }
+    consensus_cache_entry_t *ent =
+      tor_malloc_zero(sizeof(consensus_cache_entry_t));
+    ent->magic = CCE_MAGIC;
+    ent->fname = tor_strdup(fname);
+    ent->labels = labels;
+    ent->refcnt = 1;
+    ent->in_cache = cache;
+    ent->unused_since = TIME_MAX;
+    smartlist_add(cache->entries, ent);
+    tor_munmap_file(map); /* don't actually need to keep this around */
+  } SMARTLIST_FOREACH_END(fname);
+}
+
+/**
+ * Make sure that <b>ent</b> is mapped into RAM.
+ */
+static void
+consensus_cache_entry_map(consensus_cache_t *cache,
+                          consensus_cache_entry_t *ent)
+{
+  if (ent->map)
+    return;
+
+  ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
+                                     NULL, &ent->body, &ent->bodylen);
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Unmap <b>ent</b> from RAM.
+ *
+ * Do not call this if something other than the cache is holding a reference
+ * to <b>ent</b>
+ */
+static void
+consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
+{
+  ent->unused_since = TIME_MAX;
+  if (!ent->map)
+    return;
+
+  tor_munmap_file(ent->map);
+  ent->map = NULL;
+  ent->body = NULL;
+  ent->bodylen = 0;
+  ent->unused_since = TIME_MAX;
+}
+
+HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, )
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Testing only: Return true iff <b>ent</b> is mapped into memory.
+ *
+ * (In normal operation, this information is not exposed.)
+ */
+int
+consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
+{
+  if (ent->map) {
+    tor_assert(ent->body);
+    return 1;
+  } else {
+    tor_assert(!ent->body);
+    return 0;
+  }
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/or/conscache.h b/src/or/conscache.h
new file mode 100644
index 0000000000..08a5c5a37b
--- /dev/null
+++ b/src/or/conscache.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSCACHE_H
+#define TOR_CONSCACHE_H
+
+#include "handles.h"
+
+typedef struct consensus_cache_entry_t consensus_cache_entry_t;
+typedef struct consensus_cache_t consensus_cache_t;
+
+HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, )
+#define consensus_cache_entry_handle_free(h)    \
+  FREE_AND_NULL(consensus_cache_entry_handle_t, \
+                consensus_cache_entry_handle_free_, (h))
+
+consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
+void consensus_cache_free_(consensus_cache_t *cache);
+#define consensus_cache_free(cache) \
+  FREE_AND_NULL(consensus_cache_t, consensus_cache_free_, (cache))
+struct sandbox_cfg_elem;
+int consensus_cache_may_overallocate(consensus_cache_t *cache);
+int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                          struct sandbox_cfg_elem **cfg);
+void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
+void consensus_cache_delete_pending(consensus_cache_t *cache,
+                                    int force);
+int consensus_cache_get_n_filenames_available(consensus_cache_t *cache);
+consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
+                                             const config_line_t *labels,
+                                             const uint8_t *data,
+                                             size_t datalen);
+
+consensus_cache_entry_t *consensus_cache_find_first(
+                                             consensus_cache_t *cache,
+                                             const char *key,
+                                             const char *value);
+
+void consensus_cache_find_all(smartlist_t *out,
+                              consensus_cache_t *cache,
+                              const char *key,
+                              const char *value);
+void consensus_cache_filter_list(smartlist_t *lst,
+                                 const char *key,
+                                 const char *value);
+
+const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                            const char *key);
+const config_line_t *consensus_cache_entry_get_labels(
+                                          const consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
+void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
+void consensus_cache_entry_mark_for_aggressive_release(
+                                            consensus_cache_entry_t *ent);
+int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                                   const uint8_t **body_out,
+                                   size_t *sz_out);
+
+#ifdef TOR_UNIT_TESTS
+int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
+#endif
+
+#endif /* !defined(TOR_CONSCACHE_H) */
+
diff --git a/src/or/consdiff.c b/src/or/consdiff.c
new file mode 100644
index 0000000000..deaf465fe7
--- /dev/null
+++ b/src/or/consdiff.c
@@ -0,0 +1,1415 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiff.c
+ * \brief Consensus diff implementation, including both the generation and the
+ * application of diffs in a minimal ed format.
+ *
+ * The consensus diff application is done in consdiff_apply_diff, which relies
+ * on apply_ed_diff for the main ed diff part and on some digest helper
+ * functions to check the digest hashes found in the consensus diff header.
+ *
+ * The consensus diff generation is more complex. consdiff_gen_diff generates
+ * it, relying on gen_ed_diff to generate the ed diff and some digest helper
+ * functions to generate the digest hashes.
+ *
+ * gen_ed_diff is the tricky bit. In it simplest form, it will take quadratic
+ * time and linear space to generate an ed diff given two smartlists. As shown
+ * in its comment section, calling calc_changes on the entire two consensuses
+ * will calculate what is to be added and what is to be deleted in the diff.
+ * Its comment section briefly explains how it works.
+ *
+ * In our case specific to consensuses, we take advantage of the fact that
+ * consensuses list routers sorted by their identities. We use that
+ * information to avoid running calc_changes on the whole smartlists.
+ * gen_ed_diff will navigate through the two consensuses identity by identity
+ * and will send small couples of slices to calc_changes, keeping the running
+ * time near-linear. This is explained in more detail in the gen_ed_diff
+ * comments.
+ *
+ * The allocation strategy tries to save time and memory by avoiding needless
+ * copies.  Instead of actually splitting the inputs into separate strings, we
+ * allocate cdline_t objects, each of which represents a line in the original
+ * object or in the output.  We use memarea_t allocators to manage the
+ * temporary memory we use when generating or applying diffs.
+ **/
+
+#define CONSDIFF_PRIVATE
+
+#include "or.h"
+#include "consdiff.h"
+#include "memarea.h"
+#include "routerparse.h"
+
+static const char* ns_diff_version = "network-status-diff-version 1";
+static const char* hash_token = "hash";
+
+static char *consensus_join_lines(const smartlist_t *inp);
+
+/** Return true iff a and b have the same contents. */
+STATIC int
+lines_eq(const cdline_t *a, const cdline_t *b)
+{
+  return a->len == b->len && fast_memeq(a->s, b->s, a->len);
+}
+
+/** Return true iff a has the same contents as the nul-terminated string b. */
+STATIC int
+line_str_eq(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  cdline_t bline = { b, (uint32_t)len };
+  return lines_eq(a, &bline);
+}
+
+/** Return true iff a begins with the same contents as the nul-terminated
+ * string b. */
+static int
+line_starts_with_str(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  return a->len >= len && fast_memeq(a->s, b, len);
+}
+
+/** Return a new cdline_t holding as its contents the nul-terminated
+ * string s. Use the provided memory area for storage. */
+static cdline_t *
+cdline_linecpy(memarea_t *area, const char *s)
+{
+  size_t len = strlen(s);
+  const char *ss = memarea_memdup(area, s, len);
+  cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+  line->s = ss;
+  line->len = (uint32_t)len;
+  return line;
+}
+
+/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated
+ * string s.  Use the provided memory area for storage. */
+STATIC void
+smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
+{
+  smartlist_add(lst, cdline_linecpy(area, s));
+}
+
+/** Compute the digest of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest,(const char *cons,
+                          consensus_digest_t *digest_out))
+{
+  int r = crypto_digest256((char*)digest_out->sha3_256,
+                           cons, strlen(cons), DIGEST_SHA3_256);
+  return r;
+}
+
+/** Compute the digest-as-signed of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest_as_signed,(const char *cons,
+                                    consensus_digest_t *digest_out))
+{
+  return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
+                                                    cons);
+}
+
+/** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_digest_eq,(const uint8_t *d1,
+                     const uint8_t *d2))
+{
+  return fast_memeq(d1, d2, DIGEST256_LEN);
+}
+
+/** Create (allocate) a new slice from a smartlist. Assumes that the start
+ * and the end indexes are within the bounds of the initial smartlist. The end
+ * element is not part of the resulting slice. If end is -1, the slice is to
+ * reach the end of the smartlist.
+ */
+STATIC smartlist_slice_t *
+smartlist_slice(const smartlist_t *list, int start, int end)
+{
+  int list_len = smartlist_len(list);
+  tor_assert(start >= 0);
+  tor_assert(start <= list_len);
+  if (end == -1) {
+    end = list_len;
+  }
+  tor_assert(start <= end);
+
+  smartlist_slice_t *slice = tor_malloc(sizeof(smartlist_slice_t));
+  slice->list = list;
+  slice->offset = start;
+  slice->len = end - start;
+  return slice;
+}
+
+/** Helper: Compute the longest common subsequence lengths for the two slices.
+ * Used as part of the diff generation to find the column at which to split
+ * slice2 while still having the optimal solution.
+ * If direction is -1, the navigation is reversed. Otherwise it must be 1.
+ * The length of the resulting integer array is that of the second slice plus
+ * one.
+ */
+STATIC int *
+lcs_lengths(const smartlist_slice_t *slice1, const smartlist_slice_t *slice2,
+            int direction)
+{
+  size_t a_size = sizeof(int) * (slice2->len+1);
+
+  /* Resulting lcs lengths. */
+  int *result = tor_malloc_zero(a_size);
+  /* Copy of the lcs lengths from the last iteration. */
+  int *prev = tor_malloc(a_size);
+
+  tor_assert(direction == 1 || direction == -1);
+
+  int si = slice1->offset;
+  if (direction == -1) {
+    si += (slice1->len-1);
+  }
+
+  for (int i = 0; i < slice1->len; ++i, si+=direction) {
+
+    const cdline_t *line1 = smartlist_get(slice1->list, si);
+    /* Store the last results. */
+    memcpy(prev, result, a_size);
+
+    int sj = slice2->offset;
+    if (direction == -1) {
+      sj += (slice2->len-1);
+    }
+
+    for (int j = 0; j < slice2->len; ++j, sj+=direction) {
+
+      const cdline_t *line2 = smartlist_get(slice2->list, sj);
+      if (lines_eq(line1, line2)) {
+        /* If the lines are equal, the lcs is one line longer. */
+        result[j + 1] = prev[j] + 1;
+      } else {
+        /* If not, see what lcs parent path is longer. */
+        result[j + 1] = MAX(result[j], prev[j + 1]);
+      }
+    }
+  }
+  tor_free(prev);
+  return result;
+}
+
+/** Helper: Trim any number of lines that are equally at the start or the end
+ * of both slices.
+ */
+STATIC void
+trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2)
+{
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, slice1->offset);
+    const cdline_t *line2 = smartlist_get(slice2->list, slice2->offset);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    slice1->offset++; slice1->len--;
+    slice2->offset++; slice2->len--;
+  }
+
+  int i1 = (slice1->offset+slice1->len)-1;
+  int i2 = (slice2->offset+slice2->len)-1;
+
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, i1);
+    const cdline_t *line2 = smartlist_get(slice2->list, i2);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    i1--;
+    slice1->len--;
+    i2--;
+    slice2->len--;
+  }
+}
+
+/** Like smartlist_string_pos, but uses a cdline_t, and is restricted to the
+ * bounds of the slice.
+ */
+STATIC int
+smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                           const cdline_t *string)
+{
+  int end = slice->offset + slice->len;
+  for (int i = slice->offset; i < end; ++i) {
+    const cdline_t *el = smartlist_get(slice->list, i);
+    if (lines_eq(el, string)) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/** Helper: Set all the appropriate changed booleans to true. The first slice
+ * must be of length 0 or 1. All the lines of slice1 and slice2 which are not
+ * present in the other slice will be set to changed in their bool array.
+ * The two changed bool arrays are passed in the same order as the slices.
+ */
+STATIC void
+set_changed(bitarray_t *changed1, bitarray_t *changed2,
+            const smartlist_slice_t *slice1, const smartlist_slice_t *slice2)
+{
+  int toskip = -1;
+  tor_assert(slice1->len == 0 || slice1->len == 1);
+
+  if (slice1->len == 1) {
+    const cdline_t *line_common = smartlist_get(slice1->list, slice1->offset);
+    toskip = smartlist_slice_string_pos(slice2, line_common);
+    if (toskip == -1) {
+      bitarray_set(changed1, slice1->offset);
+    }
+  }
+  int end = slice2->offset + slice2->len;
+  for (int i = slice2->offset; i < end; ++i) {
+    if (i != toskip) {
+      bitarray_set(changed2, i);
+    }
+  }
+}
+
+/*
+ * Helper: Given that slice1 has been split by half into top and bot, we want
+ * to fetch the column at which to split slice2 so that we are still on track
+ * to the optimal diff solution, i.e. the shortest one. We use lcs_lengths
+ * since the shortest diff is just another way to say the longest common
+ * subsequence.
+ */
+static int
+optimal_column_to_split(const smartlist_slice_t *top,
+                        const smartlist_slice_t *bot,
+                        const smartlist_slice_t *slice2)
+{
+  int *lens_top = lcs_lengths(top, slice2, 1);
+  int *lens_bot = lcs_lengths(bot, slice2, -1);
+  int column=0, max_sum=-1;
+
+  for (int i = 0; i < slice2->len+1; ++i) {
+    int sum = lens_top[i] + lens_bot[slice2->len-i];
+    if (sum > max_sum) {
+      column = i;
+      max_sum = sum;
+    }
+  }
+  tor_free(lens_top);
+  tor_free(lens_bot);
+
+  return column;
+}
+
+/**
+ * Helper: Figure out what elements are new or gone on the second smartlist
+ * relative to the first smartlist, and store the booleans in the bitarrays.
+ * True on the first bitarray means the element is gone, true on the second
+ * bitarray means it's new.
+ *
+ * In its base case, either of the smartlists is of length <= 1 and we can
+ * quickly see what elements are new or are gone. In the other case, we will
+ * split one smartlist by half and we'll use optimal_column_to_split to find
+ * the optimal column at which to split the second smartlist so that we are
+ * finding the smallest diff possible.
+ */
+STATIC void
+calc_changes(smartlist_slice_t *slice1,
+             smartlist_slice_t *slice2,
+             bitarray_t *changed1, bitarray_t *changed2)
+{
+  trim_slices(slice1, slice2);
+
+  if (slice1->len <= 1) {
+    set_changed(changed1, changed2, slice1, slice2);
+
+  } else if (slice2->len <= 1) {
+    set_changed(changed2, changed1, slice2, slice1);
+
+  /* Keep on splitting the slices in two. */
+  } else {
+    smartlist_slice_t *top, *bot, *left, *right;
+
+    /* Split the first slice in half. */
+    int mid = slice1->len/2;
+    top = smartlist_slice(slice1->list, slice1->offset, slice1->offset+mid);
+    bot = smartlist_slice(slice1->list, slice1->offset+mid,
+        slice1->offset+slice1->len);
+
+    /* Split the second slice by the optimal column. */
+    int mid2 = optimal_column_to_split(top, bot, slice2);
+    left = smartlist_slice(slice2->list, slice2->offset, slice2->offset+mid2);
+    right = smartlist_slice(slice2->list, slice2->offset+mid2,
+        slice2->offset+slice2->len);
+
+    calc_changes(top, left, changed1, changed2);
+    calc_changes(bot, right, changed1, changed2);
+    tor_free(top);
+    tor_free(bot);
+    tor_free(left);
+    tor_free(right);
+  }
+}
+
+/* This table is from crypto.c. The SP and PAD defines are different. */
+#define NOT_VALID_BASE64 255
+#define X NOT_VALID_BASE64
+#define SP NOT_VALID_BASE64
+#define PAD NOT_VALID_BASE64
+static const uint8_t base64_compare_table[256] = {
+  X, X, X, X, X, X, X, X, X, SP, SP, SP, X, SP, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  SP, X, X, X, X, X, X, X, X, X, X, 62, X, X, X, 63,
+  52, 53, 54, 55, 56, 57, 58, 59, 60, 61, X, X, X, PAD, X, X,
+  X, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+  15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, X, X, X, X, X,
+  X, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+  41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+};
+
+/** Helper: Get the identity hash from a router line, assuming that the line
+ * at least appears to be a router line and thus starts with "r ".
+ *
+ * If an identity hash is found, store it (without decoding it) in
+ * <b>hash_out</b>, and return 0.  On failure, return -1.
+ */
+STATIC int
+get_id_hash(const cdline_t *line, cdline_t *hash_out)
+{
+  if (line->len < 2)
+    return -1;
+
+  /* Skip the router name. */
+  const char *hash = memchr(line->s + 2, ' ', line->len - 2);
+  if (!hash) {
+    return -1;
+  }
+
+  hash++;
+  const char *hash_end = hash;
+  /* Stop when the first non-base64 character is found. Use unsigned chars to
+   * avoid negative indexes causing crashes.
+   */
+  while (base64_compare_table[*((unsigned char*)hash_end)]
+           != NOT_VALID_BASE64 &&
+         hash_end < line->s + line->len) {
+    hash_end++;
+  }
+
+  /* Empty hash. */
+  if (hash_end == hash) {
+    return -1;
+  }
+
+  hash_out->s = hash;
+  /* Always true because lines are limited to this length */
+  tor_assert(hash_end >= hash);
+  tor_assert((size_t)(hash_end - hash) <= UINT32_MAX);
+  hash_out->len = (uint32_t)(hash_end - hash);
+
+  return 0;
+}
+
+/** Helper: Check that a line is a valid router entry. We must at least be
+ * able to fetch a proper identity hash from it for it to be valid.
+ */
+STATIC int
+is_valid_router_entry(const cdline_t *line)
+{
+  if (line->len < 2 || fast_memneq(line->s, "r ", 2))
+    return 0;
+  cdline_t tmp;
+  return (get_id_hash(line, &tmp) == 0);
+}
+
+/** Helper: Find the next router line starting at the current position.
+ * Assumes that cur is lower than the length of the smartlist, i.e. it is a
+ * line within the bounds of the consensus. The only exception is when we
+ * don't want to skip the first line, in which case cur will be -1.
+ */
+STATIC int
+next_router(const smartlist_t *cons, int cur)
+{
+  int len = smartlist_len(cons);
+  tor_assert(cur >= -1 && cur < len);
+
+  if (++cur >= len) {
+    return len;
+  }
+
+  const cdline_t *line = smartlist_get(cons, cur);
+  while (!is_valid_router_entry(line)) {
+    if (++cur >= len) {
+      return len;
+    }
+    line = smartlist_get(cons, cur);
+  }
+  return cur;
+}
+
+/** Helper: compare two base64-encoded identity hashes, which may be of
+ * different lengths. Comparison ends when the first non-base64 char is found.
+ */
+STATIC int
+base64cmp(const cdline_t *hash1, const cdline_t *hash2)
+{
+  /* NULL is always lower, useful for last_hash which starts at NULL. */
+  if (!hash1->s && !hash2->s) {
+    return 0;
+  }
+  if (!hash1->s) {
+    return -1;
+  }
+  if (!hash2->s) {
+    return 1;
+  }
+
+  /* Don't index with a char; char may be signed. */
+  const unsigned char *a = (unsigned char*)hash1->s;
+  const unsigned char *b = (unsigned char*)hash2->s;
+  const unsigned char *a_end = a + hash1->len;
+  const unsigned char *b_end = b + hash2->len;
+  while (1) {
+    uint8_t av = base64_compare_table[*a];
+    uint8_t bv = base64_compare_table[*b];
+    if (av == NOT_VALID_BASE64) {
+      if (bv == NOT_VALID_BASE64) {
+        /* Both ended with exactly the same characters. */
+        return 0;
+      } else {
+        /* hash2 goes on longer than hash1 and thus hash1 is lower. */
+        return -1;
+      }
+    } else if (bv == NOT_VALID_BASE64) {
+      /* hash1 goes on longer than hash2 and thus hash1 is greater. */
+      return 1;
+    } else if (av < bv) {
+      /* The first difference shows that hash1 is lower. */
+      return -1;
+    } else if (av > bv) {
+      /* The first difference shows that hash1 is greater. */
+      return 1;
+    } else {
+      a++;
+      b++;
+      if (a == a_end) {
+        if (b == b_end) {
+          return 0;
+        } else {
+          return -1;
+        }
+      } else if (b == b_end) {
+        return 1;
+      }
+    }
+  }
+}
+
+/** Structure used to remember the previous and current identity hash of
+ * the "r " lines in a consensus, to enforce well-ordering. */
+typedef struct router_id_iterator_t {
+  cdline_t last_hash;
+  cdline_t hash;
+} router_id_iterator_t;
+
+/**
+ * Initializer for a router_id_iterator_t.
+ */
+#define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } }
+
+/** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance
+ * the index to the next router line ("r ...") in the consensus, or to
+ * an index one after the end of the list if there is no such line.
+ *
+ * Use <b>iter</b> to record the hash of the found router line, if any,
+ * and to enforce ordering on the hashes.  If the hashes are mis-ordered,
+ * return -1.  Else, return 0.
+ **/
+static int
+find_next_router_line(const smartlist_t *cons,
+                      const char *consname,
+                      int *idxp,
+                      router_id_iterator_t *iter)
+{
+  *idxp = next_router(cons, *idxp);
+  if (*idxp < smartlist_len(cons)) {
+    memcpy(&iter->last_hash, &iter->hash, sizeof(cdline_t));
+    if (get_id_hash(smartlist_get(cons, *idxp), &iter->hash) < 0 ||
+        base64cmp(&iter->hash, &iter->last_hash) <= 0) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+               "the %s consensus doesn't have its router entries sorted "
+               "properly.", consname);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Line-prefix indicating the beginning of the signatures section that we
+ * intend to delete. */
+#define START_OF_SIGNATURES_SECTION "directory-signature "
+
+/** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t)
+ * to remove the signatures from it.  If the footer is removed, return a
+ * cdline_t containing a delete command to delete the footer, allocated in
+ * <b>area</>.  If no footer is removed, return NULL.
+ *
+ * We remove the signatures here because they are not themselves signed, and
+ * as such there might be different encodings for them.
+ */
+static cdline_t *
+preprocess_consensus(memarea_t *area,
+                     smartlist_t *cons)
+{
+  int idx;
+  int dirsig_idx = -1;
+  for (idx = 0; idx < smartlist_len(cons); ++idx) {
+    cdline_t *line = smartlist_get(cons, idx);
+    if (line_starts_with_str(line, START_OF_SIGNATURES_SECTION)) {
+      dirsig_idx = idx;
+      break;
+    }
+  }
+  if (dirsig_idx >= 0) {
+    char buf[64];
+    while (smartlist_len(cons) > dirsig_idx)
+      smartlist_del(cons, dirsig_idx);
+    tor_snprintf(buf, sizeof(buf), "%d,$d", dirsig_idx+1);
+    return cdline_linecpy(area, buf);
+  } else {
+    return NULL;
+  }
+}
+
+/** Generate an ed diff as a smartlist from two consensuses, also given as
+ * smartlists. Will return NULL if the diff could not be generated, which can
+ * happen if any lines the script had to add matched "." or if the routers
+ * were not properly ordered.
+ *
+ * All cdline_t objects in the resulting object are either references to lines
+ * in one of the inputs, or are newly allocated lines in the provided memarea.
+ *
+ * This implementation is consensus-specific. To generate an ed diff for any
+ * given input in quadratic time, you can replace all the code until the
+ * navigation in reverse order with the following:
+ *
+ *   int len1 = smartlist_len(cons1);
+ *   int len2 = smartlist_len(cons2);
+ *   bitarray_t *changed1 = bitarray_init_zero(len1);
+ *   bitarray_t *changed2 = bitarray_init_zero(len2);
+ *   cons1_sl = smartlist_slice(cons1, 0, -1);
+ *   cons2_sl = smartlist_slice(cons2, 0, -1);
+ *   calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+ */
+STATIC smartlist_t *
+gen_ed_diff(const smartlist_t *cons1_orig, const smartlist_t *cons2,
+            memarea_t *area)
+{
+  smartlist_t *cons1 = smartlist_new();
+  smartlist_add_all(cons1, cons1_orig);
+  cdline_t *remove_trailer = preprocess_consensus(area, cons1);
+
+  int len1 = smartlist_len(cons1);
+  int len2 = smartlist_len(cons2);
+  smartlist_t *result = smartlist_new();
+
+  if (remove_trailer) {
+    /* There's a delete-the-trailer line at the end, so add it here. */
+    smartlist_add(result, remove_trailer);
+  }
+
+  /* Initialize the changed bitarrays to zero, so that calc_changes only needs
+   * to set the ones that matter and leave the rest untouched.
+   */
+  bitarray_t *changed1 = bitarray_init_zero(len1);
+  bitarray_t *changed2 = bitarray_init_zero(len2);
+  int i1=-1, i2=-1;
+  int start1=0, start2=0;
+
+  /* To check that hashes are ordered properly */
+  router_id_iterator_t iter1 = ROUTER_ID_ITERATOR_INIT;
+  router_id_iterator_t iter2 = ROUTER_ID_ITERATOR_INIT;
+
+  /* i1 and i2 are initialized at the first line of each consensus. They never
+   * reach past len1 and len2 respectively, since next_router doesn't let that
+   * happen. i1 and i2 are advanced by at least one line at each iteration as
+   * long as they have not yet reached len1 and len2, so the loop is
+   * guaranteed to end, and each pair of (i1,i2) will be inspected at most
+   * once.
+   */
+  while (i1 < len1 || i2 < len2) {
+
+    /* Advance each of the two navigation positions by one router entry if not
+     * yet at the end.
+     */
+    if (i1 < len1) {
+      if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    if (i2 < len2) {
+      if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    /* If we have reached the end of both consensuses, there is no need to
+     * compare hashes anymore, since this is the last iteration.
+     */
+    if (i1 < len1 || i2 < len2) {
+
+      /* Keep on advancing the lower (by identity hash sorting) position until
+       * we have two matching positions. The only other possible outcome is
+       * that a lower position reaches the end of the consensus before it can
+       * reach a hash that is no longer the lower one. Since there will always
+       * be a lower hash for as long as the loop runs, one of the two indexes
+       * will always be incremented, thus assuring that the loop must end
+       * after a finite number of iterations. If that cannot be because said
+       * consensus has already reached the end, both are extended to their
+       * respecting ends since we are done.
+       */
+      int cmp = base64cmp(&iter1.hash, &iter2.hash);
+      while (cmp != 0) {
+        if (i1 < len1 && cmp < 0) {
+          if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+            goto error_cleanup;
+          }
+          if (i1 == len1) {
+            /* We finished the first consensus, so grab all the remaining
+             * lines of the second consensus and finish up.
+             */
+            i2 = len2;
+            break;
+          }
+        } else if (i2 < len2 && cmp > 0) {
+          if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+            goto error_cleanup;
+          }
+          if (i2 == len2) {
+            /* We finished the second consensus, so grab all the remaining
+             * lines of the first consensus and finish up.
+             */
+            i1 = len1;
+            break;
+          }
+        } else {
+          i1 = len1;
+          i2 = len2;
+          break;
+        }
+        cmp = base64cmp(&iter1.hash, &iter2.hash);
+      }
+    }
+
+    /* Make slices out of these chunks (up to the common router entry) and
+     * calculate the changes for them.
+     * Error if any of the two slices are longer than 10K lines. That should
+     * never happen with any pair of real consensuses. Feeding more than 10K
+     * lines to calc_changes would be very slow anyway.
+     */
+#define MAX_LINE_COUNT (10000)
+    if (i1-start1 > MAX_LINE_COUNT || i2-start2 > MAX_LINE_COUNT) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+          "we found too few common router ids.");
+      goto error_cleanup;
+    }
+
+    smartlist_slice_t *cons1_sl = smartlist_slice(cons1, start1, i1);
+    smartlist_slice_t *cons2_sl = smartlist_slice(cons2, start2, i2);
+    calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+    tor_free(cons1_sl);
+    tor_free(cons2_sl);
+    start1 = i1, start2 = i2;
+  }
+
+  /* Navigate the changes in reverse order and generate one ed command for
+   * each chunk of changes.
+   */
+  i1=len1-1, i2=len2-1;
+  char buf[128];
+  while (i1 >= 0 || i2 >= 0) {
+
+    int start1x, start2x, end1, end2, added, deleted;
+
+    /* We are at a point were no changed bools are true, so just keep going. */
+    if (!(i1 >= 0 && bitarray_is_set(changed1, i1)) &&
+        !(i2 >= 0 && bitarray_is_set(changed2, i2))) {
+      if (i1 >= 0) {
+        i1--;
+      }
+      if (i2 >= 0) {
+        i2--;
+      }
+      continue;
+    }
+
+    end1 = i1, end2 = i2;
+
+    /* Grab all contiguous changed lines */
+    while (i1 >= 0 && bitarray_is_set(changed1, i1)) {
+      i1--;
+    }
+    while (i2 >= 0 && bitarray_is_set(changed2, i2)) {
+      i2--;
+    }
+
+    start1x = i1+1, start2x = i2+1;
+    added = end2-i2, deleted = end1-i1;
+
+    if (added == 0) {
+      if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%id", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%id", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+    } else {
+      int i;
+      if (deleted == 0) {
+        tor_snprintf(buf, sizeof(buf), "%ia", start1x);
+        smartlist_add_linecpy(result, area, buf);
+      } else if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%ic", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%ic", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+
+      for (i = start2x; i <= end2; ++i) {
+        cdline_t *line = smartlist_get(cons2, i);
+        if (line_str_eq(line, ".")) {
+          log_warn(LD_CONSDIFF, "Cannot generate consensus diff because "
+              "one of the lines to be added is \".\".");
+          goto error_cleanup;
+        }
+        smartlist_add(result, line);
+      }
+      smartlist_add_linecpy(result, area, ".");
+    }
+  }
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  return result;
+
+ error_cleanup:
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  smartlist_free(result);
+
+  return NULL;
+}
+
+/* Helper: Read a base-10 number between 0 and INT32_MAX from <b>s</b> and
+ * store it in <b>num_out</b>.  Advance <b>s</b> to the characer immediately
+ * after the number.  Return 0 on success, -1 on failure. */
+static int
+get_linenum(const char **s, int *num_out)
+{
+  int ok;
+  char *next;
+  if (!TOR_ISDIGIT(**s)) {
+    return -1;
+  }
+  *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next);
+  if (ok && next) {
+    *s = next;
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
+/** Apply the ed diff, starting at <b>diff_starting_line</b>, to the consensus
+ * and return a new consensus, also as a line-based smartlist. Will return
+ * NULL if the ed diff is not properly formatted.
+ *
+ * All cdline_t objects in the resulting object are references to lines
+ * in one of the inputs; nothing is copied.
+ */
+STATIC smartlist_t *
+apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff,
+              int diff_starting_line)
+{
+  int diff_len = smartlist_len(diff);
+  int j = smartlist_len(cons1);
+  smartlist_t *cons2 = smartlist_new();
+
+  for (int i=diff_starting_line; i<diff_len; ++i) {
+    const cdline_t *diff_cdline = smartlist_get(diff, i);
+    char diff_line[128];
+
+    if (diff_cdline->len > sizeof(diff_line) - 1) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was far too long");
+      goto error_cleanup;
+    }
+    /* Copy the line to make it nul-terminated. */
+    memcpy(diff_line, diff_cdline->s, diff_cdline->len);
+    diff_line[diff_cdline->len] = 0;
+    const char *ptr = diff_line;
+    int start = 0, end = 0;
+    int had_range = 0;
+    int end_was_eof = 0;
+    if (get_linenum(&ptr, &start) < 0) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was missing a line number.");
+      goto error_cleanup;
+    }
+    if (*ptr == ',') {
+      /* Two-item range */
+      had_range = 1;
+      ++ptr;
+      if (*ptr == '$') {
+        end_was_eof = 1;
+        end = smartlist_len(cons1);
+        ++ptr;
+      } else if (get_linenum(&ptr, &end) < 0) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an ed command was missing a range end line number.");
+        goto error_cleanup;
+      }
+      /* Incoherent range. */
+      if (end <= start) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an invalid range was found in an ed command.");
+        goto error_cleanup;
+      }
+    } else {
+      /* We'll take <n1> as <n1>,<n1> for simplicity. */
+      end = start;
+    }
+
+    if (end > j) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "its commands are not properly sorted in reverse order.");
+      goto error_cleanup;
+    }
+
+    if (*ptr == '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "a line with no ed command was found");
+      goto error_cleanup;
+    }
+
+    if (*(ptr+1) != '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "an ed command longer than one char was found.");
+      goto error_cleanup;
+    }
+
+    char action = *ptr;
+
+    switch (action) {
+      case 'a':
+      case 'c':
+      case 'd':
+        break;
+      default:
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "an unrecognised ed command was found.");
+        goto error_cleanup;
+    }
+
+    /** $ is not allowed with non-d actions. */
+    if (end_was_eof && action != 'd') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "it wanted to use $ with a command other than delete");
+      goto error_cleanup;
+    }
+
+    /* 'a' commands are not allowed to have ranges. */
+    if (had_range && action == 'a') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "it wanted to add lines after a range.");
+      goto error_cleanup;
+    }
+
+    /* Add unchanged lines. */
+    for (; j && j > end; --j) {
+      cdline_t *cons_line = smartlist_get(cons1, j-1);
+      smartlist_add(cons2, cons_line);
+    }
+
+    /* Ignore removed lines. */
+    if (action == 'c' || action == 'd') {
+      while (--j >= start) {
+        /* Skip line */
+      }
+    }
+
+    /* Add new lines in reverse order, since it will all be reversed at the
+     * end.
+     */
+    if (action == 'a' || action == 'c') {
+      int added_end = i;
+
+      i++; /* Skip the line with the range and command. */
+      while (i < diff_len) {
+        if (line_str_eq(smartlist_get(diff, i), ".")) {
+          break;
+        }
+        if (++i == diff_len) {
+          log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+              "it has lines to be inserted that don't end with a \".\".");
+          goto error_cleanup;
+        }
+      }
+
+      int added_i = i-1;
+
+      /* It would make no sense to add zero new lines. */
+      if (added_i == added_end) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "it has an ed command that tries to insert zero lines.");
+        goto error_cleanup;
+      }
+
+      while (added_i > added_end) {
+        cdline_t *added_line = smartlist_get(diff, added_i--);
+        smartlist_add(cons2, added_line);
+      }
+    }
+  }
+
+  /* Add remaining unchanged lines. */
+  for (; j > 0; --j) {
+    cdline_t *cons_line = smartlist_get(cons1, j-1);
+    smartlist_add(cons2, cons_line);
+  }
+
+  /* Reverse the whole thing since we did it from the end. */
+  smartlist_reverse(cons2);
+  return cons2;
+
+ error_cleanup:
+
+  smartlist_free(cons2);
+
+  return NULL;
+}
+
+/** Generate a consensus diff as a smartlist from two given consensuses, also
+ * as smartlists. Will return NULL if the consensus diff could not be
+ * generated. Neither of the two consensuses are modified in any way, so it's
+ * up to the caller to free their resources.
+ */
+smartlist_t *
+consdiff_gen_diff(const smartlist_t *cons1,
+                  const smartlist_t *cons2,
+                  const consensus_digest_t *digests1,
+                  const consensus_digest_t *digests2,
+                  memarea_t *area)
+{
+  smartlist_t *ed_diff = gen_ed_diff(cons1, cons2, area);
+  /* ed diff could not be generated - reason already logged by gen_ed_diff. */
+  if (!ed_diff) {
+    goto error_cleanup;
+  }
+
+  /* See that the script actually produces what we want. */
+  smartlist_t *ed_cons2 = apply_ed_diff(cons1, ed_diff, 0);
+  if (!ed_cons2) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff could not be tested to successfully generate "
+        "the target consensus.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  int cons2_eq = 1;
+  if (smartlist_len(cons2) == smartlist_len(ed_cons2)) {
+    SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) {
+      const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx);
+      if (! lines_eq(line1, line2) ) {
+        cons2_eq = 0;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(line1);
+  } else {
+    cons2_eq = 0;
+  }
+  smartlist_free(ed_cons2);
+  if (!cons2_eq) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct. */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff did not generate the target consensus "
+        "successfully when tested.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  char cons1_hash_hex[HEX_DIGEST256_LEN+1];
+  char cons2_hash_hex[HEX_DIGEST256_LEN+1];
+  base16_encode(cons1_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests1->sha3_256, DIGEST256_LEN);
+  base16_encode(cons2_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests2->sha3_256, DIGEST256_LEN);
+
+  /* Create the resulting consensus diff. */
+  char buf[160];
+  smartlist_t *result = smartlist_new();
+  tor_snprintf(buf, sizeof(buf), "%s", ns_diff_version);
+  smartlist_add_linecpy(result, area, buf);
+  tor_snprintf(buf, sizeof(buf), "%s %s %s", hash_token,
+      cons1_hash_hex, cons2_hash_hex);
+  smartlist_add_linecpy(result, area, buf);
+  smartlist_add_all(result, ed_diff);
+  smartlist_free(ed_diff);
+  return result;
+
+ error_cleanup:
+
+  if (ed_diff) {
+    /* LCOV_EXCL_START -- ed_diff is NULL except in unreachable cases above */
+    smartlist_free(ed_diff);
+    /* LCOV_EXCL_STOP */
+  }
+
+  return NULL;
+}
+
+/** Fetch the digest of the base consensus in the consensus diff, encoded in
+ * base16 as found in the diff itself. digest1_out and digest2_out must be of
+ * length DIGEST256_LEN or larger if not NULL.
+ */
+int
+consdiff_get_digests(const smartlist_t *diff,
+                     char *digest1_out,
+                     char *digest2_out)
+{
+  smartlist_t *hash_words = NULL;
+  const cdline_t *format;
+  char cons1_hash[DIGEST256_LEN], cons2_hash[DIGEST256_LEN];
+  char *cons1_hash_hex, *cons2_hash_hex;
+  if (smartlist_len(diff) < 2) {
+    log_info(LD_CONSDIFF, "The provided consensus diff is too short.");
+    goto error_cleanup;
+  }
+
+  /* Check that it's the format and version we know. */
+  format = smartlist_get(diff, 0);
+  if (!line_str_eq(format, ns_diff_version)) {
+    log_warn(LD_CONSDIFF, "The provided consensus diff format is not known.");
+    goto error_cleanup;
+  }
+
+  /* Grab the base16 digests. */
+  hash_words = smartlist_new();
+  {
+    const cdline_t *line2 = smartlist_get(diff, 1);
+    char *h = tor_memdup_nulterm(line2->s, line2->len);
+    smartlist_split_string(hash_words, h, " ", 0, 0);
+    tor_free(h);
+  }
+
+  /* There have to be three words, the first of which must be hash_token. */
+  if (smartlist_len(hash_words) != 3 ||
+      strcmp(smartlist_get(hash_words, 0), hash_token)) {
+    log_info(LD_CONSDIFF, "The provided consensus diff does not include "
+        "the necessary digests.");
+    goto error_cleanup;
+  }
+
+  /* Expected hashes as found in the consensus diff header. They must be of
+   * length HEX_DIGEST256_LEN, normally 64 hexadecimal characters.
+   * If any of the decodings fail, error to make sure that the hashes are
+   * proper base16-encoded digests.
+   */
+  cons1_hash_hex = smartlist_get(hash_words, 1);
+  cons2_hash_hex = smartlist_get(hash_words, 2);
+  if (strlen(cons1_hash_hex) != HEX_DIGEST256_LEN ||
+      strlen(cons2_hash_hex) != HEX_DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "base16-encoded digests of incorrect size.");
+    goto error_cleanup;
+  }
+
+  if (base16_decode(cons1_hash, DIGEST256_LEN,
+          cons1_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN ||
+      base16_decode(cons2_hash, DIGEST256_LEN,
+          cons2_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "malformed digests.");
+    goto error_cleanup;
+  }
+
+  if (digest1_out) {
+    memcpy(digest1_out, cons1_hash, DIGEST256_LEN);
+  }
+  if (digest2_out) {
+    memcpy(digest2_out, cons2_hash, DIGEST256_LEN);
+  }
+
+  SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+  smartlist_free(hash_words);
+  return 0;
+
+ error_cleanup:
+
+  if (hash_words) {
+    SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+    smartlist_free(hash_words);
+  }
+  return 1;
+}
+
+/** Apply the consensus diff to the given consensus and return a new
+ * consensus, also as a line-based smartlist. Will return NULL if the diff
+ * could not be applied. Neither the consensus nor the diff are modified in
+ * any way, so it's up to the caller to free their resources.
+ */
+char *
+consdiff_apply_diff(const smartlist_t *cons1,
+                    const smartlist_t *diff,
+                    const consensus_digest_t *digests1)
+{
+  smartlist_t *cons2 = NULL;
+  char *cons2_str = NULL;
+  char e_cons1_hash[DIGEST256_LEN];
+  char e_cons2_hash[DIGEST256_LEN];
+
+  if (consdiff_get_digests(diff, e_cons1_hash, e_cons2_hash) != 0) {
+    goto error_cleanup;
+  }
+
+  /* See that the consensus that was given to us matches its hash. */
+  if (!consensus_digest_eq(digests1->sha3_256,
+                           (const uint8_t*)e_cons1_hash)) {
+    char hex_digest1[HEX_DIGEST256_LEN+1];
+    char e_hex_digest1[HEX_DIGEST256_LEN+1];
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the base consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    base16_encode(hex_digest1, HEX_DIGEST256_LEN+1,
+                  (const char *)digests1->sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest1, HEX_DIGEST256_LEN+1,
+                  e_cons1_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest1, e_hex_digest1);
+    goto error_cleanup;
+  }
+
+  /* Grab the ed diff and calculate the resulting consensus. */
+  /* Skip the first two lines. */
+  cons2 = apply_ed_diff(cons1, diff, 2);
+
+  /* ed diff could not be applied - reason already logged by apply_ed_diff. */
+  if (!cons2) {
+    goto error_cleanup;
+  }
+
+  cons2_str = consensus_join_lines(cons2);
+
+  consensus_digest_t cons2_digests;
+  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+    /* LCOV_EXCL_START -- digest can't fail */
+    log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
+        "resulting from applying a consensus diff.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* See that the resulting consensus matches its hash. */
+  if (!consensus_digest_eq(cons2_digests.sha3_256,
+                           (const uint8_t*)e_cons2_hash)) {
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the resulting consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    char hex_digest2[HEX_DIGEST256_LEN+1];
+    char e_hex_digest2[HEX_DIGEST256_LEN+1];
+    base16_encode(hex_digest2, HEX_DIGEST256_LEN+1,
+        (const char *)cons2_digests.sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest2, HEX_DIGEST256_LEN+1,
+        e_cons2_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest2, e_hex_digest2);
+    goto error_cleanup;
+  }
+
+  goto done;
+
+ error_cleanup:
+  tor_free(cons2_str); /* Sets it to NULL */
+
+ done:
+  if (cons2) {
+    smartlist_free(cons2);
+  }
+
+  return cons2_str;
+}
+
+/** Any consensus line longer than this means that the input is invalid. */
+#define CONSENSUS_LINE_MAX_LEN (1<<20)
+
+/**
+ * Helper: For every NL-terminated line in <b>s</b>, add a cdline referring to
+ * that line (without trailing newline) to <b>out</b>.  Return -1 if there are
+ * any non-NL terminated lines; 0 otherwise.
+ *
+ * Unlike tor_split_lines, this function avoids ambiguity on its
+ * handling of a final line that isn't NL-terminated.
+ *
+ * All cdline_t objects are allocated in the provided memarea.  Strings
+ * are not copied: if <b>s</b> changes or becomes invalid, then all
+ * generated cdlines will become invalid.
+ */
+STATIC int
+consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+{
+  const char *end_of_str = s + strlen(s);
+  tor_assert(*end_of_str == '\0');
+
+  while (*s) {
+    const char *eol = memchr(s, '\n', end_of_str - s);
+    if (!eol) {
+      /* File doesn't end with newline. */
+      return -1;
+    }
+    if (eol - s > CONSENSUS_LINE_MAX_LEN) {
+      /* Line is far too long. */
+      return -1;
+    }
+    cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+    line->s = s;
+    line->len = (uint32_t)(eol - s);
+    smartlist_add(out, line);
+    s = eol+1;
+  }
+  return 0;
+}
+
+/** Given a list of cdline_t, return a newly allocated string containing
+ * all of the lines, terminated with NL, concatenated.
+ *
+ * Unlike smartlist_join_strings(), avoids lossy operations on empty
+ * lists.  */
+static char *
+consensus_join_lines(const smartlist_t *inp)
+{
+  size_t n = 0;
+  SMARTLIST_FOREACH(inp, const cdline_t *, cdline, n += cdline->len + 1);
+  n += 1;
+  char *result = tor_malloc(n);
+  char *out = result;
+  SMARTLIST_FOREACH_BEGIN(inp, const cdline_t *, cdline) {
+    memcpy(out, cdline->s, cdline->len);
+    out += cdline->len;
+    *out++ = '\n';
+  } SMARTLIST_FOREACH_END(cdline);
+  *out++ = '\0';
+  tor_assert(out == result+n);
+  return result;
+}
+
+/** Given two consensus documents, try to compute a diff between them.  On
+ * success, retun a newly allocated string containing that diff.  On failure,
+ * return NULL. */
+char *
+consensus_diff_generate(const char *cons1,
+                        const char *cons2)
+{
+  consensus_digest_t d1, d2;
+  smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
+  int r1, r2;
+  char *result = NULL;
+
+  r1 = consensus_compute_digest_as_signed(cons1, &d1);
+  r2 = consensus_compute_digest(cons2, &d2);
+  if (BUG(r1 < 0 || r2 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  memarea_t *area = memarea_new();
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, cons1, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, cons2, area) < 0)
+    goto done;
+
+  result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
+
+ done:
+  if (result_lines) {
+    result = consensus_join_lines(result_lines);
+    smartlist_free(result_lines);
+  }
+
+  memarea_drop_all(area);
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+
+  return result;
+}
+
+/** Given a consensus document and a diff, try to apply the diff to the
+ * consensus.  On success return a newly allocated string containing the new
+ * consensus.  On failure, return NULL. */
+char *
+consensus_diff_apply(const char *consensus,
+                     const char *diff)
+{
+  consensus_digest_t d1;
+  smartlist_t *lines1 = NULL, *lines2 = NULL;
+  int r1;
+  char *result = NULL;
+  memarea_t *area = memarea_new();
+
+  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  if (BUG(r1 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, consensus, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, diff, area) < 0)
+    goto done;
+
+  result = consdiff_apply_diff(lines1, lines2, &d1);
+
+ done:
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+  memarea_drop_all(area);
+
+  return result;
+}
+
+/** Return true iff, based on its header, <b>document</b> is likely
+ * to be a consensus diff. */
+int
+looks_like_a_consensus_diff(const char *document, size_t len)
+{
+  return (len >= strlen(ns_diff_version) &&
+          fast_memeq(document, ns_diff_version, strlen(ns_diff_version)));
+}
+
diff --git a/src/or/consdiff.h b/src/or/consdiff.h
new file mode 100644
index 0000000000..eb772c0b2b
--- /dev/null
+++ b/src/or/consdiff.h
@@ -0,0 +1,98 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFF_H
+#define TOR_CONSDIFF_H
+
+#include "or.h"
+
+char *consensus_diff_generate(const char *cons1,
+                              const char *cons2);
+char *consensus_diff_apply(const char *consensus,
+                           const char *diff);
+
+int looks_like_a_consensus_diff(const char *document, size_t len);
+
+#ifdef CONSDIFF_PRIVATE
+struct memarea_t;
+
+/** Line type used for constructing consensus diffs.  Each of these lines
+ * refers to a chunk of memory allocated elsewhere, and is not necessarily
+ * NUL-terminated: this helps us avoid copies and save memory. */
+typedef struct cdline_t {
+  const char *s;
+  uint32_t len;
+} cdline_t;
+
+typedef struct consensus_digest_t {
+  uint8_t sha3_256[DIGEST256_LEN];
+} consensus_digest_t;
+
+STATIC smartlist_t *consdiff_gen_diff(const smartlist_t *cons1,
+                                      const smartlist_t *cons2,
+                                      const consensus_digest_t *digests1,
+                                      const consensus_digest_t *digests2,
+                                      struct memarea_t *area);
+STATIC char *consdiff_apply_diff(const smartlist_t *cons1,
+                                 const smartlist_t *diff,
+                                 const consensus_digest_t *digests1);
+STATIC int consdiff_get_digests(const smartlist_t *diff,
+                                char *digest1_out,
+                                char *digest2_out);
+
+/** Data structure to define a slice of a smarltist. */
+typedef struct smartlist_slice_t {
+  /**
+   * Smartlist that this slice is made from.
+   * References the whole original smartlist that the slice was made out of.
+   * */
+  const smartlist_t *list;
+  /** Starting position of the slice in the smartlist. */
+  int offset;
+  /** Length of the slice, i.e. the number of elements it holds. */
+  int len;
+} smartlist_slice_t;
+STATIC smartlist_t *gen_ed_diff(const smartlist_t *cons1,
+                                const smartlist_t *cons2,
+                                struct memarea_t *area);
+STATIC smartlist_t *apply_ed_diff(const smartlist_t *cons1,
+                                  const smartlist_t *diff,
+                                  int start_line);
+STATIC void calc_changes(smartlist_slice_t *slice1, smartlist_slice_t *slice2,
+                         bitarray_t *changed1, bitarray_t *changed2);
+STATIC smartlist_slice_t *smartlist_slice(const smartlist_t *list,
+                                          int start, int end);
+STATIC int next_router(const smartlist_t *cons, int cur);
+STATIC int *lcs_lengths(const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2,
+                        int direction);
+STATIC void trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2);
+STATIC int base64cmp(const cdline_t *hash1, const cdline_t *hash2);
+STATIC int get_id_hash(const cdline_t *line, cdline_t *hash_out);
+STATIC int is_valid_router_entry(const cdline_t *line);
+STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                                      const cdline_t *string);
+STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
+                        const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2);
+STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+                                 struct memarea_t *area);
+STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
+                                  const char *s);
+STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
+STATIC int line_str_eq(const cdline_t *a, const char *b);
+
+MOCK_DECL(STATIC int,
+          consensus_compute_digest,(const char *cons,
+                                    consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_compute_digest_as_signed,(const char *cons,
+                                              consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_digest_eq,(const uint8_t *d1,
+                               const uint8_t *d2));
+#endif /* defined(CONSDIFF_PRIVATE) */
+
+#endif /* !defined(TOR_CONSDIFF_H) */
+
diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c
new file mode 100644
index 0000000000..323f4f9ca0
--- /dev/null
+++ b/src/or/consdiffmgr.c
@@ -0,0 +1,1940 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiffmsr.c
+ *
+ * \brief consensus diff manager functions
+ *
+ * This module is run by directory authorities and caches in order
+ * to remember a number of past consensus documents, and to generate
+ * and serve the diffs from those documents to the latest consensus.
+ */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "consdiff.h"
+#include "consdiffmgr.h"
+#include "cpuworker.h"
+#include "networkstatus.h"
+#include "routerparse.h"
+#include "workqueue.h"
+
+/**
+ * Labels to apply to items in the conscache object.
+ *
+ * @{
+ */
+/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
+#define LABEL_DOCTYPE "document-type"
+/* The valid-after time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_AFTER "consensus-valid-after"
+/* The fresh-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_FRESH_UNTIL "consensus-fresh-until"
+/* The valid-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_UNTIL "consensus-valid-until"
+/* Comma-separated list of hex-encoded identity digests for the voting
+ * authorities. */
+#define LABEL_SIGNATORIES "consensus-signatories"
+/* A hex encoded SHA3 digest of the object, as compressed (if any) */
+#define LABEL_SHA3_DIGEST "sha3-digest"
+/* A hex encoded SHA3 digest of the object before compression. */
+#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
+/* A hex encoded SHA3 digest-as-signed of a consensus */
+#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
+/* The flavor of the consensus or consensuses diff */
+#define LABEL_FLAVOR "consensus-flavor"
+/* Diff only: the SHA3 digest-as-signed of the source consensus. */
+#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
+/* Diff only: the SHA3 digest-in-full of the target consensus. */
+#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
+/* Diff only: the valid-after date of the source consensus. */
+#define LABEL_FROM_VALID_AFTER "from-valid-after"
+/* What kind of compression was used? */
+#define LABEL_COMPRESSION_TYPE "compression"
+/** @} */
+
+#define DOCTYPE_CONSENSUS "consensus"
+#define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
+
+/**
+ * Underlying directory that stores consensuses and consensus diffs.  Don't
+ * use this directly: use cdm_cache_get() instead.
+ */
+static consensus_cache_t *cons_diff_cache = NULL;
+/**
+ * If true, we have learned at least one new consensus since the
+ * consensus cache was last up-to-date.
+ */
+static int cdm_cache_dirty = 0;
+/**
+ * If true, we have scanned the cache to update our hashtable of diffs.
+ */
+static int cdm_cache_loaded = 0;
+
+/**
+ * Possible status values for cdm_diff_t.cdm_diff_status
+ **/
+typedef enum cdm_diff_status_t {
+  CDM_DIFF_PRESENT=1,
+  CDM_DIFF_IN_PROGRESS=2,
+  CDM_DIFF_ERROR=3,
+} cdm_diff_status_t;
+
+/** Which methods do we use for precompressing diffs? */
+static const compress_method_t compress_diffs_with[] = {
+  NO_METHOD,
+  GZIP_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/**
+ * Event for rescanning the cache.
+ */
+static mainloop_event_t *consdiffmgr_rescan_ev = NULL;
+
+static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg);
+static void mark_cdm_cache_dirty(void);
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_diff_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_diffs_with);
+}
+
+/** Which methods do we use for precompressing consensuses? */
+static const compress_method_t compress_consensus_with[] = {
+  ZLIB_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_consensus_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_consensus_with);
+}
+
+/** For which compression method do we retain old consensuses?  There's no
+ * need to keep all of them, since we won't be serving them.  We'll
+ * go with ZLIB_METHOD because it's pretty fast and everyone has it.
+ */
+#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
+
+/** Handles pointing to the latest consensus entries as compressed and
+ * stored. */
+static consensus_cache_entry_handle_t *
+                  latest_consensus[N_CONSENSUS_FLAVORS]
+                                  [ARRAY_LENGTH(compress_consensus_with)];
+
+/** Hashtable node used to remember the current status of the diff
+ * from a given sha3 digest to the current consensus.  */
+typedef struct cdm_diff_t {
+  HT_ENTRY(cdm_diff_t) node;
+
+  /** Consensus flavor for this diff (part of ht key) */
+  consensus_flavor_t flavor;
+  /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
+   * ht key) */
+  uint8_t from_sha3[DIGEST256_LEN];
+  /** Method by which the diff is compressed. (part of the ht key */
+  compress_method_t compress_method;
+
+  /** One of the CDM_DIFF_* values, depending on whether this diff
+   * is available, in progress, or impossible to compute. */
+  cdm_diff_status_t cdm_diff_status;
+  /** SHA3-256 digest of the consensus that this diff is _to. */
+  uint8_t target_sha3[DIGEST256_LEN];
+
+  /** Handle to the cache entry for this diff, if any.  We use a handle here
+   * to avoid thinking too hard about cache entry lifetime issues. */
+  consensus_cache_entry_handle_t *entry;
+} cdm_diff_t;
+
+/** Hashtable mapping flavor and source consensus digest to status. */
+static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
+
+/**
+ * Configuration for this module
+ */
+static consdiff_cfg_t consdiff_cfg = {
+  // XXXX I'd like to make this number bigger, but it interferes with the
+  // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
+  // XXXX rules.
+  /* .cache_max_num = */ 128
+};
+
+static int consdiffmgr_ensure_space_for_files(int n);
+static int consensus_queue_compression_work(const char *consensus,
+                                            const networkstatus_t *as_parsed);
+static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                                          consensus_cache_entry_t *diff_to);
+static void consdiffmgr_set_cache_flags(void);
+
+/* =====
+ * Hashtable setup
+ * ===== */
+
+/** Helper: hash the key of a cdm_diff_t. */
+static unsigned
+cdm_diff_hash(const cdm_diff_t *diff)
+{
+  uint8_t tmp[DIGEST256_LEN + 2];
+  memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
+  tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
+  tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
+  return (unsigned) siphash24g(tmp, sizeof(tmp));
+}
+/** Helper: compare two cdm_diff_t objects for key equality */
+static int
+cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
+{
+  return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
+    diff1->flavor == diff2->flavor &&
+    diff1->compress_method == diff2->compress_method;
+}
+
+HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
+HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+#define cdm_diff_free(diff) \
+  FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
+
+/** Release all storage held in <b>diff</b>. */
+static void
+cdm_diff_free_(cdm_diff_t *diff)
+{
+  if (!diff)
+    return;
+  consensus_cache_entry_handle_free(diff->entry);
+  tor_free(diff);
+}
+
+/** Create and return a new cdm_diff_t with the given values.  Does not
+ * add it to the hashtable. */
+static cdm_diff_t *
+cdm_diff_new(consensus_flavor_t flav,
+             const uint8_t *from_sha3,
+             const uint8_t *target_sha3,
+             compress_method_t method)
+{
+  cdm_diff_t *ent;
+  ent = tor_malloc_zero(sizeof(cdm_diff_t));
+  ent->flavor = flav;
+  memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
+  memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
+  ent->compress_method = method;
+  return ent;
+}
+
+/**
+ * Examine the diff hashtable to see whether we know anything about computing
+ * a diff of type <b>flav</b> between consensuses with the two provided
+ * SHA3-256 digests.  If a computation is in progress, or if the computation
+ * has already been tried and failed, return 1.  Otherwise, note the
+ * computation as "in progress" so that we don't reattempt it later, and
+ * return 0.
+ */
+static int
+cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
+                                   const uint8_t *from_sha3,
+                                   const uint8_t *target_sha3)
+{
+  struct cdm_diff_t search, *ent;
+  unsigned u;
+  int result = 0;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    memset(&search, 0, sizeof(cdm_diff_t));
+    search.flavor = flav;
+    search.compress_method = method;
+    memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+    ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+    if (ent) {
+      tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
+      result = 1;
+      continue;
+    }
+    ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  }
+  return result;
+}
+
+/**
+ * Update the status of the diff of type <b>flav</b> between consensuses with
+ * the two provided SHA3-256 digests, so that its status becomes
+ * <b>status</b>, and its value becomes the <b>handle</b>.  If <b>handle</b>
+ * is NULL, then the old handle (if any) is freed, and replaced with NULL.
+ */
+static void
+cdm_diff_ht_set_status(consensus_flavor_t flav,
+                       const uint8_t *from_sha3,
+                       const uint8_t *to_sha3,
+                       compress_method_t method,
+                       int status,
+                       consensus_cache_entry_handle_t *handle)
+{
+  if (handle == NULL) {
+    tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
+  }
+
+  struct cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(cdm_diff_t));
+  search.flavor = flav;
+  search.compress_method = method,
+  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+  if (!ent) {
+    ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
+    // This can happen under certain really pathological conditions
+    // if we decide we don't care about a diff before it is actually
+    // done computing.
+    return;
+  }
+
+  tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
+
+  ent->cdm_diff_status = status;
+  consensus_cache_entry_handle_free(ent->entry);
+  ent->entry = handle;
+}
+
+/**
+ * Helper: Remove from the hash table every present (actually computed) diff
+ * of type <b>flav</b> whose target digest does not match
+ * <b>unless_target_sha3_matches</b>.
+ *
+ * This function is used for the hash table to throw away references to diffs
+ * that do not lead to the most given consensus of a given flavor.
+ */
+static void
+cdm_diff_ht_purge(consensus_flavor_t flav,
+                  const uint8_t *unless_target_sha3_matches)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+
+    if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
+        flav == (*diff)->flavor) {
+
+      if (BUG((*diff)->entry == NULL) ||
+          consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
+        /* the underlying entry has gone away; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+
+      if (unless_target_sha3_matches &&
+          fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
+                      DIGEST256_LEN)) {
+        /* target hash doesn't match; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+    }
+    next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
+  }
+}
+
+/**
+ * Helper: initialize <b>cons_diff_cache</b>.
+ */
+static void
+cdm_cache_init(void)
+{
+  unsigned n_entries = consdiff_cfg.cache_max_num * 2;
+
+  tor_assert(cons_diff_cache == NULL);
+  cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
+  if (cons_diff_cache == NULL) {
+    // LCOV_EXCL_START
+    log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
+    tor_assert_unreached();
+    // LCOV_EXCL_STOP
+  } else {
+    consdiffmgr_set_cache_flags();
+  }
+  consdiffmgr_rescan_ev =
+    mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL);
+  mark_cdm_cache_dirty();
+  cdm_cache_loaded = 0;
+}
+
+/**
+ * Helper: return the consensus_cache_t * that backs this manager,
+ * initializing it if needed.
+ */
+STATIC consensus_cache_t *
+cdm_cache_get(void)
+{
+  if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
+    cdm_cache_init();
+  }
+  return cons_diff_cache;
+}
+
+/**
+ * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
+ * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
+ * with <b>label</b> as its label.
+ */
+static void
+cdm_labels_prepend_sha3(config_line_t **labels,
+                        const char *label,
+                        const uint8_t *body,
+                        size_t bodylen)
+{
+  uint8_t sha3_digest[DIGEST256_LEN];
+  char hexdigest[HEX_DIGEST256_LEN+1];
+  crypto_digest256((char *)sha3_digest,
+                   (const char *)body, bodylen, DIGEST_SHA3_256);
+  base16_encode(hexdigest, sizeof(hexdigest),
+                (const char *)sha3_digest, sizeof(sha3_digest));
+
+  config_line_prepend(labels, label, hexdigest);
+}
+
+/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
+ * given label, set <b>digest_out</b> to that value (decoded), and return 0.
+ *
+ * Return -1 if there is no such label, and -2 if it is badly formatted. */
+STATIC int
+cdm_entry_get_sha3_value(uint8_t *digest_out,
+                         consensus_cache_entry_t *ent,
+                         const char *label)
+{
+  if (ent == NULL)
+    return -1;
+
+  const char *hex = consensus_cache_entry_get_value(ent, label);
+  if (hex == NULL)
+    return -1;
+
+  int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
+  if (n != DIGEST256_LEN)
+    return -2;
+  else
+    return 0;
+}
+
+/**
+ * Helper: look for a consensus with the given <b>flavor</b> and
+ * <b>valid_after</b> time in the cache. Return that consensus if it's
+ * present, or NULL if it's missing.
+ */
+STATIC consensus_cache_entry_t *
+cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
+{
+  char formatted_time[ISO_TIME_LEN+1];
+  format_iso_time_nospace(formatted_time, valid_after);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  /* We'll filter by valid-after time first, since that should
+   * match the fewest documents. */
+  /* We could add an extra hashtable here, but since we only do this scan
+   * when adding a new consensus, it probably doesn't matter much. */
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_VALID_AFTER, formatted_time);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  consensus_cache_entry_t *result = NULL;
+  if (smartlist_len(matches)) {
+    result = smartlist_get(matches, 0);
+  }
+  smartlist_free(matches);
+
+  return result;
+}
+
+/** Return the maximum age (in seconds) of consensuses that we should consider
+ * storing. The available space in the directory may impose additional limits
+ * on how much we store. */
+static int32_t
+get_max_age_to_cache(void)
+{
+  const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
+  const int32_t MIN_MAX_AGE_TO_CACHE = 0;
+  const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
+  const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
+
+  const or_options_t *options = get_options();
+
+  if (options->MaxConsensusAgeForDiffs) {
+    const int v = options->MaxConsensusAgeForDiffs;
+    if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
+      return MAX_MAX_AGE_TO_CACHE;
+    else
+      return v;
+  }
+
+  /* The parameter is in hours, so we multiply */
+  return 3600 * networkstatus_get_param(NULL,
+                                        MAX_AGE_TO_CACHE_NAME,
+                                        DEFAULT_MAX_AGE_TO_CACHE,
+                                        MIN_MAX_AGE_TO_CACHE,
+                                        MAX_MAX_AGE_TO_CACHE);
+}
+
+/**
+ * Given a string containing a networkstatus consensus, and the results of
+ * having parsed that consensus, add that consensus to the cache if it is not
+ * already present and not too old.  Create new consensus diffs from or to
+ * that consensus as appropriate.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+int
+consdiffmgr_add_consensus(const char *consensus,
+                          const networkstatus_t *as_parsed)
+{
+  if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
+    return -1; // LCOV_EXCL_LINE
+  if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
+    return -1; // LCOV_EXCL_LINE
+
+  const consensus_flavor_t flavor = as_parsed->flavor;
+  const time_t valid_after = as_parsed->valid_after;
+
+  if (valid_after < approx_time() - get_max_age_to_cache()) {
+    log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
+             "too old.");
+    return -1;
+  }
+
+  /* Do we already have this one? */
+  consensus_cache_entry_t *entry =
+    cdm_cache_lookup_consensus(flavor, valid_after);
+  if (entry) {
+    log_info(LD_DIRSERV, "We already have a copy of that consensus");
+    return -1;
+  }
+
+  /* We don't have it. Add it to the cache. */
+  return consensus_queue_compression_work(consensus, as_parsed);
+}
+
+/**
+ * Helper: used to sort two smartlists of consensus_cache_entry_t by their
+ * LABEL_VALID_AFTER labels.
+ */
+static int
+compare_by_valid_after_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  /* We're in luck here: sorting UTC iso-encoded values lexically will work
+   * fine (until 9999). */
+  return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
+                    consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
+}
+
+/**
+ * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
+ * entry.
+ */
+static consensus_cache_entry_t *
+sort_and_find_most_recent(smartlist_t *lst)
+{
+  smartlist_sort(lst, compare_by_valid_after_);
+  if (smartlist_len(lst)) {
+    return smartlist_get(lst, smartlist_len(lst) - 1);
+  } else {
+    return NULL;
+  }
+}
+
+/** Return i such that compress_consensus_with[i] == method. Return
+ * -1 if no such i exists. */
+static int
+consensus_compression_method_pos(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < n_consensus_compression_methods(); ++i) {
+    if (compress_consensus_with[i] == method) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/**
+ * If we know a consensus with the flavor <b>flavor</b> compressed with
+ * <b>method</b>, set *<b>entry_out</b> to that value.  Return values are as
+ * for consdiffmgr_find_diff_from().
+ */
+consdiff_status_t
+consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           compress_method_t method)
+{
+  tor_assert(entry_out);
+  tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
+
+  int pos = consensus_compression_method_pos(method);
+  if (pos < 0) {
+     // We don't compress consensuses with this method.
+    return CONSDIFF_NOT_FOUND;
+  }
+  consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
+  if (!handle)
+    return CONSDIFF_NOT_FOUND;
+  *entry_out = consensus_cache_entry_handle_get(handle);
+  if (*entry_out)
+    return CONSDIFF_AVAILABLE;
+  else
+    return CONSDIFF_NOT_FOUND;
+}
+
+/**
+ * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
+ * from the source consensus with the specified digest (which must be SHA3).
+ *
+ * If the diff is present, store it into *<b>entry_out</b> and return
+ * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
+ * CONSDIFF_IN_PROGRESS.
+ */
+consdiff_status_t
+consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           compress_method_t method)
+{
+  if (BUG(digest_type != DIGEST_SHA3_256) ||
+      BUG(digestlen != DIGEST256_LEN)) {
+    return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
+  }
+
+  // Try to look up the entry in the hashtable.
+  cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(search));
+  search.flavor = flavor;
+  search.compress_method = method;
+  memcpy(search.from_sha3, digest, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+
+  if (ent == NULL ||
+      ent->cdm_diff_status == CDM_DIFF_ERROR) {
+    return CONSDIFF_NOT_FOUND;
+  } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
+    return CONSDIFF_IN_PROGRESS;
+  } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
+    return CONSDIFF_IN_PROGRESS;
+  }
+
+  if (BUG(ent->entry == NULL)) {
+    return CONSDIFF_NOT_FOUND;
+  }
+  *entry_out = consensus_cache_entry_handle_get(ent->entry);
+  return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+
+#if 0
+  // XXXX Remove this.  I'm keeping it around for now in case we need to
+  // XXXX debug issues in the hashtable.
+  char hex[HEX_DIGEST256_LEN+1];
+  base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FROM_SHA3_DIGEST, hex);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+
+  *entry_out = sort_and_find_most_recent(matches);
+  consdiff_status_t result =
+    (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+  smartlist_free(matches);
+
+  return result;
+#endif /* 0 */
+}
+
+/**
+ * Perform periodic cleanup tasks on the consensus diff cache.  Return
+ * the number of objects marked for deletion.
+ */
+int
+consdiffmgr_cleanup(void)
+{
+  smartlist_t *objects = smartlist_new();
+  smartlist_t *consensuses = smartlist_new();
+  smartlist_t *diffs = smartlist_new();
+  int n_to_delete = 0;
+
+  log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
+
+  // 1. Delete any consensus or diff or anything whose valid_after is too old.
+  const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
+
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    const char *lv_valid_after =
+      consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (! lv_valid_after) {
+      log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
+                LABEL_VALID_AFTER);
+      continue;
+    }
+    time_t valid_after = 0;
+    if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
+      log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
+                "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
+      continue;
+    }
+    if (valid_after < valid_after_cutoff) {
+      log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
+                "too old", LABEL_VALID_AFTER, lv_valid_after);
+      consensus_cache_entry_mark_for_removal(ent);
+      ++n_to_delete;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+
+  // 2. Delete all diffs that lead to a consensus whose valid-after is not the
+  // latest.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+
+    /* consider all such-flavored diffs, and look to see if they match. */
+    consensus_cache_find_all(diffs, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+    consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+    SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+      const char *this_diff_target_sha3 =
+        consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
+      if (!this_diff_target_sha3)
+        continue;
+      if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
+        consensus_cache_entry_mark_for_removal(diff);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(diff);
+    smartlist_clear(consensuses);
+    smartlist_clear(diffs);
+  }
+
+  // 3. Delete all consensuses except the most recent that are compressed with
+  // an un-preferred method.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3_uncompressed =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    const char *retain_methodname = compression_method_get_name(
+                               RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+    if (BUG(most_recent_sha3_uncompressed == NULL))
+      continue;
+    SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
+      const char *lv_sha3_uncompressed =
+        consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
+      if (BUG(! lv_sha3_uncompressed))
+        continue;
+      if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
+        continue; // This _is_ the most recent.
+      const char *lv_methodname =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
+        consensus_cache_entry_mark_for_removal(ent);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+  }
+
+  smartlist_free(objects);
+  smartlist_free(consensuses);
+  smartlist_free(diffs);
+
+  // Actually remove files, if they're not used.
+  consensus_cache_delete_pending(cdm_cache_get(), 0);
+  return n_to_delete;
+}
+
+/**
+ * Initialize the consensus diff manager and its cache, and configure
+ * its parameters based on the latest torrc and networkstatus parameters.
+ */
+void
+consdiffmgr_configure(const consdiff_cfg_t *cfg)
+{
+  if (cfg)
+    memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
+
+  (void) cdm_cache_get();
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that the consensus diff manager will need.
+ */
+int
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
+{
+  return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
+}
+
+/**
+ * Scan the consensus diff manager's cache for any grossly malformed entries,
+ * and mark them as deletable.  Return 0 if no problems were found; 1
+ * if problems were found and fixed.
+ */
+int
+consdiffmgr_validate(void)
+{
+  /* Right now, we only check for entries that have bad sha3 values */
+  int problems = 0;
+
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
+    uint8_t sha3_expected[DIGEST256_LEN];
+    uint8_t sha3_received[DIGEST256_LEN];
+    int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
+    if (r == -1) {
+      /* digest isn't there; that's allowed */
+      continue;
+    } else if (r == -2) {
+      /* digest is malformed; that's not allowed */
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+    const uint8_t *body;
+    size_t bodylen;
+    consensus_cache_entry_incref(obj);
+    r = consensus_cache_entry_get_body(obj, &body, &bodylen);
+    if (r == 0) {
+      crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
+                       DIGEST_SHA3_256);
+    }
+    consensus_cache_entry_decref(obj);
+    if (r < 0)
+      continue;
+
+    // Deconfuse coverity about the possibility of sha3_received being
+    // uninitialized
+    tor_assert(r <= 0);
+
+    if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+
+  } SMARTLIST_FOREACH_END(obj);
+  smartlist_free(objects);
+  return problems;
+}
+
+/**
+ * Helper: build new diffs of <b>flavor</b> as needed
+ */
+static void
+consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
+{
+  smartlist_t *matches = NULL;
+  smartlist_t *diffs = NULL;
+  smartlist_t *compute_diffs_from = NULL;
+  strmap_t *have_diff_from = NULL;
+
+  // look for the most recent consensus, and for all previous in-range
+  // consensuses.  Do they all have diffs to it?
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  // 1. find the most recent consensus, and the ones that we might want
+  //    to diff to it.
+  const char *methodname = compression_method_get_name(
+                             RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+  matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+  consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
+  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+  if (!most_recent) {
+    log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
+             "not making diffs", flavname);
+    goto done;
+  }
+  tor_assert(smartlist_len(matches));
+  smartlist_del(matches, smartlist_len(matches) - 1);
+
+  const char *most_recent_valid_after =
+    consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
+  if (BUG(most_recent_valid_after == NULL))
+    goto done; //LCOV_EXCL_LINE
+  uint8_t most_recent_sha3[DIGEST256_LEN];
+  if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    goto done; //LCOV_EXCL_LINE
+
+  // 2. Find all the relevant diffs _to_ this consensus. These are ones
+  //    that we don't need to compute.
+  diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_VALID_AFTER, most_recent_valid_after);
+  consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+  have_diff_from = strmap_new();
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *va = consensus_cache_entry_get_value(diff,
+                                                     LABEL_FROM_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    strmap_set(have_diff_from, va, diff);
+  } SMARTLIST_FOREACH_END(diff);
+
+  // 3. See which consensuses in 'matches' don't have diffs yet.
+  smartlist_reverse(matches); // from newest to oldest.
+  compute_diffs_from = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+    const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    if (strmap_get(have_diff_from, va) != NULL)
+      continue; /* we already have this one. */
+    smartlist_add(compute_diffs_from, ent);
+    /* Since we are not going to serve this as the most recent consensus
+     * any more, we should stop keeping it mmap'd when it's not in use.
+     */
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+
+  log_info(LD_DIRSERV,
+           "The most recent %s consensus is valid-after %s. We have diffs to "
+           "this consensus for %d/%d older %s consensuses. Generating diffs "
+           "for the other %d.",
+           flavname,
+           most_recent_valid_after,
+           smartlist_len(matches) - smartlist_len(compute_diffs_from),
+           smartlist_len(matches),
+           flavname,
+           smartlist_len(compute_diffs_from));
+
+  // 4. Update the hashtable; remove entries in this flavor to other
+  //    target consensuses.
+  cdm_diff_ht_purge(flavor, most_recent_sha3);
+
+  // 5. Actually launch the requests.
+  SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
+    if (BUG(c == most_recent))
+      continue; // LCOV_EXCL_LINE
+
+    uint8_t this_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(this_sha3, c,
+                                 LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
+      // Not actually a bug, since we might be running with a directory
+      // with stale files from before the #22143 fixes.
+      continue;
+    }
+    if (cdm_diff_ht_check_and_note_pending(flavor,
+                                           this_sha3, most_recent_sha3)) {
+      // This is already pending, or we encountered an error.
+      continue;
+    }
+    consensus_diff_queue_diff_work(c, most_recent);
+  } SMARTLIST_FOREACH_END(c);
+
+ done:
+  smartlist_free(matches);
+  smartlist_free(diffs);
+  smartlist_free(compute_diffs_from);
+  strmap_free(have_diff_from, NULL);
+}
+
+/**
+ * Scan the cache for the latest consensuses and add their handles to
+ * latest_consensus
+ */
+static void
+consdiffmgr_consensus_load(void)
+{
+  smartlist_t *matches = smartlist_new();
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    smartlist_clear(matches);
+    consensus_cache_find_all(matches, cdm_cache_get(),
+                             LABEL_FLAVOR, flavname);
+    consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+    if (! most_recent)
+      continue; // no consensuses.
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+    consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                                most_recent_sha3);
+
+    // Everything that remains matches the most recent consensus of this
+    // flavor.
+    SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+      const char *lv_compression =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      compress_method_t method =
+        compression_method_get_by_name(lv_compression);
+      int pos = consensus_compression_method_pos(method);
+      if (pos < 0)
+        continue;
+      consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
+      latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
+    } SMARTLIST_FOREACH_END(ent);
+  }
+  smartlist_free(matches);
+}
+
+/**
+ * Scan the cache for diffs, and add them to the hashtable.
+ */
+static void
+consdiffmgr_diffs_load(void)
+{
+  smartlist_t *diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *lv_flavor =
+      consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
+    if (!lv_flavor)
+      continue;
+    int flavor = networkstatus_parse_flavor_name(lv_flavor);
+    if (flavor < 0)
+      continue;
+    const char *lv_compression =
+      consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
+    compress_method_t method = NO_METHOD;
+    if (lv_compression) {
+      method = compression_method_get_by_name(lv_compression);
+      if (method == UNKNOWN_METHOD) {
+        continue;
+      }
+    }
+
+    uint8_t from_sha3[DIGEST256_LEN];
+    uint8_t to_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
+      continue;
+    if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
+      continue;
+
+    cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
+                           method,
+                           CDM_DIFF_PRESENT,
+                           consensus_cache_entry_handle_new(diff));
+  } SMARTLIST_FOREACH_END(diff);
+  smartlist_free(diffs);
+}
+
+/**
+ * Build new diffs as needed.
+ */
+void
+consdiffmgr_rescan(void)
+{
+  if (cdm_cache_dirty == 0)
+    return;
+
+  // Clean up here to make room for new diffs, and to ensure that older
+  // consensuses do not have any entries.
+  consdiffmgr_cleanup();
+
+  if (cdm_cache_loaded == 0) {
+    consdiffmgr_diffs_load();
+    consdiffmgr_consensus_load();
+    cdm_cache_loaded = 1;
+  }
+
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
+  }
+
+  cdm_cache_dirty = 0;
+}
+
+/** Callback wrapper for consdiffmgr_rescan */
+static void
+consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  consdiffmgr_rescan();
+}
+
+/** Mark the cache as dirty, and schedule a rescan event. */
+static void
+mark_cdm_cache_dirty(void)
+{
+  cdm_cache_dirty = 1;
+  tor_assert(consdiffmgr_rescan_ev);
+  mainloop_event_activate(consdiffmgr_rescan_ev);
+}
+
+/**
+ * Helper: compare two files by their from-valid-after and valid-after labels,
+ * trying to sort in ascending order by from-valid-after (when present) and
+ * valid-after (when not).  Place everything that has neither label first in
+ * the list.
+ */
+static int
+compare_by_staleness_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  const char *va1, *fva1, *va2, *fva2;
+  va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
+  va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
+  fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
+  fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
+
+  if (fva1)
+    va1 = fva1;
+  if (fva2)
+    va2 = fva2;
+
+  /* See note about iso-encoded values in compare_by_valid_after_.  Also note
+   * that missing dates will get placed first. */
+  return strcmp_opt(va1, va2);
+}
+
+/** If there are not enough unused filenames to store <b>n</b> files, then
+ * delete old consensuses until there are.  (We have to keep track of the
+ * number of filenames because of the way that the seccomp2 cache works.)
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+consdiffmgr_ensure_space_for_files(int n)
+{
+  consensus_cache_t *cache = cdm_cache_get();
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // there are already enough unused filenames.
+    return 0;
+  }
+  // Try a cheap deletion of stuff that's waiting to get deleted.
+  consensus_cache_delete_pending(cache, 0);
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // okay, _that_ made enough filenames available.
+    return 0;
+  }
+  // Let's get more assertive: clean out unused stuff, and force-remove
+  // the files that we can.
+  consdiffmgr_cleanup();
+  consensus_cache_delete_pending(cache, 1);
+  const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
+  if (n_to_remove <= 0) {
+    // okay, finally!
+    return 0;
+  }
+
+  // At this point, we're going to have to throw out objects that will be
+  // missed.  Too bad!
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cache, NULL, NULL);
+  smartlist_sort(objects, compare_by_staleness_);
+  int n_marked = 0;
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_removal(ent);
+    if (++n_marked >= n_to_remove)
+      break;
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+
+  consensus_cache_delete_pending(cache, 1);
+
+  if (consensus_cache_may_overallocate(cache)) {
+    /* If we're allowed to throw extra files into the cache, let's do so
+     * rather getting upset.
+     */
+    return 0;
+  }
+
+  if (BUG(n_marked < n_to_remove))
+    return -1;
+  else
+    return 0;
+}
+
+/**
+ * Set consensus cache flags on the objects in this consdiffmgr.
+ */
+static void
+consdiffmgr_set_cache_flags(void)
+{
+  /* Right now, we just mark the consensus objects for aggressive release,
+   * so that they get mmapped for as little time as possible. */
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
+                           DOCTYPE_CONSENSUS);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+}
+
+/**
+ * Called before shutdown: drop all storage held by the consdiffmgr.c module.
+ */
+void
+consdiffmgr_free_all(void)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+    next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+    cdm_diff_free(this);
+  }
+  int i;
+  unsigned j;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    for (j = 0; j < n_consensus_compression_methods(); ++j) {
+      consensus_cache_entry_handle_free(latest_consensus[i][j]);
+    }
+  }
+  memset(latest_consensus, 0, sizeof(latest_consensus));
+  consensus_cache_free(cons_diff_cache);
+  cons_diff_cache = NULL;
+  mainloop_event_free(consdiffmgr_rescan_ev);
+}
+
+/* =====
+   Thread workers
+   =====*/
+
+typedef struct compressed_result_t {
+  config_line_t *labels;
+  /**
+   * Output: Body of the diff, as compressed.
+   */
+  uint8_t *body;
+  /**
+   * Output: length of body_out
+   */
+  size_t bodylen;
+} compressed_result_t;
+
+/**
+ * Compress the bytestring <b>input</b> of length <b>len</b> using the
+ * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
+ *
+ * For each successful compression, set the fields in the <b>results_out</b>
+ * array in the position corresponding to the compression method. Use
+ * <b>labels_in</b> as a basis for the labels of the result.
+ *
+ * Return 0 if all compression succeeded; -1 if any failed.
+ */
+static int
+compress_multiple(compressed_result_t *results_out, int n_methods,
+                  const compress_method_t *methods,
+                  const uint8_t *input, size_t len,
+                  const config_line_t *labels_in)
+{
+  int rv = 0;
+  int i;
+  for (i = 0; i < n_methods; ++i) {
+    compress_method_t method = methods[i];
+    const char *methodname = compression_method_get_name(method);
+    char *result;
+    size_t sz;
+    if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
+      results_out[i].body = (uint8_t*)result;
+      results_out[i].bodylen = sz;
+      results_out[i].labels = config_lines_dup(labels_in);
+      cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
+                              results_out[i].body,
+                              results_out[i].bodylen);
+      config_line_prepend(&results_out[i].labels,
+                          LABEL_COMPRESSION_TYPE,
+                          methodname);
+    } else {
+      rv = -1;
+    }
+  }
+  return rv;
+}
+
+/**
+ * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
+ * as produced by compress_multiple, store them all into the
+ * consdiffmgr, and store handles to them in the <b>handles_out</b>
+ * array.
+ *
+ * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
+ * was stored.
+ */
+static cdm_diff_status_t
+store_multiple(consensus_cache_entry_handle_t **handles_out,
+               int n,
+               const compress_method_t *methods,
+               const compressed_result_t *results,
+               const char *description)
+{
+  cdm_diff_status_t status = CDM_DIFF_ERROR;
+  consdiffmgr_ensure_space_for_files(n);
+
+  int i;
+  for (i = 0; i < n; ++i) {
+    compress_method_t method = methods[i];
+    uint8_t *body_out = results[i].body;
+    size_t bodylen_out = results[i].bodylen;
+    config_line_t *labels = results[i].labels;
+    const char *methodname = compression_method_get_name(method);
+    if (body_out && bodylen_out && labels) {
+      /* Success! Store the results */
+      log_info(LD_DIRSERV, "Adding %s, compressed with %s",
+               description, methodname);
+
+      consensus_cache_entry_t *ent =
+        consensus_cache_add(cdm_cache_get(),
+                            labels,
+                            body_out,
+                            bodylen_out);
+      if (ent == NULL) {
+        static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
+        log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
+                       "Unable to store object %s compressed with %s.",
+                       description, methodname);
+        continue;
+      }
+
+      status = CDM_DIFF_PRESENT;
+      handles_out[i] = consensus_cache_entry_handle_new(ent);
+      consensus_cache_entry_decref(ent);
+    }
+  }
+  return status;
+}
+
+/**
+ * An object passed to a worker thread that will try to produce a consensus
+ * diff.
+ */
+typedef struct consensus_diff_worker_job_t {
+  /**
+   * Input: The consensus to compute the diff from.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_from;
+  /**
+   * Input: The consensus to compute the diff to.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_to;
+
+  /** Output: labels and bodies */
+  compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
+} consensus_diff_worker_job_t;
+
+/** Given a consensus_cache_entry_t, check whether it has a label claiming
+ * that it was compressed.  If so, uncompress its contents into <b>out</b> and
+ * set <b>outlen</b> to hold their size.  If not, just copy the body into
+ * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
+ * -1 on failure.
+ *
+ * In all cases, the output is nul-terminated. */
+STATIC int
+uncompress_or_copy(char **out, size_t *outlen,
+                   consensus_cache_entry_t *ent)
+{
+  const uint8_t *body;
+  size_t bodylen;
+
+  if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
+    return -1;
+
+  const char *lv_compression =
+    consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+  compress_method_t method = NO_METHOD;
+
+  if (lv_compression)
+    method = compression_method_get_by_name(lv_compression);
+
+  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+                        method, 1, LOG_WARN);
+}
+
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_diff_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_diff_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_diff_worker_job_t *job = work_;
+  const uint8_t *diff_from, *diff_to;
+  size_t len_from, len_to;
+  int r;
+  /* We need to have the body already mapped into RAM here.
+   */
+  r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+
+  const char *lv_to_valid_after =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
+  const char *lv_to_fresh_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
+  const char *lv_to_valid_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
+  const char *lv_to_signatories =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
+  const char *lv_from_valid_after =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_from_flavor =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
+  const char *lv_to_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+
+  if (! lv_from_digest) {
+    /* This isn't a bug right now, since it can happen if you're migrating
+     * from an older version of master to a newer one.  The older ones didn't
+     * annotate their stored consensus objects with sha3-digest-as-signed.
+    */
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  /* All these values are mandatory on the input */
+  if (BUG(!lv_to_valid_after) ||
+      BUG(!lv_from_valid_after) ||
+      BUG(!lv_from_flavor) ||
+      BUG(!lv_to_flavor)) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+  /* The flavors need to match */
+  if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  char *consensus_diff;
+  {
+    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    size_t diff_from_nt_len, diff_to_nt_len;
+
+    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
+                           job->diff_from) < 0) {
+      return WQ_RPL_REPLY;
+    }
+    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
+                           job->diff_to) < 0) {
+      tor_free(diff_from_nt);
+      return WQ_RPL_REPLY;
+    }
+    tor_assert(diff_from_nt);
+    tor_assert(diff_to_nt);
+
+    // XXXX ugh; this is going to calculate the SHA3 of both its
+    // XXXX inputs again, even though we already have that. Maybe it's time
+    // XXXX to change the API here?
+    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
+    tor_free(diff_from_nt);
+    tor_free(diff_to_nt);
+  }
+  if (!consensus_diff) {
+    /* Couldn't generate consensus; we'll leave the reply blank. */
+    return WQ_RPL_REPLY;
+  }
+
+  /* Compress the results and send the reply */
+  tor_assert(compress_diffs_with[0] == NO_METHOD);
+  size_t difflen = strlen(consensus_diff);
+  job->out[0].body = (uint8_t *) consensus_diff;
+  job->out[0].bodylen = difflen;
+
+  config_line_t *common_labels = NULL;
+  if (lv_to_valid_until)
+    config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
+  if (lv_to_fresh_until)
+    config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
+  if (lv_to_signatories)
+    config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
+  cdm_labels_prepend_sha3(&common_labels,
+                          LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+  config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
+                      lv_from_valid_after);
+  config_line_prepend(&common_labels, LABEL_VALID_AFTER,
+                      lv_to_valid_after);
+  config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
+  config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
+                      lv_from_digest);
+  config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
+                      lv_to_digest);
+  config_line_prepend(&common_labels, LABEL_DOCTYPE,
+                      DOCTYPE_CONSENSUS_DIFF);
+
+  job->out[0].labels = config_lines_dup(common_labels);
+  cdm_labels_prepend_sha3(&job->out[0].labels,
+                          LABEL_SHA3_DIGEST,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+
+  compress_multiple(job->out+1,
+                    n_diff_compression_methods()-1,
+                    compress_diffs_with+1,
+                    (const uint8_t*)consensus_diff, difflen, common_labels);
+
+  config_free_lines(common_labels);
+  return WQ_RPL_REPLY;
+}
+
+#define consensus_diff_worker_job_free(job)             \
+  FREE_AND_NULL(consensus_diff_worker_job_t,            \
+                consensus_diff_worker_job_free_, (job))
+
+/**
+ * Helper: release all storage held in <b>job</b>.
+ */
+static void
+consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)
+{
+  if (!job)
+    return;
+  unsigned u;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  consensus_cache_entry_decref(job->diff_from);
+  consensus_cache_entry_decref(job->diff_to);
+  tor_free(job);
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_worker_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_diff_worker_replyfn(void *work_)
+{
+  tor_assert(in_main_thread());
+  tor_assert(work_);
+
+  consensus_diff_worker_job_t *job = work_;
+
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+  const char *lv_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  if (BUG(lv_from_digest == NULL))
+    lv_from_digest = "???"; // LCOV_EXCL_LINE
+  if (BUG(lv_to_digest == NULL))
+    lv_to_digest = "???"; // LCOV_EXCL_LINE
+
+  uint8_t from_sha3[DIGEST256_LEN];
+  uint8_t to_sha3[DIGEST256_LEN];
+  int flav = -1;
+  int cache = 1;
+  if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
+                                   LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
+    cache = 0;
+  if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    cache = 0;
+  if (BUG(lv_flavor == NULL)) {
+    cache = 0;
+  } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
+    cache = 0;
+  }
+
+  consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
+  memset(handles, 0, sizeof(handles));
+
+  char description[128];
+  tor_snprintf(description, sizeof(description),
+               "consensus diff from %s to %s",
+               lv_from_digest, lv_to_digest);
+
+  int status = store_multiple(handles,
+                              n_diff_compression_methods(),
+                              compress_diffs_with,
+                              job->out,
+                              description);
+
+  if (status != CDM_DIFF_PRESENT) {
+    /* Failure! Nothing to do but complain */
+    log_warn(LD_DIRSERV,
+             "Worker was unable to compute consensus diff "
+             "from %s to %s", lv_from_digest, lv_to_digest);
+    /* Cache this error so we don't try to compute this one again. */
+    status = CDM_DIFF_ERROR;
+  }
+
+  unsigned u;
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    if (cache) {
+      consensus_cache_entry_handle_t *h = handles[u];
+      int this_status = status;
+      if (h == NULL) {
+        this_status = CDM_DIFF_ERROR;
+      }
+      tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
+      cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
+    } else {
+      consensus_cache_entry_handle_free(handles[u]);
+    }
+  }
+
+  consensus_diff_worker_job_free(job);
+}
+
+/**
+ * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
+ * in a worker thread.
+ */
+static int
+consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                               consensus_cache_entry_t *diff_to)
+{
+  tor_assert(in_main_thread());
+
+  consensus_cache_entry_incref(diff_from);
+  consensus_cache_entry_incref(diff_to);
+
+  consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->diff_from = diff_from;
+  job->diff_to = diff_to;
+
+  /* Make sure body is mapped. */
+  const uint8_t *body;
+  size_t bodylen;
+  int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
+  int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
+  if (r1 < 0 || r2 < 0)
+    goto err;
+
+  workqueue_entry_t *work;
+  work = cpuworker_queue_work(WQ_PRI_LOW,
+                              consensus_diff_worker_threadfn,
+                              consensus_diff_worker_replyfn,
+                              job);
+  if (!work)
+    goto err;
+
+  return 0;
+ err:
+  consensus_diff_worker_job_free(job); // includes decrefs.
+  return -1;
+}
+
+/**
+ * Holds requests and replies for consensus_compress_workers.
+ */
+typedef struct consensus_compress_worker_job_t {
+  char *consensus;
+  size_t consensus_len;
+  consensus_flavor_t flavor;
+  config_line_t *labels_in;
+  compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
+} consensus_compress_worker_job_t;
+
+#define consensus_compress_worker_job_free(job) \
+  FREE_AND_NULL(consensus_compress_worker_job_t, \
+                consensus_compress_worker_job_free_, (job))
+
+/**
+ * Free all resources held in <b>job</b>
+ */
+static void
+consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)
+{
+  if (!job)
+    return;
+  tor_free(job->consensus);
+  config_free_lines(job->labels_in);
+  unsigned u;
+  for (u = 0; u < n_consensus_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  tor_free(job);
+}
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_compress_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_compress_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_compress_worker_job_t *job = work_;
+  consensus_flavor_t flavor = job->flavor;
+  const char *consensus = job->consensus;
+  size_t bodylen = job->consensus_len;
+
+  config_line_t *labels = config_lines_dup(job->labels_in);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          (const uint8_t *)consensus, bodylen);
+  {
+    const char *start, *end;
+    if (router_get_networkstatus_v3_signed_boundaries(consensus,
+                                                        &start, &end) < 0) {
+      start = consensus;
+      end = consensus+bodylen;
+    }
+    cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
+                            (const uint8_t *)start,
+                            end - start);
+  }
+  config_line_prepend(&labels, LABEL_FLAVOR, flavname);
+  config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  compress_multiple(job->out,
+                    n_consensus_compression_methods(),
+                    compress_consensus_with,
+                    (const uint8_t*)consensus, bodylen, labels);
+  config_free_lines(labels);
+  return WQ_RPL_REPLY;
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_compress_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_compress_worker_replyfn(void *work_)
+{
+  consensus_compress_worker_job_t *job = work_;
+
+  consensus_cache_entry_handle_t *handles[
+                               ARRAY_LENGTH(compress_consensus_with)];
+  memset(handles, 0, sizeof(handles));
+
+  store_multiple(handles,
+                 n_consensus_compression_methods(),
+                 compress_consensus_with,
+                 job->out,
+                 "consensus");
+  mark_cdm_cache_dirty();
+
+  unsigned u;
+  consensus_flavor_t f = job->flavor;
+  tor_assert((int)f < N_CONSENSUS_FLAVORS);
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    if (handles[u] == NULL)
+      continue;
+    consensus_cache_entry_handle_free(latest_consensus[f][u]);
+    latest_consensus[f][u] = handles[u];
+  }
+
+  consensus_compress_worker_job_free(job);
+}
+
+/**
+ * If true, we compress in worker threads.
+ */
+static int background_compression = 0;
+
+/**
+ * Queue a job to compress <b>consensus</b> and store its compressed
+ * text in the cache.
+ */
+static int
+consensus_queue_compression_work(const char *consensus,
+                                 const networkstatus_t *as_parsed)
+{
+  tor_assert(consensus);
+  tor_assert(as_parsed);
+
+  consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->consensus = tor_strdup(consensus);
+  job->consensus_len = strlen(consensus);
+  job->flavor = as_parsed->flavor;
+
+  char va_str[ISO_TIME_LEN+1];
+  char vu_str[ISO_TIME_LEN+1];
+  char fu_str[ISO_TIME_LEN+1];
+  format_iso_time_nospace(va_str, as_parsed->valid_after);
+  format_iso_time_nospace(fu_str, as_parsed->fresh_until);
+  format_iso_time_nospace(vu_str, as_parsed->valid_until);
+  config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
+  config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
+  config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
+  if (as_parsed->voters) {
+    smartlist_t *hexvoters = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
+                            networkstatus_voter_info_t *, vi) {
+      if (smartlist_len(vi->sigs) == 0)
+        continue; // didn't sign.
+      char d[HEX_DIGEST_LEN+1];
+      base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
+      smartlist_add_strdup(hexvoters, d);
+    } SMARTLIST_FOREACH_END(vi);
+    char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
+    config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
+    tor_free(signers);
+    SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
+    smartlist_free(hexvoters);
+  }
+
+  if (background_compression) {
+    workqueue_entry_t *work;
+    work = cpuworker_queue_work(WQ_PRI_LOW,
+                                consensus_compress_worker_threadfn,
+                                consensus_compress_worker_replyfn,
+                                job);
+    if (!work) {
+      consensus_compress_worker_job_free(job);
+      return -1;
+    }
+
+    return 0;
+  } else {
+    consensus_compress_worker_threadfn(NULL, job);
+    consensus_compress_worker_replyfn(job);
+    return 0;
+  }
+}
+
+/**
+ * Tell the consdiffmgr backend to compress consensuses in worker threads.
+ */
+void
+consdiffmgr_enable_background_compression(void)
+{
+  // This isn't the default behavior because it would break unit tests.
+  background_compression = 1;
+}
+
+/** Read the set of voters from the cached object <b>ent</b> into
+ * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
+ * -1 if no signatories were recorded. */
+int
+consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
+                                           smartlist_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
+  if (s == NULL)
+    return -1;
+  smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
+  return 0;
+}
+
+/** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
+ * and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid until timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid after timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h
new file mode 100644
index 0000000000..df569c8e23
--- /dev/null
+++ b/src/or/consdiffmgr.h
@@ -0,0 +1,74 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFFMGR_H
+#define TOR_CONSDIFFMGR_H
+
+/**
+ * Possible outcomes from trying to look up a given consensus diff.
+ */
+typedef enum consdiff_status_t {
+  CONSDIFF_AVAILABLE,
+  CONSDIFF_NOT_FOUND,
+  CONSDIFF_IN_PROGRESS,
+} consdiff_status_t;
+
+typedef struct consdiff_cfg_t {
+  int32_t cache_max_num;
+} consdiff_cfg_t;
+
+struct consensus_cache_entry_t; // from conscache.h
+
+int consdiffmgr_add_consensus(const char *consensus,
+                              const networkstatus_t *as_parsed);
+
+consdiff_status_t consdiffmgr_find_consensus(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           compress_method_t method);
+
+consdiff_status_t consdiffmgr_find_diff_from(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           compress_method_t method);
+
+int consensus_cache_entry_get_voter_id_digests(
+                                  const struct consensus_cache_entry_t *ent,
+                                  smartlist_t *out);
+int consensus_cache_entry_get_fresh_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_after(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+
+void consdiffmgr_rescan(void);
+int consdiffmgr_cleanup(void);
+void consdiffmgr_enable_background_compression(void);
+void consdiffmgr_configure(const consdiff_cfg_t *cfg);
+struct sandbox_cfg_elem;
+int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);
+void consdiffmgr_free_all(void);
+int consdiffmgr_validate(void);
+
+#ifdef CONSDIFFMGR_PRIVATE
+STATIC unsigned n_diff_compression_methods(void);
+STATIC unsigned n_consensus_compression_methods(void);
+STATIC consensus_cache_t *cdm_cache_get(void);
+STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
+                          consensus_flavor_t flavor, time_t valid_after);
+STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
+                                    consensus_cache_entry_t *ent,
+                                    const char *label);
+STATIC int uncompress_or_copy(char **out, size_t *outlen,
+                              consensus_cache_entry_t *ent);
+#endif /* defined(CONSDIFFMGR_PRIVATE) */
+
+#endif /* !defined(TOR_CONSDIFFMGR_H) */
+
diff --git a/src/or/control.c b/src/or/control.c
index ff7f2e8b85..1115ef7b1d 100644
--- a/src/or/control.c
+++ b/src/or/control.c
@@ -1,5 +1,6 @@
+
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,6 +37,7 @@
 
 #include "or.h"
 #include "addressmap.h"
+#include "bridges.h"
 #include "buffers.h"
 #include "channel.h"
 #include "channeltls.h"
@@ -51,16 +53,24 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "directory.h"
 #include "dirserv.h"
 #include "dnsserv.h"
 #include "entrynodes.h"
 #include "geoip.h"
 #include "hibernate.h"
+#include "hs_cache.h"
+#include "hs_common.h"
+#include "hs_control.h"
 #include "main.h"
+#include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "proto_control0.h"
+#include "proto_http.h"
 #include "reasons.h"
 #include "rendclient.h"
 #include "rendcommon.h"
@@ -69,14 +79,13 @@
 #include "router.h"
 #include "routerlist.h"
 #include "routerparse.h"
+#include "shared_random_client.h"
 
 #ifndef _WIN32
 #include <pwd.h>
 #include <sys/resource.h>
 #endif
 
-#include <event2/event.h>
-
 #include "crypto_s2k.h"
 #include "procmon.h"
 
@@ -104,6 +113,10 @@ static int disable_log_messages = 0;
 #define EVENT_IS_INTERESTING(e) \
   (!! (global_event_mask & EVENT_MASK_(e)))
 
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
 /** If we're using cookie-type authentication, how long should our cookies be?
  */
 #define AUTHENTICATION_COOKIE_LEN 32
@@ -208,9 +221,10 @@ static void orconn_target_get_name(char *buf, size_t len,
 static int get_cached_network_liveness(void);
 static void set_cached_network_liveness(int liveness);
 
-static void flush_queued_events_cb(evutil_socket_t fd, short what, void *arg);
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
 
 static char * download_status_to_string(const download_status_t *dl);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
 
 /** Given a control event code for a message event, return the corresponding
  * log severity. */
@@ -251,6 +265,8 @@ clear_circ_bw_fields(void)
       continue;
     ocirc = TO_ORIGIN_CIRCUIT(circ);
     ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
   }
   SMARTLIST_FOREACH_END(circ);
 }
@@ -263,6 +279,7 @@ control_update_global_event_mask(void)
   smartlist_t *conns = get_connection_array();
   event_mask_t old_mask, new_mask;
   old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
 
   global_event_mask = 0;
   SMARTLIST_FOREACH(conns, connection_t *, _conn,
@@ -280,10 +297,13 @@ control_update_global_event_mask(void)
    * we want to hear...*/
   control_adjust_event_log_severity();
 
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
   /* ...then, if we've started logging stream or circ bw, clear the
    * appropriate fields. */
-  if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) &&
-      (new_mask & EVENT_STREAM_BANDWIDTH_USED)) {
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
     SMARTLIST_FOREACH(conns, connection_t *, conn,
     {
       if (conn->type == CONN_TYPE_AP) {
@@ -292,10 +312,18 @@ control_update_global_event_mask(void)
       }
     });
   }
-  if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) &&
-      (new_mask & EVENT_CIRC_BANDWIDTH_USED)) {
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
     clear_circ_bw_fields();
   }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    reschedule_per_second_timer();
+  }
+
+#undef NEWLY_ENABLED
 }
 
 /** Adjust the log severities that result in control_event_logmsg being called
@@ -344,6 +372,65 @@ control_event_is_interesting(int event)
   return EVENT_IS_INTERESTING(event);
 }
 
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
 /** Append a NUL-terminated string <b>s</b> to the end of
  * <b>conn</b>-\>outbuf.
  */
@@ -351,7 +438,7 @@ static inline void
 connection_write_str_to_buf(const char *s, control_connection_t *conn)
 {
   size_t len = strlen(s);
-  connection_write_to_buf(s, len, TO_CONN(conn));
+  connection_buf_add(s, len, TO_CONN(conn));
 }
 
 /** Given a <b>len</b>-character string in <b>data</b>, made of lines
@@ -364,16 +451,23 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn)
 STATIC size_t
 write_escaped_data(const char *data, size_t len, char **out)
 {
-  size_t sz_out = len+8;
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
   char *outp;
   const char *start = data, *end;
-  int i;
+  size_t i;
   int start_of_line;
-  for (i=0; i<(int)len; ++i) {
-    if (data[i]== '\n')
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
       sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
   }
-  *out = outp = tor_malloc(sz_out+1);
+  *out = outp = tor_malloc(sz_out);
   end = data+len;
   start_of_line = 1;
   while (data < end) {
@@ -399,7 +493,8 @@ write_escaped_data(const char *data, size_t len, char **out)
   *outp++ = '\r';
   *outp++ = '\n';
   *outp = '\0'; /* NUL-terminate just in case. */
-  tor_assert((outp - *out) <= (int)sz_out);
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
   return outp - *out;
 }
 
@@ -535,6 +630,49 @@ decode_escaped_string(const char *start, size_t in_len_max,
   return end+1;
 }
 
+/** Create and add a new controller connection on <b>sock</b>.  If
+ * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
+ * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
+ * is set, then the connection does not need to authenticate.
+ */
+int
+control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
+{
+  if (BUG(! SOCKET_OK(sock)))
+    return -1;
+  const int is_owner = !!(flags & CC_LOCAL_FD_IS_OWNER);
+  const int is_authenticated = !!(flags & CC_LOCAL_FD_IS_AUTHENTICATED);
+  control_connection_t *control_conn = control_connection_new(AF_UNSPEC);
+  connection_t *conn = TO_CONN(control_conn);
+  conn->s = sock;
+  tor_addr_make_unspec(&conn->addr);
+  conn->port = 1;
+  conn->address = tor_strdup("<local socket>");
+
+  /* We take ownership of this socket so that later, when we close it,
+   * we don't freak out. */
+  tor_take_socket_ownership(sock);
+
+  if (set_socket_nonblocking(sock) < 0 ||
+      connection_add(conn) < 0) {
+    connection_free(conn);
+    return -1;
+  }
+
+  control_conn->is_owning_control_connection = is_owner;
+
+  if (connection_init_accepted_conn(conn, NULL) < 0) {
+    connection_mark_for_close(conn);
+    return -1;
+  }
+
+  if (is_authenticated) {
+    conn->state = CONTROL_CONN_STATE_OPEN;
+  }
+
+  return 0;
+}
+
 /** Acts like sprintf, but writes its formatted string to the end of
  * <b>conn</b>-\>outbuf. */
 static void
@@ -553,7 +691,7 @@ connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
     tor_assert(0);
   }
 
-  connection_write_to_buf(buf, (size_t)len, TO_CONN(conn));
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
 
   tor_free(buf);
 }
@@ -579,7 +717,7 @@ control_ports_write_to_file(void)
       smartlist_add_asprintf(lines, "UNIX_PORT=%s\n", conn->address);
       continue;
     }
-#endif
+#endif /* defined(AF_UNIX) */
     smartlist_add_asprintf(lines, "PORT=%s:%d\n", conn->address, conn->port);
   } SMARTLIST_FOREACH_END(conn);
 
@@ -596,7 +734,7 @@ control_ports_write_to_file(void)
                options->ControlPortWriteToFile);
     }
   }
-#endif
+#endif /* !defined(_WIN32) */
   tor_free(joined);
   SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
   smartlist_free(lines);
@@ -632,7 +770,7 @@ static tor_mutex_t *queued_control_events_lock = NULL;
 
 /** An event that should fire in order to flush the contents of
  * queued_control_events. */
-static struct event *flush_queued_events_event = NULL;
+static mainloop_event_t *flush_queued_events_event = NULL;
 
 void
 control_initialize_event_queue(void)
@@ -644,9 +782,8 @@ control_initialize_event_queue(void)
   if (flush_queued_events_event == NULL) {
     struct event_base *b = tor_libevent_get_base();
     if (b) {
-      flush_queued_events_event = tor_event_new(b,
-                                              -1, 0, flush_queued_events_cb,
-                                              NULL);
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
       tor_assert(flush_queued_events_event);
     }
   }
@@ -722,13 +859,16 @@ queue_control_event_string,(uint16_t event, char *msg))
    */
   if (activate_event) {
     tor_assert(flush_queued_events_event);
-    event_active(flush_queued_events_event, EV_READ, 1);
+    mainloop_event_activate(flush_queued_events_event);
   }
 }
 
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
 /** Release all storage held by <b>ev</b>. */
 static void
-queued_event_free(queued_event_t *ev)
+queued_event_free_(queued_event_t *ev)
 {
   if (ev == NULL)
     return;
@@ -744,6 +884,9 @@ queued_event_free(queued_event_t *ev)
 static void
 queued_events_flush_all(int force)
 {
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
   if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
     return;
   }
@@ -778,7 +921,7 @@ queued_events_flush_all(int force)
     SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
                             control_conn) {
       if (control_conn->event_mask & bit) {
-        connection_write_to_buf(ev->msg, msg_len, TO_CONN(control_conn));
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
       }
     } SMARTLIST_FOREACH_END(control_conn);
 
@@ -799,12 +942,11 @@ queued_events_flush_all(int force)
 }
 
 /** Libevent callback: Flushes pending events to controllers that are
- * interested in them */
+ * interested in them. */
 static void
-flush_queued_events_cb(evutil_socket_t fd, short what, void *arg)
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
 {
-  (void) fd;
-  (void) what;
+  (void) event;
   (void) arg;
   queued_events_flush_all(0);
 }
@@ -942,7 +1084,7 @@ control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
       ++body;
   }
 
-  smartlist_add(entries, tor_strdup(""));
+  smartlist_add_strdup(entries, "");
   config = smartlist_join_strings(entries, "\n", 0, NULL);
   SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
   smartlist_free(entries);
@@ -1060,7 +1202,7 @@ handle_control_getconf(control_connection_t *conn, uint32_t body_len,
     tor_assert(strlen(tmp)>4);
     tmp[3] = ' ';
     msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_write_to_buf(msg, msg_len, TO_CONN(conn));
+    connection_buf_add(msg, msg_len, TO_CONN(conn));
   } else {
     connection_write_str_to_buf("250 OK\r\n", conn);
   }
@@ -1142,7 +1284,6 @@ static const struct control_event_t control_event_table[] = {
   { EVENT_ERR_MSG, "ERR" },
   { EVENT_NEW_DESC, "NEWDESC" },
   { EVENT_ADDRMAP, "ADDRMAP" },
-  { EVENT_AUTHDIR_NEWDESCS, "AUTHDIR_NEWDESCS" },
   { EVENT_DESCCHANGED, "DESCCHANGED" },
   { EVENT_NS, "NS" },
   { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
@@ -1157,7 +1298,6 @@ static const struct control_event_t control_event_table[] = {
   { EVENT_CONF_CHANGED, "CONF_CHANGED"},
   { EVENT_CONN_BW, "CONN_BW" },
   { EVENT_CELL_STATS, "CELL_STATS" },
-  { EVENT_TB_EMPTY, "TB_EMPTY" },
   { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
   { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
   { EVENT_HS_DESC, "HS_DESC" },
@@ -1182,7 +1322,10 @@ handle_control_setevents(control_connection_t *conn, uint32_t len,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
     {
-      if (!strcasecmp(ev, "EXTENDED")) {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
         continue;
       } else {
         int i;
@@ -1459,8 +1602,10 @@ handle_control_saveconf(control_connection_t *conn, uint32_t len,
                         const char *body)
 {
   (void) len;
-  (void) body;
-  if (options_save_current()<0) {
+
+  int force = !strcmpstart(body, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
     connection_write_str_to_buf(
       "551 Unable to write configuration to disk.\r\n", conn);
   } else {
@@ -1640,12 +1785,12 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len,
   if (smartlist_len(reply)) {
     ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
     r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_write_to_buf(r, sz, TO_CONN(conn));
+    connection_buf_add(r, sz, TO_CONN(conn));
     tor_free(r);
   } else {
     const char *response =
       "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_write_to_buf(response, strlen(response), TO_CONN(conn));
+    connection_buf_add(response, strlen(response), TO_CONN(conn));
   }
 
   SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
@@ -1674,6 +1819,8 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
       *answer = tor_strdup(a);
   } else if (!strcmp(question, "config-text")) {
     *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
   } else if (!strcmp(question, "info/names")) {
     *answer = list_getinfo_options();
   } else if (!strcmp(question, "dormant")) {
@@ -1716,24 +1863,24 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
   } else if (!strcmp(question, "process/pid")) {
     int myPid = -1;
 
-    #ifdef _WIN32
+#ifdef _WIN32
       myPid = _getpid();
-    #else
+#else
       myPid = getpid();
-    #endif
+#endif
 
     tor_asprintf(answer, "%d", myPid);
   } else if (!strcmp(question, "process/uid")) {
-    #ifdef _WIN32
+#ifdef _WIN32
       *answer = tor_strdup("-1");
-    #else
+#else
       int myUid = geteuid();
       tor_asprintf(answer, "%d", myUid);
-    #endif
+#endif /* defined(_WIN32) */
   } else if (!strcmp(question, "process/user")) {
-    #ifdef _WIN32
+#ifdef _WIN32
       *answer = tor_strdup("");
-    #else
+#else
       int myUid = geteuid();
       const struct passwd *myPwEntry = tor_getpwuid(myUid);
 
@@ -1742,7 +1889,7 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
       } else {
         *answer = tor_strdup("");
       }
-    #endif
+#endif /* defined(_WIN32) */
   } else if (!strcmp(question, "process/descriptor-limit")) {
     int max_fds = get_max_sockets();
     tor_asprintf(answer, "%d", max_fds);
@@ -1828,6 +1975,8 @@ getinfo_helper_listeners(control_connection_t *control_conn,
 
   if (!strcmp(question, "net/listeners/or"))
     type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/extor"))
+    type = CONN_TYPE_EXT_OR_LISTENER;
   else if (!strcmp(question, "net/listeners/dir"))
     type = CONN_TYPE_DIR_LISTENER;
   else if (!strcmp(question, "net/listeners/socks"))
@@ -1836,6 +1985,8 @@ getinfo_helper_listeners(control_connection_t *control_conn,
     type = CONN_TYPE_AP_TRANS_LISTENER;
   else if (!strcmp(question, "net/listeners/natd"))
     type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/httptunnel"))
+    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
   else if (!strcmp(question, "net/listeners/dns"))
     type = CONN_TYPE_AP_DNS_LISTENER;
   else if (!strcmp(question, "net/listeners/control"))
@@ -1868,9 +2019,34 @@ getinfo_helper_listeners(control_connection_t *control_conn,
   return 0;
 }
 
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTF forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
 /** Implementation helper for GETINFO: knows the answers for questions about
  * directory information. */
-static int
+STATIC int
 getinfo_helper_dir(control_connection_t *control_conn,
                    const char *question, char **answer,
                    const char **errmsg)
@@ -1878,27 +2054,42 @@ getinfo_helper_dir(control_connection_t *control_conn,
   (void) control_conn;
   if (!strcmpstart(question, "desc/id/")) {
     const routerinfo_t *ri = NULL;
-    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"));
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
     if (node)
       ri = node->ri;
     if (ri) {
       const char *body = signed_descriptor_get_body(&ri->cache_info);
       if (body)
         *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
     }
   } else if (!strcmpstart(question, "desc/name/")) {
     const routerinfo_t *ri = NULL;
     /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
      * warning goes to the user, not to the controller. */
     const node_t *node =
-      node_get_by_nickname(question+strlen("desc/name/"), 1);
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
     if (node)
       ri = node->ri;
     if (ri) {
       const char *body = signed_descriptor_get_body(&ri->cache_info);
       if (body)
         *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
     }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
   } else if (!strcmp(question, "desc/all-recent")) {
     routerlist_t *routerlist = router_get_routerlist();
     smartlist_t *sl = smartlist_new();
@@ -1936,39 +2127,92 @@ getinfo_helper_dir(control_connection_t *control_conn,
     SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
     smartlist_free(sl);
   } else if (!strcmpstart(question, "hs/client/desc/id/")) {
-    rend_cache_entry_t *e = NULL;
+    hostname_type_t addr_type;
 
     question += strlen("hs/client/desc/id/");
-    if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) {
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
       *errmsg = "Invalid address";
       return -1;
     }
 
-    if (!rend_cache_lookup_entry(question, -1, &e)) {
-      /* Descriptor found in cache */
-      *answer = tor_strdup(e->desc);
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     } else {
-      *errmsg = "Not found in cache";
-      return -1;
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     }
   } else if (!strcmpstart(question, "hs/service/desc/id/")) {
-    rend_cache_entry_t *e = NULL;
+    hostname_type_t addr_type;
 
     question += strlen("hs/service/desc/id/");
-    if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) {
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
       *errmsg = "Invalid address";
       return -1;
     }
+    rend_cache_entry_t *e = NULL;
 
-    if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
-      /* Descriptor found in cache */
-      *answer = tor_strdup(e->desc);
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     } else {
-      *errmsg = "Not found in cache";
-      return -1;
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
     }
   } else if (!strcmpstart(question, "md/id/")) {
-    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"));
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
     const microdesc_t *md = NULL;
     if (node) md = node->md;
     if (md && md->body) {
@@ -1977,17 +2221,20 @@ getinfo_helper_dir(control_connection_t *control_conn,
   } else if (!strcmpstart(question, "md/name/")) {
     /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
      * warning goes to the user, not to the controller. */
-    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 1);
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
     /* XXXX duplicated code */
     const microdesc_t *md = NULL;
     if (node) md = node->md;
     if (md && md->body) {
       *answer = tor_strndup(md->body, md->bodylen);
     }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
   } else if (!strcmpstart(question, "desc-annotations/id/")) {
     const routerinfo_t *ri = NULL;
     const node_t *node =
-      node_get_by_hex_id(question+strlen("desc-annotations/id/"));
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
     if (node)
       ri = node->ri;
     if (ri) {
@@ -2028,13 +2275,13 @@ getinfo_helper_dir(control_connection_t *control_conn,
   } else if (!strcmpstart(question, "dir/status/")) {
     *answer = tor_strdup("");
   } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (directory_caches_dir_info(get_options())) {
+    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
       const cached_dir_t *consensus = dirserv_get_consensus("ns");
       if (consensus)
         *answer = tor_strdup(consensus->dir);
     }
     if (!*answer) { /* try loading it from disk */
-      char *filename = get_datadir_fname("cached-consensus");
+      char *filename = get_cachedir_fname("cached-consensus");
       *answer = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
       tor_free(filename);
       if (!*answer) { /* generate an error */
@@ -2044,6 +2291,12 @@ getinfo_helper_dir(control_connection_t *control_conn,
       }
     }
   } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
     routerlist_t *routerlist = router_get_routerlist();
     if (!routerlist || !routerlist->routers ||
         list_server_status_v1(routerlist->routers, answer, 1) < 0) {
@@ -2107,14 +2360,14 @@ digest_list_to_string(const smartlist_t *sl)
 static char *
 download_status_to_string(const download_status_t *dl)
 {
-  char *rv = NULL, *tmp;
+  char *rv = NULL;
   char tbuf[ISO_TIME_LEN+1];
   const char *schedule_str, *want_authority_str;
   const char *increment_on_str, *backoff_str;
 
   if (dl) {
     /* Get some substrings of the eventual output ready */
-    format_iso_time(tbuf, dl->next_attempt_at);
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
 
     switch (dl->schedule) {
       case DL_SCHED_GENERIC:
@@ -2155,49 +2408,28 @@ download_status_to_string(const download_status_t *dl)
         break;
     }
 
-    switch (dl->backoff) {
-      case DL_SCHED_DETERMINISTIC:
-        backoff_str = "DL_SCHED_DETERMINISTIC";
-        break;
-      case DL_SCHED_RANDOM_EXPONENTIAL:
-        backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
-        break;
-      default:
-        backoff_str = "unknown";
-        break;
-    }
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
 
     /* Now assemble them */
-    tor_asprintf(&tmp,
+    tor_asprintf(&rv,
                  "next-attempt-at %s\n"
                  "n-download-failures %u\n"
                  "n-download-attempts %u\n"
                  "schedule %s\n"
                  "want-authority %s\n"
                  "increment-on %s\n"
-                 "backoff %s\n",
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
                  tbuf,
                  dl->n_download_failures,
                  dl->n_download_attempts,
                  schedule_str,
                  want_authority_str,
                  increment_on_str,
-                 backoff_str);
-
-    if (dl->backoff == DL_SCHED_RANDOM_EXPONENTIAL) {
-      /* Additional fields become relevant in random-exponential mode */
-      tor_asprintf(&rv,
-                   "%s"
-                   "last-backoff-position %u\n"
-                   "last-delay-used %d\n",
-                   tmp,
-                   dl->last_backoff_position,
-                   dl->last_delay_used);
-      tor_free(tmp);
-    } else {
-      /* That was it */
-      rv = tmp;
-    }
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
   }
 
   return rv;
@@ -2537,9 +2769,16 @@ circuit_describe_status_for_controller(origin_circuit_t *circ)
     }
   }
 
-  if (circ->rend_data != NULL) {
-    smartlist_add_asprintf(descparts, "REND_QUERY=%s",
-                 circ->rend_data->onion_address);
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
   }
 
   {
@@ -2594,6 +2833,8 @@ getinfo_helper_events(control_connection_t *control_conn,
 
       if (circ->base_.state == CIRCUIT_STATE_OPEN)
         state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
       else if (circ->cpath)
         state = "EXTENDED";
       else
@@ -2819,12 +3060,13 @@ getinfo_helper_events(control_connection_t *control_conn,
 
 /** Implementation helper for GETINFO: knows how to enumerate hidden services
  * created via the control port. */
-static int
+STATIC int
 getinfo_helper_onions(control_connection_t *control_conn,
                       const char *question, char **answer,
                       const char **errmsg)
 {
   smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
 
   if (control_conn && !strcmp(question, "onions/current")) {
     onion_list = control_conn->ephemeral_onion_services;
@@ -2834,13 +3076,13 @@ getinfo_helper_onions(control_connection_t *control_conn,
     return 0;
   }
   if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (errmsg) {
-        *errmsg = "No onion services of the specified type.";
+    if (answer) {
+      *answer = tor_strdup("");
     }
-    return -1;
-  }
-  if (answer) {
+  } else {
+    if (answer) {
       *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
   }
 
   return 0;
@@ -2866,12 +3108,33 @@ getinfo_helper_liveness(control_connection_t *control_conn,
   return 0;
 }
 
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
 /** Callback function for GETINFO: on a given control connection, try to
  * answer the question <b>q</b> and store the newly-allocated answer in
  * *<b>a</b>. If an internal error occurs, return -1 and optionally set
  * *<b>error_out</b> to point to an error message to be delivered to the
  * controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized.
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
  */
 typedef int (*getinfo_helper_t)(control_connection_t *,
                                 const char *q, char **a,
@@ -2899,6 +3162,8 @@ static const getinfo_item_t getinfo_items[] = {
   ITEM("config-defaults-file", misc, "Current location of the defaults file."),
   ITEM("config-text", misc,
        "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
   ITEM("accounting/bytes", accounting,
        "Number of bytes read/written so far in the accounting interval."),
   ITEM("accounting/bytes-left", accounting,
@@ -2921,6 +3186,9 @@ static const getinfo_item_t getinfo_items[] = {
   DOC("config/defaults",
       "List of default values for configuration options. "
       "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
   PREFIX("downloads/networkstatus/", downloads,
          "Download statuses for networkstatus objects"),
   DOC("downloads/networkstatus/ns",
@@ -2974,9 +3242,13 @@ static const getinfo_item_t getinfo_items[] = {
   PREFIX("desc/name/", dir, "Router descriptors by nickname."),
   ITEM("desc/all-recent", dir,
        "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
   ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
   PREFIX("md/id/", dir, "Microdescriptors by ID"),
   PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
   PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
   PREFIX("hs/client/desc/id", dir,
          "Hidden Service descriptor in client's cache by onion."),
@@ -3058,6 +3330,8 @@ static const getinfo_item_t getinfo_items[] = {
        "Onion services owned by the current control connection."),
   ITEM("onions/detached", onions,
        "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
   { NULL, NULL, NULL, 0 }
 };
 
@@ -3122,7 +3396,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
   smartlist_t *questions = smartlist_new();
   smartlist_t *answers = smartlist_new();
   smartlist_t *unrecognized = smartlist_new();
-  char *msg = NULL, *ans = NULL;
+  char *ans = NULL;
   int i;
   (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
 
@@ -3137,20 +3411,26 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
       goto done;
     }
     if (!ans) {
-      smartlist_add(unrecognized, (char*)q);
+      if (errmsg) /* use provided error message */
+        smartlist_add_strdup(unrecognized, errmsg);
+      else /* use default error message */
+        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
     } else {
-      smartlist_add(answers, tor_strdup(q));
+      smartlist_add_strdup(answers, q);
       smartlist_add(answers, ans);
     }
   } SMARTLIST_FOREACH_END(q);
+
   if (smartlist_len(unrecognized)) {
+    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
     for (i=0; i < smartlist_len(unrecognized)-1; ++i)
       connection_printf_to_buf(conn,
-                               "552-Unrecognized key \"%s\"\r\n",
-                               (char*)smartlist_get(unrecognized, i));
+                               "552-%s\r\n",
+                               (char *)smartlist_get(unrecognized, i));
+
     connection_printf_to_buf(conn,
-                             "552 Unrecognized key \"%s\"\r\n",
-                             (char*)smartlist_get(unrecognized, i));
+                             "552 %s\r\n",
+                             (char *)smartlist_get(unrecognized, i));
     goto done;
   }
 
@@ -3166,7 +3446,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
       size_t esc_len;
       esc_len = write_escaped_data(v, strlen(v), &esc);
       connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_write_to_buf(esc, esc_len, TO_CONN(conn));
+      connection_buf_add(esc, esc_len, TO_CONN(conn));
       tor_free(esc);
     }
   }
@@ -3177,8 +3457,8 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
   smartlist_free(answers);
   SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
   smartlist_free(questions);
+  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
   smartlist_free(unrecognized);
-  tor_free(msg);
 
   return 0;
 }
@@ -3323,17 +3603,19 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
   smartlist_free(args);
 
   nodes = smartlist_new();
+  int first_node = zero_circ;
   SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
-    const node_t *node = node_get_by_nickname(n, 1);
+    const node_t *node = node_get_by_nickname(n, 0);
     if (!node) {
       connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
       goto done;
     }
-    if (!node_has_descriptor(node)) {
+    if (!node_has_preferred_descriptor(node, first_node)) {
       connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
       goto done;
     }
     smartlist_add(nodes, (void*)node);
+    first_node = 0;
   } SMARTLIST_FOREACH_END(n);
   if (!smartlist_len(nodes)) {
     connection_write_str_to_buf("512 No router names provided\r\n", conn);
@@ -3346,22 +3628,20 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
   }
 
   /* now circ refers to something that is ready to be extended */
-  int first_node = zero_circ;
+  first_node = zero_circ;
   SMARTLIST_FOREACH(nodes, const node_t *, node,
   {
     extend_info_t *info = extend_info_from_node(node, first_node);
-    if (first_node && !info) {
+    if (!info) {
+      tor_assert_nonfatal(first_node);
       log_warn(LD_CONTROL,
-               "controller tried to connect to a node that doesn't have any "
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
                "addresses that are allowed by the firewall configuration; "
                "circuit marked for closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
       connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
       goto done;
-    } else {
-      /* True, since node_has_descriptor(node) == true and we are extending
-       * to the node's primary address */
-      tor_assert(info);
     }
     circuit_append_new_exit(circ, info);
     if (circ->build_state->desired_path_len > 1) {
@@ -3380,7 +3660,8 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
       goto done;
     }
   } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN) {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
       int err_reason = 0;
       circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
       if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
@@ -3522,24 +3803,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len,
   }
   /* Is this a single hop circuit? */
   if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    const node_t *node = NULL;
-    char *exit_digest = NULL;
-    if (circ->build_state &&
-        circ->build_state->chosen_exit &&
-        !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) {
-      exit_digest = circ->build_state->chosen_exit->identity_digest;
-      node = node_get_by_id(exit_digest);
-    }
-    /* Do both the client and relay allow one-hop exit circuits? */
-    if (!node ||
-        !node_allows_single_hop_exits(node) ||
-        !get_options()->AllowSingleHopCircuits) {
-      connection_write_str_to_buf(
-      "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-      return 0;
-    }
-    tor_assert(exit_digest);
-    ap_conn->chosen_exit_name = tor_strdup(hex_str(exit_digest, DIGEST_LEN));
+    connection_write_str_to_buf(
+               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
+    return 0;
   }
 
   if (circ && hop>0) {
@@ -4042,6 +4308,14 @@ handle_control_dropguards(control_connection_t *conn,
   smartlist_split_string(args, body, " ",
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
 
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
   if (smartlist_len(args)) {
     connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
   } else {
@@ -4077,7 +4351,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
   /* Extract the first argument (either HSAddress or DescID). */
   arg1 = smartlist_get(args, 0);
   /* Test if it's an HS address without the .onion part. */
-  if (rend_valid_service_id(arg1)) {
+  if (rend_valid_v2_service_id(arg1)) {
     hsaddress = arg1;
   } else if (strcmpstart(arg1, v2_str) == 0 &&
              rend_valid_descriptor_id(arg1 + v2_str_len) &&
@@ -4087,7 +4361,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
      * of the id. */
     desc_id = digest;
   } else {
-    connection_printf_to_buf(conn, "513 Unrecognized \"%s\"\r\n",
+    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
                              arg1);
     goto done;
   }
@@ -4103,7 +4377,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
       const char *server;
 
       server = arg + strlen(opt_server);
-      node = node_get_by_hex_id(server);
+      node = node_get_by_hex_id(server, 0);
       if (!node) {
         connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
                                  server);
@@ -4164,9 +4438,11 @@ handle_control_hspost(control_connection_t *conn,
                       const char *body)
 {
   static const char *opt_server = "SERVER=";
+  static const char *opt_hsaddress = "HSADDRESS=";
   smartlist_t *hs_dirs = NULL;
   const char *encoded_desc = body;
   size_t encoded_desc_len = len;
+  const char *onion_address = NULL;
 
   char *cp = memchr(body, '\n', len);
   if (cp == NULL) {
@@ -4177,8 +4453,10 @@ handle_control_hspost(control_connection_t *conn,
 
   smartlist_t *args = smartlist_new();
 
-  /* If any SERVER= options were specified, try parse the options line */
-  if (!strcasecmpstart(argline, opt_server)) {
+  /* If any SERVER= or HSADDRESS= options were specified, try to parse
+   * the options line. */
+  if (!strcasecmpstart(argline, opt_server) ||
+      !strcasecmpstart(argline, opt_hsaddress)) {
     /* encoded_desc begins after a newline character */
     cp = cp + 1;
     encoded_desc = cp;
@@ -4189,22 +4467,24 @@ handle_control_hspost(control_connection_t *conn,
     SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
       if (!strcasecmpstart(arg, opt_server)) {
         const char *server = arg + strlen(opt_server);
-        const node_t *node = node_get_by_hex_id(server);
+        const node_t *node = node_get_by_hex_id(server, 0);
 
         if (!node || !node->rs) {
           connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
                                    server);
           goto done;
         }
-        if (!node->rs->is_hs_dir) {
-          connection_printf_to_buf(conn, "552 Server \"%s\" is not a HSDir"
-                                         "\r\n", server);
-          goto done;
-        }
         /* Valid server, add it to our local list. */
         if (!hs_dirs)
           hs_dirs = smartlist_new();
         smartlist_add(hs_dirs, node->rs);
+      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
+        const char *address = arg + strlen(opt_hsaddress);
+        if (!hs_address_is_valid(address)) {
+          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
+          goto done;
+        }
+        onion_address = address;
       } else {
         connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
                                  arg);
@@ -4213,6 +4493,21 @@ handle_control_hspost(control_connection_t *conn,
     } SMARTLIST_FOREACH_END(arg);
   }
 
+  /* Handle the v3 case. */
+  if (onion_address) {
+    char *desc_str = NULL;
+    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
+    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
+      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
+    } else {
+      send_control_done(conn);
+    }
+    tor_free(desc_str);
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
   /* Read the dot encoded descriptor, and parse it. */
   rend_encoded_v2_service_descriptor_t *desc =
       tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
@@ -4257,6 +4552,52 @@ handle_control_hspost(control_connection_t *conn,
   return 0;
 }
 
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
 /** Called when we get a ADD_ONION command; parse the body, and set up
  * the new ephemeral Onion Service. */
 static int
@@ -4265,7 +4606,7 @@ handle_control_add_onion(control_connection_t *conn,
                          const char *body)
 {
   smartlist_t *args;
-  size_t arg_len;
+  int arg_len;
   (void) len; /* body is nul-terminated; it's safe to ignore the length */
   args = getargs_helper("ADD_ONION", conn, body, 2, -1);
   if (!args)
@@ -4286,7 +4627,7 @@ handle_control_add_onion(control_connection_t *conn,
   rend_auth_type_t auth_type = REND_NO_AUTH;
   /* Default to adding an anonymous hidden service if no flag is given */
   int non_anonymous = 0;
-  for (size_t i = 1; i < arg_len; i++) {
+  for (int i = 1; i < arg_len; i++) {
     static const char *port_prefix = "Port=";
     static const char *flags_prefix = "Flags=";
     static const char *max_s_prefix = "MaxStreams=";
@@ -4438,15 +4779,15 @@ handle_control_add_onion(control_connection_t *conn,
   }
 
   /* Parse the "keytype:keyblob" argument. */
-  crypto_pk_t *pk = NULL;
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
   char *err_msg = NULL;
 
-  pk = add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
-                               &key_new_alg, &key_new_blob,
-                               &err_msg);
-  if (!pk) {
+  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              &err_msg) < 0) {
     if (err_msg) {
       connection_write_str_to_buf(err_msg, conn);
       tor_free(err_msg);
@@ -4455,16 +4796,23 @@ handle_control_add_onion(control_connection_t *conn,
   }
   tor_assert(!err_msg);
 
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
+    goto out;
+  }
+
   /* Create the HS, using private key pk, client authentication auth_type,
    * the list of auth_clients, and port config port_cfg.
    * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
    * regardless of success/failure.
    */
   char *service_id = NULL;
-  int ret = rend_service_add_ephemeral(pk, port_cfgs, max_streams,
-                                       max_streams_close_circuit,
-                                       auth_type, auth_clients,
-                                       &service_id);
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
   port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
   auth_clients = NULL; /* so is auth_clients */
   switch (ret) {
@@ -4557,9 +4905,10 @@ handle_control_add_onion(control_connection_t *conn,
  * Note: The error messages returned are deliberately vague to avoid echoing
  * key material.
  */
-STATIC crypto_pk_t *
+STATIC int
 add_onion_helper_keyarg(const char *arg, int discard_pk,
                         const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
                         char **err_msg_out)
 {
   smartlist_t *key_args = smartlist_new();
@@ -4567,7 +4916,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
   char *err_msg = NULL;
-  int ok = 0;
+  int ret = -1;
 
   smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
   if (smartlist_len(key_args) != 2) {
@@ -4579,6 +4928,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   static const char *key_type_new = "NEW";
   static const char *key_type_best = "BEST";
   static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
 
   const char *key_type = smartlist_get(key_args, 0);
   const char *key_blob = smartlist_get(key_args, 1);
@@ -4591,9 +4941,23 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
       goto err;
     }
     if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
       err_msg = tor_strdup("512 Invalid RSA key size\r\n");
       goto err;
     }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
   } else if (!strcasecmp(key_type_new, key_type)) {
     /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
     if (!strcasecmp(key_type_rsa1024, key_blob) ||
@@ -4607,12 +4971,38 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
       }
       if (!discard_pk) {
         if (crypto_pk_base64_encode(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
           tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
                        key_type_rsa1024);
           goto err;
         }
         key_new_alg = key_type_rsa1024;
       }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
+                     key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
+                       key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
     } else {
       err_msg = tor_strdup("513 Invalid key type\r\n");
       goto err;
@@ -4622,9 +5012,8 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
     goto err;
   }
 
-  /* Succeded in loading or generating a private key. */
-  tor_assert(pk);
-  ok = 1;
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
 
  err:
   SMARTLIST_FOREACH(key_args, char *, cp, {
@@ -4633,10 +5022,6 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   });
   smartlist_free(key_args);
 
-  if (!ok) {
-    crypto_pk_free(pk);
-    pk = NULL;
-  }
   if (err_msg_out) {
     *err_msg_out = err_msg;
   } else {
@@ -4645,7 +5030,7 @@ add_onion_helper_keyarg(const char *arg, int discard_pk,
   *key_new_alg_out = key_new_alg;
   *key_new_blob_out = key_new_blob;
 
-  return pk;
+  return ret;
 }
 
 /** Helper function to handle parsing a ClientAuth argument to the
@@ -4714,6 +5099,7 @@ handle_control_del_onion(control_connection_t *conn,
                           uint32_t len,
                           const char *body)
 {
+  int hs_version = 0;
   smartlist_t *args;
   (void) len; /* body is nul-terminated; it's safe to ignore the length */
   args = getargs_helper("DEL_ONION", conn, body, 1, 1);
@@ -4721,7 +5107,11 @@ handle_control_del_onion(control_connection_t *conn,
     return 0;
 
   const char *service_id = smartlist_get(args, 0);
-  if (!rend_valid_service_id(service_id)) {
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
     connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
     goto out;
   }
@@ -4748,8 +5138,20 @@ handle_control_del_onion(control_connection_t *conn,
   if (onion_services == NULL) {
     connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
   } else {
-    int ret = rend_service_del_ephemeral(service_id);
-    if (ret) {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
       /* This should *NEVER* fail, since the service is on either the
        * per-control connection list, or the global one.
        */
@@ -4819,9 +5221,16 @@ connection_control_closed(control_connection_t *conn)
    * The list and it's contents are scrubbed/freed in connection_free_.
    */
   if (conn->ephemeral_onion_services) {
-    SMARTLIST_FOREACH(conn->ephemeral_onion_services, char *, cp, {
-      rend_service_del_ephemeral(cp);
-    });
+    SMARTLIST_FOREACH_BEGIN(conn->ephemeral_onion_services, char *, cp) {
+      if (rend_valid_v2_service_id(cp)) {
+        rend_service_del_ephemeral(cp);
+      } else if (hs_address_is_valid(cp)) {
+        hs_service_del_ephemeral(cp);
+      } else {
+        /* An invalid .onion in our list should NEVER happen */
+        tor_fragile_assert();
+      }
+    } SMARTLIST_FOREACH_END(cp);
   }
 
   if (conn->is_owning_control_connection) {
@@ -4862,6 +5271,38 @@ peek_connection_has_control0_command(connection_t *conn)
   return peek_buf_has_control0_command(conn->inbuf);
 }
 
+static int
+peek_connection_has_http_command(connection_t *conn)
+{
+  return peek_buf_has_http_command(conn->inbuf);
+}
+
+static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
+  "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>Tor's ControlPort is not an HTTP proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>Tor's ControlPort is not an HTTP proxy</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use Tor's control port"
+  " as an HTTP proxy.\n"
+  "This is not correct: Tor's default SOCKS proxy port is 9050.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "<!-- Plus this comment, to make the body response more than 512 bytes, so "
+  "     IE will be willing to display it. Comment comment comment comment "
+  "     comment comment comment comment comment comment comment comment.-->\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
 /** Called when data has arrived on a v1 control connection: Try to fetch
  * commands from conn->inbuf, and execute them.
  */
@@ -4895,8 +5336,17 @@ connection_control_process_inbuf(control_connection_t *conn)
             sizeof(buf)-6);
     body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
     set_uint16(buf+0, htons(body_len));
-    connection_write_to_buf(buf, 4+body_len, TO_CONN(conn));
+    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
+
+    connection_mark_and_flush(TO_CONN(conn));
+    return 0;
+  }
 
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
   }
@@ -4908,7 +5358,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* First, fetch a line. */
     do {
       data_len = conn->incoming_cmd_len - conn->incoming_cmd_cur_len;
-      r = connection_fetch_from_buf_line(TO_CONN(conn),
+      r = connection_buf_get_line(TO_CONN(conn),
                               conn->incoming_cmd+conn->incoming_cmd_cur_len,
                               &data_len);
       if (r == 0)
@@ -5474,24 +5924,21 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
 int
 control_event_stream_bandwidth(edge_connection_t *edge_conn)
 {
-  circuit_t *circ;
-  origin_circuit_t *ocirc;
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
   if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
     if (!edge_conn->n_read && !edge_conn->n_written)
       return 0;
 
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
     send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                       "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n",
+                       "650 STREAM_BW "U64_FORMAT" %lu %lu %s\r\n",
                        U64_PRINTF_ARG(edge_conn->base_.global_identifier),
                        (unsigned long)edge_conn->n_read,
-                       (unsigned long)edge_conn->n_written);
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
 
-    circ = circuit_get_by_edge_conn(edge_conn);
-    if (circ && CIRCUIT_IS_ORIGIN(circ)) {
-      ocirc = TO_ORIGIN_CIRCUIT(circ);
-      ocirc->n_read_circ_bw += edge_conn->n_read;
-      ocirc->n_written_circ_bw += edge_conn->n_written;
-    }
     edge_conn->n_written = edge_conn->n_read = 0;
   }
 
@@ -5506,6 +5953,8 @@ control_event_stream_bandwidth_used(void)
   if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
     smartlist_t *conns = get_connection_array();
     edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
 
     SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
     {
@@ -5515,11 +5964,14 @@ control_event_stream_bandwidth_used(void)
         if (!edge_conn->n_read && !edge_conn->n_written)
           continue;
 
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
         send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                           "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n",
+                           "650 STREAM_BW "U64_FORMAT" %lu %lu %s\r\n",
                            U64_PRINTF_ARG(edge_conn->base_.global_identifier),
                            (unsigned long)edge_conn->n_read,
-                           (unsigned long)edge_conn->n_written);
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
 
         edge_conn->n_written = edge_conn->n_read = 0;
     }
@@ -5535,6 +5987,8 @@ int
 control_event_circ_bandwidth_used(void)
 {
   origin_circuit_t *ocirc;
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
   if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
     return 0;
 
@@ -5544,12 +5998,23 @@ control_event_circ_bandwidth_used(void)
     ocirc = TO_ORIGIN_CIRCUIT(circ);
     if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
       continue;
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
     send_control_event(EVENT_CIRC_BANDWIDTH_USED,
-                       "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu\r\n",
+                       "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                       "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                       "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
                        ocirc->global_identifier,
                        (unsigned long)ocirc->n_read_circ_bw,
-                       (unsigned long)ocirc->n_written_circ_bw);
+                       (unsigned long)ocirc->n_written_circ_bw,
+                       tbuf,
+                       (unsigned long)ocirc->n_delivered_read_circ_bw,
+                       (unsigned long)ocirc->n_overhead_read_circ_bw,
+                       (unsigned long)ocirc->n_delivered_written_circ_bw,
+                       (unsigned long)ocirc->n_overhead_written_circ_bw);
     ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
   }
   SMARTLIST_FOREACH_END(circ);
 
@@ -5719,7 +6184,7 @@ control_event_circuit_cell_stats(void)
   if (!get_options()->TestingEnableCellStatsEvent ||
       !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
     return 0;
-  cell_stats = tor_malloc(sizeof(cell_stats_t));;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (!circ->testing_cell_stats)
       continue;
@@ -5734,28 +6199,6 @@ control_event_circuit_cell_stats(void)
   return 0;
 }
 
-/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty
- * for <b>read_empty_time</b> millis, the write bucket was empty for
- * <b>write_empty_time</b> millis, and buckets were last refilled
- * <b>milliseconds_elapsed</b> millis ago.  Only emit TB_EMPTY event if
- * either read or write bucket have been empty before. */
-int
-control_event_tb_empty(const char *bucket, uint32_t read_empty_time,
-                       uint32_t write_empty_time,
-                       int milliseconds_elapsed)
-{
-  if (get_options()->TestingEnableTbEmptyEvent &&
-      EVENT_IS_INTERESTING(EVENT_TB_EMPTY) &&
-      (read_empty_time > 0 || write_empty_time > 0)) {
-    send_control_event(EVENT_TB_EMPTY,
-                       "650 TB_EMPTY %s READ=%d WRITTEN=%d "
-                       "LAST=%d\r\n",
-                       bucket, read_empty_time, write_empty_time,
-                       milliseconds_elapsed);
-  }
-  return 0;
-}
-
 /* about 5 minutes worth. */
 #define N_BW_EVENTS_TO_CACHE 300
 /* Index into cached_bw_events to next write. */
@@ -5843,7 +6286,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg)
   int event;
 
   /* Don't even think of trying to add stuff to a buffer from a cpuworker
-   * thread. */
+   * thread. (See #25987 for plan to fix.) */
   if (! in_main_thread())
     return;
 
@@ -5889,6 +6332,23 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg)
   }
 }
 
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
 /** Called whenever we receive new router descriptors: tell any
  * interested control connections.  <b>routers</b> is a list of
  * routerinfo_t's.
@@ -5954,47 +6414,6 @@ control_event_address_mapped(const char *from, const char *to, time_t expires,
   return 0;
 }
 
-/** The authoritative dirserver has received a new descriptor that
- * has passed basic syntax checks and is properly self-signed.
- *
- * Notify any interested party of the new descriptor and what has
- * been done with it, and also optionally give an explanation/reason. */
-int
-control_event_or_authdir_new_descriptor(const char *action,
-                                        const char *desc, size_t desclen,
-                                        const char *msg)
-{
-  char firstline[1024];
-  char *buf;
-  size_t totallen;
-  char *esc = NULL;
-  size_t esclen;
-
-  if (!EVENT_IS_INTERESTING(EVENT_AUTHDIR_NEWDESCS))
-    return 0;
-
-  tor_snprintf(firstline, sizeof(firstline),
-               "650+AUTHDIR_NEWDESC=\r\n%s\r\n%s\r\n",
-               action,
-               msg ? msg : "");
-
-  /* Escape the server descriptor properly */
-  esclen = write_escaped_data(desc, desclen, &esc);
-
-  totallen = strlen(firstline) + esclen + 1;
-  buf = tor_malloc(totallen);
-  strlcpy(buf, firstline, totallen);
-  strlcpy(buf+strlen(firstline), esc, totallen);
-  send_control_event_string(EVENT_AUTHDIR_NEWDESCS,
-                            buf);
-  send_control_event_string(EVENT_AUTHDIR_NEWDESCS,
-                            "650 OK\r\n");
-  tor_free(esc);
-  tor_free(buf);
-
-  return 0;
-}
-
 /** Cached liveness for network liveness events and GETINFO
  */
 
@@ -6056,9 +6475,9 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses,
     return 0;
 
   strs = smartlist_new();
-  smartlist_add(strs, tor_strdup("650+"));
-  smartlist_add(strs, tor_strdup(event_string));
-  smartlist_add(strs, tor_strdup("\r\n"));
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
   SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
     {
       s = networkstatus_getinfo_helper_single(rs);
@@ -6485,8 +6904,7 @@ monitor_owning_controller_process(const char *process_spec)
             "owning controller: %s.  Exiting.",
             msg);
     owning_controller_process_spec = NULL;
-    tor_cleanup();
-    exit(0);
+    tor_shutdown_event_loop_and_exit(1);
   }
 }
 
@@ -6684,80 +7102,114 @@ control_event_bootstrap(bootstrap_status_t status, int progress)
 }
 
 /** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a hint as to why, and
- * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
- * is the connection that caused this problem.
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
  */
-MOCK_IMPL(void,
-          control_event_bootstrap_problem, (const char *warn, int reason,
-                                            or_connection_t *or_conn))
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
 {
   int status = bootstrap_percent;
   const char *tag = "", *summary = "";
   char buf[BOOTSTRAP_MSG_LEN];
   const char *recommendation = "ignore";
   int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
 
   /* bootstrap_percent must not be in "undefined" state here. */
   tor_assert(status >= 0);
 
-  if (or_conn->have_noted_bootstrap_problem)
-    return;
-
-  or_conn->have_noted_bootstrap_problem = 1;
-
   if (bootstrap_percent == 100)
     return; /* already bootstrapped; nothing to be done here. */
 
   bootstrap_problems++;
 
   if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
-    recommendation = "warn";
-
-  if (reason == END_OR_CONN_REASON_NO_ROUTE)
-    recommendation = "warn";
-
-  /* If we are using bridges and all our OR connections are now
-     closed, it means that we totally failed to connect to our
-     bridges. Throw a warning. */
-  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
-    recommendation = "warn";
+    dowarn = 1;
 
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
   if (we_are_hibernating())
-    recommendation = "ignore";
+    dowarn = 0;
 
   while (status>=0 && bootstrap_status_to_string(status, &tag, &summary) < 0)
     status--; /* find a recognized status string based on current progress */
   status = bootstrap_percent; /* set status back to the actual number */
 
-  severity = !strcmp(recommendation, "warn") ? LOG_WARN : LOG_INFO;
+  severity = dowarn ? LOG_WARN : LOG_INFO;
+
+  if (dowarn)
+    recommendation = "warn";
+
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
 
   log_fn(severity,
          LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
-         "count %d; recommendation %s; host %s at %s:%d)",
-         status, summary, warn,
-         orconn_end_reason_to_control_string(reason),
+         "count %d; recommendation %s; host %s at %s)",
+         status, summary, warn, reason,
          bootstrap_problems, recommendation,
-         hex_str(or_conn->identity_digest, DIGEST_LEN),
-         or_conn->base_.address,
-         or_conn->base_.port);
+         or_id, hostaddr);
 
   connection_or_report_broken_states(severity, LD_HANDSHAKE);
 
   tor_snprintf(buf, sizeof(buf),
       "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
-      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s:%d\"",
-      bootstrap_percent, tag, summary, warn,
-      orconn_end_reason_to_control_string(reason), bootstrap_problems,
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
       recommendation,
-      hex_str(or_conn->identity_digest, DIGEST_LEN),
-      or_conn->base_.address,
-      (int)or_conn->base_.port);
+      or_id, hostaddr);
 
   tor_snprintf(last_sent_bootstrap_message,
                sizeof(last_sent_bootstrap_message),
                "WARN %s", buf);
   control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
 }
 
 /** We just generated a new summary of which countries we've seen clients
@@ -6852,25 +7304,33 @@ rend_hsaddress_str_or_unknown(const char *onion_address)
  * <b>rend_query</b> is used to fetch requested onion address and auth type.
  * <b>hs_dir</b> is the description of contacting hs directory.
  * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
  */
 void
-control_event_hs_descriptor_requested(const rend_data_t *rend_query,
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
                                       const char *id_digest,
-                                      const char *desc_id_base32)
+                                      const char *desc_id,
+                                      const char *hsdir_index)
 {
-  if (!id_digest || !rend_query || !desc_id_base32) {
-    log_warn(LD_BUG, "Called with rend_query==%p, "
-             "id_digest==%p, desc_id_base32==%p",
-             rend_query, id_digest, desc_id_base32);
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
     return;
   }
 
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
   send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC REQUESTED %s %s %s %s\r\n",
-                     rend_hsaddress_str_or_unknown(rend_query->onion_address),
-                     rend_auth_type_to_string(rend_query->auth_type),
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
                      node_describe_longname_by_id(id_digest),
-                     desc_id_base32);
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
 }
 
 /** For an HS descriptor query <b>rend_data</b>, using the
@@ -6884,19 +7344,25 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
 {
   int replica;
   const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
 
   /* Possible if the fetch was done using a descriptor ID. This means that
    * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
-    desc_id = rend_data->desc_id_fetch;
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
     goto end;
   }
 
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
   /* OK, we have an onion address so now let's find which descriptor ID
    * is the one associated with the HSDir fingerprint. */
   for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
        replica++) {
-    const char *digest = rend_data->descriptor_id[replica];
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
 
     SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
       if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
@@ -6913,90 +7379,87 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
 
 /** send HS_DESC CREATED event when a local service generates a descriptor.
  *
- * <b>service_id</b> is the descriptor onion address.
- * <b>desc_id_base32</b> is the descriptor ID.
- * <b>replica</b> is the the descriptor replica number.
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
  */
 void
-control_event_hs_descriptor_created(const char *service_id,
-                                    const char *desc_id_base32,
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
                                     int replica)
 {
-  if (!service_id || !desc_id_base32) {
-    log_warn(LD_BUG, "Called with service_digest==%p, "
-             "desc_id_base32==%p", service_id, desc_id_base32);
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
     return;
   }
 
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
   send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s "
-                     "REPLICA=%d\r\n",
-                     service_id,
-                     desc_id_base32,
-                     replica);
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
 }
 
 /** send HS_DESC upload event.
  *
- * <b>service_id</b> is the descriptor onion address.
+ * <b>onion_address</b> is service address.
  * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
  */
 void
-control_event_hs_descriptor_upload(const char *service_id,
+control_event_hs_descriptor_upload(const char *onion_address,
                                    const char *id_digest,
-                                   const char *desc_id_base32)
+                                   const char *desc_id,
+                                   const char *hsdir_index)
 {
-  if (!service_id || !id_digest || !desc_id_base32) {
-    log_warn(LD_BUG, "Called with service_digest==%p, "
-             "desc_id_base32==%p, id_digest==%p", service_id,
-             desc_id_base32, id_digest);
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
     return;
   }
 
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
   send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s\r\n",
-                     service_id,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
                      node_describe_longname_by_id(id_digest),
-                     desc_id_base32);
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
 }
 
 /** send HS_DESC event after got response from hs directory.
  *
  * NOTE: this is an internal function used by following functions:
- * control_event_hs_descriptor_received
- * control_event_hs_descriptor_failed
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
  *
  * So do not call this function directly.
  */
-void
-control_event_hs_descriptor_receive_end(const char *action,
-                                        const char *onion_address,
-                                        const rend_data_t *rend_data,
-                                        const char *id_digest,
-                                        const char *reason)
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
 {
-  char *desc_id_field = NULL;
   char *reason_field = NULL;
-  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-  const char *desc_id = NULL;
 
-  if (!action || !id_digest || !rend_data || !onion_address) {
-    log_warn(LD_BUG, "Called with action==%p, id_digest==%p, "
-             "rend_data==%p, onion_address==%p", action, id_digest,
-             rend_data, onion_address);
+  if (BUG(!action || !onion_address)) {
     return;
   }
 
-  desc_id = get_desc_id_from_query(rend_data, id_digest);
-  if (desc_id != NULL) {
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
   if (reason) {
     tor_asprintf(&reason_field, " REASON=%s", reason);
   }
@@ -7005,12 +7468,13 @@ control_event_hs_descriptor_receive_end(const char *action,
                      "650 HS_DESC %s %s %s %s%s%s\r\n",
                      action,
                      rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(rend_data->auth_type),
-                     node_describe_longname_by_id(id_digest),
-                     desc_id_field ? desc_id_field : "",
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
                      reason_field ? reason_field : "");
 
-  tor_free(desc_id_field);
   tor_free(reason_field);
 }
 
@@ -7030,9 +7494,7 @@ control_event_hs_descriptor_upload_end(const char *action,
 {
   char *reason_field = NULL;
 
-  if (!action || !id_digest) {
-    log_warn(LD_BUG, "Called with action==%p, id_digest==%p", action,
-             id_digest);
+  if (BUG(!action || !id_digest)) {
     return;
   }
 
@@ -7055,17 +7517,54 @@ control_event_hs_descriptor_upload_end(const char *action,
  * called when we successfully received a hidden service descriptor.
  */
 void
-control_event_hs_descriptor_received(const char *onion_address,
-                                     const rend_data_t *rend_data,
-                                     const char *id_digest)
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
 {
-  if (!rend_data || !id_digest || !onion_address) {
-    log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p, "
-             "onion_address==%p", rend_data, id_digest, onion_address);
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
     return;
   }
-  control_event_hs_descriptor_receive_end("RECEIVED", onion_address,
-                                          rend_data, id_digest, NULL);
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
 }
 
 /** send HS_DESC UPLOADED event
@@ -7076,9 +7575,7 @@ void
 control_event_hs_descriptor_uploaded(const char *id_digest,
                                      const char *onion_address)
 {
-  if (!id_digest) {
-    log_warn(LD_BUG, "Called with id_digest==%p",
-             id_digest);
+  if (BUG(!id_digest)) {
     return;
   }
 
@@ -7086,28 +7583,71 @@ control_event_hs_descriptor_uploaded(const char *id_digest,
                                          id_digest, NULL);
 }
 
-/** Send HS_DESC event to inform controller that query <b>rend_query</b>
- * failed to retrieve hidden service descriptor identified by
- * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON=
- * field.
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
  */
 void
-control_event_hs_descriptor_failed(const rend_data_t *rend_data,
-                                   const char *id_digest,
-                                   const char *reason)
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
 {
-  if (!rend_data || !id_digest) {
-    log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p",
-             rend_data, id_digest);
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
     return;
   }
-  control_event_hs_descriptor_receive_end("FAILED",
-                                          rend_data->onion_address,
-                                          rend_data, id_digest, reason);
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
 }
 
-/** send HS_DESC_CONTENT event after completion of a successful fetch from
- * hs directory. */
+/** Send HS_DESC_CONTENT event after completion of a successful fetch from hs
+ * directory. If <b>hsdir_id_digest</b> is NULL, it is replaced by "UNKNOWN".
+ * If <b>content</b> is NULL, it is replaced by an empty string. The
+ * <b>onion_address</b> or <b>desc_id</b> set to NULL will no trigger the
+ * control event. */
 void
 control_event_hs_descriptor_content(const char *onion_address,
                                     const char *desc_id,
@@ -7117,9 +7657,9 @@ control_event_hs_descriptor_content(const char *onion_address,
   static const char *event_name = "HS_DESC_CONTENT";
   char *esc_content = NULL;
 
-  if (!onion_address || !desc_id || !hsdir_id_digest) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, "
-             "hsdir_id_digest==%p", onion_address, desc_id, hsdir_id_digest);
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
     return;
   }
 
@@ -7134,7 +7674,9 @@ control_event_hs_descriptor_content(const char *onion_address,
                      event_name,
                      rend_hsaddress_str_or_unknown(onion_address),
                      desc_id,
-                     node_describe_longname_by_id(hsdir_id_digest),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
                      esc_content);
   tor_free(esc_content);
 }
@@ -7148,12 +7690,10 @@ control_event_hs_descriptor_upload_failed(const char *id_digest,
                                           const char *onion_address,
                                           const char *reason)
 {
-  if (!id_digest) {
-    log_warn(LD_BUG, "Called with id_digest==%p",
-             id_digest);
+  if (BUG(!id_digest)) {
     return;
   }
-  control_event_hs_descriptor_upload_end("UPLOAD_FAILED", onion_address,
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
                                          id_digest, reason);
 }
 
@@ -7161,22 +7701,40 @@ control_event_hs_descriptor_upload_failed(const char *id_digest,
 void
 control_free_all(void)
 {
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
   if (authentication_cookie) /* Free the auth cookie */
     tor_free(authentication_cookie);
   if (detached_onion_services) { /* Free the detached onion services */
     SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
     smartlist_free(detached_onion_services);
   }
-  if (queued_control_events) {
-    SMARTLIST_FOREACH(queued_control_events, queued_event_t *, ev,
-                      queued_event_free(ev));
-    smartlist_free(queued_control_events);
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
     queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
   }
   if (flush_queued_events_event) {
-    tor_event_free(flush_queued_events_event);
+    mainloop_event_free(flush_queued_events_event);
     flush_queued_events_event = NULL;
   }
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  authentication_cookie_is_set = 0;
+  global_event_mask = 0;
+  disable_log_messages = 0;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -7186,5 +7744,5 @@ control_testing_set_global_event_mask(uint64_t mask)
 {
   global_event_mask = mask;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/or/control.h b/src/or/control.h
index 6330c85571..92cbf866dd 100644
--- a/src/or/control.h
+++ b/src/or/control.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,16 +27,22 @@ void control_ports_write_to_file(void);
 #define LOG_FN_CONN(conn, args)                 \
   CONN_LOG_PROTECT(conn, log_fn args)
 
+#define CC_LOCAL_FD_IS_OWNER (1u<<0)
+#define CC_LOCAL_FD_IS_AUTHENTICATED (1u<<1)
+int control_connection_add_local_fd(tor_socket_t sock, unsigned flags);
+
 int connection_control_finished_flushing(control_connection_t *conn);
 int connection_control_reached_eof(control_connection_t *conn);
 void connection_control_closed(control_connection_t *conn);
 
 int connection_control_process_inbuf(control_connection_t *conn);
 
-#define EVENT_AUTHDIR_NEWDESCS 0x000D
 #define EVENT_NS 0x000F
 int control_event_is_interesting(int event);
 
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
 int control_event_circuit_status(origin_circuit_t *circ,
                                  circuit_status_event_t e, int reason);
 int control_event_circuit_purpose_changed(origin_circuit_t *circ,
@@ -56,18 +62,12 @@ int control_event_circ_bandwidth_used(void);
 int control_event_conn_bandwidth(connection_t *conn);
 int control_event_conn_bandwidth_used(void);
 int control_event_circuit_cell_stats(void);
-int control_event_tb_empty(const char *bucket, uint32_t read_empty_time,
-                           uint32_t write_empty_time,
-                           int milliseconds_elapsed);
 void control_event_logmsg(int severity, uint32_t domain, const char *msg);
+void control_event_logmsg_pending(void);
 int control_event_descriptors_changed(smartlist_t *routers);
 int control_event_address_mapped(const char *from, const char *to,
                                  time_t expires, const char *error,
                                  const int cached);
-int control_event_or_authdir_new_descriptor(const char *action,
-                                            const char *desc,
-                                            size_t desclen,
-                                            const char *msg);
 int control_event_my_descriptor_changed(void);
 int control_event_network_liveness_update(int liveness);
 int control_event_networkstatus_changed(smartlist_t *statuses);
@@ -104,9 +104,11 @@ void enable_control_logging(void);
 void monitor_owning_controller_process(const char *process_spec);
 
 int control_event_bootstrap(bootstrap_status_t status, int progress);
-MOCK_DECL(void, control_event_bootstrap_problem,(const char *warn,
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
                                                  int reason,
                                                  or_connection_t *or_conn));
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
 
 void control_event_clients_seen(const char *controller_str);
 void control_event_transport_launched(const char *mode,
@@ -114,32 +116,39 @@ void control_event_transport_launched(const char *mode,
                                       tor_addr_t *addr, uint16_t port);
 const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
 MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
-void control_event_hs_descriptor_requested(const rend_data_t *rend_query,
-                                           const char *desc_id_base32,
-                                           const char *hs_dir);
-void control_event_hs_descriptor_created(const char *service_id,
-                                         const char *desc_id_base32,
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
                                          int replica);
-void control_event_hs_descriptor_upload(const char *service_id,
-                                        const char *desc_id_base32,
-                                        const char *hs_dir);
-void control_event_hs_descriptor_receive_end(const char *action,
-                                             const char *onion_address,
-                                             const rend_data_t *rend_data,
-                                             const char *id_digest,
-                                             const char *reason);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
 void control_event_hs_descriptor_upload_end(const char *action,
                                             const char *onion_address,
                                             const char *hs_dir,
                                             const char *reason);
-void control_event_hs_descriptor_received(const char *onion_address,
-                                          const rend_data_t *rend_data,
-                                          const char *id_digest);
 void control_event_hs_descriptor_uploaded(const char *hs_dir,
                                           const char *onion_address);
-void control_event_hs_descriptor_failed(const rend_data_t *rend_data,
-                                        const char *id_digest,
-                                        const char *reason);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
 void control_event_hs_descriptor_upload_failed(const char *hs_dir,
                                                const char *onion_address,
                                                const char *reason);
@@ -169,8 +178,8 @@ void control_free_all(void);
 #define EVENT_WARN_MSG                0x000A
 #define EVENT_ERR_MSG                 0x000B
 #define EVENT_ADDRMAP                 0x000C
-/* Exposed above */
-// #define EVENT_AUTHDIR_NEWDESCS     0x000D
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
 #define EVENT_DESCCHANGED             0x000E
 /* Exposed above */
 // #define EVENT_NS                   0x000F
@@ -186,7 +195,7 @@ void control_free_all(void);
 #define EVENT_CONF_CHANGED            0x0019
 #define EVENT_CONN_BW                 0x001A
 #define EVENT_CELL_STATS              0x001B
-#define EVENT_TB_EMPTY                0x001C
+/* UNUSED :                           0x001C */
 #define EVENT_CIRC_BANDWIDTH_USED     0x001D
 #define EVENT_TRANSPORT_LAUNCHED      0x0020
 #define EVENT_HS_DESC                 0x0021
@@ -228,7 +237,7 @@ MOCK_DECL(STATIC void,
           queue_control_event_string,(uint16_t event, char *msg));
 
 void control_testing_set_global_event_mask(uint64_t mask);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /** Helper structure: temporarily stores cell statistics for a circuit. */
 typedef struct cell_stats_t {
@@ -255,13 +264,30 @@ void format_cell_stats(char **event_string, circuit_t *circ,
                        cell_stats_t *cell_stats);
 STATIC char *get_bw_samples(void);
 
-STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk,
-                                            const char **key_new_alg_out,
-                                            char **key_new_blob_out,
-                                            char **err_msg_out);
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version, char **err_msg_out);
+
 STATIC rend_authorized_client_t *
 add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
 
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
 STATIC void getinfo_helper_downloads_networkstatus(
     const char *flavor,
     download_status_t **dl_to_emit,
@@ -285,8 +311,16 @@ STATIC int getinfo_helper_downloads(
     control_connection_t *control_conn,
     const char *question, char **answer,
     const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
 
-#endif
+#endif /* defined(CONTROL_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_CONTROL_H) */
 
diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c
index fd6de6ea7c..15ef6869cf 100644
--- a/src/or/cpuworker.c
+++ b/src/or/cpuworker.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,8 +11,11 @@
  * The multithreading backend for this module is in workqueue.c; this module
  * specializes workqueue.c.
  *
- * Right now, we only use this for processing onionskins, and invoke it mostly
- * from onion.c.
+ * Right now, we use this infrastructure
+ *  <ul><li>for processing onionskins in onion.c
+ *      <li>for compressing consensuses in consdiffmgr.c,
+ *      <li>and for calculating diffs and compressing them in consdiffmgr.c.
+ *  </ul>
  **/
 #include "or.h"
 #include "channel.h"
@@ -21,14 +24,14 @@
 #include "connection_or.h"
 #include "config.h"
 #include "cpuworker.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "main.h"
 #include "onion.h"
 #include "rephist.h"
 #include "router.h"
 #include "workqueue.h"
 
-#include <event2/event.h>
-
 static void queue_pending_tasks(void);
 
 typedef struct worker_state_s {
@@ -45,32 +48,33 @@ worker_state_new(void *arg)
   ws->onion_keys = server_onion_keys_new();
   return ws;
 }
+
+#define worker_state_free(ws) \
+  FREE_AND_NULL(worker_state_t, worker_state_free_, (ws))
+
 static void
-worker_state_free(void *arg)
+worker_state_free_(worker_state_t *ws)
 {
-  worker_state_t *ws = arg;
+  if (!ws)
+    return;
   server_onion_keys_free(ws->onion_keys);
   tor_free(ws);
 }
 
+static void
+worker_state_free_void(void *arg)
+{
+  worker_state_free_(arg);
+}
+
 static replyqueue_t *replyqueue = NULL;
 static threadpool_t *threadpool = NULL;
-static struct event *reply_event = NULL;
 
 static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT;
 
 static int total_pending_tasks = 0;
 static int max_pending_tasks = 128;
 
-static void
-replyqueue_process_cb(evutil_socket_t sock, short events, void *arg)
-{
-  replyqueue_t *rq = arg;
-  (void) sock;
-  (void) events;
-  replyqueue_process(rq);
-}
-
 /** Initialize the cpuworker subsystem. It is OK to call this more than once
  * during Tor's lifetime.
  */
@@ -80,21 +84,25 @@ cpu_init(void)
   if (!replyqueue) {
     replyqueue = replyqueue_new(0);
   }
-  if (!reply_event) {
-    reply_event = tor_event_new(tor_libevent_get_base(),
-                                replyqueue_get_socket(replyqueue),
-                                EV_READ|EV_PERSIST,
-                                replyqueue_process_cb,
-                                replyqueue);
-    event_add(reply_event, NULL);
-  }
   if (!threadpool) {
-    threadpool = threadpool_new(get_num_cpus(get_options()),
+    /*
+      In our threadpool implementation, half the threads are permissive and
+      half are strict (when it comes to running lower-priority tasks). So we
+      always make sure we have at least two threads, so that there will be at
+      least one thread of each kind.
+    */
+    const int n_threads = get_num_cpus(get_options()) + 1;
+    threadpool = threadpool_new(n_threads,
                                 replyqueue,
                                 worker_state_new,
-                                worker_state_free,
+                                worker_state_free_void,
                                 NULL);
+
+    int r = threadpool_register_reply_event(threadpool, NULL);
+
+    tor_assert(r == 0);
   }
+
   /* Total voodoo. Can we make this more sensible? */
   max_pending_tasks = get_num_cpus(get_options()) * 64;
   crypto_seed_weak_rng(&request_sample_rng);
@@ -188,7 +196,7 @@ cpuworkers_rotate_keyinfo(void)
   if (threadpool_queue_update(threadpool,
                               worker_state_new,
                               update_state_threadfn,
-                              worker_state_free,
+                              worker_state_free_void,
                               NULL)) {
     log_warn(LD_OR, "Failed to queue key update for worker threads.");
   }
@@ -229,7 +237,7 @@ should_time_request(uint16_t onionskin_type)
 }
 
 /** Return an estimate of how many microseconds we will need for a single
- * cpuworker to to process <b>n_requests</b> onionskins of type
+ * cpuworker to process <b>n_requests</b> onionskins of type
  * <b>onionskin_type</b>. */
 uint64_t
 estimated_usec_for_onionskins(uint32_t n_requests, uint16_t onionskin_type)
@@ -372,7 +380,7 @@ cpuworker_onion_handshake_replyfn(void *work_)
 
   if (onionskin_answer(circ,
                        &rpl.created_cell,
-                       (const char*)rpl.keys,
+                       (const char*)rpl.keys, sizeof(rpl.keys),
                        rpl.rend_auth_material) < 0) {
     log_warn(LD_OR,"onionskin_answer failed. Closing.");
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
@@ -474,11 +482,27 @@ queue_pending_tasks(void)
     if (!circ)
       return;
 
-    if (assign_onionskin_to_cpuworker(circ, onionskin))
-      log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring.");
+    if (assign_onionskin_to_cpuworker(circ, onionskin) < 0)
+      log_info(LD_OR,"assign_to_cpuworker failed. Ignoring.");
   }
 }
 
+/** DOCDOC */
+MOCK_IMPL(workqueue_entry_t *,
+cpuworker_queue_work,(workqueue_priority_t priority,
+                      workqueue_reply_t (*fn)(void *, void *),
+                      void (*reply_fn)(void *),
+                      void *arg))
+{
+  tor_assert(threadpool);
+
+  return threadpool_queue_work_priority(threadpool,
+                                        priority,
+                                        fn,
+                                        reply_fn,
+                                        arg);
+}
+
 /** Try to tell a cpuworker to perform the public key operations necessary to
  * respond to <b>onionskin</b> for the circuit <b>circ</b>.
  *
@@ -510,7 +534,7 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ,
     return 0;
   }
 
-  if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest))
+  if (!channel_is_client(circ->p_chan))
     rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type);
 
   should_time = should_time_request(onionskin->handshake_type);
@@ -531,7 +555,8 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ,
   memwipe(&req, 0, sizeof(req));
 
   ++total_pending_tasks;
-  queue_entry = threadpool_queue_work(threadpool,
+  queue_entry = threadpool_queue_work_priority(threadpool,
+                                      WQ_PRI_HIGH,
                                       cpuworker_onion_handshake_threadfn,
                                       cpuworker_onion_handshake_replyfn,
                                       job);
diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h
index 62cf0eb164..d39851325f 100644
--- a/src/or/cpuworker.h
+++ b/src/or/cpuworker.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,14 @@
 
 void cpu_init(void);
 void cpuworkers_rotate_keyinfo(void);
+struct workqueue_entry_s;
+enum workqueue_reply_t;
+enum workqueue_priority_t;
+MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, (
+                    enum workqueue_priority_t priority,
+                    enum workqueue_reply_t (*fn)(void *, void *),
+                    void (*reply_fn)(void *),
+                    void *arg));
 
 struct create_cell_t;
 int assign_onionskin_to_cpuworker(or_circuit_t *circ,
@@ -25,5 +33,5 @@ void cpuworker_log_onionskin_overhead(int severity, int onionskin_type,
                                       const char *onionskin_type_name);
 void cpuworker_cancel_circ_handshake(or_circuit_t *circ);
 
-#endif
+#endif /* !defined(TOR_CPUWORKER_H) */
 
diff --git a/src/or/dircollate.c b/src/or/dirauth/dircollate.c
index 033a7afe0f..dec6f75154 100644
--- a/src/or/dircollate.c
+++ b/src/or/dirauth/dircollate.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,6 @@
 #include "dircollate.h"
 #include "dirvote.h"
 
-static void dircollator_collate_by_rsa(dircollator_t *dc);
 static void dircollator_collate_by_ed25519(dircollator_t *dc);
 
 /** Hashtable entry mapping a pair of digests (actually an ed25519 key and an
@@ -41,9 +40,12 @@ typedef struct ddmap_entry_s {
   vote_routerstatus_t *vrs_lst[FLEXIBLE_ARRAY_MEMBER];
 } ddmap_entry_t;
 
+#define ddmap_entry_free(e) \
+  FREE_AND_NULL(ddmap_entry_t, ddmap_entry_free_, (e))
+
 /** Release all storage held by e. */
 static void
-ddmap_entry_free(ddmap_entry_t *e)
+ddmap_entry_free_(ddmap_entry_t *e)
 {
   tor_free(e);
 }
@@ -53,7 +55,7 @@ ddmap_entry_free(ddmap_entry_t *e)
 static ddmap_entry_t *
 ddmap_entry_new(int n_votes)
 {
-  return tor_malloc_zero(STRUCT_OFFSET(ddmap_entry_t, vrs_lst) +
+  return tor_malloc_zero(offsetof(ddmap_entry_t, vrs_lst) +
                          sizeof(vote_routerstatus_t *) * n_votes);
 }
 
@@ -158,7 +160,7 @@ dircollator_new(int n_votes, int n_authorities)
 
 /** Release all storage held by <b>dc</b>. */
 void
-dircollator_free(dircollator_t *dc)
+dircollator_free_(dircollator_t *dc)
 {
   if (!dc)
     return;
@@ -205,49 +207,18 @@ dircollator_add_vote(dircollator_t *dc, networkstatus_t *v)
 void
 dircollator_collate(dircollator_t *dc, int consensus_method)
 {
+  (void) consensus_method;
+
   tor_assert(!dc->is_collated);
   dc->all_rsa_sha1_lst = smartlist_new();
 
-  if (consensus_method < MIN_METHOD_FOR_ED25519_ID_VOTING)
-    dircollator_collate_by_rsa(dc);
-  else
-    dircollator_collate_by_ed25519(dc);
+  dircollator_collate_by_ed25519(dc);
 
   smartlist_sort_digests(dc->all_rsa_sha1_lst);
   dc->is_collated = 1;
 }
 
 /**
- * Collation function for RSA-only consensuses: collate the votes for each
- * entry in <b>dc</b> by their RSA keys.
- *
- * The rule is:
- *    If an RSA identity key is listed by more than half of the authorities,
- *    include that identity, and treat all descriptors with that RSA identity
- *    as describing the same router.
- */
-static void
-dircollator_collate_by_rsa(dircollator_t *dc)
-{
-  const int total_authorities = dc->n_authorities;
-
-  DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) {
-    int n = 0, i;
-    for (i = 0; i < dc->n_votes; ++i) {
-      if (vrs_lst[i] != NULL)
-        ++n;
-    }
-
-    if (n <= total_authorities / 2)
-      continue;
-
-    smartlist_add(dc->all_rsa_sha1_lst, (char *)k);
-  } DIGESTMAP_FOREACH_END;
-
-  dc->by_collated_rsa_sha1 = dc->by_rsa_sha1;
-}
-
-/**
  * Collation function for ed25519 consensuses: collate the votes for each
  * entry in <b>dc</b> by ed25519 key and by RSA key.
  *
diff --git a/src/or/dircollate.h b/src/or/dirauth/dircollate.h
index 358c730cbb..0584b2fe06 100644
--- a/src/or/dircollate.h
+++ b/src/or/dirauth/dircollate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,7 +18,9 @@
 typedef struct dircollator_s dircollator_t;
 
 dircollator_t *dircollator_new(int n_votes, int n_authorities);
-void dircollator_free(dircollator_t *obj);
+void dircollator_free_(dircollator_t *obj);
+#define dircollator_free(c) \
+  FREE_AND_NULL(dircollator_t, dircollator_free_, (c))
 void dircollator_add_vote(dircollator_t *dc, networkstatus_t *v);
 
 void dircollator_collate(dircollator_t *dc, int consensus_method);
@@ -62,7 +64,7 @@ struct dircollator_s {
    * identity digests .*/
   smartlist_t *all_rsa_sha1_lst;
 };
-#endif
+#endif /* defined(DIRCOLLATE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_DIRCOLLATE_H) */
 
diff --git a/src/or/dirvote.c b/src/or/dirauth/dirvote.c
index e6a745612e..bf05286b3d 100644
--- a/src/or/dirvote.c
+++ b/src/or/dirauth/dirvote.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRVOTE_PRIVATE
@@ -9,9 +9,10 @@
 #include "dircollate.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "microdesc.h"
 #include "networkstatus.h"
+#include "nodelist.h"
+#include "parsecommon.h"
 #include "policies.h"
 #include "protover.h"
 #include "rephist.h"
@@ -21,11 +22,48 @@
 #include "routerparse.h"
 #include "entrynodes.h" /* needed for guardfraction methods */
 #include "torcert.h"
-#include "shared_random_state.h"
+#include "voting_schedule.h"
+
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
+#include "dirauth/shared_random_state.h"
 
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
+ *
+ * This module is the center of the consensus-voting based directory
+ * authority system.  With this system, a set of authorities first
+ * publish vote based on their opinions of the network, and then compute
+ * a consensus from those votes.  Each authority signs the consensus,
+ * and clients trust the consensus if enough known authorities have
+ * signed it.
+ *
+ * The code in this module is only invoked on directory authorities.  It's
+ * responsible for:
+ *
+ * <ul>
+ *   <li>Generating this authority's vote networkstatus, based on the
+ *       authority's view of the network as represented in dirserv.c
+ *   <li>Formatting the vote networkstatus objects.
+ *   <li>Generating the microdescriptors that correspond to our own
+ *       vote.
+ *   <li>Sending votes to all the other authorities.
+ *   <li>Trying to fetch missing votes from other authorities.
+ *   <li>Computing the consensus from a set of votes, as well as
+ *       a "detached signature" object for other authorities to fetch.
+ *   <li>Collecting other authorities' signatures on the same consensus,
+ *       until there are enough.
+ *   <li>Publishing the consensus to the reset of the directory system.
+ *   <li>Scheduling all of the above operations.
+ * </ul>
+ *
+ * The main entry points are in dirvote_act(), which handles scheduled
+ * actions; and dirvote_add_vote() and dirvote_add_signatures(), which
+ * handle uploaded and downloaded votes and signatures.
+ *
+ * (See dir-spec.txt from torspec.git for a complete specification of
+ * the directory protocol and voting algorithms.)
  **/
 
 /** A consensus that we have built and are appending signatures to.  Once it's
@@ -59,6 +97,30 @@ static int dirvote_compute_consensuses(void);
 static int dirvote_publish_consensus(void);
 
 /* =====
+ * Certificate functions
+ * ===== */
+
+/** Allocate and return a new authority_cert_t with the same contents as
+ * <b>cert</b>. */
+STATIC authority_cert_t *
+authority_cert_dup(authority_cert_t *cert)
+{
+  authority_cert_t *out = tor_malloc(sizeof(authority_cert_t));
+  tor_assert(cert);
+
+  memcpy(out, cert, sizeof(authority_cert_t));
+  /* Now copy pointed-to things. */
+  out->cache_info.signed_descriptor_body =
+    tor_strndup(cert->cache_info.signed_descriptor_body,
+                cert->cache_info.signed_descriptor_len);
+  out->cache_info.saved_location = SAVED_NOWHERE;
+  out->identity_key = crypto_pk_dup_key(cert->identity_key);
+  out->signing_key = crypto_pk_dup_key(cert->signing_key);
+
+  return out;
+}
+
+/* =====
  * Voting
  * =====*/
 
@@ -245,16 +307,18 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     vote_microdesc_hash_t *h;
     rsf = routerstatus_format_entry(&vrs->status,
                                     vrs->version, vrs->protocols,
-                                    NS_V3_VOTE, vrs);
+                                    NS_V3_VOTE,
+                                    ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                    vrs);
     if (rsf)
       smartlist_add(chunks, rsf);
 
     for (h = vrs->microdesc; h; h = h->next) {
-      smartlist_add(chunks, tor_strdup(h->microdesc_hash_line));
+      smartlist_add_strdup(chunks, h->microdesc_hash_line);
     }
   } SMARTLIST_FOREACH_END(vrs);
 
-  smartlist_add(chunks, tor_strdup("directory-footer\n"));
+  smartlist_add_strdup(chunks, "directory-footer\n");
 
   /* The digest includes everything up through the space after
    * directory-signature.  (Yuck.) */
@@ -273,7 +337,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                            signing_key_fingerprint);
   }
 
-  note_crypto_pk_op(SIGN_DIR);
   {
     char *sig = router_get_dirobj_signature(digest, DIGEST_LEN,
                                             private_signing_key);
@@ -313,10 +376,73 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
   return status;
 }
 
+/** Set *<b>timing_out</b> to the intervals at which we would like to vote.
+ * Note that these aren't the intervals we'll use to vote; they're the ones
+ * that we'll vote to use. */
+static void
+dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out)
+{
+  const or_options_t *options = get_options();
+
+  tor_assert(timing_out);
+
+  timing_out->vote_interval = options->V3AuthVotingInterval;
+  timing_out->n_intervals_valid = options->V3AuthNIntervalsValid;
+  timing_out->vote_delay = options->V3AuthVoteDelay;
+  timing_out->dist_delay = options->V3AuthDistDelay;
+}
+
 /* =====
  * Consensus generation
  * ===== */
 
+/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with
+ * the digest algorithm <b>alg</b>, decode it and copy it into
+ * <b>digest256_out</b> and return 0.  Otherwise return -1. */
+static int
+vote_routerstatus_find_microdesc_hash(char *digest256_out,
+                                      const vote_routerstatus_t *vrs,
+                                      int method,
+                                      digest_algorithm_t alg)
+{
+  /* XXXX only returns the sha256 method. */
+  const vote_microdesc_hash_t *h;
+  char mstr[64];
+  size_t mlen;
+  char dstr[64];
+
+  tor_snprintf(mstr, sizeof(mstr), "%d", method);
+  mlen = strlen(mstr);
+  tor_snprintf(dstr, sizeof(dstr), " %s=",
+               crypto_digest_algorithm_get_name(alg));
+
+  for (h = vrs->microdesc; h; h = h->next) {
+    const char *cp = h->microdesc_hash_line;
+    size_t num_len;
+    /* cp looks like \d+(,\d+)* (digesttype=val )+ .  Let's hunt for mstr in
+     * the first part. */
+    while (1) {
+      num_len = strspn(cp, "1234567890");
+      if (num_len == mlen && fast_memeq(mstr, cp, mlen)) {
+        /* This is the line. */
+        char buf[BASE64_DIGEST256_LEN+1];
+        /* XXXX ignores extraneous stuff if the digest is too long.  This
+         * seems harmless enough, right? */
+        cp = strstr(cp, dstr);
+        if (!cp)
+          return -1;
+        cp += strlen(dstr);
+        strlcpy(buf, cp, sizeof(buf));
+        return digest256_from_base64(digest256_out, buf);
+      }
+      if (num_len == 0 || cp[num_len] != ',')
+        break;
+      cp += num_len + 1;
+    }
+  }
+  return -1;
+}
+
 /** Given a vote <b>vote</b> (not a consensus!), return its associated
  * networkstatus_voter_info_t. */
 static networkstatus_voter_info_t *
@@ -329,20 +455,6 @@ get_voter(const networkstatus_t *vote)
   return smartlist_get(vote->voters, 0);
 }
 
-/** Return the signature made by <b>voter</b> using the algorithm
- * <b>alg</b>, or NULL if none is found. */
-document_signature_t *
-voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter,
-                           digest_algorithm_t alg)
-{
-  if (!voter->sigs)
-    return NULL;
-  SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
-    if (sig->alg == alg)
-      return sig);
-  return NULL;
-}
-
 /** Temporary structure used in constructing a list of dir-source entries
  * for a consensus.  One of these is generated for every vote, and one more
  * for every legacy key in each vote. */
@@ -487,7 +599,7 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
 
   /* compare_vote_rs_() sorts the items by identity digest (all the same),
    * then by SD digest.  That way, if we have a tie that the published_on
-   * date cannot tie, we use the descriptor with the smaller digest.
+   * date cannot break, we use the descriptor with the smaller digest.
    */
   smartlist_sort(votes, compare_vote_rs_);
   SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) {
@@ -509,18 +621,18 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
   if (cur_n > most_n ||
       (cur && cur_n == most_n && cur->status.published_on > most_published)) {
     most = cur;
-    most_n = cur_n;
-    most_published = cur->status.published_on;
+    // most_n = cur_n; // unused after this point.
+    // most_published = cur->status.published_on; // unused after this point.
   }
 
   tor_assert(most);
 
-  /* If we're producing "a" lines, vote on potential alternative (sets
-   * of) OR port(s) in the winning routerstatuses.
+  /* Vote on potential alternative (sets of) OR port(s) in the winning
+   * routerstatuses.
    *
    * XXX prop186 There's at most one alternative OR port (_the_ IPv6
    * port) for now. */
-  if (consensus_method >= MIN_METHOD_FOR_A_LINES && best_alt_orport_out) {
+  if (best_alt_orport_out) {
     smartlist_t *alt_orports = smartlist_new();
     const tor_addr_port_t *most_alt_orport = NULL;
 
@@ -630,13 +742,6 @@ compute_consensus_method(smartlist_t *votes)
 static int
 consensus_method_is_supported(int method)
 {
-  if (method == MIN_METHOD_FOR_ED25519_ID_IN_MD) {
-    /* This method was broken due to buggy code accidently left in
-     * dircollate.c; do not actually use it.
-     */
-    return 0;
-  }
-
   return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) &&
     (method <= MAX_SUPPORTED_CONSENSUS_METHOD);
 }
@@ -712,12 +817,12 @@ dirvote_get_intermediate_param_value(const smartlist_t *param_list,
     }
   } SMARTLIST_FOREACH_END(k_v_pair);
 
-  if (n_found == 1)
+  if (n_found == 1) {
     return value;
-  else if (BUG(n_found > 1))
-    return default_val;
-  else
+  } else {
+    tor_assert_nonfatal(n_found == 0);
     return default_val;
+  }
 }
 
 /** Minimum number of directory authorities voting for a parameter to
@@ -771,6 +876,9 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities)
   output = smartlist_new();
 
   SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) {
+    /* resolve spurious clang shallow analysis null pointer errors */
+    tor_assert(param);
+
     const char *next_param;
     int ok=0;
     eq = strchr(param, '=');
@@ -783,8 +891,7 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities)
       next_param = NULL;
     else
       next_param = smartlist_get(param_list, param_sl_idx+1);
-    /* resolve spurious clang shallow analysis null pointer errors */
-    tor_assert(param);
+
     if (!next_param || strncmp(next_param, param, cur_param_len)) {
       /* We've reached the end of a series. */
       /* Make sure enough authorities voted on this param, unless the
@@ -902,7 +1009,7 @@ networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg,
  *
  * It returns true if weights could be computed, false otherwise.
  */
-static int
+int
 networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
                                      int64_t M, int64_t E, int64_t D,
                                      int64_t T, int64_t weight_scale)
@@ -984,7 +1091,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
         Wgd = weight_scale;
       }
     } else { // Subcase b: R+D >= S
-      casename = "Case 2b1 (Wgg=1, Wmd=Wgd)";
+      casename = "Case 2b1 (Wgg=weight_scale, Wmd=Wgd)";
       Wee = (weight_scale*(E - G + M))/E;
       Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D);
       Wme = (weight_scale*(G-M))/E;
@@ -997,7 +1104,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
                                        weight_scale, G, M, E, D, T, 10, 1);
 
       if (berr) {
-        casename = "Case 2b2 (Wgg=1, Wee=1)";
+        casename = "Case 2b2 (Wgg=weight_scale, Wee=weight_scale)";
         Wgg = weight_scale;
         Wee = weight_scale;
         Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
@@ -1066,7 +1173,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
     } else { // Subcase b: S+D >= T/3
       // D != 0 because S+D >= T/3
       if (G < E) {
-        casename = "Case 3bg (G scarce, Wgg=1, Wmd == Wed)";
+        casename = "Case 3bg (G scarce, Wgg=weight_scale, Wmd == Wed)";
         Wgg = weight_scale;
         Wgd = (weight_scale*(D - 2*G + E + M))/(3*D);
         Wmg = 0;
@@ -1078,7 +1185,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
         berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee,
                     Wed, weight_scale, G, M, E, D, T, 10, 1);
       } else { // G >= E
-        casename = "Case 3be (E scarce, Wee=1, Wmd == Wgd)";
+        casename = "Case 3be (E scarce, Wee=weight_scale, Wmd == Wgd)";
         Wee = weight_scale;
         Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
         Wme = 0;
@@ -1112,7 +1219,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
   tor_assert(0 < weight_scale && weight_scale <= INT32_MAX);
 
   /*
-   * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine
+   * Provide Wgm=Wgg, Wmm=weight_scale, Wem=Wee, Weg=Wed. May later determine
    * that middle nodes need different bandwidth weights for dirport traffic,
    * or that weird exit policies need special weight, or that bridges
    * need special weight.
@@ -1291,12 +1398,23 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes)
 /** Given a list of vote networkstatus_t in <b>votes</b>, our public
  * authority <b>identity_key</b>, our private authority <b>signing_key</b>,
  * and the number of <b>total_authorities</b> that we believe exist in our
- * voting quorum, generate the text of a new v3 consensus vote, and return the
- * value in a newly allocated string.
+ * voting quorum, generate the text of a new v3 consensus or microdescriptor
+ * consensus (depending on <b>flavor</b>), and return the value in a newly
+ * allocated string.
  *
  * Note: this function DOES NOT check whether the votes are from
- * recognized authorities.   (dirvote_add_vote does that.) */
-char *
+ * recognized authorities.   (dirvote_add_vote does that.)
+ *
+ * <strong>WATCH OUT</strong>: You need to think before you change the
+ * behavior of this function, or of the functions it calls! If some
+ * authorities compute the consensus with a different algorithm than
+ * others, they will not reach the same result, and they will not all
+ * sign the same thing!  If you really need to change the algorithm
+ * here, you should allocate a new "consensus_method" for the new
+ * behavior, and make the new behavior conditional on a new-enough
+ * consensus_method.
+ **/
+STATIC char *
 networkstatus_compute_consensus(smartlist_t *votes,
                                 int total_authorities,
                                 crypto_pk_t *identity_key,
@@ -1314,7 +1432,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   smartlist_t *flags;
   const char *flavor_name;
   uint32_t max_unmeasured_bw_kb = DEFAULT_MAX_UNMEASURED_BW_KB;
-  int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */
+  int64_t G, M, E, D, T; /* For bandwidth weights */
   const routerstatus_format_type_t rs_format =
     flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
   char *params = NULL;
@@ -1346,6 +1464,16 @@ networkstatus_compute_consensus(smartlist_t *votes,
     consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD;
   }
 
+  if (consensus_method >= MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE) {
+    /* It's smarter to initialize these weights to 1, so that later on,
+     * we can't accidentally divide by zero. */
+    G = M = E = D = 1;
+    T = 4;
+  } else {
+    /* ...but originally, they were set to zero. */
+    G = M = E = D = T = 0;
+  }
+
   /* Compute medians of time-related things, and figure out how many
    * routers we might need to talk about. */
   {
@@ -1385,7 +1513,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         smartlist_free(sv); /* elements get freed later. */
       }
       SMARTLIST_FOREACH(v->known_flags, const char *, cp,
-                        smartlist_add(flags, tor_strdup(cp)));
+                        smartlist_add_strdup(flags, cp));
     } SMARTLIST_FOREACH_END(v);
     valid_after = median_time(va_times, n_votes);
     fresh_until = median_time(fu_times, n_votes);
@@ -1406,19 +1534,14 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                                       n_versioning_servers);
     client_versions = compute_consensus_versions_list(combined_client_versions,
                                                       n_versioning_clients);
-    if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) {
-      packages = compute_consensus_package_lines(votes);
-    } else {
-      packages = tor_strdup("");
-    }
+    packages = compute_consensus_package_lines(votes);
 
     SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp));
     SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp));
     smartlist_free(combined_server_versions);
     smartlist_free(combined_client_versions);
 
-    if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING)
-      smartlist_add(flags, tor_strdup("NoEdConsensus"));
+    smartlist_add_strdup(flags, "NoEdConsensus");
 
     smartlist_sort_strings(flags);
     smartlist_uniq_strings(flags);
@@ -1467,7 +1590,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     tor_free(flaglist);
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS) {
+  {
     int num_dirauth = get_n_authorities(V3_DIRINFO);
     int idx;
     for (idx = 0; idx < 4; ++idx) {
@@ -1482,12 +1605,12 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                       total_authorities);
   if (smartlist_len(param_list)) {
     params = smartlist_join_strings(param_list, " ", 0, NULL);
-    smartlist_add(chunks, tor_strdup("params "));
+    smartlist_add_strdup(chunks, "params ");
     smartlist_add(chunks, params);
-    smartlist_add(chunks, tor_strdup("\n"));
+    smartlist_add_strdup(chunks, "\n");
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_SHARED_RANDOM) {
+  {
     int num_dirauth = get_n_authorities(V3_DIRINFO);
     /* Default value of this is 2/3 of the total number of authorities. For
      * instance, if we have 9 dirauth, the default value is 6. The following
@@ -1552,7 +1675,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_free(dir_sources);
   }
 
-  if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW) {
+  {
     char *max_unmeasured_param = NULL;
     /* XXXX Extract this code into a common function.  Or don't!  see #19011 */
     if (params) {
@@ -1814,7 +1937,6 @@ networkstatus_compute_consensus(smartlist_t *votes,
         continue;
 
       if (ed_consensus > 0) {
-        tor_assert(consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING);
         if (ed_consensus <= total_authorities / 2) {
           log_warn(LD_BUG, "Not enough entries had ed_consensus set; how "
                    "can we have a consensus of %d?", ed_consensus);
@@ -1841,10 +1963,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
       rs_out.published_on = rs->status.published_on;
       rs_out.dir_port = rs->status.dir_port;
       rs_out.or_port = rs->status.or_port;
-      if (consensus_method >= MIN_METHOD_FOR_A_LINES) {
-        tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr);
-        rs_out.ipv6_orport = alt_orport.port;
-      }
+      tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr);
+      rs_out.ipv6_orport = alt_orport.port;
       rs_out.has_bandwidth = 0;
       rs_out.has_exitsummary = 0;
 
@@ -1874,8 +1994,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         } else if (!strcmp(fl, "Unnamed")) {
           if (is_unnamed)
             smartlist_add(chosen_flags, (char*)fl);
-        } else if (!strcmp(fl, "NoEdConsensus") &&
-                   consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) {
+        } else if (!strcmp(fl, "NoEdConsensus")) {
           if (ed_consensus <= total_authorities/2)
             smartlist_add(chosen_flags, (char*)fl);
         } else {
@@ -1902,8 +2021,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
       /* Starting with consensus method 24, we don't list servers
        * that are not valid in a consensus.  See Proposal 272 */
-      if (!is_valid &&
-          consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES)
+      if (!is_valid)
         continue;
 
       /* Pick the version. */
@@ -1924,8 +2042,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
       /* If it's a guard and we have enough guardfraction votes,
          calculate its consensus guardfraction value. */
-      if (is_guard && num_guardfraction_inputs > 2 &&
-          consensus_method >= MIN_METHOD_FOR_GUARDFRACTION) {
+      if (is_guard && num_guardfraction_inputs > 2) {
         rs_out.has_guardfraction = 1;
         rs_out.guardfraction_percentage = median_uint32(measured_guardfraction,
                                                      num_guardfraction_inputs);
@@ -1942,8 +2059,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         rs_out.has_bandwidth = 1;
         rs_out.bw_is_unmeasured = 1;
         rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths);
-        if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW &&
-            n_authorities_measuring_bandwidth > 2) {
+        if (n_authorities_measuring_bandwidth > 2) {
           /* Cap non-measured bandwidths. */
           if (rs_out.bandwidth_kb > max_unmeasured_bw_kb) {
             rs_out.bandwidth_kb = max_unmeasured_bw_kb;
@@ -2055,7 +2171,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
         char *buf;
         /* Okay!! Now we can write the descriptor... */
         /*     First line goes into "buf". */
-        buf = routerstatus_format_entry(&rs_out, NULL, NULL, rs_format, NULL);
+        buf = routerstatus_format_entry(&rs_out, NULL, NULL,
+                                        rs_format, consensus_method, NULL);
         if (buf)
           smartlist_add(chunks, buf);
       }
@@ -2071,10 +2188,10 @@ networkstatus_compute_consensus(smartlist_t *votes,
                     smartlist_join_strings(chosen_flags, " ", 0, NULL));
       /*     Now the version line. */
       if (chosen_version) {
-        smartlist_add(chunks, tor_strdup("\nv "));
-        smartlist_add(chunks, tor_strdup(chosen_version));
+        smartlist_add_strdup(chunks, "\nv ");
+        smartlist_add_strdup(chunks, chosen_version);
       }
-      smartlist_add(chunks, tor_strdup("\n"));
+      smartlist_add_strdup(chunks, "\n");
       if (chosen_protocol_list &&
           consensus_method >= MIN_METHOD_FOR_RS_PROTOCOLS) {
         smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list);
@@ -2082,8 +2199,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
       /*     Now the weight line. */
       if (rs_out.has_bandwidth) {
         char *guardfraction_str = NULL;
-        int unmeasured = rs_out.bw_is_unmeasured &&
-          consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW;
+        int unmeasured = rs_out.bw_is_unmeasured;
 
         /* If we have guardfraction info, include it in the 'w' line. */
         if (rs_out.has_guardfraction) {
@@ -2127,7 +2243,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   }
 
   /* Mark the directory footer region */
-  smartlist_add(chunks, tor_strdup("directory-footer\n"));
+  smartlist_add_strdup(chunks, "directory-footer\n");
 
   {
     int64_t weight_scale = BW_WEIGHT_SCALE;
@@ -2178,7 +2294,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     const char *algname = crypto_digest_algorithm_get_name(digest_alg);
     char *signature;
 
-    smartlist_add(chunks, tor_strdup("directory-signature "));
+    smartlist_add_strdup(chunks, "directory-signature ");
 
     /* Compute the hash of the chunks. */
     crypto_digest_smartlist(digest, digest_len, chunks, "", digest_alg);
@@ -2205,7 +2321,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_add(chunks, signature);
 
     if (legacy_id_key_digest && legacy_signing_key) {
-      smartlist_add(chunks, tor_strdup("directory-signature "));
+      smartlist_add_strdup(chunks, "directory-signature ");
       base16_encode(fingerprint, sizeof(fingerprint),
                     legacy_id_key_digest, DIGEST_LEN);
       crypto_pk_get_fingerprint(legacy_signing_key,
@@ -2342,7 +2458,7 @@ compute_consensus_package_lines(smartlist_t *votes)
  * new signature is verifiable.)  Return the number of signatures added or
  * changed, or -1 if the document signed by <b>sigs</b> isn't the same
  * document as <b>target</b>. */
-int
+STATIC int
 networkstatus_add_detached_signatures(networkstatus_t *target,
                                       ns_detached_signatures_t *sigs,
                                       const char *source,
@@ -2426,7 +2542,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target,
       continue;
     }
 
-    old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg);
+    old_sig = networkstatus_get_voter_sig_by_alg(target_voter, sig->alg);
 
     /* If the target already has a good signature from this voter, then skip
      * this one. */
@@ -2518,7 +2634,7 @@ networkstatus_format_signatures(networkstatus_t *consensus,
       base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len,
                     BASE64_ENCODE_MULTILINE);
       strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf));
-      smartlist_add(elements, tor_strdup(buf));
+      smartlist_add_strdup(elements, buf);
     } SMARTLIST_FOREACH_END(sig);
   } SMARTLIST_FOREACH_END(v);
 
@@ -2534,7 +2650,7 @@ networkstatus_format_signatures(networkstatus_t *consensus,
  * corresponding to the signatures on <b>consensuses</b>, which must contain
  * exactly one FLAV_NS consensus, and no more than one consensus for each
  * other flavor. */
-char *
+STATIC char *
 networkstatus_get_detached_signatures(smartlist_t *consensuses)
 {
   smartlist_t *elements;
@@ -2639,256 +2755,66 @@ get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending,
   return signatures;
 }
 
-/** Release all storage held in <b>s</b>. */
-void
-ns_detached_signatures_free(ns_detached_signatures_t *s)
-{
-  if (!s)
-    return;
-  if (s->signatures) {
-    STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) {
-      SMARTLIST_FOREACH(sigs, document_signature_t *, sig,
-                        document_signature_free(sig));
-      smartlist_free(sigs);
-    } STRMAP_FOREACH_END;
-    strmap_free(s->signatures, NULL);
-    strmap_free(s->digests, tor_free_);
-  }
-
-  tor_free(s);
-}
-
-/* =====
- * Certificate functions
- * ===== */
-
-/** Allocate and return a new authority_cert_t with the same contents as
- * <b>cert</b>. */
-authority_cert_t *
-authority_cert_dup(authority_cert_t *cert)
-{
-  authority_cert_t *out = tor_malloc(sizeof(authority_cert_t));
-  tor_assert(cert);
-
-  memcpy(out, cert, sizeof(authority_cert_t));
-  /* Now copy pointed-to things. */
-  out->cache_info.signed_descriptor_body =
-    tor_strndup(cert->cache_info.signed_descriptor_body,
-                cert->cache_info.signed_descriptor_len);
-  out->cache_info.saved_location = SAVED_NOWHERE;
-  out->identity_key = crypto_pk_dup_key(cert->identity_key);
-  out->signing_key = crypto_pk_dup_key(cert->signing_key);
-
-  return out;
-}
-
-/* =====
- * Vote scheduling
- * ===== */
-
-/** Set *<b>timing_out</b> to the intervals at which we would like to vote.
- * Note that these aren't the intervals we'll use to vote; they're the ones
- * that we'll vote to use. */
-void
-dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out)
-{
-  const or_options_t *options = get_options();
-
-  tor_assert(timing_out);
-
-  timing_out->vote_interval = options->V3AuthVotingInterval;
-  timing_out->n_intervals_valid = options->V3AuthNIntervalsValid;
-  timing_out->vote_delay = options->V3AuthVoteDelay;
-  timing_out->dist_delay = options->V3AuthDistDelay;
-}
-
-/** Return the start of the next interval of size <b>interval</b> (in
- * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
- * starts a fresh interval, and if the last interval of a day would be
- * truncated to less than half its size, it is rolled into the
- * previous interval. */
-time_t
-dirvote_get_start_of_next_interval(time_t now, int interval, int offset)
-{
-  struct tm tm;
-  time_t midnight_today=0;
-  time_t midnight_tomorrow;
-  time_t next;
-
-  tor_gmtime_r(&now, &tm);
-  tm.tm_hour = 0;
-  tm.tm_min = 0;
-  tm.tm_sec = 0;
-
-  if (tor_timegm(&tm, &midnight_today) < 0) {
-    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
-  }
-  midnight_tomorrow = midnight_today + (24*60*60);
-
-  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
-
-  /* Intervals never cross midnight. */
-  if (next > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  /* If the interval would only last half as long as it's supposed to, then
-   * skip over to the next day. */
-  if (next + interval/2 > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  next += offset;
-  if (next - interval > now)
-    next -= interval;
-
-  return next;
-}
-
-/* Using the time <b>now</b>, return the next voting valid-after time. */
+/**
+ * Entry point: Take whatever voting actions are pending as of <b>now</b>.
+ *
+ * Return the time at which the next action should be taken.
+ */
 time_t
-get_next_valid_after_time(time_t now)
-{
-  time_t next_valid_after_time;
-  const or_options_t *options = get_options();
-  voting_schedule_t *new_voting_schedule =
-    get_voting_schedule(options, now, LOG_INFO);
-  tor_assert(new_voting_schedule);
-
-  next_valid_after_time = new_voting_schedule->interval_starts;
-  voting_schedule_free(new_voting_schedule);
-
-  return next_valid_after_time;
-}
-
-static voting_schedule_t voting_schedule;
-
-/** Set voting_schedule to hold the timing for the next vote we should be
- * doing. */
-void
-dirvote_recalculate_timing(const or_options_t *options, time_t now)
-{
-  voting_schedule_t *new_voting_schedule;
-
-  if (!authdir_mode_v3(options)) {
-    return;
-  }
-
-  /* get the new voting schedule */
-  new_voting_schedule = get_voting_schedule(options, now, LOG_NOTICE);
-  tor_assert(new_voting_schedule);
-
-  /* Fill in the global static struct now */
-  memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
-  voting_schedule_free(new_voting_schedule);
-}
-
-/* Populate and return a new voting_schedule_t that can be used to schedule
- * voting. The object is allocated on the heap and it's the responsibility of
- * the caller to free it. Can't fail. */
-voting_schedule_t *
-get_voting_schedule(const or_options_t *options, time_t now, int severity)
-{
-  int interval, vote_delay, dist_delay;
-  time_t start;
-  time_t end;
-  networkstatus_t *consensus;
-  voting_schedule_t *new_voting_schedule;
-
-  new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t));
-
-  consensus = networkstatus_get_live_consensus(now);
-
-  if (consensus) {
-    interval = (int)( consensus->fresh_until - consensus->valid_after );
-    vote_delay = consensus->vote_seconds;
-    dist_delay = consensus->dist_seconds;
-  } else {
-    interval = options->TestingV3AuthInitialVotingInterval;
-    vote_delay = options->TestingV3AuthInitialVoteDelay;
-    dist_delay = options->TestingV3AuthInitialDistDelay;
-  }
-
-  tor_assert(interval > 0);
-
-  if (vote_delay + dist_delay > interval/2)
-    vote_delay = dist_delay = interval / 4;
-
-  start = new_voting_schedule->interval_starts =
-    dirvote_get_start_of_next_interval(now,interval,
-                                      options->TestingV3AuthVotingStartOffset);
-  end = dirvote_get_start_of_next_interval(start+1, interval,
-                                      options->TestingV3AuthVotingStartOffset);
-
-  tor_assert(end > start);
-
-  new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2);
-  new_voting_schedule->voting_ends = start - dist_delay;
-  new_voting_schedule->fetch_missing_votes =
-    start - dist_delay - (vote_delay/2);
-  new_voting_schedule->voting_starts = start - dist_delay - vote_delay;
-
-  {
-    char tbuf[ISO_TIME_LEN+1];
-    format_iso_time(tbuf, new_voting_schedule->interval_starts);
-    tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: "
-            "consensus_set=%d, interval=%d",
-            tbuf, consensus?1:0, interval);
-  }
-
-  return new_voting_schedule;
-}
-
-/** Frees a voting_schedule_t. This should be used instead of the generic
- * tor_free. */
-void
-voting_schedule_free(voting_schedule_t *voting_schedule_to_free)
-{
-  if (!voting_schedule_to_free)
-    return;
-  tor_free(voting_schedule_to_free);
-}
-
-/** Entry point: Take whatever voting actions are pending as of <b>now</b>. */
-void
 dirvote_act(const or_options_t *options, time_t now)
 {
   if (!authdir_mode_v3(options))
-    return;
-  if (!voting_schedule.voting_starts) {
+    return TIME_MAX;
+  tor_assert_nonfatal(voting_schedule.voting_starts);
+  /* If we haven't initialized this object through this codeflow, we need to
+   * recalculate the timings to match our vote. The reason to do that is if we
+   * have a voting schedule initialized 1 minute ago, the voting timings might
+   * not be aligned to what we should expect with "now". This is especially
+   * true for TestingTorNetwork using smaller timings.  */
+  if (voting_schedule.created_on_demand) {
     char *keys = list_v3_auth_ids();
     authority_cert_t *c = get_my_v3_authority_cert();
     log_notice(LD_DIR, "Scheduling voting.  Known authority IDs are %s. "
                "Mine is %s.",
                keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN));
     tor_free(keys);
-    dirvote_recalculate_timing(options, now);
+    voting_schedule_recalculate_timing(options, now);
+  }
+
+#define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \
+  if (! voting_schedule.done_field) {                   \
+    if (voting_schedule.when_field > now) {             \
+      return voting_schedule.when_field;                \
+    } else {
+#define ENDIF \
+    }           \
   }
-  if (voting_schedule.voting_starts < now && !voting_schedule.have_voted) {
+
+  IF_TIME_FOR_NEXT_ACTION(voting_starts, have_voted) {
     log_notice(LD_DIR, "Time to vote.");
     dirvote_perform_vote();
     voting_schedule.have_voted = 1;
-  }
-  if (voting_schedule.fetch_missing_votes < now &&
-      !voting_schedule.have_fetched_missing_votes) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(fetch_missing_votes, have_fetched_missing_votes) {
     log_notice(LD_DIR, "Time to fetch any votes that we're missing.");
     dirvote_fetch_missing_votes();
     voting_schedule.have_fetched_missing_votes = 1;
-  }
-  if (voting_schedule.voting_ends < now &&
-      !voting_schedule.have_built_consensus) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(voting_ends, have_built_consensus) {
     log_notice(LD_DIR, "Time to compute a consensus.");
     dirvote_compute_consensuses();
     /* XXXX We will want to try again later if we haven't got enough
      * votes yet.  Implement this if it turns out to ever happen. */
     voting_schedule.have_built_consensus = 1;
-  }
-  if (voting_schedule.fetch_missing_signatures < now &&
-      !voting_schedule.have_fetched_missing_signatures) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(fetch_missing_signatures,
+                          have_fetched_missing_signatures) {
     log_notice(LD_DIR, "Time to fetch any signatures that we're missing.");
     dirvote_fetch_missing_signatures();
     voting_schedule.have_fetched_missing_signatures = 1;
-  }
-  if (voting_schedule.interval_starts < now &&
-      !voting_schedule.have_published_consensus) {
+  } ENDIF
+  IF_TIME_FOR_NEXT_ACTION(interval_starts,
+                          have_published_consensus) {
     log_notice(LD_DIR, "Time to publish the consensus and discard old votes");
     dirvote_publish_consensus();
     dirvote_clear_votes(0);
@@ -2898,8 +2824,15 @@ dirvote_act(const or_options_t *options, time_t now)
                 networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
     /* XXXX We will want to try again later if we haven't got enough
      * signatures yet.  Implement this if it turns out to ever happen. */
-    dirvote_recalculate_timing(options, now);
-  }
+    voting_schedule_recalculate_timing(options, now);
+    return voting_schedule.voting_starts;
+  } ENDIF
+
+  tor_assert_nonfatal_unreached();
+  return now + 1;
+
+#undef ENDIF
+#undef IF_TIME_FOR_NEXT_ACTION
 }
 
 /** A vote networkstatus_t and its unparsed body: held around so we can
@@ -3505,7 +3438,13 @@ dirvote_add_signatures_to_pending_consensus(
   }
   r = networkstatus_add_detached_signatures(pc->consensus, sigs,
                                             source, severity, msg_out);
-  log_info(LD_DIR,"Added %d signatures to consensus.", r);
+  if (r >= 0) {
+    log_info(LD_DIR,"Added %d signatures to consensus.", r);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "Unable to add signatures to consensus: %s",
+           *msg_out ? *msg_out : "(unknown)");
+  }
 
   if (r >= 1) {
     char *new_signatures =
@@ -3628,8 +3567,8 @@ dirvote_add_signatures(const char *detached_signatures_body,
                        "Queuing it for the next consensus.", source);
     if (!pending_consensus_signature_list)
       pending_consensus_signature_list = smartlist_new();
-    smartlist_add(pending_consensus_signature_list,
-                  tor_strdup(detached_signatures_body));
+    smartlist_add_strdup(pending_consensus_signature_list,
+                  detached_signatures_body);
     *msg = "Signature queued";
     return 0;
   }
@@ -3757,7 +3696,7 @@ dirvote_get_vote(const char *fp, int flags)
 /** Construct and return a new microdescriptor from a routerinfo <b>ri</b>
  * according to <b>consensus_method</b>.
  **/
-microdesc_t *
+STATIC microdesc_t *
 dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
 {
   microdesc_t *result = NULL;
@@ -3774,8 +3713,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
 
   smartlist_add_asprintf(chunks, "onion-key\n%s", key);
 
-  if (consensus_method >= MIN_METHOD_FOR_NTOR_KEY &&
-      ri->onion_curve25519_pkey) {
+  if (ri->onion_curve25519_pkey) {
     char kbuf[128];
     base64_encode(kbuf, sizeof(kbuf),
                   (const char*)ri->onion_curve25519_pkey->public_key,
@@ -3783,7 +3721,9 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_A_LINES &&
+  /* We originally put a lines in the micrdescriptors, but then we worked out
+   * that we needed them in the microdesc consensus. See #20916. */
+  if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC &&
       !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport)
     smartlist_add_asprintf(chunks, "a %s\n",
                            fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
@@ -3794,8 +3734,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
   if (summary && strcmp(summary, "reject 1-65535"))
     smartlist_add_asprintf(chunks, "p %s\n", summary);
 
-  if (consensus_method >= MIN_METHOD_FOR_P6_LINES &&
-      ri->ipv6_exit_policy) {
+  if (ri->ipv6_exit_policy) {
     /* XXXX+++ This doesn't match proposal 208, which says these should
      * be taken unchanged from the routerinfo.  That's bogosity, IMO:
      * the proposal should have said to do this instead.*/
@@ -3805,11 +3744,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     tor_free(p6);
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) {
+  {
     char idbuf[ED25519_BASE64_LEN+1];
     const char *keytype;
-    if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_IN_MD &&
-        ri->cache_info.signing_key_cert &&
+    if (ri->cache_info.signing_key_cert &&
         ri->cache_info.signing_key_cert->signing_key_included) {
       keytype = "ed25519";
       ed25519_public_to_base64(idbuf,
@@ -3853,7 +3791,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
  * in a consensus vote document.  Write it into the <b>out_len</b>-byte buffer
  * in <b>out</b>.  Return -1 on failure and the number of characters written
  * on success. */
-ssize_t
+static ssize_t
 dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
                                    const microdesc_t *md,
                                    int consensus_method_low,
@@ -3887,12 +3825,8 @@ static const struct consensus_method_range_t {
   int low;
   int high;
 } microdesc_consensus_methods[] = {
-  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_A_LINES - 1},
-  {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1},
-  {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1},
-  {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1},
-  {MIN_METHOD_FOR_ID_HASH_IN_MD, MIN_METHOD_FOR_ED25519_ID_IN_MD - 1},
-  {MIN_METHOD_FOR_ED25519_ID_IN_MD, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
+  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
   {-1, -1}
 };
 
@@ -3949,14 +3883,15 @@ dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now,
   while ((ep = entries)) {
     char buf[128];
     vote_microdesc_hash_t *h;
-    dirvote_format_microdesc_vote_line(buf, sizeof(buf), ep->md,
-                                       ep->low, ep->high);
-    h = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
-    h->microdesc_hash_line = tor_strdup(buf);
-    h->next = result;
-    result = h;
-    ep->md->last_listed = now;
-    smartlist_add(microdescriptors_out, ep->md);
+    if (dirvote_format_microdesc_vote_line(buf, sizeof(buf), ep->md,
+                                           ep->low, ep->high) >= 0) {
+      h = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
+      h->microdesc_hash_line = tor_strdup(buf);
+      h->next = result;
+      result = h;
+      ep->md->last_listed = now;
+      smartlist_add(microdescriptors_out, ep->md);
+    }
     entries = ep->next;
     tor_free(ep);
   }
@@ -3964,49 +3899,692 @@ dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now,
   return result;
 }
 
-/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with
- * the digest algorithm <b>alg</b>, decode it and copy it into
- * <b>digest256_out</b> and return 0.  Otherwise return -1. */
-int
-vote_routerstatus_find_microdesc_hash(char *digest256_out,
-                                      const vote_routerstatus_t *vrs,
-                                      int method,
-                                      digest_algorithm_t alg)
+/** Parse and extract all SR commits from <b>tokens</b> and place them in
+ *  <b>ns</b>. */
+static void
+extract_shared_random_commits(networkstatus_t *ns, const smartlist_t *tokens)
 {
-  /* XXXX only returns the sha256 method. */
-  const vote_microdesc_hash_t *h;
-  char mstr[64];
-  size_t mlen;
-  char dstr[64];
+  smartlist_t *chunks = NULL;
 
-  tor_snprintf(mstr, sizeof(mstr), "%d", method);
-  mlen = strlen(mstr);
-  tor_snprintf(dstr, sizeof(dstr), " %s=",
-               crypto_digest_algorithm_get_name(alg));
+  tor_assert(ns);
+  tor_assert(tokens);
+  /* Commits are only present in a vote. */
+  tor_assert(ns->type == NS_TYPE_VOTE);
 
-  for (h = vrs->microdesc; h; h = h->next) {
-    const char *cp = h->microdesc_hash_line;
-    size_t num_len;
-    /* cp looks like \d+(,\d+)* (digesttype=val )+ .  Let's hunt for mstr in
-     * the first part. */
-    while (1) {
-      num_len = strspn(cp, "1234567890");
-      if (num_len == mlen && fast_memeq(mstr, cp, mlen)) {
-        /* This is the line. */
-        char buf[BASE64_DIGEST256_LEN+1];
-        /* XXXX ignores extraneous stuff if the digest is too long.  This
-         * seems harmless enough, right? */
-        cp = strstr(cp, dstr);
-        if (!cp)
-          return -1;
-        cp += strlen(dstr);
-        strlcpy(buf, cp, sizeof(buf));
-        return digest256_from_base64(digest256_out, buf);
+  ns->sr_info.commits = smartlist_new();
+
+  smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT);
+  /* It's normal that a vote might contain no commits even if it participates
+   * in the SR protocol. Don't treat it as an error. */
+  if (commits == NULL) {
+    goto end;
+  }
+
+  /* Parse the commit. We do NO validation of number of arguments or ordering
+   * for forward compatibility, it's the parse commit job to inform us if it's
+   * supported or not. */
+  chunks = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) {
+    /* Extract all arguments and put them in the chunks list. */
+    for (int i = 0; i < tok->n_args; i++) {
+      smartlist_add(chunks, tok->args[i]);
+    }
+    sr_commit_t *commit = sr_parse_commit(chunks);
+    smartlist_clear(chunks);
+    if (commit == NULL) {
+      /* Get voter identity so we can warn that this dirauth vote contains
+       * commit we can't parse. */
+      networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0);
+      tor_assert(voter);
+      log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.",
+               escaped(tok->object_body),
+               hex_str(voter->identity_digest,
+                       sizeof(voter->identity_digest)));
+      /* Commitment couldn't be parsed. Continue onto the next commit because
+       * this one could be unsupported for instance. */
+      continue;
+    }
+    /* Add newly created commit object to the vote. */
+    smartlist_add(ns->sr_info.commits, commit);
+  } SMARTLIST_FOREACH_END(tok);
+
+ end:
+  smartlist_free(chunks);
+  smartlist_free(commits);
+}
+
+/* Using the given directory tokens in tokens, parse the shared random commits
+ * and put them in the given vote document ns.
+ *
+ * This also sets the SR participation flag if present in the vote. */
+void
+dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  /* Does this authority participates in the SR protocol? */
+  directory_token_t *tok = find_opt_by_keyword(tokens, K_SR_FLAG);
+  if (tok) {
+    ns->sr_info.participate = 1;
+    /* Get the SR commitments and reveals from the vote. */
+    extract_shared_random_commits(ns, tokens);
+  }
+}
+
+/* For the given vote, free the shared random commits if any. */
+void
+dirvote_clear_commits(networkstatus_t *ns)
+{
+  tor_assert(ns->type == NS_TYPE_VOTE);
+
+  if (ns->sr_info.commits) {
+    SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c,
+                      sr_commit_free(c));
+    smartlist_free(ns->sr_info.commits);
+  }
+}
+
+/* The given url is the /tor/status-vote GET directory request. Populates the
+ * items list with strings that we can compress on the fly and dir_items with
+ * cached_dir_t objects that have a precompressed deflated version. */
+void
+dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                               smartlist_t *dir_items)
+{
+  int current;
+
+  url += strlen("/tor/status-vote/");
+  current = !strcmpstart(url, "current/");
+  url = strchr(url, '/');
+  tor_assert(url);
+  ++url;
+  if (!strcmp(url, "consensus")) {
+    const char *item;
+    tor_assert(!current); /* we handle current consensus specially above,
+                           * since it wants to be spooled. */
+    if ((item = dirvote_get_pending_consensus(FLAV_NS)))
+      smartlist_add(items, (char*)item);
+  } else if (!current && !strcmp(url, "consensus-signatures")) {
+    /* XXXX the spec says that we should implement
+     * current/consensus-signatures too.  It doesn't seem to be needed,
+     * though. */
+    const char *item;
+    if ((item=dirvote_get_pending_detached_signatures()))
+      smartlist_add(items, (char*)item);
+  } else if (!strcmp(url, "authority")) {
+    const cached_dir_t *d;
+    int flags = DGV_BY_ID |
+      (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
+    if ((d=dirvote_get_vote(NULL, flags)))
+      smartlist_add(dir_items, (cached_dir_t*)d);
+  } else {
+    const cached_dir_t *d;
+    smartlist_t *fps = smartlist_new();
+    int flags;
+    if (!strcmpstart(url, "d/")) {
+      url += 2;
+      flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
+    } else {
+      flags = DGV_BY_ID |
+        (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
+    }
+    dir_split_resource_into_fingerprints(url, fps, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(fps, char *, fp, {
+                      if ((d = dirvote_get_vote(fp, flags)))
+                      smartlist_add(dir_items, (cached_dir_t*)d);
+                      tor_free(fp);
+                      });
+    smartlist_free(fps);
+  }
+}
+
+/** Get the best estimate of a router's bandwidth for dirauth purposes,
+ * preferring measured to advertised values if available. */
+static uint32_t
+dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri)
+{
+  uint32_t bw_kb = 0;
+  /*
+   * Yeah, measured bandwidths in measured_bw_line_t are (implicitly
+   * signed) longs and the ones router_get_advertised_bandwidth() returns
+   * are uint32_t.
+   */
+  long mbw_kb = 0;
+
+  if (ri) {
+    /*
+     * * First try to see if we have a measured bandwidth; don't bother with
+     * as_of_out here, on the theory that a stale measured bandwidth is still
+     * better to trust than an advertised one.
+     */
+    if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
+                                           &mbw_kb, NULL)) {
+      /* Got one! */
+      bw_kb = (uint32_t)mbw_kb;
+    } else {
+      /* If not, fall back to advertised */
+      bw_kb = router_get_advertised_bandwidth(ri) / 1000;
+    }
+  }
+
+  return bw_kb;
+}
+
+/** Helper for sorting: compares two routerinfos first by address, and then by
+ * descending order of "usefulness".  (An authority is more useful than a
+ * non-authority; a running router is more useful than a non-running router;
+ * and a router with more bandwidth is more useful than one with less.)
+ **/
+static int
+compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
+{
+  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
+  int first_is_auth, second_is_auth;
+  uint32_t bw_kb_first, bw_kb_second;
+  const node_t *node_first, *node_second;
+  int first_is_running, second_is_running;
+
+  /* we return -1 if first should appear before second... that is,
+   * if first is a better router. */
+  if (first->addr < second->addr)
+    return -1;
+  else if (first->addr > second->addr)
+    return 1;
+
+  /* Potentially, this next bit could cause k n lg n memeq calls.  But in
+   * reality, we will almost never get here, since addresses will usually be
+   * different. */
+
+  first_is_auth =
+    router_digest_is_trusted_dir(first->cache_info.identity_digest);
+  second_is_auth =
+    router_digest_is_trusted_dir(second->cache_info.identity_digest);
+
+  if (first_is_auth && !second_is_auth)
+    return -1;
+  else if (!first_is_auth && second_is_auth)
+    return 1;
+
+  node_first = node_get_by_id(first->cache_info.identity_digest);
+  node_second = node_get_by_id(second->cache_info.identity_digest);
+  first_is_running = node_first && node_first->is_running;
+  second_is_running = node_second && node_second->is_running;
+
+  if (first_is_running && !second_is_running)
+    return -1;
+  else if (!first_is_running && second_is_running)
+    return 1;
+
+  bw_kb_first = dirserv_get_bandwidth_for_router_kb(first);
+  bw_kb_second = dirserv_get_bandwidth_for_router_kb(second);
+
+  if (bw_kb_first > bw_kb_second)
+    return -1;
+  else if (bw_kb_first < bw_kb_second)
+    return 1;
+
+  /* They're equal! Compare by identity digest, so there's a
+   * deterministic order and we avoid flapping. */
+  return fast_memcmp(first->cache_info.identity_digest,
+                     second->cache_info.identity_digest,
+                     DIGEST_LEN);
+}
+
+/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t
+ * whose keys are the identity digests of those routers that we're going to
+ * exclude for Sybil-like appearance. */
+static digestmap_t *
+get_possible_sybil_list(const smartlist_t *routers)
+{
+  const or_options_t *options = get_options();
+  digestmap_t *omit_as_sybil;
+  smartlist_t *routers_by_ip = smartlist_new();
+  uint32_t last_addr;
+  int addr_count;
+  /* Allow at most this number of Tor servers on a single IP address, ... */
+  int max_with_same_addr = options->AuthDirMaxServersPerAddr;
+  if (max_with_same_addr <= 0)
+    max_with_same_addr = INT_MAX;
+
+  smartlist_add_all(routers_by_ip, routers);
+  smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_);
+  omit_as_sybil = digestmap_new();
+
+  last_addr = 0;
+  addr_count = 0;
+  SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) {
+    if (last_addr != ri->addr) {
+      last_addr = ri->addr;
+      addr_count = 1;
+    } else if (++addr_count > max_with_same_addr) {
+      digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  smartlist_free(routers_by_ip);
+  return omit_as_sybil;
+}
+
+/** Given a platform string as in a routerinfo_t (possibly null), return a
+ * newly allocated version string for a networkstatus document, or NULL if the
+ * platform doesn't give a Tor version. */
+static char *
+version_from_platform(const char *platform)
+{
+  if (platform && !strcmpstart(platform, "Tor ")) {
+    const char *eos = find_whitespace(platform+4);
+    if (eos && !strcmpstart(eos, " (r")) {
+      /* XXXX Unify this logic with the other version extraction
+       * logic in routerparse.c. */
+      eos = find_whitespace(eos+1);
+    }
+    if (eos) {
+      return tor_strndup(platform, eos-platform);
+    }
+  }
+  return NULL;
+}
+
+/** Given a (possibly empty) list of config_line_t, each line of which contains
+ * a list of comma-separated version numbers surrounded by optional space,
+ * allocate and return a new string containing the version numbers, in order,
+ * separated by commas.  Used to generate Recommended(Client|Server)?Versions
+ */
+char *
+format_recommended_version_list(const config_line_t *ln, int warn)
+{
+  smartlist_t *versions;
+  char *result;
+  versions = smartlist_new();
+  for ( ; ln; ln = ln->next) {
+    smartlist_split_string(versions, ln->value, ",",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  }
+
+  /* Handle the case where a dirauth operator has accidentally made some
+   * versions space-separated instead of comma-separated. */
+  smartlist_t *more_versions = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(versions, char *, v) {
+    if (strchr(v, ' ')) {
+      if (warn)
+        log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. "
+                 "(These are supposed to be comma-separated; I'll pretend you "
+                 "used commas instead.)", escaped(v));
+      SMARTLIST_DEL_CURRENT(versions, v);
+      smartlist_split_string(more_versions, v, NULL,
+                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+      tor_free(v);
+    }
+  } SMARTLIST_FOREACH_END(v);
+  smartlist_add_all(versions, more_versions);
+  smartlist_free(more_versions);
+
+  /* Check to make sure everything looks like a version. */
+  if (warn) {
+    SMARTLIST_FOREACH_BEGIN(versions, const char *, v) {
+      tor_version_t ver;
+      if (tor_version_parse(v, &ver) < 0) {
+        log_warn(LD_DIRSERV, "Recommended version %s does not look valid. "
+                 " (I'll include it anyway, since you told me to.)",
+                 escaped(v));
       }
-      if (num_len == 0 || cp[num_len] != ',')
-        break;
-      cp += num_len + 1;
+    } SMARTLIST_FOREACH_END(v);
+  }
+
+  sort_version_list(versions, 1);
+  result = smartlist_join_strings(versions,",",0,NULL);
+  SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
+  smartlist_free(versions);
+  return result;
+}
+
+/** If there are entries in <b>routers</b> with exactly the same ed25519 keys,
+ * remove the older one.  If they are exactly the same age, remove the one
+ * with the greater descriptor digest. May alter the order of the list. */
+static void
+routers_make_ed_keys_unique(smartlist_t *routers)
+{
+  routerinfo_t *ri2;
+  digest256map_t *by_ed_key = digest256map_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    ri->omit_from_vote = 0;
+    if (ri->cache_info.signing_key_cert == NULL)
+      continue; /* No ed key */
+    const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey;
+    if ((ri2 = digest256map_get(by_ed_key, pk))) {
+      /* Duplicate; must omit one.  Set the omit_from_vote flag in whichever
+       * one has the earlier published_on. */
+      const time_t ri_pub = ri->cache_info.published_on;
+      const time_t ri2_pub = ri2->cache_info.published_on;
+      if (ri2_pub < ri_pub ||
+          (ri2_pub == ri_pub &&
+           fast_memcmp(ri->cache_info.signed_descriptor_digest,
+                       ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) {
+        digest256map_set(by_ed_key, pk, ri);
+        ri2->omit_from_vote = 1;
+      } else {
+        ri->omit_from_vote = 1;
+      }
+    } else {
+      /* Add to map */
+      digest256map_set(by_ed_key, pk, ri);
     }
+  } SMARTLIST_FOREACH_END(ri);
+
+  digest256map_free(by_ed_key, NULL);
+
+  /* Now remove every router where the omit_from_vote flag got set. */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    if (ri->omit_from_vote) {
+      SMARTLIST_DEL_CURRENT(routers, ri);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
+/** Routerstatus <b>rs</b> is part of a group of routers that are on
+ * too narrow an IP-space. Clear out its flags since we don't want it be used
+ * because of its Sybil-like appearance.
+ *
+ * Leave its BadExit flag alone though, since if we think it's a bad exit,
+ * we want to vote that way in case all the other authorities are voting
+ * Running and Exit.
+ */
+static void
+clear_status_flags_on_sybil(routerstatus_t *rs)
+{
+  rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
+    rs->is_flagged_running = rs->is_named = rs->is_valid =
+    rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0;
+  /* FFFF we might want some mechanism to check later on if we
+   * missed zeroing any flags: it's easy to add a new flag but
+   * forget to add it to this clause. */
+}
+
+/** Return a new networkstatus_t* containing our current opinion. (For v3
+ * authorities) */
+networkstatus_t *
+dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
+                                        authority_cert_t *cert)
+{
+  const or_options_t *options = get_options();
+  networkstatus_t *v3_out = NULL;
+  uint32_t addr;
+  char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
+  const char *contact;
+  smartlist_t *routers, *routerstatuses;
+  char identity_digest[DIGEST_LEN];
+  char signing_key_digest[DIGEST_LEN];
+  int listbadexits = options->AuthDirListBadExits;
+  routerlist_t *rl = router_get_routerlist();
+  time_t now = time(NULL);
+  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  networkstatus_voter_info_t *voter = NULL;
+  vote_timing_t timing;
+  digestmap_t *omit_as_sybil = NULL;
+  const int vote_on_reachability = running_long_enough_to_decide_unreachable();
+  smartlist_t *microdescriptors = NULL;
+
+  tor_assert(private_key);
+  tor_assert(cert);
+
+  if (crypto_pk_get_digest(private_key, signing_key_digest)<0) {
+    log_err(LD_BUG, "Error computing signing key digest");
+    return NULL;
   }
-  return -1;
+  if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) {
+    log_err(LD_BUG, "Error computing identity key digest");
+    return NULL;
+  }
+  if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) {
+    log_warn(LD_NET, "Couldn't resolve my hostname");
+    return NULL;
+  }
+  if (!hostname || !strchr(hostname, '.')) {
+    tor_free(hostname);
+    hostname = tor_dup_ip(addr);
+  }
+
+  if (options->VersioningAuthoritativeDir) {
+    client_versions =
+      format_recommended_version_list(options->RecommendedClientVersions, 0);
+    server_versions =
+      format_recommended_version_list(options->RecommendedServerVersions, 0);
+  }
+
+  contact = get_options()->ContactInfo;
+  if (!contact)
+    contact = "(none)";
+
+  /*
+   * Do this so dirserv_compute_performance_thresholds() and
+   * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
+   */
+  if (options->V3BandwidthsFile) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
+  } else {
+    /*
+     * No bandwidths file; clear the measured bandwidth cache in case we had
+     * one last time around.
+     */
+    if (dirserv_get_measured_bw_cache_size() > 0) {
+      dirserv_clear_measured_bw_cache();
+    }
+  }
+
+  /* precompute this part, since we need it to decide what "stable"
+   * means. */
+  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
+                    dirserv_set_router_is_running(ri, now);
+                    });
+
+  routers = smartlist_new();
+  smartlist_add_all(routers, rl->routers);
+  routers_make_ed_keys_unique(routers);
+  /* After this point, don't use rl->routers; use 'routers' instead. */
+  routers_sort_by_identity(routers);
+  omit_as_sybil = get_possible_sybil_list(routers);
+
+  DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) {
+    (void) ignore;
+    rep_hist_make_router_pessimal(sybil_id, now);
+  } DIGESTMAP_FOREACH_END;
+
+  /* Count how many have measured bandwidths so we know how to assign flags;
+   * this must come before dirserv_compute_performance_thresholds() */
+  dirserv_count_measured_bws(routers);
+
+  dirserv_compute_performance_thresholds(omit_as_sybil);
+
+  routerstatuses = smartlist_new();
+  microdescriptors = smartlist_new();
+
+  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
+    /* If it has a protover list and contains a protocol name greater than
+     * MAX_PROTOCOL_NAME_LENGTH, skip it. */
+    if (ri->protocol_list &&
+        protover_contains_long_protocol_names(ri->protocol_list)) {
+      continue;
+    }
+    if (ri->cache_info.published_on >= cutoff) {
+      routerstatus_t *rs;
+      vote_routerstatus_t *vrs;
+      node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      if (!node)
+        continue;
+
+      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
+      rs = &vrs->status;
+      set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                       listbadexits);
+
+      if (ri->cache_info.signing_key_cert) {
+        memcpy(vrs->ed25519_id,
+               ri->cache_info.signing_key_cert->signing_key.pubkey,
+               ED25519_PUBKEY_LEN);
+      }
+
+      if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest))
+        clear_status_flags_on_sybil(rs);
+
+      if (!vote_on_reachability)
+        rs->is_flagged_running = 0;
+
+      vrs->version = version_from_platform(ri->platform);
+      if (ri->protocol_list) {
+        vrs->protocols = tor_strdup(ri->protocol_list);
+      } else {
+        vrs->protocols = tor_strdup(
+                                    protover_compute_for_old_tor(vrs->version));
+      }
+      vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now,
+                                                               microdescriptors);
+
+      smartlist_add(routerstatuses, vrs);
+    }
+  } SMARTLIST_FOREACH_END(ri);
+
+  {
+    smartlist_t *added =
+      microdescs_add_list_to_cache(get_microdesc_cache(),
+                                   microdescriptors, SAVED_NOWHERE, 0);
+    smartlist_free(added);
+    smartlist_free(microdescriptors);
+  }
+
+  smartlist_free(routers);
+  digestmap_free(omit_as_sybil, NULL);
+
+  /* Apply guardfraction information to routerstatuses. */
+  if (options->GuardfractionFile) {
+    dirserv_read_guardfraction_file(options->GuardfractionFile,
+                                    routerstatuses);
+  }
+
+  /* This pass through applies the measured bw lines to the routerstatuses */
+  if (options->V3BandwidthsFile) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
+                                     routerstatuses);
+  } else {
+    /*
+     * No bandwidths file; clear the measured bandwidth cache in case we had
+     * one last time around.
+     */
+    if (dirserv_get_measured_bw_cache_size() > 0) {
+      dirserv_clear_measured_bw_cache();
+    }
+  }
+
+  v3_out = tor_malloc_zero(sizeof(networkstatus_t));
+
+  v3_out->type = NS_TYPE_VOTE;
+  dirvote_get_preferred_voting_intervals(&timing);
+  v3_out->published = now;
+  {
+    char tbuf[ISO_TIME_LEN+1];
+    networkstatus_t *current_consensus =
+      networkstatus_get_live_consensus(now);
+    long last_consensus_interval; /* only used to pick a valid_after */
+    if (current_consensus)
+      last_consensus_interval = current_consensus->fresh_until -
+        current_consensus->valid_after;
+    else
+      last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
+    v3_out->valid_after =
+      voting_schedule_get_start_of_next_interval(now,
+                                   (int)last_consensus_interval,
+                                   options->TestingV3AuthVotingStartOffset);
+    format_iso_time(tbuf, v3_out->valid_after);
+    log_notice(LD_DIR,"Choosing valid-after time in vote as %s: "
+               "consensus_set=%d, last_interval=%d",
+               tbuf, current_consensus?1:0, (int)last_consensus_interval);
+  }
+  v3_out->fresh_until = v3_out->valid_after + timing.vote_interval;
+  v3_out->valid_until = v3_out->valid_after +
+    (timing.vote_interval * timing.n_intervals_valid);
+  v3_out->vote_seconds = timing.vote_delay;
+  v3_out->dist_seconds = timing.dist_delay;
+  tor_assert(v3_out->vote_seconds > 0);
+  tor_assert(v3_out->dist_seconds > 0);
+  tor_assert(timing.n_intervals_valid > 0);
+
+  v3_out->client_versions = client_versions;
+  v3_out->server_versions = server_versions;
+
+  /* These are hardwired, to avoid disaster. */
+  v3_out->recommended_relay_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 Microdesc=1-2 Relay=2");
+  v3_out->recommended_client_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 Microdesc=1-2 Relay=2");
+  v3_out->required_client_protocols =
+    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=4 Microdesc=1-2 Relay=2");
+  v3_out->required_relay_protocols =
+    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
+               "Link=3-4 Microdesc=1 Relay=1-2");
+
+  /* We are not allowed to vote to require anything we don't have. */
+  tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
+  tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL));
+
+  /* We should not recommend anything we don't have. */
+  tor_assert_nonfatal(protover_all_supported(
+                                             v3_out->recommended_relay_protocols, NULL));
+  tor_assert_nonfatal(protover_all_supported(
+                                             v3_out->recommended_client_protocols, NULL));
+
+  v3_out->package_lines = smartlist_new();
+  {
+    config_line_t *cl;
+    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
+      if (validate_recommended_package_line(cl->value))
+        smartlist_add_strdup(v3_out->package_lines, cl->value);
+    }
+  }
+
+  v3_out->known_flags = smartlist_new();
+  smartlist_split_string(v3_out->known_flags,
+                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  if (vote_on_reachability)
+    smartlist_add_strdup(v3_out->known_flags, "Running");
+  if (listbadexits)
+    smartlist_add_strdup(v3_out->known_flags, "BadExit");
+  smartlist_sort_strings(v3_out->known_flags);
+
+  if (options->ConsensusParams) {
+    v3_out->net_params = smartlist_new();
+    smartlist_split_string(v3_out->net_params,
+                           options->ConsensusParams, NULL, 0, 0);
+    smartlist_sort_strings(v3_out->net_params);
+  }
+
+  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
+  voter->nickname = tor_strdup(options->Nickname);
+  memcpy(voter->identity_digest, identity_digest, DIGEST_LEN);
+  voter->sigs = smartlist_new();
+  voter->address = hostname;
+  voter->addr = addr;
+  voter->dir_port = router_get_advertised_dir_port(options, 0);
+  voter->or_port = router_get_advertised_or_port(options);
+  voter->contact = tor_strdup(contact);
+  if (options->V3AuthUseLegacyKey) {
+    authority_cert_t *c = get_my_v3_legacy_cert();
+    if (c) {
+      if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) {
+        log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key");
+        memset(voter->legacy_id_digest, 0, DIGEST_LEN);
+      }
+    }
+  }
+
+  v3_out->voters = smartlist_new();
+  smartlist_add(v3_out->voters, voter);
+  v3_out->cert = authority_cert_dup(cert);
+  v3_out->routerstatus_list = routerstatuses;
+  /* Note: networkstatus_digest is unset; it won't get set until we actually
+   * format the vote. */
+
+  return v3_out;
 }
diff --git a/src/or/dirvote.h b/src/or/dirauth/dirvote.h
index efd233ef5f..b69bbbf5d9 100644
--- a/src/or/dirvote.h
+++ b/src/or/dirauth/dirvote.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,8 +12,6 @@
 #ifndef TOR_DIRVOTE_H
 #define TOR_DIRVOTE_H
 
-#include "testsupport.h"
-
 /*
  * Ideally, assuming synced clocks, we should only need 1 second for each of:
  *  - Vote
@@ -51,57 +49,15 @@
 #define MIN_VOTE_INTERVAL_TESTING_INITIAL \
                 ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)
 
+/* A placeholder for routerstatus_format_entry() when the consensus method
+ * argument is not applicable. */
+#define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0
+
 /** The lowest consensus method that we currently support. */
-#define MIN_SUPPORTED_CONSENSUS_METHOD 13
+#define MIN_SUPPORTED_CONSENSUS_METHOD 25
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 25
-
-/** Lowest consensus method where microdesc consensuses omit any entry
- * with no microdesc. */
-#define MIN_METHOD_FOR_MANDATORY_MICRODESC 13
-
-/** Lowest consensus method that contains "a" lines. */
-#define MIN_METHOD_FOR_A_LINES 14
-
-/** Lowest consensus method where microdescs may include a "p6" line. */
-#define MIN_METHOD_FOR_P6_LINES 15
-
-/** Lowest consensus method where microdescs may include an onion-key-ntor
- * line */
-#define MIN_METHOD_FOR_NTOR_KEY 16
-
-/** Lowest consensus method that ensures that authorities output an
- * Unmeasured=1 flag for unmeasured bandwidths */
-#define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17
-
-/** Lowest consensus method where authorities may include an "id" line in
- * microdescriptors. */
-#define MIN_METHOD_FOR_ID_HASH_IN_MD 18
-
-/** Lowest consensus method where we include "package" lines*/
-#define MIN_METHOD_FOR_PACKAGE_LINES 19
-
-/** Lowest consensus method where authorities may include
- * GuardFraction information in microdescriptors. */
-#define MIN_METHOD_FOR_GUARDFRACTION 20
-
-/** Lowest consensus method where authorities may include an "id" line for
- * ed25519 identities in microdescriptors. (Broken; see
- * consensus_method_is_supported() for more info.) */
-#define MIN_METHOD_FOR_ED25519_ID_IN_MD 21
-
-/** Lowest consensus method where authorities vote on ed25519 ids and ensure
- * ed25519 id consistency. */
-#define MIN_METHOD_FOR_ED25519_ID_VOTING 22
-
-/** Lowest consensus method where authorities may include a shared random
- * value(s). */
-#define MIN_METHOD_FOR_SHARED_RANDOM 23
-
-/** Lowest consensus method where authorities drop all nodes that don't get
- * the Valid flag. */
-#define MIN_METHOD_FOR_EXCLUDING_INVALID_NODES 24
+#define MAX_SUPPORTED_CONSENSUS_METHOD 28
 
 /** Lowest consensus method where authorities vote on required/recommended
  * protocols. */
@@ -111,75 +67,44 @@
  * entries. */
 #define MIN_METHOD_FOR_RS_PROTOCOLS 25
 
+/** Lowest consensus method where authorities initialize bandwidth weights to 1
+ * instead of 0. See #14881 */
+#define MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE 26
+
+/** Lowest consensus method where the microdesc consensus contains relay IPv6
+ * addresses. See #23826 and #20916. */
+#define MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS 27
+
+/** Lowest consensus method where microdescriptors do not contain relay IPv6
+ * addresses. See #23828 and #20916. */
+#define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
+
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
  * get confused with the above macros.) */
 #define DEFAULT_MAX_UNMEASURED_BW_KB 20
 
+/* Directory Get Vote (DGV) flags for dirvote_get_vote(). */
+#define DGV_BY_ID 1
+#define DGV_INCLUDE_PENDING 2
+#define DGV_INCLUDE_PREVIOUS 4
+
+/*
+ * Public API. Used outside of the dirauth subsystem.
+ *
+ * We need to nullify them if the module is disabled.
+ */
+#ifdef HAVE_MODULE_DIRAUTH
+
+time_t dirvote_act(const or_options_t *options, time_t now);
 void dirvote_free_all(void);
 
-/* vote manipulation */
-char *networkstatus_compute_consensus(smartlist_t *votes,
-                                      int total_authorities,
-                                      crypto_pk_t *identity_key,
-                                      crypto_pk_t *signing_key,
-                                      const char *legacy_identity_key_digest,
-                                      crypto_pk_t *legacy_signing_key,
-                                      consensus_flavor_t flavor);
-int networkstatus_add_detached_signatures(networkstatus_t *target,
-                                          ns_detached_signatures_t *sigs,
-                                          const char *source,
-                                          int severity,
-                                          const char **msg_out);
-char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
-void ns_detached_signatures_free(ns_detached_signatures_t *s);
-
-/* cert manipulation */
-authority_cert_t *authority_cert_dup(authority_cert_t *cert);
-
-/* vote scheduling */
-
-/** Scheduling information for a voting interval. */
-typedef struct {
-  /** When do we generate and distribute our vote for this interval? */
-  time_t voting_starts;
-  /** When do we send an HTTP request for any votes that we haven't
-   * been posted yet?*/
-  time_t fetch_missing_votes;
-  /** When do we give up on getting more votes and generate a consensus? */
-  time_t voting_ends;
-  /** When do we send an HTTP request for any signatures we're expecting to
-   * see on the consensus? */
-  time_t fetch_missing_signatures;
-  /** When do we publish the consensus? */
-  time_t interval_starts;
-
-  /* True iff we have generated and distributed our vote. */
-  int have_voted;
-  /* True iff we've requested missing votes. */
-  int have_fetched_missing_votes;
-  /* True iff we have built a consensus and sent the signatures around. */
-  int have_built_consensus;
-  /* True iff we've fetched missing signatures. */
-  int have_fetched_missing_signatures;
-  /* True iff we have published our consensus. */
-  int have_published_consensus;
-} voting_schedule_t;
-
-voting_schedule_t *get_voting_schedule(const or_options_t *options,
-                                       time_t now, int severity);
-
-void voting_schedule_free(voting_schedule_t *voting_schedule_to_free);
-
-void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out);
-time_t dirvote_get_start_of_next_interval(time_t now,
-                                          int interval,
-                                          int offset);
-void dirvote_recalculate_timing(const or_options_t *options, time_t now);
-void dirvote_act(const or_options_t *options, time_t now);
-time_t get_next_valid_after_time(time_t now);
-
-/* invoked on timers and by outside triggers. */
+void dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens);
+void dirvote_clear_commits(networkstatus_t *ns);
+void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                                    smartlist_t *dir_items);
+
+/* Storing signatures and votes functions */
 struct pending_vote_t * dirvote_add_vote(const char *vote_body,
                                          const char **msg_out,
                                          int *status_out);
@@ -187,15 +112,82 @@ int dirvote_add_signatures(const char *detached_signatures_body,
                            const char *source,
                            const char **msg_out);
 
+#else /* HAVE_MODULE_DIRAUTH */
+
+static inline time_t
+dirvote_act(const or_options_t *options, time_t now)
+{
+  (void) options;
+  (void) now;
+  return TIME_MAX;
+}
+
+static inline void
+dirvote_free_all(void)
+{
+}
+
+static inline void
+dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens)
+{
+  (void) ns;
+  (void) tokens;
+}
+
+static inline void
+dirvote_clear_commits(networkstatus_t *ns)
+{
+  (void) ns;
+}
+
+static inline void
+dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
+                               smartlist_t *dir_items)
+{
+  (void) url;
+  (void) items;
+  (void) dir_items;
+}
+
+static inline struct pending_vote_t *
+dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+{
+  (void) vote_body;
+  /* If the dirauth module is disabled, this should NEVER be called else we
+   * failed to safeguard the dirauth module. */
+  tor_assert_nonfatal_unreached();
+
+  /* We need to send out an error code. */
+  *status_out = 400;
+  *msg_out = "No directory authority support";
+  return NULL;
+}
+
+static inline int
+dirvote_add_signatures(const char *detached_signatures_body, const char *source,
+                       const char **msg_out)
+{
+  (void) detached_signatures_body;
+  (void) source;
+  (void) msg_out;
+  /* If the dirauth module is disabled, this should NEVER be called else we
+   * failed to safeguard the dirauth module. */
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
 /* Item access */
 MOCK_DECL(const char*, dirvote_get_pending_consensus,
           (consensus_flavor_t flav));
 MOCK_DECL(const char*, dirvote_get_pending_detached_signatures, (void));
-
-#define DGV_BY_ID 1
-#define DGV_INCLUDE_PENDING 2
-#define DGV_INCLUDE_PREVIOUS 4
 const cached_dir_t *dirvote_get_vote(const char *fp, int flags);
+
+/*
+ * API used _only_ by the dirauth subsystem.
+ */
+
 void set_routerstatus_from_routerinfo(routerstatus_t *rs,
                                       node_t *node,
                                       routerinfo_t *ri, time_t now,
@@ -204,26 +196,18 @@ networkstatus_t *
 dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                                         authority_cert_t *cert);
 
-microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
-                                            int consensus_method);
-ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len,
-                                           const microdesc_t *md,
-                                           int consensus_method_low,
-                                           int consensus_method_high);
 vote_microdesc_hash_t *dirvote_format_all_microdesc_vote_lines(
                                         const routerinfo_t *ri,
                                         time_t now,
                                         smartlist_t *microdescriptors_out);
 
-int vote_routerstatus_find_microdesc_hash(char *digest256_out,
-                                          const vote_routerstatus_t *vrs,
-                                          int method,
-                                          digest_algorithm_t alg);
-document_signature_t *voter_get_sig_by_algorithm(
-                           const networkstatus_voter_info_t *voter,
-                           digest_algorithm_t alg);
-
+/*
+ * Exposed functions for unit tests.
+ */
 #ifdef DIRVOTE_PRIVATE
+
+/* Cert manipulation */
+STATIC authority_cert_t *authority_cert_dup(authority_cert_t *cert);
 STATIC int32_t dirvote_get_intermediate_param_value(
                                    const smartlist_t *param_list,
                                    const char *keyword,
@@ -234,7 +218,30 @@ STATIC smartlist_t *dirvote_compute_params(smartlist_t *votes, int method,
                              int total_authorities);
 STATIC char *compute_consensus_package_lines(smartlist_t *votes);
 STATIC char *make_consensus_method_list(int low, int high, const char *sep);
-#endif
+STATIC int
+networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
+                                     int64_t M, int64_t E, int64_t D,
+                                     int64_t T, int64_t weight_scale);
+STATIC
+char *networkstatus_compute_consensus(smartlist_t *votes,
+                                      int total_authorities,
+                                      crypto_pk_t *identity_key,
+                                      crypto_pk_t *signing_key,
+                                      const char *legacy_identity_key_digest,
+                                      crypto_pk_t *legacy_signing_key,
+                                      consensus_flavor_t flavor);
+STATIC
+int networkstatus_add_detached_signatures(networkstatus_t *target,
+                                          ns_detached_signatures_t *sigs,
+                                          const char *source,
+                                          int severity,
+                                          const char **msg_out);
+STATIC
+char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
+STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
+                                                   int consensus_method);
+
+#endif /* defined(DIRVOTE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_DIRVOTE_H) */
 
diff --git a/src/or/dirauth/mode.h b/src/or/dirauth/mode.h
new file mode 100644
index 0000000000..8a0d3142f1
--- /dev/null
+++ b/src/or/dirauth/mode.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file mode.h
+ * \brief Standalone header file for directory authority mode.
+ **/
+
+#ifndef TOR_DIRAUTH_MODE_H
+#define TOR_DIRAUTH_MODE_H
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+#include "router.h"
+
+/* Return true iff we believe ourselves to be a v3 authoritative directory
+ * server. */
+static inline int
+authdir_mode_v3(const or_options_t *options)
+{
+  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
+}
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+/* Without the dirauth module, we can't be a v3 directory authority, ever. */
+
+static inline int
+authdir_mode_v3(const or_options_t *options)
+{
+  (void) options;
+  return 0;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+#endif /* TOR_MODE_H */
+
diff --git a/src/or/shared_random.c b/src/or/dirauth/shared_random.c
index 5f6b03f1ba..6dd1f330e0 100644
--- a/src/or/shared_random.c
+++ b/src/or/dirauth/shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -91,13 +91,19 @@
 #include "shared_random.h"
 #include "config.h"
 #include "confparse.h"
-#include "dirvote.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "networkstatus.h"
-#include "routerkeys.h"
 #include "router.h"
+#include "routerkeys.h"
 #include "routerlist.h"
+#include "shared_random_client.h"
 #include "shared_random_state.h"
 #include "util.h"
+#include "voting_schedule.h"
+
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
 
 /* String prefix of shared random values in votes/consensuses. */
 static const char previous_srv_str[] = "shared-rand-previous-value";
@@ -192,7 +198,7 @@ verify_commit_and_reveal(const sr_commit_t *commit)
     /* Use the invariant length since the encoded reveal variable has an
      * extra byte for the NUL terminated byte. */
     if (crypto_digest256(received_hashed_reveal, commit->encoded_reveal,
-                         SR_REVEAL_BASE64_LEN, commit->alg)) {
+                         SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
       /* Unable to digest the reveal blob, this is unlikely. */
       goto invalid;
     }
@@ -230,9 +236,7 @@ commit_decode(const char *encoded, sr_commit_t *commit)
 {
   int decoded_len = 0;
   size_t offset = 0;
-  /* XXX: Needs two extra bytes for the base64 decode calculation matches
-   * the binary length once decoded. #17868. */
-  char b64_decoded[SR_COMMIT_LEN + 2];
+  char b64_decoded[SR_COMMIT_LEN];
 
   tor_assert(encoded);
   tor_assert(commit);
@@ -284,9 +288,7 @@ STATIC int
 reveal_decode(const char *encoded, sr_commit_t *commit)
 {
   int decoded_len = 0;
-  /* XXX: Needs two extra bytes for the base64 decode calculation matches
-   * the binary length once decoded. #17868. */
-  char b64_decoded[SR_REVEAL_LEN + 2];
+  char b64_decoded[SR_REVEAL_LEN];
 
   tor_assert(encoded);
   tor_assert(commit);
@@ -388,7 +390,7 @@ commit_encode(const sr_commit_t *commit, char *dst, size_t len)
 static void
 sr_cleanup(void)
 {
-  sr_state_free();
+  sr_state_free_all();
 }
 
 /* Using <b>commit</b>, return a newly allocated string containing the commit
@@ -864,30 +866,9 @@ get_majority_srv_from_votes(const smartlist_t *votes, int current)
   return the_srv;
 }
 
-/* Encode the given shared random value and put it in dst. Destination
- * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
-void
-sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
-{
-  int ret;
-  /* Extra byte for the NULL terminated char. */
-  char buf[SR_SRV_VALUE_BASE64_LEN + 1];
-
-  tor_assert(dst);
-  tor_assert(srv);
-  tor_assert(dst_len >= sizeof(buf));
-
-  ret = base64_encode(buf, sizeof(buf), (const char *) srv->value,
-                      sizeof(srv->value), 0);
-  /* Always expect the full length without the NULL byte. */
-  tor_assert(ret == (sizeof(buf) - 1));
-  tor_assert(ret <= (int) dst_len);
-  strlcpy(dst, buf, dst_len);
-}
-
 /* Free a commit object. */
 void
-sr_commit_free(sr_commit_t *commit)
+sr_commit_free_(sr_commit_t *commit)
 {
   if (commit == NULL) {
     return;
@@ -932,7 +913,7 @@ sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
   /* The invariant length is used here since the encoded reveal variable
    * has an extra byte added for the NULL terminated byte. */
   if (crypto_digest256(commit->hashed_reveal, commit->encoded_reveal,
-                       SR_REVEAL_BASE64_LEN, commit->alg)) {
+                       SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
     goto error;
   }
 
@@ -1012,7 +993,7 @@ sr_compute_srv(void)
     SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
     smartlist_free(chunks);
     if (crypto_digest256(hashed_reveals, reveals, strlen(reveals),
-                         SR_DIGEST_ALG)) {
+                         SR_DIGEST_ALG) < 0) {
       goto end;
     }
     current_srv = generate_srv(hashed_reveals, reveal_num,
@@ -1026,55 +1007,6 @@ sr_compute_srv(void)
   tor_free(reveals);
 }
 
-/* Parse a list of arguments from a SRV value either from a vote, consensus
- * or from our disk state and return a newly allocated srv object. NULL is
- * returned on error.
- *
- * The arguments' order:
- *    num_reveals, value
- */
-sr_srv_t *
-sr_parse_srv(const smartlist_t *args)
-{
-  char *value;
-  int ok, ret;
-  uint64_t num_reveals;
-  sr_srv_t *srv = NULL;
-
-  tor_assert(args);
-
-  if (smartlist_len(args) < 2) {
-    goto end;
-  }
-
-  /* First argument is the number of reveal values */
-  num_reveals = tor_parse_uint64(smartlist_get(args, 0),
-                                 10, 0, UINT64_MAX, &ok, NULL);
-  if (!ok) {
-    goto end;
-  }
-  /* Second and last argument is the shared random value it self. */
-  value = smartlist_get(args, 1);
-  if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) {
-    goto end;
-  }
-
-  srv = tor_malloc_zero(sizeof(*srv));
-  srv->num_reveals = num_reveals;
-  /* We substract one byte from the srclen because the function ignores the
-   * '=' character in the given buffer. This is broken but it's a documented
-   * behavior of the implementation. */
-  ret = base64_decode((char *) srv->value, sizeof(srv->value), value,
-                      SR_SRV_VALUE_BASE64_LEN - 1);
-  if (ret != sizeof(srv->value)) {
-    tor_free(srv);
-    srv = NULL;
-    goto end;
-  }
- end:
-  return srv;
-}
-
 /* Parse a commit from a vote or from our disk state and return a newly
  * allocated commit object. NULL is returned on error.
  *
@@ -1323,13 +1255,7 @@ sr_act_post_consensus(const networkstatus_t *consensus)
   }
 
   /* Prepare our state so that it's ready for the next voting period. */
-  {
-    voting_schedule_t *voting_schedule =
-      get_voting_schedule(options,time(NULL), LOG_NOTICE);
-    time_t interval_starts = voting_schedule->interval_starts;
-    sr_state_update(interval_starts);
-    voting_schedule_free(voting_schedule);
-  }
+  sr_state_update(voting_schedule_get_next_valid_after_time());
 }
 
 /* Initialize shared random subsystem. This MUST be called early in the boot
@@ -1359,5 +1285,5 @@ set_num_srv_agreements(int32_t value)
   num_srv_agreements_from_vote = value;
 }
 
-#endif /* TOR_UNIT_TESTS */
+#endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/or/shared_random.h b/src/or/dirauth/shared_random.h
index 9885934cc7..1778ce8f09 100644
--- a/src/or/shared_random.h
+++ b/src/or/dirauth/shared_random.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_H
@@ -36,17 +36,14 @@
 
 /* Length of base64 encoded commit NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
-#define SR_COMMIT_BASE64_LEN \
-  (((SR_COMMIT_LEN - 1) / 3) * 4 + 4)
+#define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN))
 /* Length of base64 encoded reveal NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
-#define SR_REVEAL_BASE64_LEN \
-  (((SR_REVEAL_LEN - 1) / 3) * 4 + 4)
+#define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN))
 /* Length of base64 encoded shared random value. It's 32 bytes long so 44
  * bytes from the base64_encode_size formula. That includes the '='
  * character at the end. */
-#define SR_SRV_VALUE_BASE64_LEN \
-  (((DIGEST256_LEN - 1) / 3) * 4 + 4)
+#define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN))
 
 /* Assert if commit valid flag is not set. */
 #define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid)
@@ -104,20 +101,48 @@ typedef struct sr_commit_t {
 
 /* API */
 
-/* Public methods: */
+/* Public methods used _outside_ of the module.
+ *
+ * We need to nullify them if the module is disabled. */
+#ifdef HAVE_MODULE_DIRAUTH
 
 int sr_init(int save_to_disk);
 void sr_save_and_cleanup(void);
 void sr_act_post_consensus(const networkstatus_t *consensus);
+
+#else /* HAVE_MODULE_DIRAUTH */
+
+static inline int
+sr_init(int save_to_disk)
+{
+  (void) save_to_disk;
+  /* Always return success. */
+  return 0;
+}
+
+static inline void
+sr_save_and_cleanup(void)
+{
+}
+
+static inline void
+sr_act_post_consensus(const networkstatus_t *consensus)
+{
+  (void) consensus;
+}
+
+#endif /* HAVE_MODULE_DIRAUTH */
+
+/* Public methods used only by dirauth code. */
+
 void sr_handle_received_commits(smartlist_t *commits,
                                 crypto_pk_t *voter_key);
 sr_commit_t *sr_parse_commit(const smartlist_t *args);
-sr_srv_t *sr_parse_srv(const smartlist_t *args);
 char *sr_get_string_for_vote(void);
 char *sr_get_string_for_consensus(const smartlist_t *votes,
                                   int32_t num_srv_agreements);
-void sr_commit_free(sr_commit_t *commit);
-void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv);
+void sr_commit_free_(sr_commit_t *commit);
+#define sr_commit_free(sr) FREE_AND_NULL(sr_commit_t, sr_commit_free_, (sr))
 
 /* Private methods (only used by shared_random_state.c): */
 static inline
@@ -129,6 +154,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+
 #ifdef SHARED_RANDOM_PRIVATE
 
 /* Encode */
@@ -156,7 +182,7 @@ STATIC int should_keep_commit(const sr_commit_t *commit,
                               sr_phase_t phase);
 STATIC void save_commit_during_reveal_phase(const sr_commit_t *commit);
 
-#endif /* SHARED_RANDOM_PRIVATE */
+#endif /* defined(SHARED_RANDOM_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
 
@@ -164,5 +190,5 @@ void set_num_srv_agreements(int32_t value);
 
 #endif /* TOR_UNIT_TESTS */
 
-#endif /* TOR_SHARED_RANDOM_H */
+#endif /* !defined(TOR_SHARED_RANDOM_H) */
 
diff --git a/src/or/shared_random_state.c b/src/or/dirauth/shared_random_state.c
index f27eccafc7..ef9641d9a7 100644
--- a/src/or/shared_random_state.c
+++ b/src/or/dirauth/shared_random_state.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,13 +11,16 @@
 #define SHARED_RANDOM_STATE_PRIVATE
 
 #include "or.h"
-#include "shared_random.h"
 #include "config.h"
 #include "confparse.h"
-#include "dirvote.h"
+#include "crypto_util.h"
+#include "dirauth/dirvote.h"
 #include "networkstatus.h"
 #include "router.h"
+#include "shared_random.h"
+#include "shared_random_client.h"
 #include "shared_random_state.h"
+#include "voting_schedule.h"
 
 /* Default filename of the shared random state on disk. */
 static const char default_fname[] = "sr-state";
@@ -40,19 +43,19 @@ static const char dstate_commit_key[] = "Commit";
 static const char dstate_prev_srv_key[] = "SharedRandPreviousValue";
 static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
 
+/** dummy instance of sr_disk_state_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
+
 /* These next two are duplicates or near-duplicates from config.c */
 #define VAR(name, conftype, member, initvalue)                              \
-  { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(sr_disk_state_t, member), \
-    initvalue }
+  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
+      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
 /* As VAR, but the option name and member name are the same. */
 #define V(member, conftype, initvalue) \
   VAR(#member, conftype, member, initvalue)
 /* Our persistent state magic number. */
 #define SR_DISK_STATE_MAGIC 0x98AB1254
-/* Each protocol phase has 12 rounds  */
-#define SHARED_RANDOM_N_ROUNDS 12
-/* Number of phase we have in a protocol. */
-#define SHARED_RANDOM_N_PHASES 2
 
 static int
 disk_state_validate_cb(void *old_state, void *state, void *default_state,
@@ -70,21 +73,22 @@ static config_var_t state_vars[] = {
   V(SharedRandValues,           LINELIST_V, NULL),
   VAR("SharedRandPreviousValue",LINELIST_S, SharedRandValues, NULL),
   VAR("SharedRandCurrentValue", LINELIST_S, SharedRandValues, NULL),
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+  END_OF_CONFIG_VARS
 };
 
 /* "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
 static config_var_t state_extra_var = {
   "__extra", CONFIG_TYPE_LINELIST,
-  STRUCT_OFFSET(sr_disk_state_t, ExtraLines), NULL
+  offsetof(sr_disk_state_t, ExtraLines), NULL
+  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
 };
 
 /* Configuration format of sr_disk_state_t. */
 static const config_format_t state_format = {
   sizeof(sr_disk_state_t),
   SR_DISK_STATE_MAGIC,
-  STRUCT_OFFSET(sr_disk_state_t, magic_),
+  offsetof(sr_disk_state_t, magic_),
   NULL,
   NULL,
   state_vars,
@@ -110,52 +114,6 @@ get_phase_str(sr_phase_t phase)
 
   return the_string;
 }
-
-/* Return the voting interval of the tor vote subsystem. */
-static int
-get_voting_interval(void)
-{
-  int interval;
-  networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL));
-
-  if (consensus) {
-    interval = (int)(consensus->fresh_until - consensus->valid_after);
-  } else {
-    /* Same for both a testing and real network. We voluntarily ignore the
-     * InitialVotingInterval since it complexifies things and it doesn't
-     * affect the SR protocol. */
-    interval = get_options()->V3AuthVotingInterval;
-  }
-  tor_assert(interval > 0);
-  return interval;
-}
-
-/* Given the time <b>now</b>, return the start time of the current round of
- * the SR protocol. For example, if it's 23:47:08, the current round thus
- * started at 23:47:00 for a voting interval of 10 seconds. */
-static time_t
-get_start_time_of_current_round(time_t now)
-{
-  const or_options_t *options = get_options();
-  int voting_interval = get_voting_interval();
-  voting_schedule_t *new_voting_schedule =
-    get_voting_schedule(options, now, LOG_INFO);
-  tor_assert(new_voting_schedule);
-
-  /* First, get the start time of the next round */
-  time_t next_start = new_voting_schedule->interval_starts;
-  /* Now roll back next_start by a voting interval to find the start time of
-     the current round. */
-  time_t curr_start = dirvote_get_start_of_next_interval(
-                                     next_start - voting_interval - 1,
-                                     voting_interval,
-                                     options->TestingV3AuthVotingStartOffset);
-
-  voting_schedule_free(new_voting_schedule);
-
-  return curr_start;
-}
-
 /* Return the time we should expire the state file created at <b>now</b>.
  * We expire the state file in the beginning of the next protocol run. */
 STATIC time_t
@@ -167,7 +125,7 @@ get_state_valid_until_time(time_t now)
 
   voting_interval = get_voting_interval();
   /* Find the time the current round started. */
-  beginning_of_current_round = get_start_time_of_current_round(now);
+  beginning_of_current_round = get_start_time_of_current_round();
 
   /* Find how many rounds are left till the end of the protocol run */
   current_round = (now / voting_interval) % total_rounds;
@@ -237,12 +195,15 @@ commit_add_to_state(sr_commit_t *commit, sr_state_t *state)
 static void
 commit_free_(void *p)
 {
-  sr_commit_free(p);
+  sr_commit_free_(p);
 }
 
+#define state_free(val) \
+  FREE_AND_NULL(sr_state_t, state_free_, (val))
+
 /* Free a state that was allocated with state_new(). */
 static void
-state_free(sr_state_t *state)
+state_free_(sr_state_t *state)
 {
   if (state == NULL) {
     return;
@@ -284,9 +245,12 @@ state_set(sr_state_t *state)
   sr_state = state;
 }
 
+#define disk_state_free(val) \
+  FREE_AND_NULL(sr_disk_state_t, disk_state_free_, (val))
+
 /* Free an allocated disk state. */
 static void
-disk_state_free(sr_disk_state_t *state)
+disk_state_free_(sr_disk_state_t *state)
 {
   if (state == NULL) {
     return;
@@ -1062,7 +1026,7 @@ sr_state_get_previous_srv(void)
 }
 
 /* Set the current SRV value from our state. Value CAN be NULL. The srv
- * object ownership is transfered to the state object. */
+ * object ownership is transferred to the state object. */
 void
 sr_state_set_previous_srv(const sr_srv_t *srv)
 {
@@ -1081,7 +1045,7 @@ sr_state_get_current_srv(void)
 }
 
 /* Set the current SRV value from our state. Value CAN be NULL. The srv
- * object ownership is transfered to the state object. */
+ * object ownership is transferred to the state object. */
 void
 sr_state_set_current_srv(const sr_srv_t *srv)
 {
@@ -1186,7 +1150,7 @@ sr_state_get_commit(const char *rsa_identity)
 }
 
 /* Add <b>commit</b> to the permanent state. The commit object ownership is
- * transfered to the state so the caller MUST not free it. */
+ * transferred to the state so the caller MUST not free it. */
 void
 sr_state_add_commit(sr_commit_t *commit)
 {
@@ -1253,7 +1217,7 @@ sr_state_srv_is_fresh(void)
 
 /* Cleanup and free our disk and memory state. */
 void
-sr_state_free(void)
+sr_state_free_all(void)
 {
   state_free(sr_state);
   disk_state_free(sr_disk_state);
@@ -1330,7 +1294,7 @@ sr_state_init(int save_to_disk, int read_from_disk)
   /* We have a state in memory, let's make sure it's updated for the current
    * and next voting round. */
   {
-    time_t valid_after = get_next_valid_after_time(now);
+    time_t valid_after = voting_schedule_get_next_valid_after_time();
     sr_state_update(valid_after);
   }
   return 0;
@@ -1356,5 +1320,5 @@ get_sr_state(void)
   return sr_state;
 }
 
-#endif /* TOR_UNIT_TESTS */
+#endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/or/shared_random_state.h b/src/or/dirauth/shared_random_state.h
index cf027f2d35..d31983d187 100644
--- a/src/or/shared_random_state.h
+++ b/src/or/dirauth/shared_random_state.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_STATE_H
@@ -77,7 +77,7 @@ typedef struct sr_state_t {
 typedef struct sr_disk_state_t {
   uint32_t magic_;
   /* Version of the protocol. */
-  uint32_t Version;
+  int Version;
   /* Version of our running tor. */
   char *TorVersion;
   /* Creation time of this state */
@@ -119,7 +119,7 @@ void sr_state_unset_fresh_srv(void);
 int sr_state_init(int save_to_disk, int read_from_disk);
 int sr_state_is_initialized(void);
 void sr_state_save(void);
-void sr_state_free(void);
+void sr_state_free_all(void);
 
 #ifdef SHARED_RANDOM_STATE_PRIVATE
 
@@ -134,7 +134,7 @@ STATIC void new_protocol_run(time_t valid_after);
 STATIC void state_rotate_srv(void);
 STATIC int is_phase_transition(sr_phase_t next_phase);
 
-#endif /* SHARED_RANDOM_STATE_PRIVATE */
+#endif /* defined(SHARED_RANDOM_STATE_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
 
@@ -143,7 +143,7 @@ STATIC sr_state_t *get_sr_state(void);
 STATIC void state_del_previous_srv(void);
 STATIC void state_del_current_srv(void);
 
-#endif /* TOR_UNIT_TESTS */
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif /* TOR_SHARED_RANDOM_STATE_H */
+#endif /* !defined(TOR_SHARED_RANDOM_STATE_H) */
 
diff --git a/src/or/directory.c b/src/or/directory.c
index f285e4c6ed..efb6cd0e94 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -1,21 +1,33 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define DIRECTORY_PRIVATE
+
 #include "or.h"
 #include "backtrace.h"
+#include "bridges.h"
 #include "buffers.h"
 #include "circuitbuild.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
+#include "conscache.h"
+#include "consdiff.h"
+#include "consdiffmgr.h"
 #include "control.h"
+#include "compat.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "entrynodes.h"
 #include "geoip.h"
+#include "hs_cache.h"
+#include "hs_common.h"
+#include "hs_control.h"
+#include "hs_client.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
@@ -30,19 +42,51 @@
 #include "routerlist.h"
 #include "routerparse.h"
 #include "routerset.h"
-#include "shared_random.h"
 
 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
-#ifndef OPENBSD
+#if !defined(OpenBSD)
 #include <malloc.h>
 #endif
 #endif
 
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
+#include "dirauth/shared_random.h"
+
 /**
  * \file directory.c
- * \brief Code to send and fetch directories and router
- * descriptors via HTTP.  Directories use dirserv.c to generate the
- * results; clients use routers.c to parse them.
+ * \brief Code to send and fetch information from directory authorities and
+ * caches via HTTP.
+ *
+ * Directory caches and authorities use dirserv.c to generate the results of a
+ * query and stream them to the connection; clients use routerparse.c to parse
+ * them.
+ *
+ * Every directory request has a dir_connection_t on the client side and on
+ * the server side.  In most cases, the dir_connection_t object is a linked
+ * connection, tunneled through an edge_connection_t so that it can be a
+ * stream on the Tor network.  The only non-tunneled connections are those
+ * that are used to upload material (descriptors and votes) to authorities.
+ * Among tunneled connections, some use one-hop circuits, and others use
+ * multi-hop circuits for anonymity.
+ *
+ * Directory requests are launched by calling
+ * directory_initiate_request(). This
+ * launch the connection, will construct an HTTP request with
+ * directory_send_command(), send the and wait for a response.  The client
+ * later handles the response with connection_dir_client_reached_eof(),
+ * which passes the information received to another part of Tor.
+ *
+ * On the server side, requests are read in directory_handle_command(),
+ * which dispatches first on the request type (GET or POST), and then on
+ * the URL requested. GET requests are processed with a table-based
+ * dispatcher in url_table[].  The process of handling larger GET requests
+ * is complicated because we need to avoid allocating a copy of all the
+ * data to be sent to the client in one huge buffer.  Instead, we spool the
+ * data into the buffer using logic in connection_dirserv_flushed_some() in
+ * dirserv.c.  (TODO: If we extended buf.c to have a zero-copy
+ * reference-based buffer type, we could remove most of that code, at the
+ * cost of a bit more reference counting.)
  **/
 
 /* In-points to directory.c:
@@ -62,12 +106,9 @@
  *   connection_finished_connecting() in connection.c
  */
 static void directory_send_command(dir_connection_t *conn,
-                             int purpose, int direct, const char *resource,
-                             const char *payload, size_t payload_len,
-                             time_t if_modified_since);
-static int directory_handle_command(dir_connection_t *conn);
+                                   int direct,
+                                   const directory_request_t *request);
 static int body_is_plausible(const char *body, size_t body_len, int purpose);
-static char *http_get_header(const char *headers, const char *which);
 static void http_set_address_origin(const char *headers, connection_t *conn);
 static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
 static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
@@ -80,21 +121,10 @@ static void dir_routerdesc_download_failed(smartlist_t *failed,
                                            int was_extrainfo,
                                            int was_descriptor_digests);
 static void dir_microdesc_download_failed(smartlist_t *failed,
-                                          int status_code);
-static int client_likes_consensus(networkstatus_t *v, const char *want_url);
-
-static void directory_initiate_command_rend(
-                                          const tor_addr_port_t *or_addr_port,
-                                          const tor_addr_port_t *dir_addr_port,
-                                          const char *digest,
-                                          uint8_t dir_purpose,
-                                          uint8_t router_purpose,
-                                          dir_indirection_t indirection,
-                                          const char *resource,
-                                          const char *payload,
-                                          size_t payload_len,
-                                          time_t if_modified_since,
-                                          const rend_data_t *rend_query);
+                                          int status_code,
+                                          const char *dir_id);
+static int client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                                  const char *want_url);
 
 static void connection_dir_close_consensus_fetches(
                    dir_connection_t *except_this_one, const char *resource);
@@ -106,6 +136,7 @@ static void connection_dir_close_consensus_fetches(
 #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
 
 #define X_ADDRESS_HEADER "X-Your-Address-Is: "
+#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
 
 /** HTTP cache control: how long do we tell proxies they can cache each
  * kind of document we serve? */
@@ -120,29 +151,58 @@ static void connection_dir_close_consensus_fetches(
 
 /********* END VARIABLES ************/
 
-/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's
- * fetching descriptors, it's fetching them for <b>router_purpose</b>)
- * must use an anonymous connection to a directory. */
+/** Return false if the directory purpose <b>dir_purpose</b>
+ * does not require an anonymous (three-hop) connection.
+ *
+ * Return true 1) by default, 2) if all directory actions have
+ * specifically been configured to be over an anonymous connection,
+ * or 3) if the router is a bridge */
 int
-purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
+purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                        const char *resource)
 {
   if (get_options()->AllDirActionsPrivate)
     return 1;
-  if (router_purpose == ROUTER_PURPOSE_BRIDGE)
+
+  if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+    if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
+        && resource && !strcmp(resource, "authority.z")) {
+      /* We are asking a bridge for its own descriptor. That doesn't need
+         anonymity. */
+      return 0;
+    }
+    /* Assume all other bridge stuff needs anonymity. */
     return 1; /* if no circuits yet, this might break bootstrapping, but it's
                * needed to be safe. */
-  if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
-      dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
-      dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
-      dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
-      dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
-      dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
-      dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
-      dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
-      dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
-    return 0;
-  return 1;
+  }
+
+  switch (dir_purpose)
+  {
+    case DIR_PURPOSE_UPLOAD_DIR:
+    case DIR_PURPOSE_UPLOAD_VOTE:
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return 0;
+    case DIR_PURPOSE_HAS_FETCHED_HSDESC:
+    case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_HSDESC:
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return 1;
+    case DIR_PURPOSE_SERVER:
+    default:
+      log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
+               dir_purpose, router_purpose);
+      tor_assert_nonfatal_unreached();
+      return 1; /* Assume it needs anonymity; better safe than sorry. */
+  }
 }
 
 /** Return a newly allocated string describing <b>auth</b>. Only describes
@@ -193,6 +253,10 @@ dir_conn_purpose_to_string(int purpose)
       return "hidden-service v2 descriptor fetch";
     case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
       return "hidden-service v2 descriptor upload";
+    case DIR_PURPOSE_FETCH_HSDESC:
+      return "hidden-service descriptor fetch";
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      return "hidden-service descriptor upload";
     case DIR_PURPOSE_FETCH_MICRODESC:
       return "microdescriptor fetch";
     }
@@ -347,7 +411,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
         log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
                  (int) extrainfo_len);
       }
-      if (purpose_needs_anonymity(dir_purpose, router_purpose)) {
+      if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) {
         indirection = DIRIND_ANONYMOUS;
       } else if (!fascist_firewall_allows_dir_server(ds,
                                                      FIREWALL_DIR_CONNECTION,
@@ -359,10 +423,14 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
       } else {
         indirection = DIRIND_DIRECT_CONN;
       }
-      directory_initiate_command_routerstatus(rs, dir_purpose,
-                                              router_purpose,
-                                              indirection,
-                                              NULL, payload, upload_len, 0);
+
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_indirection(req, indirection);
+      directory_request_set_payload(req, payload, upload_len);
+      directory_initiate_request(req);
+      directory_request_free(req);
   } SMARTLIST_FOREACH_END(ds);
   if (!found) {
     char *s = authdir_type_to_string(type);
@@ -380,10 +448,9 @@ should_use_directory_guards(const or_options_t *options)
   /* Public (non-bridge) servers never use directory guards. */
   if (public_server_mode(options))
     return 0;
-  /* If guards are disabled, or directory guards are disabled, we can't
-   * use directory guards.
+  /* If guards are disabled, we can't use directory guards.
    */
-  if (!options->UseEntryGuards || !options->UseEntryGuardsAsDirGuards)
+  if (!options->UseEntryGuards)
     return 0;
   /* If we're configured to fetch directory info aggressively or of a
    * nonstandard type, don't use directory guards. */
@@ -398,7 +465,8 @@ should_use_directory_guards(const or_options_t *options)
  * information of type <b>type</b>, and return its routerstatus. */
 static const routerstatus_t *
 directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
-                                 uint8_t dir_purpose)
+                                 uint8_t dir_purpose,
+                                 circuit_guard_state_t **guard_state_out)
 {
   const routerstatus_t *rs = NULL;
   const or_options_t *options = get_options();
@@ -407,7 +475,7 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
     log_warn(LD_BUG, "Called when we have UseBridges set.");
 
   if (should_use_directory_guards(options)) {
-    const node_t *node = choose_random_dirguard(type);
+    const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out);
     if (node)
       rs = node->rs;
   } else {
@@ -423,13 +491,94 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
   return rs;
 }
 
+/**
+ * Set the extra fields in <b>req</b> that are used when requesting a
+ * consensus of type <b>resource</b>.
+ *
+ * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
+ */
+static void
+dir_consensus_request_set_additional_headers(directory_request_t *req,
+                                             const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+  int or_diff_from_is_set = 0;
+
+  /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
+   * period of 1 hour.
+   */
+  const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180;
+  const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72;
+  const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0;
+  const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192;
+  const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] =
+    "try-diff-for-consensus-newer-than";
+
+  int flav = FLAV_NS;
+  if (resource)
+    flav = networkstatus_parse_flavor_name(resource);
+
+  int32_t max_age_for_diff = 3600 *
+    networkstatus_get_param(NULL,
+                            TRY_DIFF_FOR_CONSENSUS_NEWER_NAME,
+                            DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MIN_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MAX_TRY_DIFF_FOR_CONSENSUS_NEWER);
+
+  if (flav != -1) {
+    /* IF we have a parsed consensus of this type, we can do an
+     * if-modified-time based on it. */
+    networkstatus_t *v;
+    v = networkstatus_get_latest_consensus_by_flavor(flav);
+    if (v) {
+      /* In networks with particularly short V3AuthVotingIntervals,
+       * ask for the consensus if it's been modified since half the
+       * V3AuthVotingInterval of the most recent consensus. */
+      time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (v->fresh_until > v->valid_after
+          && ims_delay > (v->fresh_until - v->valid_after)/2) {
+        ims_delay = (v->fresh_until - v->valid_after)/2;
+      }
+      if_modified_since = v->valid_after + ims_delay;
+      if (v->valid_after >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  } else {
+    /* Otherwise it might be a consensus we don't parse, but which we
+     * do cache.  Look at the cached copy, perhaps. */
+    cached_dir_t *cd = dirserv_get_consensus(resource);
+    /* We have no method of determining the voting interval from an
+     * unparsed consensus, so we use the default. */
+    if (cd) {
+      if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (cd->published >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  }
+
+  if (if_modified_since > 0)
+    directory_request_set_if_modified_since(req, if_modified_since);
+  if (or_diff_from_is_set) {
+    char hex[HEX_DIGEST256_LEN + 1];
+    base16_encode(hex, sizeof(hex),
+                  (const char*)or_diff_from, sizeof(or_diff_from));
+    directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
+  }
+}
+
 /** Start a connection to a random running directory server, using
  * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
  * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
  * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
  * or router_pick_trusteddirserver().
  */
-MOCK_IMPL(void, directory_get_from_dirserver, (
+MOCK_IMPL(void,
+directory_get_from_dirserver,(
                             uint8_t dir_purpose,
                             uint8_t router_purpose,
                             const char *resource,
@@ -441,52 +590,17 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
   int prefer_authority = (directory_fetches_from_authorities(options)
                           || want_authority == DL_WANT_AUTHORITY);
   int require_authority = 0;
-  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
+  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
+                                            resource);
   dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
-  time_t if_modified_since = 0;
 
   if (type == NO_DIRINFO)
     return;
 
-  if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    int flav = FLAV_NS;
-    networkstatus_t *v;
-    if (resource)
-      flav = networkstatus_parse_flavor_name(resource);
-
-    /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
-     * period of 1 hour.
-     */
-#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180)
-    if (flav != -1) {
-      /* IF we have a parsed consensus of this type, we can do an
-       * if-modified-time based on it. */
-      v = networkstatus_get_latest_consensus_by_flavor(flav);
-      if (v) {
-        /* In networks with particularly short V3AuthVotingIntervals,
-         * ask for the consensus if it's been modified since half the
-         * V3AuthVotingInterval of the most recent consensus. */
-        time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
-        if (v->fresh_until > v->valid_after
-            && ims_delay > (v->fresh_until - v->valid_after)/2) {
-          ims_delay = (v->fresh_until - v->valid_after)/2;
-        }
-        if_modified_since = v->valid_after + ims_delay;
-      }
-    } else {
-      /* Otherwise it might be a consensus we don't parse, but which we
-       * do cache.  Look at the cached copy, perhaps. */
-      cached_dir_t *cd = dirserv_get_consensus(resource);
-      /* We have no method of determining the voting interval from an
-       * unparsed consensus, so we use the default. */
-      if (cd)
-        if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
-    }
-  }
-
   if (!options->FetchServerDescriptors)
     return;
 
+  circuit_guard_state_t *guard_state = NULL;
   if (!get_via_tor) {
     if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
       /* We want to ask a running bridge for which we have a descriptor.
@@ -495,25 +609,34 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
        * sort of dir fetch we'll be doing, so it won't return a bridge
        * that can't answer our question.
        */
-      const node_t *node = choose_random_dirguard(type);
+      const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state);
       if (node && node->ri) {
         /* every bridge has a routerinfo. */
         routerinfo_t *ri = node->ri;
         /* clients always make OR connections to bridges */
         tor_addr_port_t or_ap;
+        directory_request_t *req = directory_request_new(dir_purpose);
         /* we are willing to use a non-preferred address if we need to */
         fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
                                              &or_ap);
-        directory_initiate_command(&or_ap.addr, or_ap.port,
-                                   NULL, 0, /*no dirport*/
-                                   ri->cache_info.identity_digest,
-                                   dir_purpose,
-                                   router_purpose,
-                                   DIRIND_ONEHOP,
-                                   resource, NULL, 0, if_modified_since);
-      } else
+        directory_request_set_or_addr_port(req, &or_ap);
+        directory_request_set_directory_id_digest(req,
+                                            ri->cache_info.identity_digest);
+        directory_request_set_router_purpose(req, router_purpose);
+        directory_request_set_resource(req, resource);
+        if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+          dir_consensus_request_set_additional_headers(req, resource);
+        directory_request_set_guard_state(req, guard_state);
+        directory_initiate_request(req);
+        directory_request_free(req);
+      } else {
+        if (guard_state) {
+          entry_guard_cancel(&guard_state);
+        }
         log_notice(LD_DIR, "Ignoring directory request, since no bridge "
                            "nodes are available yet.");
+      }
+
       return;
     } else {
       if (prefer_authority || (type & BRIDGE_DIRINFO)) {
@@ -544,9 +667,9 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
         }
       }
       if (!rs && !(type & BRIDGE_DIRINFO)) {
-        /* */
         rs = directory_pick_generic_dirserver(type, pds_flags,
-                                              dir_purpose);
+                                              dir_purpose,
+                                              &guard_state);
         if (!rs)
           get_via_tor = 1; /* last resort: try routing it via Tor */
       }
@@ -565,17 +688,23 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
   if (rs) {
     const dir_indirection_t indirection =
       get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
-    directory_initiate_command_routerstatus(rs, dir_purpose,
-                                            router_purpose,
-                                            indirection,
-                                            resource, NULL, 0,
-                                            if_modified_since);
+    directory_request_t *req = directory_request_new(dir_purpose);
+    directory_request_set_routerstatus(req, rs);
+    directory_request_set_router_purpose(req, router_purpose);
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+      dir_consensus_request_set_additional_headers(req, resource);
+    if (guard_state)
+      directory_request_set_guard_state(req, guard_state);
+    directory_initiate_request(req);
+    directory_request_free(req);
   } else {
     log_notice(LD_DIR,
                "While fetching directory info, "
                "no running dirservers known. Will try again later. "
                "(purpose %d)", dir_purpose);
-    if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
+    if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
       /* remember we tried them all and failed. */
       directory_all_unreachable(time(NULL));
     }
@@ -595,15 +724,17 @@ directory_get_from_all_authorities(uint8_t dir_purpose,
 
   SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
                           dir_server_t *, ds) {
-      routerstatus_t *rs;
       if (router_digest_is_me(ds->digest))
         continue;
       if (!(ds->type & V3_DIRINFO))
         continue;
-      rs = &ds->fake_status;
-      directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
-                                              DIRIND_ONEHOP, resource, NULL,
-                                              0, 0);
+      const routerstatus_t *rs = &ds->fake_status;
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_resource(req, resource);
+      directory_initiate_request(req);
+      directory_request_free(req);
   } SMARTLIST_FOREACH_END(ds);
 }
 
@@ -667,9 +798,9 @@ directory_choose_address_routerstatus(const routerstatus_t *status,
      * Use the preferred address and port if they are reachable, otherwise,
      * use the alternate address and port (if any).
      */
-    have_or = fascist_firewall_choose_address_rs(status,
-                                                 FIREWALL_OR_CONNECTION, 0,
-                                                 use_or_ap);
+    fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0,
+                                       use_or_ap);
+    have_or = tor_addr_port_is_valid_ap(use_or_ap, 0);
   }
 
   /* DirPort connections
@@ -678,9 +809,9 @@ directory_choose_address_routerstatus(const routerstatus_t *status,
       indirection == DIRIND_ANON_DIRPORT ||
       (indirection == DIRIND_ONEHOP
        && !directory_must_use_begindir(options))) {
-    have_dir = fascist_firewall_choose_address_rs(status,
-                                                  FIREWALL_DIR_CONNECTION, 0,
-                                                  use_dir_ap);
+    fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0,
+                                       use_dir_ap);
+    have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0);
   }
 
   /* We rejected all addresses in the relay's status. This means we can't
@@ -703,106 +834,6 @@ directory_choose_address_routerstatus(const routerstatus_t *status,
   return 0;
 }
 
-/** Same as directory_initiate_command_routerstatus(), but accepts
- * rendezvous data to fetch a hidden service descriptor. */
-void
-directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
-                                             uint8_t dir_purpose,
-                                             uint8_t router_purpose,
-                                             dir_indirection_t indirection,
-                                             const char *resource,
-                                             const char *payload,
-                                             size_t payload_len,
-                                             time_t if_modified_since,
-                                             const rend_data_t *rend_query)
-{
-  const or_options_t *options = get_options();
-  const node_t *node;
-  tor_addr_port_t use_or_ap, use_dir_ap;
-  const int anonymized_connection = dirind_is_anon(indirection);
-
-  tor_assert(status != NULL);
-
-  node = node_get_by_id(status->identity_digest);
-
-  /* XXX The below check is wrong: !node means it's not in the consensus,
-   * but we haven't checked if we have a descriptor for it -- and also,
-   * we only care about the descriptor if it's a begindir-style anonymized
-   * connection. */
-  if (!node && anonymized_connection) {
-    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
-             "don't have its router descriptor.",
-             routerstatus_describe(status));
-    return;
-  }
-
-  if (options->ExcludeNodes && options->StrictNodes &&
-      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
-    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
-             "it's in our ExcludedNodes list and StrictNodes is set. "
-             "Skipping. This choice might make your Tor not work.",
-             routerstatus_describe(status),
-             dir_conn_purpose_to_string(dir_purpose));
-    return;
-  }
-
-  /* At this point, if we are a client making a direct connection to a
-   * directory server, we have selected a server that has at least one address
-   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
-   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
-   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
-   * by default.)
-   *
-   * Now choose an address that we can use to connect to the directory server.
-   */
-  if (directory_choose_address_routerstatus(status, indirection, &use_or_ap,
-                                            &use_dir_ap) < 0) {
-    return;
-  }
-
-  /* We don't retry the alternate OR/Dir address for the same directory if
-   * the address we choose fails (#6772).
-   * Instead, we'll retry another directory on failure. */
-
-  directory_initiate_command_rend(&use_or_ap, &use_dir_ap,
-                                  status->identity_digest,
-                                  dir_purpose, router_purpose,
-                                  indirection, resource,
-                                  payload, payload_len, if_modified_since,
-                                  rend_query);
-}
-
-/** Launch a new connection to the directory server <b>status</b> to
- * upload or download a server or rendezvous
- * descriptor. <b>dir_purpose</b> determines what
- * kind of directory connection we're launching, and must be one of
- * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b>
- * specifies the descriptor purposes we have in mind (currently only
- * used for FETCH_DIR).
- *
- * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
- * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
- *
- * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
- * want to fetch.
- */
-MOCK_IMPL(void, directory_initiate_command_routerstatus,
-                (const routerstatus_t *status,
-                 uint8_t dir_purpose,
-                 uint8_t router_purpose,
-                 dir_indirection_t indirection,
-                 const char *resource,
-                 const char *payload,
-                 size_t payload_len,
-                 time_t if_modified_since))
-{
-  directory_initiate_command_routerstatus_rend(status, dir_purpose,
-                                          router_purpose,
-                                          indirection, resource,
-                                          payload, payload_len,
-                                          if_modified_since, NULL);
-}
-
 /** Return true iff <b>conn</b> is the client side of a directory connection
  * we launched to ourself in order to determine the reachability of our
  * dir_port. */
@@ -828,6 +859,11 @@ directory_conn_is_self_reachability_test(dir_connection_t *conn)
 static void
 connection_dir_request_failed(dir_connection_t *conn)
 {
+  if (conn->guard_state) {
+    /* We haven't seen a success on this guard state, so consider it to have
+     * failed. */
+    entry_guard_failed(&conn->guard_state);
+  }
   if (directory_conn_is_self_reachability_test(conn)) {
     return; /* this was a test fetch. don't retry. */
   }
@@ -983,6 +1019,7 @@ directory_must_use_begindir(const or_options_t *options)
 
 /** Evaluate the situation and decide if we should use an encrypted
  * "begindir-style" connection for this directory request.
+ * 0) If there is no DirPort, yes.
  * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
  *    or we're a dir mirror, no.
  * 2) If we prefer to avoid begindir conns, and we're not fetching or
@@ -993,15 +1030,24 @@ directory_must_use_begindir(const or_options_t *options)
  */
 static int
 directory_command_should_use_begindir(const or_options_t *options,
-                                      const tor_addr_t *addr,
-                                      int or_port, uint8_t router_purpose,
-                                      dir_indirection_t indirection,
+                                      const directory_request_t *req,
                                       const char **reason)
 {
-  (void) router_purpose;
+  const tor_addr_t *or_addr = &req->or_addr_port.addr;
+  //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
+  const int or_port = req->or_addr_port.port;
+  const int dir_port = req->dir_addr_port.port;
+
+  const dir_indirection_t indirection = req->indirection;
+
   tor_assert(reason);
   *reason = NULL;
 
+  /* Reasons why we must use begindir */
+  if (!dir_port) {
+    *reason = "(using begindir - directory with no DirPort)";
+    return 1; /* We don't know a DirPort -- must begindir. */
+  }
   /* Reasons why we can't possibly use begindir */
   if (!or_port) {
     *reason = "directory with unknown ORPort";
@@ -1014,7 +1060,7 @@ directory_command_should_use_begindir(const or_options_t *options,
   }
   if (indirection == DIRIND_ONEHOP) {
     /* We're firewalled and want a direct OR connection */
-    if (!fascist_firewall_allows_address_addr(addr, or_port,
+    if (!fascist_firewall_allows_address_addr(or_addr, or_port,
                                               FIREWALL_OR_CONNECTION, 0, 0)) {
       *reason = "ORPort not reachable";
       return 0;
@@ -1033,79 +1079,320 @@ directory_command_should_use_begindir(const or_options_t *options,
   return 1;
 }
 
-/** Helper for directory_initiate_command_rend: send the
- * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>,
- * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose
- * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and
- * <b>router_purpose</b>, making an (in)direct connection as specified in
- * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of
- * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>.
+/**
+ * Create and return a new directory_request_t with purpose
+ * <b>dir_purpose</b>.
+ */
+directory_request_t *
+directory_request_new(uint8_t dir_purpose)
+{
+  tor_assert(dir_purpose >= DIR_PURPOSE_MIN_);
+  tor_assert(dir_purpose <= DIR_PURPOSE_MAX_);
+  tor_assert(dir_purpose != DIR_PURPOSE_SERVER);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC);
+
+  directory_request_t *result = tor_malloc_zero(sizeof(*result));
+  tor_addr_make_null(&result->or_addr_port.addr, AF_INET);
+  result->or_addr_port.port = 0;
+  tor_addr_make_null(&result->dir_addr_port.addr, AF_INET);
+  result->dir_addr_port.port = 0;
+  result->dir_purpose = dir_purpose;
+  result->router_purpose = ROUTER_PURPOSE_GENERAL;
+  result->indirection = DIRIND_ONEHOP;
+  return result;
+}
+/**
+ * Release all resources held by <b>req</b>.
  */
 void
-directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
-                           const tor_addr_t *dir_addr, uint16_t dir_port,
-                           const char *digest,
-                           uint8_t dir_purpose, uint8_t router_purpose,
-                           dir_indirection_t indirection, const char *resource,
-                           const char *payload, size_t payload_len,
-                           time_t if_modified_since)
+directory_request_free_(directory_request_t *req)
 {
-  tor_addr_port_t or_ap, dir_ap;
+  if (req == NULL)
+    return;
+  config_free_lines(req->additional_headers);
+  tor_free(req);
+}
+/**
+ * Set the address and OR port to use for this directory request.  If there is
+ * no OR port, we'll have to connect over the dirport.  (If there are both,
+ * the indirection setting determines which to use.)
+ */
+void
+directory_request_set_or_addr_port(directory_request_t *req,
+                                   const tor_addr_port_t *p)
+{
+  memcpy(&req->or_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the address and dirport to use for this directory request.  If there
+ * is no dirport, we'll have to connect over the OR port.  (If there are both,
+ * the indirection setting determines which to use.)
+ */
+void
+directory_request_set_dir_addr_port(directory_request_t *req,
+                                    const tor_addr_port_t *p)
+{
+  memcpy(&req->dir_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the RSA identity digest of the directory to use for this directory
+ * request.
+ */
+void
+directory_request_set_directory_id_digest(directory_request_t *req,
+                                          const char *digest)
+{
+  memcpy(req->digest, digest, DIGEST_LEN);
+}
+/**
+ * Set the router purpose associated with uploaded and downloaded router
+ * descriptors and extrainfo documents in this directory request.  The purpose
+ * must be one of ROUTER_PURPOSE_GENERAL (the default) or
+ * ROUTER_PURPOSE_BRIDGE.
+ */
+void
+directory_request_set_router_purpose(directory_request_t *req,
+                                     uint8_t router_purpose)
+{
+  tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL ||
+             router_purpose == ROUTER_PURPOSE_BRIDGE);
+  // assert that it actually makes sense to set this purpose, given
+  // the dir_purpose.
+  req->router_purpose = router_purpose;
+}
+/**
+ * Set the indirection to be used for the directory request.  The indirection
+ * parameter configures whether to connect to a DirPort or ORPort, and whether
+ * to anonymize the connection.  DIRIND_ONEHOP (use ORPort, don't anonymize)
+ * is the default.  See dir_indirection_t for more information.
+ */
+void
+directory_request_set_indirection(directory_request_t *req,
+                                  dir_indirection_t indirection)
+{
+  req->indirection = indirection;
+}
 
-  /* Use the null tor_addr and 0 port if the address or port isn't valid. */
-  if (tor_addr_port_is_valid(or_addr, or_port, 0)) {
-    tor_addr_copy(&or_ap.addr, or_addr);
-    or_ap.port = or_port;
-  } else {
-    /* the family doesn't matter here, so make it IPv4 */
-    tor_addr_make_null(&or_ap.addr, AF_INET);
-    or_ap.port = or_port = 0;
-  }
+/**
+ * Set a pointer to the resource to request from a directory.  Different
+ * request types use resources to indicate different components of their URL.
+ * Note that only an alias to <b>resource</b> is stored, so the
+ * <b>resource</b> must outlive the request.
+ */
+void
+directory_request_set_resource(directory_request_t *req,
+                               const char *resource)
+{
+  req->resource = resource;
+}
+/**
+ * Set a pointer to the payload to include with this directory request, along
+ * with its length.  Note that only an alias to <b>payload</b> is stored, so
+ * the <b>payload</b> must outlive the request.
+ */
+void
+directory_request_set_payload(directory_request_t *req,
+                              const char *payload,
+                              size_t payload_len)
+{
+  tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose));
 
-  if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) {
-    tor_addr_copy(&dir_ap.addr, dir_addr);
-    dir_ap.port = dir_port;
-  } else {
-    /* the family doesn't matter here, so make it IPv4 */
-    tor_addr_make_null(&dir_ap.addr, AF_INET);
-    dir_ap.port = dir_port = 0;
+  req->payload = payload;
+  req->payload_len = payload_len;
+}
+/**
+ * Set an if-modified-since date to send along with the request.  The
+ * default is 0 (meaning, send no if-modified-since header).
+ */
+void
+directory_request_set_if_modified_since(directory_request_t *req,
+                                        time_t if_modified_since)
+{
+  req->if_modified_since = if_modified_since;
+}
+
+/** Include a header of name <b>key</b> with content <b>val</b> in the
+ * request. Neither may include newlines or other odd characters. Their
+ * ordering is not currently guaranteed.
+ *
+ * Note that, as elsewhere in this module, header keys include a trailing
+ * colon and space.
+ */
+void
+directory_request_add_header(directory_request_t *req,
+                             const char *key,
+                             const char *val)
+{
+  config_line_prepend(&req->additional_headers, key, val);
+}
+/**
+ * Set an object containing HS data to be associated with this request.  Note
+ * that only an alias to <b>query</b> is stored, so the <b>query</b> object
+ * must outlive the request.
+ */
+void
+directory_request_set_rend_query(directory_request_t *req,
+                                 const rend_data_t *query)
+{
+  if (query) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
+               req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  }
+  req->rend_query = query;
+}
+/**
+ * Set an object containing HS connection identifier to be associated with
+ * this request. Note that only an alias to <b>ident</b> is stored, so the
+ * <b>ident</b> object must outlive the request.
+ */
+void
+directory_request_upload_set_hs_ident(directory_request_t *req,
+                                      const hs_ident_dir_conn_t *ident)
+{
+  if (ident) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC);
   }
+  req->hs_ident = ident;
+}
+/**
+ * Set an object containing HS connection identifier to be associated with
+ * this fetch request. Note that only an alias to <b>ident</b> is stored, so
+ * the <b>ident</b> object must outlive the request.
+ */
+void
+directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                     const hs_ident_dir_conn_t *ident)
+{
+  if (ident) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC);
+  }
+  req->hs_ident = ident;
+}
+/** Set a static circuit_guard_state_t object to affliate with the request in
+ * <b>req</b>.  This object will receive notification when the attempt to
+ * connect to the guard either succeeds or fails. */
+void
+directory_request_set_guard_state(directory_request_t *req,
+                                  circuit_guard_state_t *state)
+{
+  req->guard_state = state;
+}
 
-  directory_initiate_command_rend(&or_ap, &dir_ap,
-                             digest, dir_purpose,
-                             router_purpose, indirection,
-                             resource, payload, payload_len,
-                             if_modified_since, NULL);
+/**
+ * Internal: Return true if any information for contacting the directory in
+ * <b>req</b> has been set, other than by the routerstatus. */
+static int
+directory_request_dir_contact_info_specified(const directory_request_t *req)
+{
+  /* We only check for ports here, since we don't use an addr unless the port
+   * is set */
+  return (req->or_addr_port.port ||
+          req->dir_addr_port.port ||
+          ! tor_digest_is_zero(req->digest));
 }
 
-/** Return non-zero iff a directory connection with purpose
- * <b>dir_purpose</b> reveals sensitive information about a Tor
- * instance's client activities.  (Such connections must be performed
- * through normal three-hop Tor circuits.) */
-int
-is_sensitive_dir_purpose(uint8_t dir_purpose)
+/**
+ * Set the routerstatus to use for the directory associated with this
+ * request.  If this option is set, then no other function to set the
+ * directory's address or identity should be called.
+ */
+void
+directory_request_set_routerstatus(directory_request_t *req,
+                                   const routerstatus_t *status)
 {
-  return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) ||
-          (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) ||
-          (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2));
+  req->routerstatus = status;
 }
+/**
+ * Helper: update the addresses, ports, and identities in <b>req</b>
+ * from the routerstatus object in <b>req</b>.  Return 0 on success.
+ * On failure, warn and return -1.
+ */
+static int
+directory_request_set_dir_from_routerstatus(directory_request_t *req)
 
-/** Same as directory_initiate_command(), but accepts rendezvous data to
- * fetch a hidden service descriptor, and takes its address & port arguments
- * as tor_addr_port_t. */
-static void
-directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
-                                const tor_addr_port_t *dir_addr_port,
-                                const char *digest,
-                                uint8_t dir_purpose, uint8_t router_purpose,
-                                dir_indirection_t indirection,
-                                const char *resource,
-                                const char *payload, size_t payload_len,
-                                time_t if_modified_since,
-                                const rend_data_t *rend_query)
 {
-  tor_assert(or_addr_port);
-  tor_assert(dir_addr_port);
+  const routerstatus_t *status = req->routerstatus;
+  if (BUG(status == NULL))
+    return -1;
+  const or_options_t *options = get_options();
+  const node_t *node;
+  tor_addr_port_t use_or_ap, use_dir_ap;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
+  tor_assert(status != NULL);
+
+  node = node_get_by_id(status->identity_digest);
+
+  /* XXX The below check is wrong: !node means it's not in the consensus,
+   * but we haven't checked if we have a descriptor for it -- and also,
+   * we only care about the descriptor if it's a begindir-style anonymized
+   * connection. */
+  if (!node && anonymized_connection) {
+    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
+             "don't have its router descriptor.",
+             routerstatus_describe(status));
+    return -1;
+  }
+
+  if (options->ExcludeNodes && options->StrictNodes &&
+      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
+    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
+             "it's in our ExcludedNodes list and StrictNodes is set. "
+             "Skipping. This choice might make your Tor not work.",
+             routerstatus_describe(status),
+             dir_conn_purpose_to_string(req->dir_purpose));
+    return -1;
+  }
+
+    /* At this point, if we are a client making a direct connection to a
+   * directory server, we have selected a server that has at least one address
+   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
+   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
+   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
+   * by default.)
+   *
+   * Now choose an address that we can use to connect to the directory server.
+   */
+  if (directory_choose_address_routerstatus(status,
+                                            req->indirection, &use_or_ap,
+                                            &use_dir_ap) < 0) {
+    return -1;
+  }
+
+  directory_request_set_or_addr_port(req, &use_or_ap);
+  directory_request_set_dir_addr_port(req, &use_dir_ap);
+  directory_request_set_directory_id_digest(req, status->identity_digest);
+  return 0;
+}
+
+/**
+ * Launch the provided directory request, configured in <b>request</b>.
+ * After this function is called, you can free <b>request</b>.
+ */
+MOCK_IMPL(void,
+directory_initiate_request,(directory_request_t *request))
+{
+  tor_assert(request);
+  if (request->routerstatus) {
+    tor_assert_nonfatal(
+               ! directory_request_dir_contact_info_specified(request));
+    if (directory_request_set_dir_from_routerstatus(request) < 0) {
+      return;
+    }
+  }
+
+  const tor_addr_port_t *or_addr_port = &request->or_addr_port;
+  const tor_addr_port_t *dir_addr_port = &request->dir_addr_port;
+  const char *digest = request->digest;
+  const uint8_t dir_purpose = request->dir_purpose;
+  const uint8_t router_purpose = request->router_purpose;
+  const dir_indirection_t indirection = request->indirection;
+  const char *resource = request->resource;
+  const rend_data_t *rend_query = request->rend_query;
+  const hs_ident_dir_conn_t *hs_ident = request->hs_ident;
+  circuit_guard_state_t *guard_state = request->guard_state;
+
   tor_assert(or_addr_port->port || dir_addr_port->port);
   tor_assert(digest);
 
@@ -1115,10 +1402,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
   const char *begindir_reason = NULL;
   /* Should the connection be to a relay's OR port (and inside that we will
    * send our directory request)? */
-  const int use_begindir = directory_command_should_use_begindir(options,
-                                     &or_addr_port->addr, or_addr_port->port,
-                                     router_purpose, indirection,
-                                     &begindir_reason);
+  const int use_begindir =
+    directory_command_should_use_begindir(options, request, &begindir_reason);
+
   /* Will the connection go via a three-hop Tor circuit? Note that this
    * is separate from whether it will use_begindir. */
   const int anonymized_connection = dirind_is_anon(indirection);
@@ -1137,7 +1423,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
 
   log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
 
-  if (is_sensitive_dir_purpose(dir_purpose)) {
+  if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
     tor_assert(anonymized_connection ||
                rend_non_anonymous_mode_enabled(options));
   }
@@ -1163,9 +1449,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
   if (!port || tor_addr_is_null(&addr)) {
     static int logged_backtrace = 0;
     log_warn(LD_DIR,
-             "Cannot make an outgoing %sconnection without %sPort.",
+             "Cannot make an outgoing %sconnection without a remote %sPort.",
              use_begindir ? "begindir " : "",
-             use_begindir ? "an OR" : "a Dir");
+             use_begindir ? "OR" : "Dir");
     if (!logged_backtrace) {
       log_backtrace(LOG_INFO, LD_BUG, "Address came from");
       logged_backtrace = 1;
@@ -1192,8 +1478,16 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
   conn->dirconn_direct = !anonymized_connection;
 
   /* copy rendezvous data, if any */
-  if (rend_query)
+  if (rend_query) {
+    /* We can't have both v2 and v3+ identifier. */
+    tor_assert_nonfatal(!hs_ident);
     conn->rend_data = rend_data_dup(rend_query);
+  }
+  if (hs_ident) {
+    /* We can't have both v2 and v3+ identifier. */
+    tor_assert_nonfatal(!rend_query);
+    conn->hs_ident = hs_ident_dir_conn_dup(hs_ident);
+  }
 
   if (!anonymized_connection && !use_begindir) {
     /* then we want to connect to dirport directly */
@@ -1203,6 +1497,11 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
       port = options->HTTPProxyPort;
     }
 
+    // In this case we should not have picked a directory guard.
+    if (BUG(guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
     switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
                                port, &socket_error)) {
       case -1:
@@ -1214,9 +1513,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
         /* fall through */
       case 0:
         /* queue the command on the outbuf */
-        directory_send_command(conn, dir_purpose, 1, resource,
-                               payload, payload_len,
-                               if_modified_since);
+        directory_send_command(conn, 1, request);
         connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
         /* writable indicates finish, readable indicates broken link,
            error indicates broken link in windowsland. */
@@ -1239,6 +1536,14 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
     else if (anonymized_connection && !use_begindir)
       rep_hist_note_used_port(time(NULL), conn->base_.port);
 
+    // In this case we should not have a directory guard; we'll
+    // get a regular guard later when we build the circuit.
+    if (BUG(anonymized_connection && guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
+    conn->guard_state = guard_state;
+
     /* make an AP connection
      * populate it and add it at the right state
      * hook up both sides
@@ -1262,9 +1567,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
     }
     conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
     /* queue the command on the outbuf */
-    directory_send_command(conn, dir_purpose, 0, resource,
-                           payload, payload_len,
-                           if_modified_since);
+    directory_send_command(conn, 0, request);
 
     connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
     connection_start_reading(ENTRY_TO_CONN(linked_conn));
@@ -1274,7 +1577,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
 /** Return true iff anything we say on <b>conn</b> is being encrypted before
  * we send it to the client/server. */
 int
-connection_dir_is_encrypted(dir_connection_t *conn)
+connection_dir_is_encrypted(const dir_connection_t *conn)
 {
   /* Right now it's sufficient to see if conn is or has been linked, since
    * the only thing it could be linked to is an edge connection on a
@@ -1367,15 +1670,23 @@ copy_ipv6_address(char* destination, const char* source, size_t len,
   }
 }
 
-/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
- * are as in directory_initiate_command().
+/** Queue an appropriate HTTP command for <b>request</b> on
+ * <b>conn</b>-\>outbuf.  If <b>direct</b> is true, we're making a
+ * non-anonymized connection to the dirport.
  */
 static void
 directory_send_command(dir_connection_t *conn,
-                       int purpose, int direct, const char *resource,
-                       const char *payload, size_t payload_len,
-                       time_t if_modified_since)
+                       const int direct,
+                       const directory_request_t *req)
 {
+  tor_assert(req);
+  const int purpose = req->dir_purpose;
+  const char *resource = req->resource;
+  const char *payload = req->payload;
+  const size_t payload_len = req->payload_len;
+  const time_t if_modified_since = req->if_modified_since;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
   char proxystring[256];
   char hoststring[128];
   /* NEEDS to be the same size hoststring.
@@ -1383,7 +1694,10 @@ directory_send_command(dir_connection_t *conn,
   char decorated_address[128];
   smartlist_t *headers = smartlist_new();
   char *url;
+  char *accept_encoding;
+  size_t url_len;
   char request[8192];
+  size_t request_len, total_request_len = 0;
   const char *httpcommand = NULL;
 
   tor_assert(conn);
@@ -1437,6 +1751,22 @@ directory_send_command(dir_connection_t *conn,
     proxystring[0] = 0;
   }
 
+  if (! anonymized_connection) {
+    /* Add Accept-Encoding. */
+    accept_encoding = accept_encoding_header();
+    smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n",
+                           accept_encoding);
+    tor_free(accept_encoding);
+  }
+
+  /* Add additional headers, if any */
+  {
+    config_line_t *h;
+    for (h = req->additional_headers; h; h = h->next) {
+      smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value);
+    }
+  }
+
   switch (purpose) {
     case DIR_PURPOSE_FETCH_CONSENSUS:
       /* resource is optional.  If present, it's a flavor name */
@@ -1485,9 +1815,10 @@ directory_send_command(dir_connection_t *conn,
       tor_assert(payload);
       httpcommand = "POST";
       url = tor_strdup("/tor/");
-      if (why) {
-        smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
+      if (!why) {
+        why = "for no reason at all";
       }
+      smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
       break;
     }
     case DIR_PURPOSE_UPLOAD_VOTE:
@@ -1509,12 +1840,25 @@ directory_send_command(dir_connection_t *conn,
       httpcommand = "GET";
       tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
       break;
+    case DIR_PURPOSE_FETCH_HSDESC:
+      tor_assert(resource);
+      tor_assert(strlen(resource) <= ED25519_BASE64_LEN);
+      tor_assert(!payload);
+      httpcommand = "GET";
+      tor_asprintf(&url, "/tor/hs/3/%s", resource);
+      break;
     case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
       tor_assert(!resource);
       tor_assert(payload);
       httpcommand = "POST";
       url = tor_strdup("/tor/rendezvous2/publish");
       break;
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      tor_assert(resource);
+      tor_assert(payload);
+      httpcommand = "POST";
+      tor_asprintf(&url, "/tor/hs/%s/publish", resource);
+      break;
     default:
       tor_assert(0);
       return;
@@ -1529,8 +1873,14 @@ directory_send_command(dir_connection_t *conn,
   }
 
   tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
-  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
-  connection_write_to_buf(url, strlen(url), TO_CONN(conn));
+
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_buf_add(request, request_len, TO_CONN(conn));
+
+  url_len = strlen(url);
+  total_request_len += url_len;
+  connection_buf_add(url, url_len, TO_CONN(conn));
   tor_free(url);
 
   if (!strcmp(httpcommand, "POST") || payload) {
@@ -1545,15 +1895,27 @@ directory_send_command(dir_connection_t *conn,
     tor_free(header);
   }
 
-  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_buf_add(request, request_len, TO_CONN(conn));
 
   if (payload) {
     /* then send the payload afterwards too */
-    connection_write_to_buf(payload, payload_len, TO_CONN(conn));
+    connection_buf_add(payload, payload_len, TO_CONN(conn));
+    total_request_len += payload_len;
   }
 
   SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
   smartlist_free(headers);
+
+  log_debug(LD_DIR,
+            "Sent request to directory server '%s:%d': "
+            "(purpose: %d, request size: " U64_FORMAT ", "
+            "payload size: " U64_FORMAT ")",
+            conn->base_.address, conn->base_.port,
+            conn->base_.purpose,
+            U64_PRINTF_ARG(total_request_len),
+            U64_PRINTF_ARG(payload ? payload_len : 0));
 }
 
 /** Parse an HTTP request string <b>headers</b> of the form
@@ -1568,15 +1930,41 @@ directory_send_command(dir_connection_t *conn,
 STATIC int
 parse_http_url(const char *headers, char **url)
 {
+  char *command = NULL;
+  if (parse_http_command(headers, &command, url) < 0) {
+    return -1;
+  }
+  if (strcmpstart(*url, "/tor/")) {
+    char *new_url = NULL;
+    tor_asprintf(&new_url, "/tor%s%s",
+                 *url[0] == '/' ? "" : "/",
+                 *url);
+    tor_free(*url);
+    *url = new_url;
+  }
+  tor_free(command);
+  return 0;
+}
+
+/** Parse an HTTP request line at the start of a headers string.  On failure,
+ * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
+ * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
+ * return 0. */
+int
+parse_http_command(const char *headers, char **command_out, char **url_out)
+{
+  const char *command, *end_of_command;
   char *s, *start, *tmp;
 
   s = (char *)eat_whitespace_no_nl(headers);
   if (!*s) return -1;
+  command = s;
   s = (char *)find_whitespace(s); /* get past GET/POST */
   if (!*s) return -1;
+  end_of_command = s;
   s = (char *)eat_whitespace_no_nl(s);
   if (!*s) return -1;
-  start = s; /* this is it, assuming it's valid */
+  start = s; /* this is the URL, assuming it's valid */
   s = (char *)find_whitespace(start);
   if (!*s) return -1;
 
@@ -1607,13 +1995,8 @@ parse_http_url(const char *headers, char **url)
       return -1;
   }
 
-  if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */
-    *url = tor_malloc(s - start + 5);
-    strlcpy(*url,"/tor", s-start+5);
-    strlcat((*url)+4, start, s-start+1);
-  } else {
-    *url = tor_strndup(start, s-start);
-  }
+  *url_out = tor_memdup_nulterm(start, s-start);
+  *command_out = tor_memdup_nulterm(command, end_of_command - command);
   return 0;
 }
 
@@ -1621,7 +2004,7 @@ parse_http_url(const char *headers, char **url)
  * <b>which</b>.  The key should be given with a terminating colon and space;
  * this function copies everything after, up to but not including the
  * following \\r\\n. */
-static char *
+char *
 http_get_header(const char *headers, const char *which)
 {
   const char *cp = headers;
@@ -1738,16 +2121,15 @@ parse_http_response(const char *headers, int *code, time_t *date,
       if (!strcmpstart(s, "Content-Encoding: ")) {
         enc = s+18; break;
       });
-    if (!enc || !strcmp(enc, "identity")) {
+
+    if (enc == NULL)
       *compression = NO_METHOD;
-    } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
-      *compression = ZLIB_METHOD;
-    } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
-      *compression = GZIP_METHOD;
-    } else {
-      log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
-               escaped(enc));
-      *compression = UNKNOWN_METHOD;
+    else {
+      *compression = compression_method_get_by_name(enc);
+
+      if (*compression == UNKNOWN_METHOD)
+        log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
+                 escaped(enc));
     }
   }
   SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
@@ -1770,15 +2152,15 @@ body_is_plausible(const char *body, size_t len, int purpose)
   if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
     return (!strcmpstart(body,"onion-key"));
   }
-  if (1) {
-    if (!strcmpstart(body,"router") ||
-        !strcmpstart(body,"network-status"))
-      return 1;
-    for (i=0;i<32;++i) {
-      if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
-        return 0;
-    }
+
+  if (!strcmpstart(body,"router") ||
+      !strcmpstart(body,"network-status"))
+    return 1;
+  for (i=0;i<32;++i) {
+    if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
+      return 0;
   }
+
   return 1;
 }
 
@@ -1820,6 +2202,144 @@ load_downloaded_routers(const char *body, smartlist_t *which,
   return added;
 }
 
+static int handle_response_fetch_certificate(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_status_vote(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_detached_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_desc(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_dir(dir_connection_t *,
+                                      const response_handler_args_t *);
+static int handle_response_upload_vote(dir_connection_t *,
+                                       const response_handler_args_t *);
+static int handle_response_upload_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_renddesc_v2(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_renddesc_v2(dir_connection_t *,
+                                              const response_handler_args_t *);
+static int handle_response_upload_hsdesc(dir_connection_t *,
+                                         const response_handler_args_t *);
+
+static int
+dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
+                                    dir_connection_t *conn,
+                                    compress_method_t compression,
+                                    int anonymized_connection)
+{
+  int rv = 0;
+  const char *body = *bodyp;
+  size_t body_len = *bodylenp;
+  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  int plausible = body_is_plausible(body, body_len, conn->base_.purpose);
+
+  if (plausible && compression == NO_METHOD) {
+    return 0;
+  }
+
+  int severity = LOG_DEBUG;
+  char *new_body = NULL;
+  size_t new_len = 0;
+  const char *description1, *description2;
+  int want_to_try_both = 0;
+  int tried_both = 0;
+  compress_method_t guessed = detect_compression_method(body, body_len);
+
+  description1 = compression_method_get_human_name(compression);
+
+  if (BUG(description1 == NULL))
+    description1 = compression_method_get_human_name(UNKNOWN_METHOD);
+
+  if (guessed == UNKNOWN_METHOD && !plausible)
+    description2 = "confusing binary junk";
+  else
+    description2 = compression_method_get_human_name(guessed);
+
+  /* Tell the user if we don't believe what we're told about compression.*/
+  want_to_try_both = (compression == UNKNOWN_METHOD ||
+                      guessed != compression);
+  if (want_to_try_both) {
+    severity = LOG_PROTOCOL_WARN;
+  }
+
+  tor_log(severity, LD_HTTP,
+          "HTTP body from server '%s:%d' was labeled as %s, "
+          "%s it seems to be %s.%s",
+          conn->base_.address, conn->base_.port, description1,
+          guessed != compression?"but":"and",
+          description2,
+          (compression>0 && guessed>0 && want_to_try_both)?
+          "  Trying both.":"");
+
+  /* Try declared compression first if we can.
+   * tor_compress_supports_method() also returns true for NO_METHOD.
+   * Ensure that the server is not sending us data compressed using a
+   * compression method that is not allowed for anonymous connections. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(compression)) {
+    warn_disallowed_anonymous_compression_method(compression);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(compression)) {
+    tor_uncompress(&new_body, &new_len, body, body_len, compression,
+                   !allow_partial, LOG_PROTOCOL_WARN);
+    if (new_body) {
+      /* We succeeded with the declared compression method. Great! */
+      rv = 0;
+      goto done;
+    }
+  }
+
+  /* Okay, if that didn't work, and we think that it was compressed
+   * differently, try that. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(guessed)) {
+    warn_disallowed_anonymous_compression_method(guessed);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(guessed) &&
+      compression != guessed) {
+    tor_uncompress(&new_body, &new_len, body, body_len, guessed,
+                   !allow_partial, LOG_INFO);
+    tried_both = 1;
+  }
+  /* If we're pretty sure that we have a compressed directory, and
+   * we didn't manage to uncompress it, then warn and bail. */
+  if (!plausible && !new_body) {
+    log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+           "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
+           description1,
+           tried_both?" and ":"",
+           tried_both?description2:"",
+           conn->base_.address, conn->base_.port);
+    rv = -1;
+    goto done;
+  }
+
+ done:
+  if (new_body) {
+    if (rv == 0) {
+      /* success! */
+      tor_free(*bodyp);
+      *bodyp = new_body;
+      *bodylenp = new_len;
+    } else {
+      tor_free(new_body);
+    }
+  }
+
+  return rv;
+}
+
 /** We are a client, and we've finished reading the server's
  * response. Parse it and act appropriately.
  *
@@ -1832,21 +2352,26 @@ load_downloaded_routers(const char *body, smartlist_t *which,
 static int
 connection_dir_client_reached_eof(dir_connection_t *conn)
 {
-  char *body;
-  char *headers;
+  char *body = NULL;
+  char *headers = NULL;
   char *reason = NULL;
   size_t body_len = 0;
   int status_code;
   time_t date_header = 0;
   long apparent_skew;
   compress_method_t compression;
-  int plausible;
   int skewed = 0;
+  int rv;
   int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
                        conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
                        conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
-  time_t now = time(NULL);
-  int src_code;
+  size_t received_bytes;
+  const int anonymized_connection =
+    purpose_needs_anonymity(conn->base_.purpose,
+                            conn->router_purpose,
+                            conn->requested_resource);
+
+  received_bytes = connection_get_inbuf_len(TO_CONN(conn));
 
   switch (connection_fetch_from_buf_http(TO_CONN(conn),
                               &headers, MAX_HEADERS_SIZE,
@@ -1868,17 +2393,39 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
                           &compression, &reason) < 0) {
     log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
              conn->base_.address, conn->base_.port);
-    tor_free(body); tor_free(headers);
-    return -1;
+
+    rv = -1;
+    goto done;
   }
   if (!reason) reason = tor_strdup("[no reason given]");
 
-  log_debug(LD_DIR,
+  tor_log(LOG_DEBUG, LD_DIR,
             "Received response from directory server '%s:%d': %d %s "
-            "(purpose: %d)",
+            "(purpose: %d, response size: " U64_FORMAT
+#ifdef MEASUREMENTS_21206
+            ", data cells received: %d, data cells sent: %d"
+#endif
+            ", compression: %d)",
             conn->base_.address, conn->base_.port, status_code,
-            escaped(reason),
-            conn->base_.purpose);
+            escaped(reason), conn->base_.purpose,
+            U64_PRINTF_ARG(received_bytes),
+#ifdef MEASUREMENTS_21206
+            conn->data_cells_received, conn->data_cells_sent,
+#endif
+            compression);
+
+  if (conn->guard_state) {
+    /* we count the connection as successful once we can read from it.  We do
+     * not, however, delay use of the circuit here, since it's just for a
+     * one-hop directory request. */
+    /* XXXXprop271 note that this will not do the right thing for other
+     * waiting circuits that would be triggered by this circuit becoming
+     * complete/usable. But that's ok, I think.
+     */
+    entry_guard_succeeded(&conn->guard_state);
+    circuit_guard_state_free(conn->guard_state);
+    conn->guard_state = NULL;
+  }
 
   /* now check if it's got any hints for us about our IP address. */
   if (conn->dirconn_direct) {
@@ -1895,7 +2442,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
      * and the date header.  (We used to check now-date_header, but that's
      * inaccurate if we spend a lot of time downloading.)
      */
-    apparent_skew = conn->base_.timestamp_lastwritten - date_header;
+    apparent_skew = conn->base_.timestamp_last_write_allowed - date_header;
     if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) {
       int trusted = router_digest_is_trusted_dir(conn->identity_digest);
       clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP,
@@ -1916,540 +2463,886 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
              "'%s:%d'. I'll try again soon.",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
+    time_t now = approx_time();
     if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
       rs->last_dir_503_at = now;
     if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
       ds->fake_status.last_dir_503_at = now;
 
-    tor_free(body); tor_free(headers); tor_free(reason);
-    return -1;
+    rv = -1;
+    goto done;
   }
 
-  plausible = body_is_plausible(body, body_len, conn->base_.purpose);
-  if (compression != NO_METHOD || !plausible) {
-    char *new_body = NULL;
-    size_t new_len = 0;
-    compress_method_t guessed = detect_compression_method(body, body_len);
-    if (compression == UNKNOWN_METHOD || guessed != compression) {
-      /* Tell the user if we don't believe what we're told about compression.*/
-      const char *description1, *description2;
-      if (compression == ZLIB_METHOD)
-        description1 = "as deflated";
-      else if (compression == GZIP_METHOD)
-        description1 = "as gzipped";
-      else if (compression == NO_METHOD)
-        description1 = "as uncompressed";
-      else
-        description1 = "with an unknown Content-Encoding";
-      if (guessed == ZLIB_METHOD)
-        description2 = "deflated";
-      else if (guessed == GZIP_METHOD)
-        description2 = "gzipped";
-      else if (!plausible)
-        description2 = "confusing binary junk";
-      else
-        description2 = "uncompressed";
+  if (dir_client_decompress_response_body(&body, &body_len,
+                             conn, compression, anonymized_connection) < 0) {
+    rv = -1;
+    goto done;
+  }
 
-      log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, "
-               "but it seems to be %s.%s",
-               conn->base_.address, conn->base_.port, description1,
-               description2,
-               (compression>0 && guessed>0)?"  Trying both.":"");
-    }
-    /* Try declared compression first if we can. */
-    if (compression == GZIP_METHOD  || compression == ZLIB_METHOD)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression,
-                          !allow_partial, LOG_PROTOCOL_WARN);
-    /* Okay, if that didn't work, and we think that it was compressed
-     * differently, try that. */
-    if (!new_body &&
-        (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) &&
-        compression != guessed)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed,
-                          !allow_partial, LOG_PROTOCOL_WARN);
-    /* If we're pretty sure that we have a compressed directory, and
-     * we didn't manage to uncompress it, then warn and bail. */
-    if (!plausible && !new_body) {
-      log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
-             "Unable to decompress HTTP body (server '%s:%d').",
-             conn->base_.address, conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    if (new_body) {
-      tor_free(body);
-      body = new_body;
-      body_len = new_len;
-    }
+  response_handler_args_t args;
+  memset(&args, 0, sizeof(args));
+  args.status_code = status_code;
+  args.reason = reason;
+  args.body = body;
+  args.body_len = body_len;
+  args.headers = headers;
+
+  switch (conn->base_.purpose) {
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      rv = handle_response_fetch_consensus(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      rv = handle_response_fetch_certificate(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      rv = handle_response_fetch_status_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      rv = handle_response_fetch_detached_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      rv = handle_response_fetch_desc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      rv = handle_response_fetch_microdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      rv = handle_response_fetch_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_DIR:
+      rv = handle_response_upload_dir(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      rv = handle_response_upload_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      rv = handle_response_upload_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      rv = handle_response_upload_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_HSDESC:
+      rv = handle_response_upload_hsdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_HSDESC:
+      rv = handle_response_fetch_hsdesc_v3(conn, &args);
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      rv = -1;
+      break;
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    int r;
-    const char *flavname = conn->requested_resource;
-    if (status_code != 200) {
-      int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
-      tor_log(severity, LD_DIR,
-          "Received http status code %d (%s) from server "
-          "'%s:%d' while fetching consensus directory.",
-           status_code, escaped(reason), conn->base_.address,
-           conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(status_code, flavname);
-      return -1;
+ done:
+  tor_free(body);
+  tor_free(headers);
+  tor_free(reason);
+  return rv;
+}
+
+/**
+ * Handler function: processes a response to a request for a networkstatus
+ * consensus document by checking the consensus, storing it, and marking
+ * router requests as reachable.
+ **/
+STATIC int
+handle_response_fetch_consensus(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS);
+  const int status_code = args->status_code;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+  const char *reason = args->reason;
+  const time_t now = approx_time();
+
+  const char *consensus;
+  char *new_consensus = NULL;
+  const char *sourcename;
+
+  int r;
+  const char *flavname = conn->requested_resource;
+  if (status_code != 200) {
+    int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
+    tor_log(severity, LD_DIR,
+            "Received http status code %d (%s) from server "
+            "'%s:%d' while fetching consensus directory.",
+            status_code, escaped(reason), conn->base_.address,
+            conn->base_.port);
+    networkstatus_consensus_download_failed(status_code, flavname);
+    return -1;
+  }
+
+  if (looks_like_a_consensus_diff(body, body_len)) {
+    /* First find our previous consensus. Maybe it's in ram, maybe not. */
+    cached_dir_t *cd = dirserv_get_consensus(flavname);
+    const char *consensus_body;
+    char *owned_consensus = NULL;
+    if (cd) {
+      consensus_body = cd->dir;
+    } else {
+      owned_consensus = networkstatus_read_cached_consensus(flavname);
+      consensus_body = owned_consensus;
     }
-    log_info(LD_DIR,"Received consensus directory (size %d) from server "
-             "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
-    if ((r=networkstatus_set_current_consensus(body, flavname, 0,
-                                               conn->identity_digest))<0) {
-      log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
-             "Unable to load %s consensus directory downloaded from "
-             "server '%s:%d'. I'll try again soon.",
-             flavname, conn->base_.address, conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
+    if (!consensus_body) {
+      log_warn(LD_DIR, "Received a consensus diff, but we can't find "
+               "any %s-flavored consensus in our current cache.",flavname);
       networkstatus_consensus_download_failed(0, flavname);
+      // XXXX if this happens too much, see below
       return -1;
     }
 
-    /* If we launched other fetches for this consensus, cancel them. */
-    connection_dir_close_consensus_fetches(conn, flavname);
-
-    /* launches router downloads as needed */
-    routers_update_all_from_networkstatus(now, 3);
-    update_microdescs_from_networkstatus(now);
-    update_microdesc_downloads(now);
-    directory_info_has_arrived(now, 0, 0);
-    if (authdir_mode_v3(get_options())) {
-      sr_act_post_consensus(
-                   networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
-    }
-    log_info(LD_DIR, "Successfully loaded consensus.");
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
-    if (status_code != 200) {
-      log_warn(LD_DIR,
-          "Received http status code %d (%s) from server "
-          "'%s:%d' while fetching \"/tor/keys/%s\".",
-           status_code, escaped(reason), conn->base_.address,
-           conn->base_.port, conn->requested_resource);
-      connection_dir_download_cert_failed(conn, status_code);
-      tor_free(body); tor_free(headers); tor_free(reason);
+    new_consensus = consensus_diff_apply(consensus_body, body);
+    tor_free(owned_consensus);
+    if (new_consensus == NULL) {
+      log_warn(LD_DIR, "Could not apply consensus diff received from server "
+               "'%s:%d'", conn->base_.address, conn->base_.port);
+      // XXXX If this happens too many times, we should maybe not use
+      // XXXX this directory for diffs any more?
+      networkstatus_consensus_download_failed(0, flavname);
       return -1;
     }
-    log_info(LD_DIR,"Received authority certificates (size %d) from server "
+    log_info(LD_DIR, "Applied consensus diff (size %d) from server "
+             "'%s:%d', resulting in a new consensus document (size %d).",
+             (int)body_len, conn->base_.address, conn->base_.port,
+             (int)strlen(new_consensus));
+    consensus = new_consensus;
+    sourcename = "generated based on a diff";
+  } else {
+    log_info(LD_DIR,"Received consensus directory (body size %d) from server "
              "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
+    consensus = body;
+    sourcename = "downloaded";
+  }
 
-    /*
-     * Tell trusted_dirs_load_certs_from_string() whether it was by fp
-     * or fp-sk pair.
-     */
-    src_code = -1;
-    if (!strcmpstart(conn->requested_resource, "fp/")) {
-      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
-    } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
-      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
-    }
+  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+                                             conn->identity_digest))<0) {
+    log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
+           "Unable to load %s consensus directory %s from "
+           "server '%s:%d'. I'll try again soon.",
+           flavname, sourcename, conn->base_.address, conn->base_.port);
+    networkstatus_consensus_download_failed(0, flavname);
+    tor_free(new_consensus);
+    return -1;
+  }
 
-    if (src_code != -1) {
-      if (trusted_dirs_load_certs_from_string(body, src_code, 1,
-                                              conn->identity_digest)<0) {
-        log_warn(LD_DIR, "Unable to parse fetched certificates");
-        /* if we fetched more than one and only some failed, the successful
-         * ones got flushed to disk so it's safe to call this on them */
-        connection_dir_download_cert_failed(conn, status_code);
-      } else {
-        directory_info_has_arrived(now, 0, 0);
-        log_info(LD_DIR, "Successfully loaded certificates from fetch.");
-      }
-    } else {
-      log_warn(LD_DIR,
-               "Couldn't figure out what to do with fetched certificates for "
-               "unknown resource %s",
-               conn->requested_resource);
+  /* If we launched other fetches for this consensus, cancel them. */
+  connection_dir_close_consensus_fetches(conn, flavname);
+
+  /* update the list of routers and directory guards */
+  routers_update_all_from_networkstatus(now, 3);
+  update_microdescs_from_networkstatus(now);
+  directory_info_has_arrived(now, 0, 0);
+
+  if (authdir_mode_v3(get_options())) {
+    sr_act_post_consensus(
+                     networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
+  }
+  log_info(LD_DIR, "Successfully loaded consensus.");
+
+  tor_free(new_consensus);
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for one or more
+ * authority certificates
+ **/
+static int
+handle_response_fetch_certificate(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+             "Received http status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/keys/%s\".",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port, conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
+    return -1;
+  }
+  log_info(LD_DIR,"Received authority certificates (body size %d) from "
+           "server '%s:%d'",
+           (int)body_len, conn->base_.address, conn->base_.port);
+
+  /*
+   * Tell trusted_dirs_load_certs_from_string() whether it was by fp
+   * or fp-sk pair.
+   */
+  int src_code = -1;
+  if (!strcmpstart(conn->requested_resource, "fp/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
+  } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
+  }
+
+  if (src_code != -1) {
+    if (trusted_dirs_load_certs_from_string(body, src_code, 1,
+                                            conn->identity_digest)<0) {
+      log_warn(LD_DIR, "Unable to parse fetched certificates");
+      /* if we fetched more than one and only some failed, the successful
+       * ones got flushed to disk so it's safe to call this on them */
       connection_dir_download_cert_failed(conn, status_code);
+    } else {
+      time_t now = approx_time();
+      directory_info_has_arrived(now, 0, 0);
+      log_info(LD_DIR, "Successfully loaded certificates from fetch.");
     }
+  } else {
+    log_warn(LD_DIR,
+             "Couldn't figure out what to do with fetched certificates for "
+             "unknown resource %s",
+             conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
   }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
-    const char *msg;
-    int st;
-    log_info(LD_DIR,"Got votes (size %d) from server %s:%d",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (status_code != 200) {
-      log_warn(LD_DIR,
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for an authority's
+ * current networkstatus vote.
+ **/
+static int
+handle_response_fetch_status_vote(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg;
+  int st;
+  log_info(LD_DIR,"Got votes (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
              "Received http status code %d (%s) from server "
              "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port, conn->requested_resource);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    dirvote_add_vote(body, &msg, &st);
-    if (st > 299) {
-      log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
-    } else {
-      log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
-    }
+    return -1;
   }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
-    const char *msg = NULL;
-    log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (status_code != 200) {
-      log_warn(LD_DIR,
+  dirvote_add_vote(body, &msg, &st);
+  if (st > 299) {
+    log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
+  } else {
+    log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for the signatures
+ * that an authority knows about on a given consensus.
+ **/
+static int
+handle_response_fetch_detached_signatures(dir_connection_t *conn,
+                                          const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg = NULL;
+  log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
         "Received http status code %d (%s) from server '%s:%d' while fetching "
         "\"/tor/status-vote/next/consensus-signatures.z\".",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
-      log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
-               conn->base_.address, conn->base_.port, msg?msg:"???");
-    }
+        status_code, escaped(reason), conn->base_.address,
+        conn->base_.port);
+    return -1;
+  }
+  if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
+    log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
+             conn->base_.address, conn->base_.port, msg?msg:"???");
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
-    int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
-    smartlist_t *which = NULL;
-    int n_asked_for = 0;
-    int descriptor_digests = conn->requested_resource &&
-                             !strcmpstart(conn->requested_resource,"d/");
-    log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'",
-             was_ei ? "extra server info" : "server info",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (conn->requested_resource &&
-        (!strcmpstart(conn->requested_resource,"d/") ||
-         !strcmpstart(conn->requested_resource,"fp/"))) {
-      which = smartlist_new();
-      dir_split_resource_into_fingerprints(conn->requested_resource +
-                                             (descriptor_digests ? 2 : 3),
-                                           which, NULL, 0);
-      n_asked_for = smartlist_len(which);
-    }
-    if (status_code != 200) {
-      int dir_okay = status_code == 404 ||
-        (status_code == 400 && !strcmp(reason, "Servers unavailable."));
-      /* 404 means that it didn't have them; no big deal.
-       * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
-      log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
-             "Received http status code %d (%s) from server '%s:%d' "
-             "while fetching \"/tor/server/%s\". I'll try again soon.",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port, conn->requested_resource);
-      if (!which) {
-        connection_dir_download_routerdesc_failed(conn);
-      } else {
-        dir_routerdesc_download_failed(which, status_code,
-                                       conn->router_purpose,
-                                       was_ei, descriptor_digests);
-        SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-        smartlist_free(which);
-      }
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return dir_okay ? 0 : -1;
-    }
-    /* Learn the routers, assuming we requested by fingerprint or "all"
-     * or "authority".
-     *
-     * We use "authority" to fetch our own descriptor for
-     * testing, and to fetch bridge descriptors for bootstrapping. Ignore
-     * the output of "authority" requests unless we are using bridges,
-     * since otherwise they'll be the response from reachability tests,
-     * and we don't really want to add that to our routerlist. */
-    if (which || (conn->requested_resource &&
-                  (!strcmpstart(conn->requested_resource, "all") ||
-                   (!strcmpstart(conn->requested_resource, "authority") &&
-                    get_options()->UseBridges)))) {
-      /* as we learn from them, we remove them from 'which' */
-      if (was_ei) {
-        router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
-                                          descriptor_digests);
-      } else {
-        //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
-        //                       descriptor_digests, conn->router_purpose);
-        if (load_downloaded_routers(body, which, descriptor_digests,
-                                conn->router_purpose,
-                                conn->base_.address))
-          directory_info_has_arrived(now, 0, 0);
-      }
-    }
-    if (which) { /* mark remaining ones as failed */
-      log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
-               n_asked_for-smartlist_len(which), n_asked_for,
-               was_ei ? "extra-info documents" : "router descriptors",
-               conn->base_.address, (int)conn->base_.port);
-      if (smartlist_len(which)) {
-        dir_routerdesc_download_failed(which, status_code,
-                                       conn->router_purpose,
-                                       was_ei, descriptor_digests);
-      }
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-    }
-    if (directory_conn_is_self_reachability_test(conn))
-      router_dirport_found_reachable();
-  }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
-    smartlist_t *which = NULL;
-    log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
-             "size %d) from server '%s:%d'",
-             status_code, (int)body_len, conn->base_.address,
-             conn->base_.port);
-    tor_assert(conn->requested_resource &&
-               !strcmpstart(conn->requested_resource, "d/"));
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of server
+ * descriptors or an extrainfo documents.
+ **/
+static int
+handle_response_fetch_desc(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
+  smartlist_t *which = NULL;
+  int n_asked_for = 0;
+  int descriptor_digests = conn->requested_resource &&
+    !strcmpstart(conn->requested_resource,"d/");
+  log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'",
+           was_ei ? "extra server info" : "server info",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (conn->requested_resource &&
+      (!strcmpstart(conn->requested_resource,"d/") ||
+       !strcmpstart(conn->requested_resource,"fp/"))) {
     which = smartlist_new();
-    dir_split_resource_into_fingerprints(conn->requested_resource+2,
-                                         which, NULL,
-                                         DSR_DIGEST256|DSR_BASE64);
-    if (status_code != 200) {
-      log_info(LD_DIR, "Received status code %d (%s) from server "
-               "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
-               "soon.",
-               status_code, escaped(reason), conn->base_.address,
-               (int)conn->base_.port, conn->requested_resource);
-      dir_microdesc_download_failed(which, status_code);
+    dir_split_resource_into_fingerprints(conn->requested_resource +
+                                         (descriptor_digests ? 2 : 3),
+                                         which, NULL, 0);
+    n_asked_for = smartlist_len(which);
+  }
+  if (status_code != 200) {
+    int dir_okay = status_code == 404 ||
+      (status_code == 400 && !strcmp(reason, "Servers unavailable."));
+    /* 404 means that it didn't have them; no big deal.
+     * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
+    log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
+           "Received http status code %d (%s) from server '%s:%d' "
+           "while fetching \"/tor/server/%s\". I'll try again soon.",
+           status_code, escaped(reason), conn->base_.address,
+           conn->base_.port, conn->requested_resource);
+    if (!which) {
+      connection_dir_download_routerdesc_failed(conn);
+    } else {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
       SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
       smartlist_free(which);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return 0;
+    }
+    return dir_okay ? 0 : -1;
+  }
+  /* Learn the routers, assuming we requested by fingerprint or "all"
+   * or "authority".
+   *
+   * We use "authority" to fetch our own descriptor for
+   * testing, and to fetch bridge descriptors for bootstrapping. Ignore
+   * the output of "authority" requests unless we are using bridges,
+   * since otherwise they'll be the response from reachability tests,
+   * and we don't really want to add that to our routerlist. */
+  if (which || (conn->requested_resource &&
+                (!strcmpstart(conn->requested_resource, "all") ||
+                 (!strcmpstart(conn->requested_resource, "authority") &&
+                  get_options()->UseBridges)))) {
+    /* as we learn from them, we remove them from 'which' */
+    if (was_ei) {
+      router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
+                                        descriptor_digests);
     } else {
-      smartlist_t *mds;
-      mds = microdescs_add_to_cache(get_microdesc_cache(),
-                                    body, body+body_len, SAVED_NOWHERE, 0,
-                                    now, which);
-      if (smartlist_len(which)) {
-        /* Mark remaining ones as failed. */
-        dir_microdesc_download_failed(which, status_code);
-      }
-      if (mds && smartlist_len(mds)) {
-        control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
-                                count_loading_descriptors_progress());
+      //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
+      //                       descriptor_digests, conn->router_purpose);
+      if (load_downloaded_routers(body, which, descriptor_digests,
+                                  conn->router_purpose,
+                                  conn->base_.address)) {
+        time_t now = approx_time();
         directory_info_has_arrived(now, 0, 1);
       }
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-      smartlist_free(mds);
     }
   }
+  if (which) { /* mark remaining ones as failed */
+    log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
+             n_asked_for-smartlist_len(which), n_asked_for,
+             was_ei ? "extra-info documents" : "router descriptors",
+             conn->base_.address, (int)conn->base_.port);
+    if (smartlist_len(which)) {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+  }
+  if (directory_conn_is_self_reachability_test(conn))
+    router_dirport_found_reachable();
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) {
-    switch (status_code) {
-      case 200: {
-          dir_server_t *ds =
-            router_get_trusteddirserver_by_digest(conn->identity_digest);
-          char *rejected_hdr = http_get_header(headers,
-                                               "X-Descriptor-Not-New: ");
-          if (rejected_hdr) {
-            if (!strcmp(rejected_hdr, "Yes")) {
-              log_info(LD_GENERAL,
-                       "Authority '%s' declined our descriptor (not new)",
-                       ds->nickname);
-              /* XXXX use this information; be sure to upload next one
-               * sooner. -NM */
-              /* XXXX++ On further thought, the task above implies that we're
-               * basing our regenerate-descriptor time on when we uploaded the
-               * last descriptor, not on the published time of the last
-               * descriptor.  If those are different, that's a bad thing to
-               * do. -NM */
-            }
-            tor_free(rejected_hdr);
-          }
-          log_info(LD_GENERAL,"eof (status 200) after uploading server "
-                   "descriptor: finished.");
-          control_event_server_status(
-                      LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
-                      conn->base_.address, conn->base_.port);
-
-          ds->has_accepted_serverdesc = 1;
-          if (directories_have_accepted_server_descriptor())
-            control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
-        }
-        break;
-      case 400:
-        log_warn(LD_GENERAL,"http status 400 (%s) response from "
-                 "dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        control_event_server_status(LOG_WARN,
-                      "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
-                      conn->base_.address, conn->base_.port, escaped(reason));
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "descriptor to server '%s:%d').",
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of
+ * microdescriptors
+ **/
+STATIC int
+handle_response_fetch_microdesc(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  smartlist_t *which = NULL;
+  log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
+           "body size %d) from server '%s:%d'",
+           status_code, (int)body_len, conn->base_.address,
+           conn->base_.port);
+  tor_assert(conn->requested_resource &&
+             !strcmpstart(conn->requested_resource, "d/"));
+  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  which = smartlist_new();
+  dir_split_resource_into_fingerprints(conn->requested_resource+2,
+                                       which, NULL,
+                                       DSR_DIGEST256|DSR_BASE64);
+  if (status_code != 200) {
+    log_info(LD_DIR, "Received status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
+             "soon.",
              status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-        break;
+             (int)conn->base_.port, conn->requested_resource);
+    dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    return 0;
+  } else {
+    smartlist_t *mds;
+    time_t now = approx_time();
+    mds = microdescs_add_to_cache(get_microdesc_cache(),
+                                  body, body+body_len, SAVED_NOWHERE, 0,
+                                  now, which);
+    if (smartlist_len(which)) {
+      /* Mark remaining ones as failed. */
+      dir_microdesc_download_failed(which, status_code, conn->identity_digest);
     }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
+    if (mds && smartlist_len(mds)) {
+      control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                              count_loading_descriptors_progress());
+      directory_info_has_arrived(now, 0, 1);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    smartlist_free(mds);
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE) {
-    switch (status_code) {
-      case 200: {
-        log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload our
+ * router descriptor.
+ **/
+static int
+handle_response_upload_dir(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *headers = args->headers;
+
+  switch (status_code) {
+  case 200: {
+    dir_server_t *ds =
+      router_get_trusteddirserver_by_digest(conn->identity_digest);
+    char *rejected_hdr = http_get_header(headers,
+                                         "X-Descriptor-Not-New: ");
+    if (rejected_hdr) {
+      if (!strcmp(rejected_hdr, "Yes")) {
+        log_info(LD_GENERAL,
+                 "Authority '%s' declined our descriptor (not new)",
+                 ds->nickname);
+        /* XXXX use this information; be sure to upload next one
+         * sooner. -NM */
+        /* XXXX++ On further thought, the task above implies that we're
+         * basing our regenerate-descriptor time on when we uploaded the
+         * last descriptor, not on the published time of the last
+         * descriptor.  If those are different, that's a bad thing to
+         * do. -NM */
+      }
+      tor_free(rejected_hdr);
+    }
+    log_info(LD_GENERAL,"eof (status 200) after uploading server "
+             "descriptor: finished.");
+    control_event_server_status(
+                   LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
                    conn->base_.address, conn->base_.port);
-        }
-        break;
-      case 400:
-        log_warn(LD_DIR,"http status 400 (%s) response after uploading "
-                 "vote to dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "vote to server '%s:%d').",
+
+    ds->has_accepted_serverdesc = 1;
+    if (directories_have_accepted_server_descriptor())
+      control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
+  }
+    break;
+  case 400:
+    log_warn(LD_GENERAL,"http status 400 (%s) response from "
+             "dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    control_event_server_status(LOG_WARN,
+                    "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
+                    conn->base_.address, conn->base_.port, escaped(reason));
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "descriptor to server '%s:%d'. Possibly the server is "
+             "misconfigured?",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
-        break;
-    }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
+    break;
   }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) {
-    switch (status_code) {
-      case 200: {
-        log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
-                   conn->base_.address, conn->base_.port);
-        }
-        break;
-      case 400:
-        log_warn(LD_DIR,"http status 400 (%s) response after uploading "
-                 "signatures to dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "signatures to server '%s:%d').",
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * own networkstatus vote.
+ **/
+static int
+handle_response_upload_vote(dir_connection_t *conn,
+                            const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "vote to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "vote to server '%s:%d'.",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
-        break;
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * view of the signatures on the current consensus.
+ **/
+static int
+handle_response_upload_signatures(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "signatures to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "signatures to server '%s:%d'.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v3 hidden service
+ * descriptor.
+ **/
+STATIC int
+handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  tor_assert(conn->hs_ident);
+
+  log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
+           (int)body_len, status_code, escaped(reason));
+
+  switch (status_code) {
+  case 200:
+    /* We got something: Try storing it in the cache. */
+    if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
+      log_info(LD_REND, "Failed to store hidden service descriptor");
+      /* Fire control port FAILED event. */
+      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                   "BAD_DESC");
+      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                    NULL);
+    } else {
+      log_info(LD_REND, "Stored hidden service descriptor successfully.");
+      TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
+      hs_client_desc_has_arrived(conn->hs_ident);
+      /* Fire control port RECEIVED event. */
+      hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
+      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                    body);
     }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) {
-    #define SEND_HS_DESC_FAILED_EVENT(reason) ( \
-      control_event_hs_descriptor_failed(conn->rend_data, \
-                                         conn->identity_digest, \
-                                         reason) )
-    #define SEND_HS_DESC_FAILED_CONTENT() ( \
-      control_event_hs_descriptor_content(conn->rend_data->onion_address, \
-                                          conn->requested_resource, \
-                                          conn->identity_digest, \
-                                          NULL) )
-    tor_assert(conn->rend_data);
-    log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d "
-             "(%s))",
-             (int)body_len, status_code, escaped(reason));
-    switch (status_code) {
-      case 200:
-      {
-        rend_cache_entry_t *entry = NULL;
-
-        if (rend_cache_store_v2_desc_as_client(body,
-            conn->requested_resource, conn->rend_data, &entry) < 0) {
-          log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
-                   "Retrying at another directory.");
-          /* We'll retry when connection_about_to_close_connection()
-           * cleans this dir conn up. */
-          SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
-          SEND_HS_DESC_FAILED_CONTENT();
-        } else {
-          char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
-          /* Should never be NULL here if we found the descriptor. */
-          tor_assert(entry);
-          rend_get_service_id(entry->parsed->pk, service_id);
-
-          /* success. notify pending connections about this. */
-          log_info(LD_REND, "Successfully fetched v2 rendezvous "
-                   "descriptor.");
-          control_event_hs_descriptor_received(service_id,
+    break;
+  case 404:
+    /* Not there. We'll retry when connection_about_to_close_connection()
+     * tries to clean this conn up. */
+    log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
+                      "Retrying at another directory.");
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "NOT_FOUND");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                      "http status 400 (%s). Dirserver didn't like our "
+                      "query? Retrying at another directory.",
+             escaped(reason));
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "QUERY_REJECTED");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+             "http status %d (%s) response unexpected from HSDir server "
+             "'%s:%d'. Retrying at another directory.",
+             status_code, escaped(reason), TO_CONN(conn)->address,
+             TO_CONN(conn)->port);
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UNEXPECTED");
+    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
+                                  NULL);
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v2 hidden service
+ * descriptor.
+ **/
+static int
+handle_response_fetch_renddesc_v2(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+#define SEND_HS_DESC_FAILED_EVENT(reason)                               \
+  (control_event_hsv2_descriptor_failed(conn->rend_data,                \
+                                        conn->identity_digest,          \
+                                        reason))
+#define SEND_HS_DESC_FAILED_CONTENT()                                   \
+  (control_event_hs_descriptor_content(                                 \
+                                rend_data_get_address(conn->rend_data), \
+                                conn->requested_resource,               \
+                                conn->identity_digest,                  \
+                                NULL))
+
+  tor_assert(conn->rend_data);
+  log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d "
+           "(%s))",
+           (int)body_len, status_code, escaped(reason));
+  switch (status_code) {
+  case 200:
+    {
+      rend_cache_entry_t *entry = NULL;
+
+      if (rend_cache_store_v2_desc_as_client(body,
+                                             conn->requested_resource,
+                                             conn->rend_data, &entry) < 0) {
+        log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
+                 "Retrying at another directory.");
+        /* We'll retry when connection_about_to_close_connection()
+         * cleans this dir conn up. */
+        SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
+        SEND_HS_DESC_FAILED_CONTENT();
+      } else {
+        char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
+        /* Should never be NULL here if we found the descriptor. */
+        tor_assert(entry);
+        rend_get_service_id(entry->parsed->pk, service_id);
+
+        /* success. notify pending connections about this. */
+        log_info(LD_REND, "Successfully fetched v2 rendezvous "
+                 "descriptor.");
+        control_event_hsv2_descriptor_received(service_id,
                                                conn->rend_data,
                                                conn->identity_digest);
-          control_event_hs_descriptor_content(service_id,
-                                              conn->requested_resource,
-                                              conn->identity_digest,
-                                              body);
-          conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
-          rend_client_desc_trynow(service_id);
-          memwipe(service_id, 0, sizeof(service_id));
-        }
-        break;
+        control_event_hs_descriptor_content(service_id,
+                                            conn->requested_resource,
+                                            conn->identity_digest,
+                                            body);
+        conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
+        rend_client_desc_trynow(service_id);
+        memwipe(service_id, 0, sizeof(service_id));
       }
-      case 404:
-        /* Not there. We'll retry when
-         * connection_about_to_close_connection() cleans this conn up. */
-        log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
-                         "Retrying at another directory.");
-        SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-      case 400:
-        log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
-                 "http status 400 (%s). Dirserver didn't like our "
-                 "v2 rendezvous query? Retrying at another directory.",
-                 escaped(reason));
-        SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-      default:
-        log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
-                 "http status %d (%s) response unexpected while "
-                 "fetching v2 hidden service descriptor (server '%s:%d'). "
-                 "Retrying at another directory.",
-                 status_code, escaped(reason), conn->base_.address,
-                 conn->base_.port);
-        SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
+      break;
     }
+  case 404:
+    /* Not there. We'll retry when
+     * connection_about_to_close_connection() cleans this conn up. */
+    log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
+             "Retrying at another directory.");
+    SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status 400 (%s). Dirserver didn't like our "
+             "v2 rendezvous query? Retrying at another directory.",
+             escaped(reason));
+    SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status %d (%s) response unexpected while "
+             "fetching v2 hidden service descriptor (server '%s:%d'). "
+             "Retrying at another directory.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) {
-    #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \
-      control_event_hs_descriptor_upload_failed( \
-        conn->identity_digest, \
-        conn->rend_data->onion_address, \
-        reason) )
-    log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
-             "(%s))",
-             status_code, escaped(reason));
-    /* Without the rend data, we'll have a problem identifying what has been
-     * uploaded for which service. */
-    tor_assert(conn->rend_data);
-    switch (status_code) {
-      case 200:
-        log_info(LD_REND,
-                 "Uploading rendezvous descriptor: finished with status "
-                 "200 (%s)", escaped(reason));
-        control_event_hs_descriptor_uploaded(conn->identity_digest,
-                                             conn->rend_data->onion_address);
-        rend_service_desc_has_uploaded(conn->rend_data);
-        break;
-      case 400:
-        log_warn(LD_REND,"http status 400 (%s) response from dirserver "
-                 "'%s:%d'. Malformed rendezvous descriptor?",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
-        break;
-      default:
-        log_warn(LD_REND,"http status %d (%s) response unexpected (server "
-                 "'%s:%d').",
-                 status_code, escaped(reason), conn->base_.address,
-                 conn->base_.port);
-        SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
-        break;
-    }
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload a v2
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_renddesc_v2(dir_connection_t *conn,
+                                   const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason)                        \
+    (control_event_hs_descriptor_upload_failed(                         \
+                                conn->identity_digest,                  \
+                                rend_data_get_address(conn->rend_data), \
+                                reason))
+
+  log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
+           "(%s))",
+           status_code, escaped(reason));
+  /* Without the rend data, we'll have a problem identifying what has been
+   * uploaded for which service. */
+  tor_assert(conn->rend_data);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND,
+             "Uploading rendezvous descriptor: finished with status "
+             "200 (%s)", escaped(reason));
+    control_event_hs_descriptor_uploaded(conn->identity_digest,
+                                   rend_data_get_address(conn->rend_data));
+    rend_service_desc_has_uploaded(conn->rend_data);
+    break;
+  case 400:
+    log_warn(LD_REND,"http status 400 (%s) response from dirserver "
+             "'%s:%d'. Malformed rendezvous descriptor?",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND,"http status %d (%s) response unexpected (server "
+             "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
+    break;
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload an
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_hsdesc(dir_connection_t *conn,
+                              const response_handler_args_t *args)
+{
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  tor_assert(conn);
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC);
+
+  log_info(LD_REND, "Uploaded hidden service descriptor (status %d "
+                    "(%s))",
+           status_code, escaped(reason));
+  /* For this directory response, it MUST have an hidden service identifier on
+   * this connection. */
+  tor_assert(conn->hs_ident);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND, "Uploading hidden service descriptor: "
+                      "finished with status 200 (%s)", escaped(reason));
+    hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest);
+    break;
+  case 400:
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Uploading hidden service descriptor: http "
+           "status 400 (%s) response from dirserver "
+           "'%s:%d'. Malformed hidden service descriptor?",
+           escaped(reason), conn->base_.address, conn->base_.port);
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND, "Uploading hidden service descriptor: http "
+                      "status %d (%s) response unexpected (server "
+                      "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
+                                 "UNEXPECTED");
+    break;
   }
-  tor_free(body); tor_free(headers); tor_free(reason);
+
   return 0;
 }
 
@@ -2524,6 +3417,33 @@ connection_dir_process_inbuf(dir_connection_t *conn)
   return 0;
 }
 
+/** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
+ *  that tried to fetch an HS descriptor, check if it successfully fetched it,
+ *  or if we need to try again. */
+static void
+refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
+{
+  connection_t *conn = TO_CONN(dir_conn);
+
+  /* If we were trying to fetch a v2 rend desc and did not succeed, retry as
+   * needed. (If a fetch is successful, the connection state is changed to
+   * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to
+   * mark that refetching is unnecessary.) */
+  if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
+      dir_conn->rend_data &&
+      rend_valid_v2_service_id(
+           rend_data_get_address(dir_conn->rend_data))) {
+    rend_client_refetch_v2_renddesc(dir_conn->rend_data);
+  }
+
+  /* Check for v3 rend desc fetch */
+  if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC &&
+      dir_conn->hs_ident &&
+      !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) {
+    hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk);
+  }
+}
+
 /** Called when we're about to finally unlink and free a directory connection:
  * perform necessary accounting and cleanup */
 void
@@ -2536,31 +3456,39 @@ connection_dir_about_to_close(dir_connection_t *dir_conn)
      * failed: forget about this router, and maybe try again. */
     connection_dir_request_failed(dir_conn);
   }
-  /* If we were trying to fetch a v2 rend desc and did not succeed,
-   * retry as needed. (If a fetch is successful, the connection state
-   * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that
-   * refetching is unnecessary.) */
-  if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
-      dir_conn->rend_data &&
-      strlen(dir_conn->rend_data->onion_address) == REND_SERVICE_ID_LEN_BASE32)
-    rend_client_refetch_v2_renddesc(dir_conn->rend_data);
+
+  refetch_hsdesc_if_needed(dir_conn);
 }
 
 /** Create an http response for the client <b>conn</b> out of
  * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
  */
 static void
-write_http_status_line(dir_connection_t *conn, int status,
+write_short_http_response(dir_connection_t *conn, int status,
                        const char *reason_phrase)
 {
-  char buf[256];
-  if (tor_snprintf(buf, sizeof(buf), "HTTP/1.0 %d %s\r\n\r\n",
-      status, reason_phrase ? reason_phrase : "OK") < 0) {
-    log_warn(LD_BUG,"status line too long.");
-    return;
+  char *buf = NULL;
+  char *datestring = NULL;
+
+  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
+    reason_phrase = "unspecified";
+  }
+
+  if (server_mode(get_options())) {
+    /* include the Date: header, but only if we're a relay or bridge */
+    char datebuf[RFC1123_TIME_LEN+1];
+    format_rfc1123_time(datebuf, time(NULL));
+    tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
   }
+
+  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
+               status, reason_phrase, datestring?datestring:"");
+
   log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
-  connection_write_to_buf(buf, strlen(buf), TO_CONN(conn));
+  connection_buf_add(buf, strlen(buf), TO_CONN(conn));
+
+  tor_free(datestring);
+  tor_free(buf);
 }
 
 /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
@@ -2577,76 +3505,160 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
                            long cache_lifetime)
 {
   char date[RFC1123_TIME_LEN+1];
-  char tmp[1024];
-  char *cp;
   time_t now = time(NULL);
+  buf_t *buf = buf_new_with_capacity(1024);
 
   tor_assert(conn);
 
   format_rfc1123_time(date, now);
-  cp = tmp;
-  tor_snprintf(cp, sizeof(tmp),
-               "HTTP/1.0 200 OK\r\nDate: %s\r\n",
-               date);
-  cp += strlen(tmp);
+
+  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
   if (type) {
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp), "Content-Type: %s\r\n", type);
-    cp += strlen(cp);
+    buf_add_printf(buf, "Content-Type: %s\r\n", type);
   }
   if (!is_local_addr(&conn->base_.addr)) {
     /* Don't report the source address for a nearby/private connection.
      * Otherwise we tend to mis-report in cases where incoming ports are
      * being forwarded to a Tor server running behind the firewall. */
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
-    cp += strlen(cp);
+    buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
   }
   if (encoding) {
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 "Content-Encoding: %s\r\n", encoding);
-    cp += strlen(cp);
+    buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
   }
   if (length >= 0) {
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 "Content-Length: %ld\r\n", (long)length);
-    cp += strlen(cp);
+    buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
   }
   if (cache_lifetime > 0) {
     char expbuf[RFC1123_TIME_LEN+1];
     format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
     /* We could say 'Cache-control: max-age=%d' here if we start doing
      * http/1.1 */
-    tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
-                 "Expires: %s\r\n", expbuf);
-    cp += strlen(cp);
+    buf_add_printf(buf, "Expires: %s\r\n", expbuf);
   } else if (cache_lifetime == 0) {
     /* We could say 'Cache-control: no-cache' here if we start doing
      * http/1.1 */
-    strlcpy(cp, "Pragma: no-cache\r\n", sizeof(tmp)-(cp-tmp));
-    cp += strlen(cp);
+    buf_add_string(buf, "Pragma: no-cache\r\n");
   }
   if (extra_headers) {
-    strlcpy(cp, extra_headers, sizeof(tmp)-(cp-tmp));
-    cp += strlen(cp);
+    buf_add_string(buf, extra_headers);
   }
-  if (sizeof(tmp)-(cp-tmp) > 3)
-    memcpy(cp, "\r\n", 3);
-  else
-    tor_assert(0);
-  connection_write_to_buf(tmp, strlen(tmp), TO_CONN(conn));
+  buf_add_string(buf, "\r\n");
+
+  connection_buf_add_buf(TO_CONN(conn), buf);
+  buf_free(buf);
 }
 
 /** As write_http_response_header_impl, but sets encoding and content-typed
  * based on whether the response will be <b>compressed</b> or not. */
 static void
-write_http_response_header(dir_connection_t *conn, ssize_t length,
-                           int compressed, long cache_lifetime)
+write_http_response_headers(dir_connection_t *conn, ssize_t length,
+                            compress_method_t method,
+                            const char *extra_headers, long cache_lifetime)
 {
+  const char *methodname = compression_method_get_name(method);
+  const char *doctype;
+  if (method == NO_METHOD)
+    doctype = "text/plain";
+  else
+    doctype = "application/octet-stream";
   write_http_response_header_impl(conn, length,
-                          compressed?"application/octet-stream":"text/plain",
-                          compressed?"deflate":"identity",
-                             NULL,
-                             cache_lifetime);
+                                  doctype,
+                                  methodname,
+                                  extra_headers,
+                                  cache_lifetime);
+}
+
+/** As write_http_response_headers, but assumes extra_headers is NULL */
+static void
+write_http_response_header(dir_connection_t *conn, ssize_t length,
+                           compress_method_t method,
+                           long cache_lifetime)
+{
+  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
+}
+
+/** Array of compression methods to use (if supported) for serving
+ * precompressed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_precompressed[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of compression methods to use (if supported) for serving
+ * streamed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_streaming_compression[] = {
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of allowed compression methods to use (if supported) when receiving a
+ * response from a request that was required to be anonymous. */
+static compress_method_t client_meth_allowed_anonymous_compression[] = {
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
+ * and convert them to a bitfield where compression method x is supported if
+ * and only if 1 &lt;&lt; x is set in the bitfield. */
+STATIC unsigned
+parse_accept_encoding_header(const char *h)
+{
+  unsigned result = (1u << NO_METHOD);
+  smartlist_t *methods = smartlist_new();
+  smartlist_split_string(methods, h, ",",
+             SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
+    compress_method_t method = compression_method_get_by_name(m);
+    if (method != UNKNOWN_METHOD) {
+      tor_assert(((unsigned)method) < 8*sizeof(unsigned));
+      result |= (1u << method);
+    }
+  } SMARTLIST_FOREACH_END(m);
+  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
+    tor_free(m);
+  } SMARTLIST_FOREACH_END(m);
+  smartlist_free(methods);
+  return result;
+}
+
+/** Array of compression methods to use (if supported) for requesting
+ * compressed data, ordered from best to worst. */
+static compress_method_t client_meth_pref[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Return a newly allocated string containing a comma separated list of
+ * supported encodings. */
+STATIC char *
+accept_encoding_header(void)
+{
+  smartlist_t *methods = smartlist_new();
+  char *header = NULL;
+  compress_method_t method;
+  unsigned i;
+
+  for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
+    method = client_meth_pref[i];
+    if (tor_compress_supports_method(method))
+      smartlist_add(methods, (char *)compression_method_get_name(method));
+  }
+
+  header = smartlist_join_strings(methods, ", ", 0, NULL);
+  smartlist_free(methods);
+
+  return header;
 }
 
 /** Decide whether a client would accept the consensus we have.
@@ -2664,48 +3676,46 @@ write_http_response_header(dir_connection_t *conn, ssize_t length,
  * consensus, 0 otherwise.
  */
 int
-client_likes_consensus(networkstatus_t *v, const char *want_url)
+client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                       const char *want_url)
 {
-  smartlist_t *want_authorities = smartlist_new();
+  smartlist_t *voters = smartlist_new();
   int need_at_least;
   int have = 0;
 
+  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
+    smartlist_free(voters);
+    return 1; // We don't know the voters; assume the client won't mind. */
+  }
+
+  smartlist_t *want_authorities = smartlist_new();
   dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
   need_at_least = smartlist_len(want_authorities)/2+1;
-  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) {
-    char want_digest[DIGEST_LEN];
-    size_t want_len = strlen(d)/2;
-    if (want_len > DIGEST_LEN)
-      want_len = DIGEST_LEN;
-
-    if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2)
-                      != (int) want_len) {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-             "Failed to decode requested authority digest %s.", escaped(d));
-      continue;
-    };
 
-    SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) {
-      if (smartlist_len(vi->sigs) &&
-          tor_memeq(vi->identity_digest, want_digest, want_len)) {
+  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
+
+    SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
+      if (!strcasecmpstart(digest, want_digest)) {
         have++;
         break;
       };
-    } SMARTLIST_FOREACH_END(vi);
+    } SMARTLIST_FOREACH_END(digest);
 
     /* early exit, if we already have enough */
     if (have >= need_at_least)
       break;
-  } SMARTLIST_FOREACH_END(d);
+  } SMARTLIST_FOREACH_END(want_digest);
 
   SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
   smartlist_free(want_authorities);
+  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
+  smartlist_free(voters);
   return (have >= need_at_least);
 }
 
 /** Return the compression level we should use for sending a compressed
  * response of size <b>n_bytes</b>. */
-STATIC zlib_compression_level_t
+STATIC compression_level_t
 choose_compression_level(ssize_t n_bytes)
 {
   if (! have_been_under_memory_pressure()) {
@@ -2723,8 +3733,9 @@ choose_compression_level(ssize_t n_bytes)
 
 /** Information passed to handle a GET request. */
 typedef struct get_handler_args_t {
-  /** True if the client asked for compressed data. */
-  int compressed;
+  /** Bitmask of compression methods that the client said (or implied) it
+   * supported. */
+  unsigned compression_supported;
   /** If nonzero, the time included an if-modified-since header with this
    * value. */
   time_t if_modified_since;
@@ -2762,8 +3773,8 @@ static int handle_get_descriptor(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_keys(dir_connection_t *conn,
                                 const get_handler_args_t *args);
-static int handle_get_rendezvous2(dir_connection_t *conn,
-                                const get_handler_args_t *args);
+static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                                       const get_handler_args_t *args);
 static int handle_get_robots(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
@@ -2779,7 +3790,8 @@ static const url_table_ent_t url_table[] = {
   { "/tor/server/", 1, handle_get_descriptor },
   { "/tor/extra/", 1, handle_get_descriptor },
   { "/tor/keys/", 1, handle_get_keys },
-  { "/tor/rendezvous2/", 1, handle_get_rendezvous2 },
+  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
+  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
   { "/tor/robots.txt", 0, handle_get_robots },
   { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
   { NULL, 0, NULL },
@@ -2791,14 +3803,14 @@ static const url_table_ent_t url_table[] = {
  * conn-\>outbuf.  If the request is unrecognized, send a 404.
  * Return 0 if we handled this successfully, or -1 if we need to close
  * the connection. */
-STATIC int
-directory_handle_command_get(dir_connection_t *conn, const char *headers,
-                             const char *req_body, size_t req_body_len)
+MOCK_IMPL(STATIC int,
+directory_handle_command_get,(dir_connection_t *conn, const char *headers,
+                              const char *req_body, size_t req_body_len))
 {
   char *url, *url_mem, *header;
   time_t if_modified_since = 0;
-  int compressed;
-  size_t url_len;
+  int zlib_compressed_in_url;
+  unsigned compression_methods_supported;
 
   /* We ignore the body of a GET request. */
   (void)req_body;
@@ -2809,7 +3821,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
   conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
 
   if (parse_http_url(headers, &url) < 0) {
-    write_http_status_line(conn, 400, "Bad request");
+    write_short_http_response(conn, 400, "Bad request");
     return 0;
   }
   if ((header = http_get_header(headers, "If-Modified-Since: "))) {
@@ -2828,18 +3840,33 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
   log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
 
   url_mem = url;
-  url_len = strlen(url);
-  compressed = url_len > 2 && !strcmp(url+url_len-2, ".z");
-  if (compressed) {
-    url[url_len-2] = '\0';
-    url_len -= 2;
+  {
+    size_t url_len = strlen(url);
+
+    zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
+    if (zlib_compressed_in_url) {
+      url[url_len-2] = '\0';
+    }
+  }
+
+  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
+    compression_methods_supported = parse_accept_encoding_header(header);
+    tor_free(header);
+  } else {
+    compression_methods_supported = (1u << NO_METHOD);
   }
+  if (zlib_compressed_in_url) {
+    compression_methods_supported |= (1u << ZLIB_METHOD);
+  }
+
+  /* Remove all methods that we don't both support. */
+  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
 
   get_handler_args_t args;
   args.url = url;
   args.headers = headers;
   args.if_modified_since = if_modified_since;
-  args.compressed = compressed;
+  args.compression_supported = compression_methods_supported;
 
   int i, result = -1;
   for (i = 0; url_table[i].string; ++i) {
@@ -2856,7 +3883,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
   }
 
   /* we didn't recognize the url */
-  write_http_status_line(conn, 404, "Not found");
+  write_short_http_response(conn, 404, "Not found");
   result = 0;
 
  done:
@@ -2882,141 +3909,530 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
      *  this page no matter what.] */
     write_http_response_header_impl(conn, dlen, "text/html", "identity",
                                     NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
-    connection_write_to_buf(frontpage, dlen, TO_CONN(conn));
+    connection_buf_add(frontpage, dlen, TO_CONN(conn));
   } else {
-    write_http_status_line(conn, 404, "Not found");
+    write_short_http_response(conn, 404, "Not found");
   }
   return 0;
 }
 
-/** Helper function for GET /tor/status-vote/current/consensus
+/** Warn that the cached consensus <b>consensus</b> of type
+ * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
+ * warning to avoid logging an entry on every request.
  */
-static int
-handle_get_current_consensus(dir_connection_t *conn,
-                             const get_handler_args_t *args)
+static void
+warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now)
 {
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  const time_t if_modified_since = args->if_modified_since;
+#define TOO_OLD_WARNING_INTERVAL (60*60)
+  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+  char timestamp[ISO_TIME_LEN+1];
+  time_t valid_until;
+  char *dupes;
 
-  {
-    /* v3 network status fetch. */
-    smartlist_t *dir_fps = smartlist_new();
-    long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
-
-    if (1) {
-      networkstatus_t *v;
-      time_t now = time(NULL);
-      const char *want_fps = NULL;
-      char *flavor = NULL;
-      int flav = FLAV_NS;
-      #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
-      #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
-      /* figure out the flavor if any, and who we wanted to sign the thing */
-      if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
-        const char *f, *cp;
-        f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
-        cp = strchr(f, '/');
-        if (cp) {
-          want_fps = cp+1;
-          flavor = tor_strndup(f, cp-f);
-        } else {
-          flavor = tor_strdup(f);
-        }
-        flav = networkstatus_parse_flavor_name(flavor);
-        if (flav < 0)
-          flav = FLAV_NS;
-      } else {
-        if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
-          want_fps = url+strlen(CONSENSUS_URL_PREFIX);
-      }
+  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
+    return;
 
-      v = networkstatus_get_latest_consensus_by_flavor(flav);
+  if ((dupes = rate_limit_log(&warned, now))) {
+    format_local_iso_time(timestamp, valid_until);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
+             "serve it to clients. It was valid until %s local time and we "
+             "continued to serve it for up to 24 hours after it expired.%s",
+             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+    tor_free(dupes);
+  }
+}
 
-      if (v && want_fps &&
-          !client_likes_consensus(v, want_fps)) {
-        write_http_status_line(conn, 404, "Consensus not signed by sufficient "
-                                          "number of requested authorities");
-        smartlist_free(dir_fps);
-        geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
-        tor_free(flavor);
-        goto done;
-      }
+/**
+ * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
+ * <b>digest</b>. Return 0 on success.  On failure, report that the hash came
+ * from <b>location</b>, report that we are taking <b>action</b> with it, and
+ * return -1.
+ */
+static int
+parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
+                    const char *action)
+{
+  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
+      DIGEST256_LEN) {
+    return 0;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "%s contained bogus digest %s; %s.",
+           location, escaped(hex), action);
+    return -1;
+  }
+}
 
-      {
-        char *fp = tor_malloc_zero(DIGEST_LEN);
-        if (flavor)
-          strlcpy(fp, flavor, DIGEST_LEN);
-        tor_free(flavor);
-        smartlist_add(dir_fps, fp);
+/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
+ * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * hex-encoded digest listed in that header and return 0.  Otherwise return
+ * -1.  */
+static int
+parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
+{
+  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
+  if (hdr == NULL) {
+    return -1;
+  }
+  smartlist_t *hex_digests = smartlist_new();
+  *digests_out = smartlist_new();
+  smartlist_split_string(hex_digests, hdr, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
+    uint8_t digest[DIGEST256_LEN];
+    if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
+                             "ignoring")) {
+      smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
+    }
+  } SMARTLIST_FOREACH_END(hex);
+  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
+  smartlist_free(hex_digests);
+  tor_free(hdr);
+  return 0;
+}
+
+/** Fallback compression method.  The fallback compression method is used in
+ * case a client requests a non-compressed document. We only store compressed
+ * documents, so we use this compression method to fetch the document and let
+ * the spooling system do the streaming decompression.
+ */
+#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
+
+/**
+ * Try to find the best consensus diff possible in order to serve a client
+ * request for a diff from one of the consensuses in <b>digests</b> to the
+ * current consensus of flavor <b>flav</b>.  The client supports the
+ * compression methods listed in the <b>compression_methods</b> bitfield:
+ * place the method chosen (if any) into <b>compression_used_out</b>.
+ */
+static struct consensus_cache_entry_t *
+find_best_diff(const smartlist_t *digests, int flav,
+               unsigned compression_methods,
+               compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    unsigned u;
+    for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+      compress_method_t method = srv_meth_pref_precompressed[u];
+      if (0 == (compression_methods & (1u<<method)))
+        continue; // client doesn't like this one, or we don't have it.
+      if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
+                                     diff_from, DIGEST256_LEN,
+                                     method) == CONSDIFF_AVAILABLE) {
+        tor_assert_nonfatal(result);
+        *compression_used_out = method;
+        return result;
       }
-      lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0;
     }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
+          DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = FALLBACK_COMPRESS_METHOD;
+      return result;
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
 
-    if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
-      write_http_status_line(conn, 503, "Network status object unavailable");
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE);
-      goto done;
+  return NULL;
+}
+
+/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
+ * The preferred set of compression methods should be listed in the
+ * <b>compression_methods</b> bitfield. The compression method chosen (if any)
+ * is stored in <b>compression_used_out</b>. */
+static struct consensus_cache_entry_t *
+find_best_consensus(int flav,
+                    unsigned compression_methods,
+                    compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+    compress_method_t method = srv_meth_pref_precompressed[u];
+
+    if (0 == (compression_methods & (1u<<method)))
+      continue;
+
+    if (consdiffmgr_find_consensus(&result, flav,
+                                   method) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = method;
+      return result;
     }
+  }
 
-    if (!dirserv_remove_old_statuses(dir_fps, if_modified_since)) {
-      write_http_status_line(conn, 404, "Not found");
-      SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
-      goto done;
-    } else if (!smartlist_len(dir_fps)) {
-      write_http_status_line(conn, 304, "Not modified");
-      SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
-      goto done;
+  if (consdiffmgr_find_consensus(&result, flav,
+        FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+    tor_assert_nonfatal(result);
+    *compression_used_out = FALLBACK_COMPRESS_METHOD;
+    return result;
+  }
+
+  return NULL;
+}
+
+/** Try to find the best supported compression method possible from a given
+ * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
+ * compression method could be found. */
+static compress_method_t
+find_best_compression_method(unsigned compression_methods, int stream)
+{
+  unsigned u;
+  compress_method_t *methods;
+  size_t length;
+
+  if (stream) {
+    methods = srv_meth_pref_streaming_compression;
+    length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
+  } else {
+    methods = srv_meth_pref_precompressed;
+    length = ARRAY_LENGTH(srv_meth_pref_precompressed);
+  }
+
+  for (u = 0; u < length; ++u) {
+    compress_method_t method = methods[u];
+    if (compression_methods & (1u<<method))
+      return method;
+  }
+
+  return NO_METHOD;
+}
+
+/** Check if any of the digests in <b>digests</b> matches the latest consensus
+ *  flavor (given in <b>flavor</b>) that we have available. */
+static int
+digest_list_contains_best_consensus(consensus_flavor_t flavor,
+                                    const smartlist_t *digests)
+{
+  const networkstatus_t *ns = NULL;
+
+  if (digests == NULL)
+    return 0;
+
+  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
+
+  if (ns == NULL)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
+    if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
+      return 1;
+  } SMARTLIST_FOREACH_END(digest);
+
+  return 0;
+}
+
+/** Check if the given compression method is allowed for a connection that is
+ * supposed to be anonymous. Returns 1 if the compression method is allowed,
+ * otherwise 0. */
+STATIC int
+allowed_anonymous_connection_compression_method(compress_method_t method)
+{
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
+       ++u) {
+    compress_method_t allowed_method =
+      client_meth_allowed_anonymous_compression[u];
+
+    if (! tor_compress_supports_method(allowed_method))
+      continue;
+
+    if (method == allowed_method)
+      return 1;
+  }
+
+  return 0;
+}
+
+/** Log a warning when a remote server has sent us a document using a
+ * compression method that is not allowed for anonymous directory requests. */
+STATIC void
+warn_disallowed_anonymous_compression_method(compress_method_t method)
+{
+  log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+         "Received a %s HTTP response, which is not "
+         "allowed for anonymous directory requests.",
+         compression_method_get_human_name(method));
+}
+
+/** Encodes the results of parsing a consensus request to figure out what
+ * consensus, and possibly what diffs, the user asked for. */
+typedef struct {
+  /** name of the flavor to retrieve. */
+  char *flavor;
+  /** flavor to retrive, as enum. */
+  consensus_flavor_t flav;
+  /** plus-separated list of authority fingerprints; see
+   * client_likes_consensus(). Aliases the URL in the request passed to
+   * parse_consensus_request(). */
+  const char *want_fps;
+  /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
+   * to return a diff from. */
+  smartlist_t *diff_from_digests;
+  /** If true, never send a full consensus. If there is no diff, send
+   * a 404 instead. */
+  int diff_only;
+} parsed_consensus_request_t;
+
+/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
+ * it is stack-allocated. */
+static void
+parsed_consensus_request_clear(parsed_consensus_request_t *req)
+{
+  if (!req)
+    return;
+  tor_free(req->flavor);
+  if (req->diff_from_digests) {
+    SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(req->diff_from_digests);
+  }
+  memset(req, 0, sizeof(parsed_consensus_request_t));
+}
+
+/**
+ * Parse the URL and relevant headers of <b>args</b> for a current-consensus
+ * request to learn what flavor of consensus we want, what keys it must be
+ * signed with, and what diffs we would accept (or demand) instead. Return 0
+ * on success and -1 on failure.
+ */
+static int
+parse_consensus_request(parsed_consensus_request_t *out,
+                        const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  memset(out, 0, sizeof(parsed_consensus_request_t));
+  out->flav = FLAV_NS;
+
+  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
+  const char CONSENSUS_FLAVORED_PREFIX[] =
+    "/tor/status-vote/current/consensus-";
+
+  /* figure out the flavor if any, and who we wanted to sign the thing */
+  const char *after_flavor = NULL;
+
+  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+    const char *f, *cp;
+    f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+    cp = strchr(f, '/');
+    if (cp) {
+      after_flavor = cp+1;
+      out->flavor = tor_strndup(f, cp-f);
+    } else {
+      out->flavor = tor_strdup(f);
     }
+    int flav = networkstatus_parse_flavor_name(out->flavor);
+    if (flav < 0)
+      flav = FLAV_NS;
+    out->flav = flav;
+  } else {
+    if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+      after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
+  }
 
-    size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed);
-    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
-      log_debug(LD_DIRSERV,
-               "Client asked for network status lists, but we've been "
-               "writing too many bytes lately. Sending 503 Dir busy.");
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp));
-      smartlist_free(dir_fps);
+  /* see whether we've been asked explicitly for a diff from an older
+   * consensus. (The user might also have said that a diff would be okay,
+   * via X-Or-Diff-From-Consensus */
+  const char DIFF_COMPONENT[] = "diff/";
+  char *diff_hash_in_url = NULL;
+  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
+    after_flavor += strlen(DIFF_COMPONENT);
+    const char *cp = strchr(after_flavor, '/');
+    if (cp) {
+      diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
+      out->want_fps = cp+1;
+    } else {
+      diff_hash_in_url = tor_strdup(after_flavor);
+      out->want_fps = NULL;
+    }
+  } else {
+    out->want_fps = after_flavor;
+  }
 
-      geoip_note_ns_response(GEOIP_REJECT_BUSY);
-      goto done;
+  if (diff_hash_in_url) {
+    uint8_t diff_from[DIGEST256_LEN];
+    out->diff_from_digests = smartlist_new();
+    out->diff_only = 1;
+    int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
+                                  "rejecting");
+    tor_free(diff_hash_in_url);
+    if (ok) {
+      smartlist_add(out->diff_from_digests,
+                    tor_memdup(diff_from, DIGEST256_LEN));
+    } else {
+      return -1;
     }
+  } else {
+    parse_or_diff_from_header(&out->diff_from_digests, args->headers);
+  }
 
-    if (1) {
-      tor_addr_t addr;
-      if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
-        geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
-                               &addr, NULL,
-                               time(NULL));
-        geoip_note_ns_response(GEOIP_SUCCESS);
-        /* Note that a request for a network status has started, so that we
-         * can measure the download time later on. */
-        if (conn->dirreq_id)
-          geoip_start_dirreq(conn->dirreq_id, dlen, DIRREQ_TUNNELED);
-        else
-          geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen,
-                             DIRREQ_DIRECT);
-      }
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/current/consensus
+ */
+static int
+handle_get_current_consensus(dir_connection_t *conn,
+                             const get_handler_args_t *args)
+{
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 0);
+  const time_t if_modified_since = args->if_modified_since;
+  int clear_spool = 0;
+
+  /* v3 network status fetch. */
+  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
+  time_t now = time(NULL);
+  parsed_consensus_request_t req;
+
+  if (parse_consensus_request(&req, args) < 0) {
+    write_short_http_response(conn, 404, "Couldn't parse request");
+    goto done;
+  }
+
+  if (digest_list_contains_best_consensus(req.flav,
+                                          req.diff_from_digests)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  struct consensus_cache_entry_t *cached_consensus = NULL;
+
+  compress_method_t compression_used = NO_METHOD;
+  if (req.diff_from_digests) {
+    cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
+                                      args->compression_supported,
+                                      &compression_used);
+  }
+
+  if (req.diff_only && !cached_consensus) {
+    write_short_http_response(conn, 404, "No such diff available");
+    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (! cached_consensus) {
+    cached_consensus = find_best_consensus(req.flav,
+                                           args->compression_supported,
+                                           &compression_used);
+  }
+
+  time_t fresh_until, valid_until;
+  int have_fresh_until = 0, have_valid_until = 0;
+  if (cached_consensus) {
+    have_fresh_until =
+      !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
+    have_valid_until =
+      !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
+  }
+
+  if (cached_consensus && have_valid_until &&
+      !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
+    write_short_http_response(conn, 404, "Consensus is too old");
+    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (cached_consensus && req.want_fps &&
+      !client_likes_consensus(cached_consensus, req.want_fps)) {
+    write_short_http_response(conn, 404, "Consensus not signed by sufficient "
+                           "number of requested authorities");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
+    goto done;
+  }
+
+  conn->spool = smartlist_new();
+  clear_spool = 1;
+  {
+    spooled_resource_t *spooled;
+    if (cached_consensus) {
+      spooled = spooled_resource_new_from_cache_entry(cached_consensus);
+      smartlist_add(conn->spool, spooled);
     }
+  }
+
+  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
 
-    write_http_response_header(conn, -1, compressed,
-                               smartlist_len(dir_fps) == 1 ? lifetime : 0);
-    conn->fingerprint_stack = dir_fps;
-    if (! compressed)
-      conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
+  size_t size_guess = 0;
+  int n_expired = 0;
+  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
+                                              compress_method != NO_METHOD,
+                                              &size_guess,
+                                              &n_expired);
+
+  if (!smartlist_len(conn->spool) && !n_expired) {
+    write_short_http_response(conn, 404, "Not found");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (!smartlist_len(conn->spool)) {
+    write_short_http_response(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
 
-    /* Prime the connection with some data. */
-    conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
-    connection_dirserv_flushed_some(conn);
+  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    log_debug(LD_DIRSERV,
+              "Client asked for network status lists, but we've been "
+              "writing too many bytes lately. Sending 503 Dir busy.");
+    write_short_http_response(conn, 503, "Directory busy, try again later");
+    geoip_note_ns_response(GEOIP_REJECT_BUSY);
     goto done;
   }
 
+  tor_addr_t addr;
+  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
+    geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
+                           &addr, NULL,
+                           time(NULL));
+    geoip_note_ns_response(GEOIP_SUCCESS);
+    /* Note that a request for a network status has started, so that we
+     * can measure the download time later on. */
+    if (conn->dirreq_id)
+      geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
+    else
+      geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
+                         DIRREQ_DIRECT);
+  }
+
+  /* Use this header to tell caches that the response depends on the
+   * X-Or-Diff-From-Consensus header (or lack thereof). */
+  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
+
+  clear_spool = 0;
+
+  // The compress_method might have been NO_METHOD, but we store the data
+  // compressed. Decompress them using `compression_used`. See fallback code in
+  // find_best_consensus() and find_best_diff().
+  write_http_response_headers(conn, -1,
+                             compress_method == NO_METHOD ?
+                               NO_METHOD : compression_used,
+                             vary_header,
+                             smartlist_len(conn->spool) == 1 ? lifetime : 0);
+
+  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
+    conn->compress_state = tor_compress_new(0, compression_used,
+                                            HIGH_COMPRESSION);
+
+  /* Prime the connection with some data. */
+  const int initial_flush_result = connection_dirserv_flushed_some(conn);
+  tor_assert_nonfatal(initial_flush_result == 0);
+  goto done;
+
  done:
+  parsed_consensus_request_clear(&req);
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
   return 0;
 }
 
@@ -3026,68 +4442,47 @@ static int
 handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
   {
-    int current;
     ssize_t body_len = 0;
     ssize_t estimated_len = 0;
+    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
+    /* This smartlist holds strings that we can compress on the fly. */
     smartlist_t *items = smartlist_new();
+    /* This smartlist holds cached_dir_t objects that have a precompressed
+     * deflated version. */
     smartlist_t *dir_items = smartlist_new();
-    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
-    url += strlen("/tor/status-vote/");
-    current = !strcmpstart(url, "current/");
-    url = strchr(url, '/');
-    tor_assert(url);
-    ++url;
-    if (!strcmp(url, "consensus")) {
-      const char *item;
-      tor_assert(!current); /* we handle current consensus specially above,
-                             * since it wants to be spooled. */
-      if ((item = dirvote_get_pending_consensus(FLAV_NS)))
-        smartlist_add(items, (char*)item);
-    } else if (!current && !strcmp(url, "consensus-signatures")) {
-      /* XXXX the spec says that we should implement
-       * current/consensus-signatures too.  It doesn't seem to be needed,
-       * though. */
-      const char *item;
-      if ((item=dirvote_get_pending_detached_signatures()))
-        smartlist_add(items, (char*)item);
-    } else if (!strcmp(url, "authority")) {
-      const cached_dir_t *d;
-      int flags = DGV_BY_ID |
-        (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
-      if ((d=dirvote_get_vote(NULL, flags)))
-        smartlist_add(dir_items, (cached_dir_t*)d);
-    } else {
-      const cached_dir_t *d;
-      smartlist_t *fps = smartlist_new();
-      int flags;
-      if (!strcmpstart(url, "d/")) {
-        url += 2;
-        flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
-      } else {
-        flags = DGV_BY_ID |
-          (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
-      }
-      dir_split_resource_into_fingerprints(url, fps, NULL,
-                                           DSR_HEX|DSR_SORT_UNIQ);
-      SMARTLIST_FOREACH(fps, char *, fp, {
-          if ((d = dirvote_get_vote(fp, flags)))
-            smartlist_add(dir_items, (cached_dir_t*)d);
-          tor_free(fp);
-        });
-      smartlist_free(fps);
-    }
+    dirvote_dirreq_get_status_vote(url, items, dir_items);
     if (!smartlist_len(dir_items) && !smartlist_len(items)) {
-      write_http_status_line(conn, 404, "Not found");
+      write_short_http_response(conn, 404, "Not found");
       goto vote_done;
     }
+
+    /* We're sending items from at most one kind of source */
+    tor_assert_nonfatal(smartlist_len(items) == 0 ||
+                        smartlist_len(dir_items) == 0);
+
+    int streaming;
+    unsigned mask;
+    if (smartlist_len(items)) {
+      /* We're taking strings and compressing them on the fly. */
+      streaming = 1;
+      mask = ~0u;
+    } else {
+      /* We're taking cached_dir_t objects. We only have them uncompressed
+       * or deflated. */
+      streaming = 0;
+      mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
+    }
+    const compress_method_t compress_method = find_best_compression_method(
+                              args->compression_supported&mask, streaming);
+
     SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-                      body_len += compressed ? d->dir_z_len : d->dir_len);
+                      body_len += compress_method != NO_METHOD ?
+                        d->dir_compressed_len : d->dir_len);
     estimated_len += body_len;
     SMARTLIST_FOREACH(items, const char *, item, {
         size_t ln = strlen(item);
-        if (compressed) {
+        if (compress_method != NO_METHOD) {
           estimated_len += ln/2;
         } else {
           body_len += ln; estimated_len += ln;
@@ -3095,27 +4490,30 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       });
 
     if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
-      write_http_status_line(conn, 503, "Directory busy, try again later");
+      write_short_http_response(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
-    write_http_response_header(conn, body_len ? body_len : -1, compressed,
+    write_http_response_header(conn, body_len ? body_len : -1,
+                 compress_method,
                  lifetime);
 
     if (smartlist_len(items)) {
-      if (compressed) {
-        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                    choose_compression_level(estimated_len));
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                           choose_compression_level(estimated_len));
         SMARTLIST_FOREACH(items, const char *, c,
-                 connection_write_to_buf_zlib(c, strlen(c), conn, 0));
-        connection_write_to_buf_zlib("", 0, conn, 1);
+                 connection_buf_add_compress(c, strlen(c), conn, 0));
+        connection_buf_add_compress("", 0, conn, 1);
       } else {
         SMARTLIST_FOREACH(items, const char *, c,
-                         connection_write_to_buf(c, strlen(c), TO_CONN(conn)));
+                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
       }
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-          connection_write_to_buf(compressed ? d->dir_z : d->dir,
-                                  compressed ? d->dir_z_len : d->dir_len,
+          connection_buf_add(compress_method != NO_METHOD ?
+                                    d->dir_compressed : d->dir,
+                                  compress_method != NO_METHOD ?
+                                    d->dir_compressed_len : d->dir_len,
                                   TO_CONN(conn)));
     }
   vote_done:
@@ -3133,44 +4531,51 @@ static int
 handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  int clear_spool = 1;
   {
-    smartlist_t *fps = smartlist_new();
+    conn->spool = smartlist_new();
 
-    dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"),
-                                      fps, NULL,
+    dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
+                                      DIR_SPOOL_MICRODESC,
+                                      conn->spool, NULL,
                                       DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
 
-    if (!dirserv_have_any_microdesc(fps)) {
-      write_http_status_line(conn, 404, "Not found");
-      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
-      smartlist_free(fps);
+    size_t size_guess = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, 0,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, NULL);
+    if (smartlist_len(conn->spool) == 0) {
+      write_short_http_response(conn, 404, "Not found");
       goto done;
     }
-    size_t dlen = dirserv_estimate_microdesc_size(fps, compressed);
-    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
+    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
-      smartlist_free(fps);
+      write_short_http_response(conn, 503, "Directory busy, try again later");
       goto done;
     }
 
-    write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME);
-    conn->dir_spool_src = DIR_SPOOL_MICRODESC;
-    conn->fingerprint_stack = fps;
+    clear_spool = 0;
+    write_http_response_header(conn, -1,
+                               compress_method,
+                               MICRODESC_CACHE_LIFETIME);
 
-    if (compressed)
-      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                      choose_compression_level(dlen));
+    if (compress_method != NO_METHOD)
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                      choose_compression_level(size_guess));
 
-    connection_dirserv_flushed_some(conn);
+    const int initial_flush_result = connection_dirserv_flushed_some(conn);
+    tor_assert_nonfatal(initial_flush_result == 0);
     goto done;
   }
 
  done:
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
   return 0;
 }
 
@@ -3180,71 +4585,93 @@ static int
 handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
   const or_options_t *options = get_options();
+  int clear_spool = 1;
   if (!strcmpstart(url,"/tor/server/") ||
       (!options->BridgeAuthoritativeDir &&
        !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
-    size_t dlen;
     int res;
-    const char *msg;
+    const char *msg = NULL;
     int cache_lifetime = 0;
     int is_extra = !strcmpstart(url,"/tor/extra/");
     url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
-    conn->fingerprint_stack = smartlist_new();
-    res = dirserv_get_routerdesc_fingerprints(conn->fingerprint_stack, url,
-                                          &msg,
-                                          !connection_dir_is_encrypted(conn),
-                                          is_extra);
-
-    if (!strcmpstart(url, "fp/")) {
-      if (smartlist_len(conn->fingerprint_stack) == 1)
-        cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "authority")) {
-      cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "all")) {
-      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "d/")) {
-      if (smartlist_len(conn->fingerprint_stack) == 1)
-        cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
-    }
-    if (!strcmpstart(url, "d/"))
-      conn->dir_spool_src =
+    dir_spool_source_t source;
+    time_t publish_cutoff = 0;
+    if (!strcmpstart(url, "d/")) {
+      source =
         is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
-    else
-      conn->dir_spool_src =
+    } else {
+      source =
         is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
+      /* We only want to apply a publish cutoff when we're requesting
+       * resources by fingerprint. */
+      publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    }
+
+    conn->spool = smartlist_new();
+    res = dirserv_get_routerdesc_spool(conn->spool, url,
+                                       source,
+                                       connection_dir_is_encrypted(conn),
+                                       &msg);
+
+    if (!strcmpstart(url, "all")) {
+      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
+    } else if (smartlist_len(conn->spool) == 1) {
+      cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
+    }
 
-    if (!dirserv_have_any_serverdesc(conn->fingerprint_stack,
-                                     conn->dir_spool_src)) {
-      res = -1;
-      msg = "Not found";
+    size_t size_guess = 0;
+    int n_expired = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, &n_expired);
+
+    /* If we are the bridge authority and the descriptor is a bridge
+     * descriptor, remember that we served this descriptor for desc stats. */
+    /* XXXX it's a bit of a kludge to have this here. */
+    if (get_options()->BridgeAuthoritativeDir &&
+        source == DIR_SPOOL_SERVER_BY_FP) {
+      SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
+        const routerinfo_t *router =
+          router_get_by_id_digest((const char *)spooled->digest);
+        /* router can be NULL here when the bridge auth is asked for its own
+         * descriptor. */
+        if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
+          rep_hist_note_desc_served(router->cache_info.identity_digest);
+      } SMARTLIST_FOREACH_END(spooled);
     }
 
-    if (res < 0)
-      write_http_status_line(conn, 404, msg);
-    else {
-      dlen = dirserv_estimate_data_size(conn->fingerprint_stack,
-                                        1, compressed);
-      if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
+    if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
+      if (msg == NULL)
+        msg = "Not found";
+      write_short_http_response(conn, 404, msg);
+    } else {
+      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
-        write_http_status_line(conn, 503, "Directory busy, try again later");
-        conn->dir_spool_src = DIR_SPOOL_NONE;
+        write_short_http_response(conn, 503,
+                                  "Directory busy, try again later");
+        dir_conn_clear_spool(conn);
         goto done;
       }
-      write_http_response_header(conn, -1, compressed, cache_lifetime);
-      if (compressed)
-        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                        choose_compression_level(dlen));
+      write_http_response_header(conn, -1, compress_method, cache_lifetime);
+      if (compress_method != NO_METHOD)
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(size_guess));
+      clear_spool = 0;
       /* Prime the connection with some data. */
-      connection_dirserv_flushed_some(conn);
+      int initial_flush_result = connection_dirserv_flushed_some(conn);
+      tor_assert_nonfatal(initial_flush_result == 0);
     }
     goto done;
   }
  done:
- return 0;
+  if (clear_spool)
+    dir_conn_clear_spool(conn);
+  return 0;
 }
 
 /** Helper function for GET /tor/keys/...
@@ -3253,7 +4680,8 @@ static int
 handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
   const time_t if_modified_since = args->if_modified_since;
   {
     smartlist_t *certs = smartlist_new();
@@ -3298,41 +4726,47 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
       });
       smartlist_free(fp_sks);
     } else {
-      write_http_status_line(conn, 400, "Bad request");
+      write_short_http_response(conn, 400, "Bad request");
       goto keys_done;
     }
     if (!smartlist_len(certs)) {
-      write_http_status_line(conn, 404, "Not found");
+      write_short_http_response(conn, 404, "Not found");
       goto keys_done;
     }
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
       if (c->cache_info.published_on < if_modified_since)
         SMARTLIST_DEL_CURRENT(certs, c));
     if (!smartlist_len(certs)) {
-      write_http_status_line(conn, 304, "Not modified");
+      write_short_http_response(conn, 304, "Not modified");
       goto keys_done;
     }
     len = 0;
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) {
-      write_http_status_line(conn, 503, "Directory busy, try again later");
+    if (global_write_bucket_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len,
+                                2)) {
+      write_short_http_response(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
 
-    write_http_response_header(conn, compressed?-1:len, compressed, 60*60);
-    if (compressed) {
-      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                      choose_compression_level(len));
+    write_http_response_header(conn,
+                               compress_method != NO_METHOD ? -1 : len,
+                               compress_method,
+                               60*60);
+    if (compress_method != NO_METHOD) {
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                              choose_compression_level(len));
       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,
-                                         c->cache_info.signed_descriptor_len,
-                                         conn, 0));
-      connection_write_to_buf_zlib("", 0, conn, 1);
+            connection_buf_add_compress(
+                c->cache_info.signed_descriptor_body,
+                c->cache_info.signed_descriptor_len,
+                conn, 0));
+      connection_buf_add_compress("", 0, conn, 1);
     } else {
       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf(c->cache_info.signed_descriptor_body,
+            connection_buf_add(c->cache_info.signed_descriptor_body,
                                     c->cache_info.signed_descriptor_len,
                                     TO_CONN(conn)));
     }
@@ -3347,7 +4781,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
 /** Helper function for GET /tor/rendezvous2/
  */
 static int
-handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args)
+handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                            const get_handler_args_t *args)
 {
   const char *url = args->url;
   if (connection_dir_is_encrypted(conn)) {
@@ -3359,24 +4794,61 @@ handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args)
                safe_str(escaped(query)));
       switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
         case 1: /* valid */
-          write_http_response_header(conn, strlen(descp), 0, 0);
-          connection_write_to_buf(descp, strlen(descp), TO_CONN(conn));
+          write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
+          connection_buf_add(descp, strlen(descp), TO_CONN(conn));
           break;
         case 0: /* well-formed but not present */
-          write_http_status_line(conn, 404, "Not found");
+          write_short_http_response(conn, 404, "Not found");
           break;
         case -1: /* not well-formed */
-          write_http_status_line(conn, 400, "Bad request");
+          write_short_http_response(conn, 400, "Bad request");
           break;
       }
     } else { /* not well-formed */
-      write_http_status_line(conn, 400, "Bad request");
+      write_short_http_response(conn, 400, "Bad request");
     }
     goto done;
   } else {
     /* Not encrypted! */
-    write_http_status_line(conn, 404, "Not found");
+    write_short_http_response(conn, 404, "Not found");
+  }
+ done:
+  return 0;
+}
+
+/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+ */
+STATIC int
+handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  int retval;
+  const char *desc_str = NULL;
+  const char *pubkey_str = NULL;
+  const char *url = args->url;
+
+  /* Reject unencrypted dir connections */
+  if (!connection_dir_is_encrypted(conn)) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
   }
+
+  /* After the path prefix follows the base64 encoded blinded pubkey which we
+   * use to get the descriptor from the cache. Skip the prefix and get the
+   * pubkey. */
+  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
+  pubkey_str = url + strlen("/tor/hs/3/");
+  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
+                                  pubkey_str, &desc_str);
+  if (retval <= 0 || desc_str == NULL) {
+    write_short_http_response(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* Found requested descriptor! Pass it to this nice client. */
+  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
+  connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
+
  done:
   return 0;
 }
@@ -3404,7 +4876,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
     if (!header ||
         tor_memneq(digest,
                    options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
-      write_http_status_line(conn, 404, "Not found");
+      write_short_http_response(conn, 404, "Not found");
       tor_free(header);
       goto done;
     }
@@ -3413,8 +4885,8 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
     /* all happy now. send an answer. */
     status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
     size_t dlen = strlen(status);
-    write_http_response_header(conn, dlen, 0, 0);
-    connection_write_to_buf(status, dlen, TO_CONN(conn));
+    write_http_response_header(conn, dlen, NO_METHOD, 0);
+    connection_buf_add(status, dlen, TO_CONN(conn));
     tor_free(status);
     goto done;
   }
@@ -3430,20 +4902,104 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
   {
     const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
     size_t len = strlen(robots);
-    write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME);
-    connection_write_to_buf(robots, len, TO_CONN(conn));
+    write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
+    connection_buf_add(robots, len, TO_CONN(conn));
   }
   return 0;
 }
 
+/* Given the <b>url</b> from a POST request, try to extract the version number
+ * using the provided <b>prefix</b>. The version should be after the prefix and
+ * ending with the separator "/". For instance:
+ *      /tor/hs/3/publish
+ *
+ * On success, <b>end_pos</b> points to the position right after the version
+ * was found. On error, it is set to NULL.
+ *
+ * Return version on success else negative value. */
+STATIC int
+parse_hs_version_from_post(const char *url, const char *prefix,
+                           const char **end_pos)
+{
+  int ok;
+  unsigned long version;
+  const char *start;
+  char *end = NULL;
+
+  tor_assert(url);
+  tor_assert(prefix);
+  tor_assert(end_pos);
+
+  /* Check if the prefix does start the url. */
+  if (strcmpstart(url, prefix)) {
+    goto err;
+  }
+  /* Move pointer to the end of the prefix string. */
+  start = url + strlen(prefix);
+  /* Try this to be the HS version and if we are still at the separator, next
+   * will be move to the right value. */
+  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
+  if (!ok) {
+    goto err;
+  }
+
+  *end_pos = end;
+  return (int) version;
+ err:
+  *end_pos = NULL;
+  return -1;
+}
+
+/* Handle the POST request for a hidden service descripror. The request is in
+ * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
+ * else return 400 indicating a bad request. */
+STATIC int
+handle_post_hs_descriptor(const char *url, const char *body)
+{
+  int version;
+  const char *end_pos;
+
+  tor_assert(url);
+  tor_assert(body);
+
+  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
+  if (version < 0) {
+    goto err;
+  }
+
+  /* We have a valid version number, now make sure it's a publish request. Use
+   * the end position just after the version and check for the command. */
+  if (strcmpstart(end_pos, "/publish")) {
+    goto err;
+  }
+
+  switch (version) {
+  case HS_VERSION_THREE:
+    if (hs_cache_store_as_dir(body) < 0) {
+      goto err;
+    }
+    log_info(LD_REND, "Publish request for HS descriptor handled "
+                      "successfully.");
+    break;
+  default:
+    /* Unsupported version, return a bad request. */
+    goto err;
+  }
+
+  return 200;
+ err:
+  /* Bad request. */
+  return 400;
+}
+
 /** Helper function: called when a dirserver gets a complete HTTP POST
  * request.  Look for an uploaded server descriptor or rendezvous
  * service descriptor.  On finding one, process it and write a
  * response into conn-\>outbuf.  If the request is unrecognized, send a
  * 400.  Always return 0. */
-static int
-directory_handle_command_post(dir_connection_t *conn, const char *headers,
-                              const char *body, size_t body_len)
+MOCK_IMPL(STATIC int,
+directory_handle_command_post,(dir_connection_t *conn, const char *headers,
+                               const char *body, size_t body_len))
 {
   char *url = NULL;
   const or_options_t *options = get_options();
@@ -3455,12 +5011,12 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
   if (!public_server_mode(options)) {
     log_info(LD_DIR, "Rejected dir post request from %s "
              "since we're not a public relay.", conn->base_.address);
-    write_http_status_line(conn, 503, "Not acting as a public relay");
+    write_short_http_response(conn, 503, "Not acting as a public relay");
     goto done;
   }
 
   if (parse_http_url(headers, &url) < 0) {
-    write_http_status_line(conn, 400, "Bad request");
+    write_short_http_response(conn, 400, "Bad request");
     return 0;
   }
   log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
@@ -3469,26 +5025,41 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
   if (connection_dir_is_encrypted(conn) &&
       !strcmpstart(url,"/tor/rendezvous2/publish")) {
     if (rend_cache_store_v2_desc_as_dir(body) < 0) {
-      log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.",
+      log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
                (int)body_len, conn->base_.address);
-      write_http_status_line(conn, 400,
+      write_short_http_response(conn, 400,
                              "Invalid v2 service descriptor rejected");
     } else {
-      write_http_status_line(conn, 200, "Service descriptor (v2) stored");
+      write_short_http_response(conn, 200, "Service descriptor (v2) stored");
       log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
     }
     goto done;
   }
 
+  /* Handle HS descriptor publish request. */
+  /* XXX: This should be disabled with a consensus param until we want to
+   * the prop224 be deployed and thus use. */
+  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+    const char *msg = "HS descriptor stored successfully.";
+
+    /* We most probably have a publish request for an HS descriptor. */
+    int code = handle_post_hs_descriptor(url, body);
+    if (code != 200) {
+      msg = "Invalid HS descriptor. Rejected.";
+    }
+    write_short_http_response(conn, code, msg);
+    goto done;
+  }
+
   if (!authdir_mode(options)) {
     /* we just provide cached directories; we don't want to
      * receive anything. */
-    write_http_status_line(conn, 400, "Nonauthoritative directory does not "
+    write_short_http_response(conn, 400, "Nonauthoritative directory does not "
                            "accept posted server descriptors");
     goto done;
   }
 
-  if (authdir_mode_handles_descs(options, -1) &&
+  if (authdir_mode(options) &&
       !strcmp(url,"/tor/")) { /* server descriptor post */
     const char *msg = "[None]";
     uint8_t purpose = authdir_mode_bridge(options) ?
@@ -3497,15 +5068,8 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
                                              conn->base_.address, &msg);
     tor_assert(msg);
 
-    if (r == ROUTER_ADDED_NOTIFY_GENERATOR) {
-      /* Accepted with a message. */
-      log_info(LD_DIRSERV,
-               "Problematic router descriptor or extra-info from %s "
-               "(\"%s\").",
-               conn->base_.address, msg);
-      write_http_status_line(conn, 400, msg);
-    } else if (r == ROUTER_ADDED_SUCCESSFULLY) {
-      write_http_status_line(conn, 200, msg);
+    if (r == ROUTER_ADDED_SUCCESSFULLY) {
+      write_short_http_response(conn, 200, msg);
     } else if (WRA_WAS_OUTDATED(r)) {
       write_http_response_header_impl(conn, -1, NULL, NULL,
                                       "X-Descriptor-Not-New: Yes\r\n", -1);
@@ -3514,7 +5078,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
                "Rejected router descriptor or extra-info from %s "
                "(\"%s\").",
                conn->base_.address, msg);
-      write_http_status_line(conn, 400, msg);
+      write_short_http_response(conn, 400, msg);
     }
     goto done;
   }
@@ -3524,12 +5088,12 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
     const char *msg = "OK";
     int status;
     if (dirvote_add_vote(body, &msg, &status)) {
-      write_http_status_line(conn, status, "Vote stored");
+      write_short_http_response(conn, status, "Vote stored");
     } else {
       tor_assert(msg);
       log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
                conn->base_.address, msg);
-      write_http_status_line(conn, status, msg);
+      write_short_http_response(conn, status, msg);
     }
     goto done;
   }
@@ -3538,17 +5102,18 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
       !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
     const char *msg = NULL;
     if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
-      write_http_status_line(conn, 200, msg?msg:"Signatures stored");
+      write_short_http_response(conn, 200, msg?msg:"Signatures stored");
     } else {
       log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
                conn->base_.address, msg?msg:"???");
-      write_http_status_line(conn, 400, msg?msg:"Unable to store signatures");
+      write_short_http_response(conn, 400,
+                                msg?msg:"Unable to store signatures");
     }
     goto done;
   }
 
   /* we didn't recognize the url */
-  write_http_status_line(conn, 404, "Not found");
+  write_short_http_response(conn, 404, "Not found");
 
  done:
   tor_free(url);
@@ -3560,7 +5125,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
  * from the inbuf, try to process it; otherwise, leave it on the
  * buffer.  Return a 0 on success, or -1 on error.
  */
-static int
+STATIC int
 directory_handle_command(dir_connection_t *conn)
 {
   char *headers=NULL, *body=NULL;
@@ -3614,6 +5179,9 @@ connection_dir_finished_flushing(dir_connection_t *conn)
   tor_assert(conn);
   tor_assert(conn->base_.type == CONN_TYPE_DIR);
 
+  if (conn->base_.marked_for_close)
+    return 0;
+
   /* Note that we have finished writing the directory response. For direct
    * connections this means we're done; for tunneled connections it's only
    * an intermediate step. */
@@ -3631,7 +5199,7 @@ connection_dir_finished_flushing(dir_connection_t *conn)
       conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
       return 0;
     case DIR_CONN_STATE_SERVER_WRITING:
-      if (conn->dir_spool_src != DIR_SPOOL_NONE) {
+      if (conn->spool) {
         log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
         connection_mark_for_close(TO_CONN(conn));
       } else {
@@ -3697,154 +5265,141 @@ connection_dir_finished_connecting(dir_connection_t *conn)
 
 /** Decide which download schedule we want to use based on descriptor type
  * in <b>dls</b> and <b>options</b>.
- * Then return a list of int pointers defining download delays in seconds.
+ *
+ * Then, return the initial delay for that download schedule, in seconds.
+ *
  * Helper function for download_status_increment_failure(),
  * download_status_reset(), and download_status_increment_attempt(). */
-STATIC const smartlist_t *
-find_dl_schedule(download_status_t *dls, const or_options_t *options)
+STATIC int
+find_dl_min_delay(const download_status_t *dls, const or_options_t *options)
 {
+  tor_assert(dls);
+  tor_assert(options);
+
   switch (dls->schedule) {
     case DL_SCHED_GENERIC:
       /* Any other directory document */
       if (dir_server_mode(options)) {
         /* A directory authority or directory mirror */
-        return options->TestingServerDownloadSchedule;
+        return options->TestingServerDownloadInitialDelay;
       } else {
-        return options->TestingClientDownloadSchedule;
+        return options->TestingClientDownloadInitialDelay;
       }
     case DL_SCHED_CONSENSUS:
       if (!networkstatus_consensus_can_use_multiple_directories(options)) {
         /* A public relay */
-        return options->TestingServerConsensusDownloadSchedule;
+        return options->TestingServerConsensusDownloadInitialDelay;
       } else {
         /* A client or bridge */
         if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
           /* During bootstrapping */
           if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
             /* A bootstrapping client without extra fallback directories */
-            return
-             options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule;
+            return options->
+              ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
           } else if (dls->want_authority) {
             /* A bootstrapping client with extra fallback directories, but
              * connecting to an authority */
             return
-             options->ClientBootstrapConsensusAuthorityDownloadSchedule;
+             options->ClientBootstrapConsensusAuthorityDownloadInitialDelay;
           } else {
             /* A bootstrapping client connecting to extra fallback directories
              */
             return
-              options->ClientBootstrapConsensusFallbackDownloadSchedule;
+              options->ClientBootstrapConsensusFallbackDownloadInitialDelay;
           }
         } else {
           /* A client with a reasonably live consensus, with or without
            * certificates */
-          return options->TestingClientConsensusDownloadSchedule;
+          return options->TestingClientConsensusDownloadInitialDelay;
         }
       }
     case DL_SCHED_BRIDGE:
-      return options->TestingBridgeDownloadSchedule;
+      if (options->UseBridges && num_bridges_usable(0) > 0) {
+        /* A bridge client that is sure that one or more of its bridges are
+         * running can afford to wait longer to update bridge descriptors. */
+        return options->TestingBridgeDownloadInitialDelay;
+      } else {
+        /* A bridge client which might have no running bridges, must try to
+         * get bridge descriptors straight away. */
+        return options->TestingBridgeBootstrapDownloadInitialDelay;
+      }
     default:
       tor_assert(0);
   }
 
   /* Impossible, but gcc will fail with -Werror without a `return`. */
-  return NULL;
+  return 0;
 }
 
-/** Decide which minimum and maximum delay step we want to use based on
- * descriptor type in <b>dls</b> and <b>options</b>.
- * Helper function for download_status_schedule_get_delay(). */
+/** As next_random_exponential_delay() below, but does not compute a random
+ * value. Instead, compute the range of values that
+ * next_random_exponential_delay() should use when computing its random value.
+ * Store the low bound into *<b>low_bound_out</b>, and the high bound into
+ * *<b>high_bound_out</b>.  Guarantees that the low bound is strictly less
+ * than the high bound. */
 STATIC void
-find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options,
-                          int *min, int *max)
+next_random_exponential_delay_range(int *low_bound_out,
+                                    int *high_bound_out,
+                                    int delay,
+                                    int base_delay)
 {
-  tor_assert(dls);
-  tor_assert(options);
-  tor_assert(min);
-  tor_assert(max);
-
-  /*
-   * For now, just use the existing schedule config stuff and pick the
-   * first/last entries off to get min/max delay for backoff purposes
-   */
-  const smartlist_t *schedule = find_dl_schedule(dls, options);
-  tor_assert(schedule != NULL && smartlist_len(schedule) >= 2);
-  *min = *((int *)(smartlist_get(schedule, 0)));
-  if (dls->backoff == DL_SCHED_DETERMINISTIC)
-    *max = *((int *)((smartlist_get(schedule, smartlist_len(schedule) - 1))));
-  else
-    *max = INT_MAX;
+  // This is the "decorrelated jitter" approach, from
+  //    https://www.awsarchitectureblog.com/2015/03/backoff.html
+  // The formula is
+  //    sleep = min(cap, random_between(base, sleep * 3))
+
+  const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
+  *low_bound_out = base_delay;
+  if (delay_times_3 > base_delay) {
+    *high_bound_out = delay_times_3;
+  } else {
+    *high_bound_out = base_delay+1;
+  }
 }
 
-/** Advance one delay step.  The algorithm is to use the previous delay to
- * compute an increment, we construct a value uniformly at random between
- * delay and MAX(delay*2,delay+1).  We then clamp that value to be no larger
- * than max_delay, and return it.
+/** Advance one delay step.  The algorithm will generate a random delay,
+ * such that each failure is possibly (random) longer than the ones before.
  *
- * Requires that delay is less than INT_MAX, and delay is in [0,max_delay].
+ * We then clamp that value to be no larger than max_delay, and return it.
+ *
+ * The <b>base_delay</b> parameter is lowest possible delay time (can't be
+ * zero); the <b>backoff_position</b> parameter is the number of times we've
+ * generated a delay; and the <b>delay</b> argument is the most recently used
+ * delay.
  */
 STATIC int
-next_random_exponential_delay(int delay, int max_delay)
+next_random_exponential_delay(int delay,
+                              int base_delay)
 {
   /* Check preconditions */
-  if (BUG(max_delay < 0))
-    max_delay = 0;
-  if (BUG(delay > max_delay))
-    delay = max_delay;
-  if (delay == INT_MAX)
-    return INT_MAX; /* prevent overflow */
   if (BUG(delay < 0))
     delay = 0;
 
-  /* How much are we willing to add to the delay? */
-  int max_increment;
-  int multiplier = 3; /* no more than quadruple the previous delay */
-  if (get_options()->TestingTorNetwork) {
-    /* Decrease the multiplier in testing networks. This reduces the variance,
-     * so that bootstrap is more reliable. */
-    multiplier = 2; /* no more than triple the previous delay */
-  }
-
-  if (delay && delay < (INT_MAX-1) / multiplier) {
-    max_increment = delay * multiplier;
-  } else if (delay) {
-    max_increment = INT_MAX-1;
-  } else {
-    max_increment = 1;
-  }
+  if (base_delay < 1)
+    base_delay = 1;
 
-  if (BUG(max_increment < 1))
-    max_increment = 1;
+  int low_bound=0, high_bound=INT_MAX;
 
-  /* the + 1 here is so that we always wait longer than last time. */
-  int increment = crypto_rand_int(max_increment)+1;
+  next_random_exponential_delay_range(&low_bound, &high_bound,
+                                      delay, base_delay);
 
-  if (increment < max_delay - delay)
-    return delay + increment;
-  else
-    return max_delay;
+  return crypto_rand_int_range(low_bound, high_bound);
 }
 
-/** Find the current delay for dls based on schedule or min_delay/
- * max_delay if we're using exponential backoff.  If dls->backoff is
- * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <=
- * INT_MAX, but schedule may be set to NULL; otherwise schedule is required.
+/** Find the current delay for dls based on min_delay.
+ *
  * This function sets dls->next_attempt_at based on now, and returns the delay.
  * Helper for download_status_increment_failure and
  * download_status_increment_attempt. */
 STATIC int
 download_status_schedule_get_delay(download_status_t *dls,
-                                   const smartlist_t *schedule,
-                                   int min_delay, int max_delay,
+                                   int min_delay,
                                    time_t now)
 {
   tor_assert(dls);
-  /* We don't need a schedule if we're using random exponential backoff */
-  tor_assert(dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL ||
-             schedule != NULL);
   /* If we're using random exponential backoff, we do need min/max delay */
-  tor_assert(dls->backoff != DL_SCHED_RANDOM_EXPONENTIAL ||
-             (min_delay >= 0 && max_delay >= min_delay));
+  tor_assert(min_delay >= 0);
 
   int delay = INT_MAX;
   uint8_t dls_schedule_position = (dls->increment_on
@@ -3852,42 +5407,32 @@ download_status_schedule_get_delay(download_status_t *dls,
                                    ? dls->n_download_attempts
                                    : dls->n_download_failures);
 
-  if (dls->backoff == DL_SCHED_DETERMINISTIC) {
-    if (dls_schedule_position < smartlist_len(schedule))
-      delay = *(int *)smartlist_get(schedule, dls_schedule_position);
-    else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD)
-      delay = INT_MAX;
-    else
-      delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1);
-  } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) {
-    /* Check if we missed a reset somehow */
-    if (dls->last_backoff_position > dls_schedule_position) {
-      dls->last_backoff_position = 0;
-      dls->last_delay_used = 0;
-    }
+  /* Check if we missed a reset somehow */
+  IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
+    dls->last_backoff_position = 0;
+    dls->last_delay_used = 0;
+  }
 
-    if (dls_schedule_position > 0) {
-      delay = dls->last_delay_used;
+  if (dls_schedule_position > 0) {
+    delay = dls->last_delay_used;
 
-      while (dls->last_backoff_position < dls_schedule_position) {
-        /* Do one increment step */
-        delay = next_random_exponential_delay(delay, max_delay);
-        /* Update our position */
-        ++(dls->last_backoff_position);
-      }
-    } else {
-      /* If we're just starting out, use the minimum delay */
-      delay = min_delay;
+    while (dls->last_backoff_position < dls_schedule_position) {
+      /* Do one increment step */
+      delay = next_random_exponential_delay(delay, min_delay);
+      /* Update our position */
+      ++(dls->last_backoff_position);
     }
+  } else {
+    /* If we're just starting out, use the minimum delay */
+    delay = min_delay;
+  }
 
-    /* Clamp it within min/max if we have them */
-    if (min_delay >= 0 && delay < min_delay) delay = min_delay;
-    if (max_delay != INT_MAX && delay > max_delay) delay = max_delay;
+  /* Clamp it within min/max if we have them */
+  if (min_delay >= 0 && delay < min_delay) delay = min_delay;
 
-    /* Store it for next time */
-    dls->last_backoff_position = dls_schedule_position;
-    dls->last_delay_used = delay;
-  }
+  /* Store it for next time */
+  dls->last_backoff_position = dls_schedule_position;
+  dls->last_delay_used = delay;
 
   /* A negative delay makes no sense. Knowing that delay is
    * non-negative allows us to safely do the wrapping check below. */
@@ -3950,10 +5495,15 @@ download_status_increment_failure(download_status_t *dls, int status_code,
   (void) status_code; // XXXX no longer used.
   (void) server; // XXXX no longer used.
   int increment = -1;
-  int min_delay = 0, max_delay = INT_MAX;
+  int min_delay = 0;
 
   tor_assert(dls);
 
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
   /* count the failure */
   if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
     ++dls->n_download_failures;
@@ -3970,22 +5520,22 @@ download_status_increment_failure(download_status_t *dls, int status_code,
 
     /* only return a failure retry time if this schedule increments on failures
      */
-    const smartlist_t *schedule = find_dl_schedule(dls, get_options());
-    find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay);
-    increment = download_status_schedule_get_delay(dls, schedule,
-                                                   min_delay, max_delay, now);
+    min_delay = find_dl_min_delay(dls, get_options());
+    increment = download_status_schedule_get_delay(dls, min_delay, now);
   }
 
   download_status_log_helper(item, !dls->increment_on, "failed",
                              "concurrently", dls->n_download_failures,
-                             increment, dls->next_attempt_at, now);
+                             increment,
+                             download_status_get_next_attempt_at(dls),
+                             now);
 
   if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
     /* stop this schedule retrying on failure, it will launch concurrent
      * connections instead */
     return TIME_MAX;
   } else {
-    return dls->next_attempt_at;
+    return download_status_get_next_attempt_at(dls);
   }
 }
 
@@ -4002,10 +5552,15 @@ download_status_increment_attempt(download_status_t *dls, const char *item,
                                   time_t now)
 {
   int delay = -1;
-  int min_delay = 0, max_delay = INT_MAX;
+  int min_delay = 0;
 
   tor_assert(dls);
 
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
+  }
+
   if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
     /* this schedule should retry on failure, and not launch any concurrent
      attempts */
@@ -4017,16 +5572,23 @@ download_status_increment_attempt(download_status_t *dls, const char *item,
   if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
     ++dls->n_download_attempts;
 
-  const smartlist_t *schedule = find_dl_schedule(dls, get_options());
-  find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay);
-  delay = download_status_schedule_get_delay(dls, schedule,
-                                             min_delay, max_delay, now);
+  min_delay = find_dl_min_delay(dls, get_options());
+  delay = download_status_schedule_get_delay(dls, min_delay, now);
 
   download_status_log_helper(item, dls->increment_on, "attempted",
                              "on failure", dls->n_download_attempts,
-                             delay, dls->next_attempt_at, now);
+                             delay, download_status_get_next_attempt_at(dls),
+                             now);
 
-  return dls->next_attempt_at;
+  return download_status_get_next_attempt_at(dls);
+}
+
+static time_t
+download_status_get_initial_delay_from_now(const download_status_t *dls)
+{
+  /* We use constant initial delays, even in exponential backoff
+   * schedules. */
+  return time(NULL) + find_dl_min_delay(dls, get_options());
 }
 
 /** Reset <b>dls</b> so that it will be considered downloadable
@@ -4038,8 +5600,8 @@ download_status_increment_attempt(download_status_t *dls, const char *item,
  * (We find the zeroth element of the download schedule, and set
  * next_attempt_at to be the appropriate offset from 'now'. In most
  * cases this means setting it to 'now', so the item will be immediately
- * downloadable; in the case of bridge descriptors, the zeroth element
- * is an hour from now.) */
+ * downloadable; when using authorities with fallbacks, there is a few seconds'
+ * delay.) */
 void
 download_status_reset(download_status_t *dls)
 {
@@ -4047,11 +5609,9 @@ download_status_reset(download_status_t *dls)
       || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
     return; /* Don't reset this. */
 
-  const smartlist_t *schedule = find_dl_schedule(dls, get_options());
-
   dls->n_download_failures = 0;
   dls->n_download_attempts = 0;
-  dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0);
+  dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
   dls->last_backoff_position = 0;
   dls->last_delay_used = 0;
   /* Don't reset dls->want_authority or dls->increment_on */
@@ -4078,6 +5638,12 @@ download_status_get_n_attempts(const download_status_t *dls)
 time_t
 download_status_get_next_attempt_at(const download_status_t *dls)
 {
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    /* so give the answer we would have given if it had been */
+    return download_status_get_initial_delay_from_now(dls);
+  }
+
   return dls->next_attempt_at;
 }
 
@@ -4115,8 +5681,7 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
     } else {
       dls = router_get_dl_status_by_descriptor_digest(digest);
     }
-    if (!dls || dls->n_download_failures >=
-                get_options()->TestingDescriptorMaxDownloadTries)
+    if (!dls)
       continue;
     download_status_increment_failure(dls, status_code, cp, server, now);
   } SMARTLIST_FOREACH_END(cp);
@@ -4125,13 +5690,14 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
    * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
 }
 
-/** Called when a connection to download microdescriptors has failed in whole
- * or in part. <b>failed</b> is a list of every microdesc digest we didn't
- * get. <b>status_code</b> is the http status code we received. Reschedule the
- * microdesc downloads as appropriate. */
+/** Called when a connection to download microdescriptors from relay with
+ * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
+ * of every microdesc digest we didn't get. <b>status_code</b> is the http
+ * status code we received. Reschedule the microdesc downloads as
+ * appropriate. */
 static void
 dir_microdesc_download_failed(smartlist_t *failed,
-                              int status_code)
+                              int status_code, const char *dir_id)
 {
   networkstatus_t *consensus
     = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
@@ -4142,17 +5708,22 @@ dir_microdesc_download_failed(smartlist_t *failed,
 
   if (! consensus)
     return;
+
+  /* We failed to fetch a microdescriptor from 'dir_id', note it down
+   * so that we don't try the same relay next time... */
+  microdesc_note_outdated_dirserver(dir_id);
+
   SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
     rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
     if (!rs)
       continue;
     dls = &rs->dl_status;
-    if (dls->n_download_failures >=
-        get_options()->TestingMicrodescMaxDownloadTries)
-      continue;
-    {
+
+    { /* Increment the failure count for this md fetch */
       char buf[BASE64_DIGEST256_LEN+1];
       digest256_to_base64(buf, d);
+      log_info(LD_DIR, "Failed to download md %s from %s",
+               buf, hex_str(dir_id, DIGEST_LEN));
       download_status_increment_failure(dls, status_code, buf,
                                         server, now);
     }
@@ -4314,3 +5885,33 @@ dir_split_resource_into_fingerprints(const char *resource,
   return 0;
 }
 
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}
diff --git a/src/or/directory.h b/src/or/directory.h
index 629b3ead90..5f5ff7eca6 100644
--- a/src/or/directory.h
+++ b/src/or/directory.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,8 @@
 #ifndef TOR_DIRECTORY_H
 #define TOR_DIRECTORY_H
 
+#include "hs_ident.h"
+
 int directories_have_accepted_server_descriptor(void);
 void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                                   dirinfo_type_t type, const char *payload,
@@ -41,43 +43,62 @@ typedef enum {
 
 int directory_must_use_begindir(const or_options_t *options);
 
-MOCK_DECL(void, directory_initiate_command_routerstatus,
-                (const routerstatus_t *status,
-                 uint8_t dir_purpose,
-                 uint8_t router_purpose,
-                 dir_indirection_t indirection,
-                 const char *resource,
-                 const char *payload,
-                 size_t payload_len,
-                 time_t if_modified_since));
-
-void directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
-                                                  uint8_t dir_purpose,
-                                                  uint8_t router_purpose,
-                                                 dir_indirection_t indirection,
-                                                  const char *resource,
-                                                  const char *payload,
-                                                  size_t payload_len,
-                                                  time_t if_modified_since,
-                                                const rend_data_t *rend_query);
+/**
+ * A directory_request_t describes the information about a directory request
+ * at the client side.  It describes what we're going to ask for, which
+ * directory we're going to ask for it, how we're going to contact that
+ * directory, and (in some cases) what to do with it when we're done.
+ */
+typedef struct directory_request_t directory_request_t;
+directory_request_t *directory_request_new(uint8_t dir_purpose);
+void directory_request_free_(directory_request_t *req);
+#define directory_request_free(req) \
+  FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
+void directory_request_set_or_addr_port(directory_request_t *req,
+                                        const tor_addr_port_t *p);
+void directory_request_set_dir_addr_port(directory_request_t *req,
+                                         const tor_addr_port_t *p);
+void directory_request_set_directory_id_digest(directory_request_t *req,
+                                               const char *digest);
+void directory_request_set_guard_state(directory_request_t *req,
+                                       struct circuit_guard_state_t *state);
+void directory_request_set_router_purpose(directory_request_t *req,
+                                          uint8_t router_purpose);
+void directory_request_set_indirection(directory_request_t *req,
+                                       dir_indirection_t indirection);
+void directory_request_set_resource(directory_request_t *req,
+                                    const char *resource);
+void directory_request_set_payload(directory_request_t *req,
+                                   const char *payload,
+                                   size_t payload_len);
+void directory_request_set_if_modified_since(directory_request_t *req,
+                                             time_t if_modified_since);
+void directory_request_set_rend_query(directory_request_t *req,
+                                      const rend_data_t *query);
+void directory_request_upload_set_hs_ident(directory_request_t *req,
+                                           const hs_ident_dir_conn_t *ident);
+void directory_request_fetch_set_hs_ident(directory_request_t *req,
+                                          const hs_ident_dir_conn_t *ident);
+
+void directory_request_set_routerstatus(directory_request_t *req,
+                                        const routerstatus_t *rs);
+void directory_request_add_header(directory_request_t *req,
+                                  const char *key,
+                                  const char *val);
+MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
 
 int parse_http_response(const char *headers, int *code, time_t *date,
                         compress_method_t *compression, char **response);
+int parse_http_command(const char *headers,
+                       char **command_out, char **url_out);
+char *http_get_header(const char *headers, const char *which);
 
-int connection_dir_is_encrypted(dir_connection_t *conn);
+int connection_dir_is_encrypted(const dir_connection_t *conn);
 int connection_dir_reached_eof(dir_connection_t *conn);
 int connection_dir_process_inbuf(dir_connection_t *conn);
 int connection_dir_finished_flushing(dir_connection_t *conn);
 int connection_dir_finished_connecting(dir_connection_t *conn);
 void connection_dir_about_to_close(dir_connection_t *dir_conn);
-void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
-                                const tor_addr_t *dir_addr, uint16_t dir_port,
-                                const char *digest,
-                                uint8_t dir_purpose, uint8_t router_purpose,
-                                dir_indirection_t indirection,
-                                const char *resource,
-                                const char *payload, size_t payload_len,
-                                time_t if_modified_since);
 
 #define DSR_HEX       (1<<0)
 #define DSR_BASE64    (1<<1)
@@ -86,7 +107,12 @@ void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
 int dir_split_resource_into_fingerprints(const char *resource,
                                      smartlist_t *fp_out, int *compressed_out,
                                      int flags);
-
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
 int dir_split_resource_into_fingerprint_pairs(const char *res,
                                               smartlist_t *pairs_out);
 char *directory_dump_request_log(void);
@@ -106,23 +132,20 @@ time_t download_status_increment_attempt(download_status_t *dls,
                                     time(NULL))
 
 void download_status_reset(download_status_t *dls);
-static int download_status_is_ready(download_status_t *dls, time_t now,
-                                    int max_failures);
+static int download_status_is_ready(download_status_t *dls, time_t now);
+time_t download_status_get_next_attempt_at(const download_status_t *dls);
+
 /** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
  * ready to get its download reattempted. */
 static inline int
-download_status_is_ready(download_status_t *dls, time_t now,
-                         int max_failures)
+download_status_is_ready(download_status_t *dls, time_t now)
 {
-  if (dls->backoff == DL_SCHED_DETERMINISTIC) {
-    /* Deterministic schedules can hit an endpoint; exponential backoff
-     * schedules just wait longer and longer. */
-    int under_failure_limit = (dls->n_download_failures <= max_failures
-                               && dls->n_download_attempts <= max_failures);
-    if (!under_failure_limit)
-      return 0;
+  /* dls wasn't reset before it was used */
+  if (dls->next_attempt_at == 0) {
+    download_status_reset(dls);
   }
-  return dls->next_attempt_at <= now;
+
+  return download_status_get_next_attempt_at(dls) <= now;
 }
 
 static void download_status_mark_impossible(download_status_t *dl);
@@ -136,40 +159,132 @@ download_status_mark_impossible(download_status_t *dl)
 
 int download_status_get_n_failures(const download_status_t *dls);
 int download_status_get_n_attempts(const download_status_t *dls);
-time_t download_status_get_next_attempt_at(const download_status_t *dls);
 
-/* Yes, these two functions are confusingly similar.
- * Let's sort that out in #20077. */
-int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose);
-int is_sensitive_dir_purpose(uint8_t dir_purpose);
+int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                            const char *resource);
+
+#ifdef DIRECTORY_PRIVATE
+
+/** A structure to hold arguments passed into each directory response
+ * handler */
+typedef struct response_handler_args_t {
+  int status_code;
+  const char *reason;
+  const char *body;
+  size_t body_len;
+  const char *headers;
+} response_handler_args_t;
+
+struct directory_request_t {
+  /**
+   * These fields specify which directory we're contacting.  Routerstatus,
+   * if present, overrides the other fields.
+   *
+   * @{ */
+  tor_addr_port_t or_addr_port;
+  tor_addr_port_t dir_addr_port;
+  char digest[DIGEST_LEN];
+
+  const routerstatus_t *routerstatus;
+  /** @} */
+  /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
+   * kind of operation we'll be doing (upload/download), and of what kind
+   * of document. */
+  uint8_t dir_purpose;
+  /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
+   * and extrainfo docs.  */
+  uint8_t router_purpose;
+  /** Enum: determines whether to anonymize, and whether to use dirport or
+   * orport. */
+  dir_indirection_t indirection;
+  /** Alias to the variable part of the URL for this request */
+  const char *resource;
+  /** Alias to the payload to upload (if any) */
+  const char *payload;
+  /** Number of bytes to upload from payload</b> */
+  size_t payload_len;
+  /** Value to send in an if-modified-since header, or 0 for none. */
+  time_t if_modified_since;
+  /** Hidden-service-specific information v2. */
+  const rend_data_t *rend_query;
+  /** Extra headers to append to the request */
+  config_line_t *additional_headers;
+  /** Hidden-service-specific information for v3+. */
+  const hs_ident_dir_conn_t *hs_ident;
+  /** Used internally to directory.c: gets informed when the attempt to
+   * connect to the directory succeeds or fails, if that attempt bears on the
+   * directory's usability as a directory guard. */
+  struct circuit_guard_state_t *guard_state;
+};
+
+struct get_handler_args_t;
+STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                                       const struct get_handler_args_t *args);
+STATIC int directory_handle_command(dir_connection_t *conn);
+STATIC char *accept_encoding_header(void);
+STATIC int allowed_anonymous_connection_compression_method(compress_method_t);
+STATIC void warn_disallowed_anonymous_compression_method(compress_method_t);
+
+STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+                                          const response_handler_args_t *args);
+STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
+                                 const response_handler_args_t *args);
+
+STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
+                                         const response_handler_args_t *args);
+
+#endif /* defined(DIRECTORY_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
-/* Used only by directory.c and test_dir.c */
+/* Used only by test_dir.c and test_hs_cache.c */
 
 STATIC int parse_http_url(const char *headers, char **url);
 STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
                                      const char *resource);
-STATIC int directory_handle_command_get(dir_connection_t *conn,
-                                        const char *headers,
-                                        const char *req_body,
-                                        size_t req_body_len);
+MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
+                                                    const char *headers,
+                                                    const char *req_body,
+                                                    size_t req_body_len));
+MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
+                                                     const char *headers,
+                                                     const char *body,
+                                                     size_t body_len));
 STATIC int download_status_schedule_get_delay(download_status_t *dls,
-                                              const smartlist_t *schedule,
-                                              int min_delay, int max_delay,
+                                              int min_delay,
                                               time_t now);
 
+STATIC int handle_post_hs_descriptor(const char *url, const char *body);
+
 STATIC char* authdir_type_to_string(dirinfo_type_t auth);
 STATIC const char * dir_conn_purpose_to_string(int purpose);
 STATIC int should_use_directory_guards(const or_options_t *options);
-STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes);
-STATIC const smartlist_t *find_dl_schedule(download_status_t *dls,
-                                           const or_options_t *options);
-STATIC void find_dl_min_and_max_delay(download_status_t *dls,
-                                      const or_options_t *options,
-                                      int *min, int *max);
-STATIC int next_random_exponential_delay(int delay, int max_delay);
+STATIC compression_level_t choose_compression_level(ssize_t n_bytes);
+STATIC int find_dl_min_delay(const download_status_t *dls,
+                             const or_options_t *options);
+
+STATIC int next_random_exponential_delay(int delay,
+                                         int base_delay);
+
+STATIC void next_random_exponential_delay_range(int *low_bound_out,
+                                                int *high_bound_out,
+                                                int delay,
+                                                int base_delay);
+
+STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
+                                      const char **end_pos);
+
+STATIC unsigned parse_accept_encoding_header(const char *h);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#if defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE)
+/* Used only by directory.c and test_dir.c */
+
+/* no more than quadruple the previous delay (multiplier + 1) */
+#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
+/* no more than triple the previous delay */
+#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
 
-#endif
+#endif /* defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_DIRECTORY_H) */
 
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index 177009208d..ae1975f166 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRSERV_PRIVATE
@@ -13,10 +13,11 @@
 #include "command.h"
 #include "connection.h"
 #include "connection_or.h"
+#include "conscache.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "hibernate.h"
 #include "keypin.h"
 #include "main.h"
@@ -31,6 +32,9 @@
 #include "routerparse.h"
 #include "routerset.h"
 #include "torcert.h"
+#include "voting_schedule.h"
+
+#include "dirauth/dirvote.h"
 
 /**
  * \file dirserv.c
@@ -41,7 +45,7 @@
  * the directory authority functionality.  The directory.c module delegates
  * here in order to handle incoming requests from clients, via
  * connection_dirserv_flushed_some() and its kin.  In order to save RAM, this
- * module is reponsible for spooling directory objects (in whole or in part)
+ * module is responsible for spooling directory objects (in whole or in part)
  * onto buf_t instances, and then closing the dir_connection_t once the
  * objects are totally flushed.
  *
@@ -80,14 +84,22 @@ dirserv_get_status_impl(const char *fp, const char *nickname,
                         int severity);
 static void clear_cached_dir(cached_dir_t *d);
 static const signed_descriptor_t *get_signed_descriptor_by_fp(
-                                                        const char *fp,
-                                                        int extrainfo,
-                                                        time_t publish_cutoff);
+                                                        const uint8_t *fp,
+                                                        int extrainfo);
 static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
                                                 const char **msg);
-static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri);
 static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri);
 
+static int spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                                        int conn_is_encrypted,
+                                        const uint8_t **body_out,
+                                        size_t *size_out,
+                                        time_t *published_out);
+static cached_dir_t *spooled_resource_lookup_cached_dir(
+                                   const spooled_resource_t *spooled,
+                                   time_t *published_out);
+static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
+
 /************** Fingerprint handling code ************/
 
 /*                 1  Historically used to indicate Named */
@@ -247,11 +259,12 @@ dirserv_load_fingerprint_file(void)
  * identity to stop doing so.  This is going to be essential for good identity
  * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could
  * just sign fake descriptors missing the Ed25519 key.  But we won't actually
- * be able to prevent that kind of thing until we're confident that there
- * isn't actually a legit reason to downgrade to 0.2.5.  So for now, we have
- * to leave this #undef.
+ * be able to prevent that kind of thing until we're confident that there isn't
+ * actually a legit reason to downgrade to 0.2.5.  Now we are not recommending
+ * 0.2.5 anymore so there is no reason to keep the #undef.
  */
-#undef DISABLE_DISABLING_ED25519
+
+#define DISABLE_DISABLING_ED25519
 
 /** Check whether <b>router</b> has a nickname/identity key combination that
  * we recognize from the fingerprint list, or an IP we automatically act on
@@ -273,6 +286,13 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
     return FP_REJECT;
   }
 
+  /* Check for the more usual versions to reject a router first. */
+  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
+                                             router->addr, router->or_port,
+                                             router->platform, msg, severity);
+  if (r)
+    return r;
+
   /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc,
    * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha.
    * But just in case a relay doesn't provide or lies about its version, or
@@ -319,13 +339,11 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
         }
         return FP_REJECT;
       }
-#endif
+#endif /* defined(DISABLE_DISABLING_ED25519) */
     }
   }
 
-  return dirserv_get_status_impl(d, router->nickname,
-                                 router->addr, router->or_port,
-                                 router->platform, msg, severity);
+  return 0;
 }
 
 /** Return true if there is no point in downloading the router described by
@@ -577,6 +595,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
                    !general ? router_purpose_to_string(purpose) : "",
                    !general ? "\n" : "")<0) {
     *msg = "Couldn't format annotations";
+    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
+     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
     return -1;
   }
 
@@ -660,9 +680,6 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
                ri->nickname, source, (int)ri->cache_info.signed_descriptor_len,
                MAX_DESCRIPTOR_UPLOAD_SIZE);
     *msg = "Router descriptor was too large.";
-    control_event_or_authdir_new_descriptor("REJECTED",
-               ri->cache_info.signed_descriptor_body,
-                                            desclen, *msg);
     r = ROUTER_AUTHDIR_REJECTS;
     goto fail;
   }
@@ -681,9 +698,6 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
              router_describe(ri), source);
     *msg = "Not replacing router descriptor; no information has changed since "
       "the last one with this identity.";
-    control_event_or_authdir_new_descriptor("DROPPED",
-                         ri->cache_info.signed_descriptor_body,
-                                            desclen, *msg);
     r = ROUTER_IS_ALREADY_KNOWN;
     goto fail;
   }
@@ -691,10 +705,19 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
   /* Do keypinning again ... this time, to add the pin if appropriate */
   int keypin_status;
   if (ri->cache_info.signing_key_cert) {
+    ed25519_public_key_t *pkey = &ri->cache_info.signing_key_cert->signing_key;
+    /* First let's validate this pubkey before pinning it */
+    if (ed25519_validate_pubkey(pkey) < 0) {
+      log_warn(LD_DIRSERV, "Received bad key from %s (source %s)",
+               router_describe(ri), source);
+      routerinfo_free(ri);
+      return ROUTER_AUTHDIR_REJECTS;
+    }
+
+    /* Now pin it! */
     keypin_status = keypin_check_and_add(
       (const uint8_t*)ri->cache_info.identity_digest,
-      ri->cache_info.signing_key_cert->signing_key.pubkey,
-      ! key_pinning);
+      pkey->pubkey, ! key_pinning);
   } else {
     keypin_status = keypin_check_lone_rsa(
       (const uint8_t*)ri->cache_info.identity_digest);
@@ -707,7 +730,12 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
     log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because "
              "its key did not match an older RSA/Ed25519 keypair",
              router_describe(ri), source);
-    *msg = "Looks like your keypair does not match its older value.";
+    *msg = "Looks like your keypair has changed? This authority previously "
+      "recorded a different RSA identity for this Ed25519 identity (or vice "
+      "versa.) Did you replace or copy some of your key files, but not "
+      "the others? You should either restore the expected keypair, or "
+      "delete your keys and restart Tor to start your relay with a new "
+      "identity.";
     r = ROUTER_AUTHDIR_REJECTS;
     goto fail;
   }
@@ -724,14 +752,11 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
   r = router_add_to_routerlist(ri, msg, 0, 0);
   if (!WRA_WAS_ADDED(r)) {
     /* unless the routerinfo was fine, just out-of-date */
-    if (WRA_WAS_REJECTED(r))
-      control_event_or_authdir_new_descriptor("REJECTED", desc, desclen, *msg);
     log_info(LD_DIRSERV,
              "Did not add descriptor from '%s' (source: %s): %s.",
              nickname, source, *msg ? *msg : "(no message)");
   } else {
     smartlist_t *changed;
-    control_event_or_authdir_new_descriptor("ACCEPTED", desc, desclen, *msg);
 
     changed = smartlist_new();
     smartlist_add(changed, ri);
@@ -834,13 +859,13 @@ directory_remove_invalid(void)
 
   SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) {
     const char *msg = NULL;
+    const char *description;
     routerinfo_t *ent = node->ri;
-    char description[NODE_DESC_BUF_LEN];
     uint32_t r;
     if (!ent)
       continue;
     r = dirserv_router_get_status(ent, &msg, LOG_INFO);
-    router_get_description(description, ent);
+    description = router_describe(ent);
     if (r & FP_REJECT) {
       log_info(LD_DIRSERV, "Router %s is now rejected: %s",
                description, msg?msg:"");
@@ -867,6 +892,9 @@ directory_remove_invalid(void)
  * Allocate and return a description of the status of the server <b>desc</b>,
  * for use in a v1-style router-status line.  The server is listed
  * as running iff <b>is_live</b> is true.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
  */
 static char *
 list_single_server_status(const routerinfo_t *desc, int is_live)
@@ -894,7 +922,7 @@ list_single_server_status(const routerinfo_t *desc, int is_live)
 }
 
 /* DOCDOC running_long_enough_to_decide_unreachable */
-static inline int
+int
 running_long_enough_to_decide_unreachable(void)
 {
   return time_of_process_start
@@ -927,7 +955,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
   tor_assert(node);
 
   if (router_is_me(router)) {
-    /* We always know if we are down ourselves. */
+    /* We always know if we are shutting down or hibernating ourselves. */
     answer = ! we_are_hibernating();
   } else if (router->is_hibernating &&
              (router->cache_info.published_on +
@@ -979,6 +1007,9 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
  * *<b>router_status_out</b>.  Return 0 on success, -1 on failure.
  *
  * If for_controller is true, include the routers with very old descriptors.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
  */
 int
 list_server_status_v1(smartlist_t *routers, char **router_status_out,
@@ -1011,7 +1042,7 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
       if (!node->is_running)
         *cp++ = '!';
       router_get_verbose_nickname(cp, ri);
-      smartlist_add(rs_entries, tor_strdup(name_buf));
+      smartlist_add_strdup(rs_entries, name_buf);
     } else if (ri->cache_info.published_on >= cutoff) {
       smartlist_add(rs_entries, list_single_server_status(ri,
                                                           node->is_running));
@@ -1026,59 +1057,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   return 0;
 }
 
-/** Given a (possibly empty) list of config_line_t, each line of which contains
- * a list of comma-separated version numbers surrounded by optional space,
- * allocate and return a new string containing the version numbers, in order,
- * separated by commas.  Used to generate Recommended(Client|Server)?Versions
- */
-char *
-format_recommended_version_list(const config_line_t *ln, int warn)
-{
-  smartlist_t *versions;
-  char *result;
-  versions = smartlist_new();
-  for ( ; ln; ln = ln->next) {
-    smartlist_split_string(versions, ln->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  }
-
-  /* Handle the case where a dirauth operator has accidentally made some
-   * versions space-separated instead of comma-separated. */
-  smartlist_t *more_versions = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(versions, char *, v) {
-    if (strchr(v, ' ')) {
-      if (warn)
-        log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. "
-                 "(These are supposed to be comma-separated; I'll pretend you "
-                 "used commas instead.)", escaped(v));
-      SMARTLIST_DEL_CURRENT(versions, v);
-      smartlist_split_string(more_versions, v, NULL,
-                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-      tor_free(v);
-    }
-  } SMARTLIST_FOREACH_END(v);
-  smartlist_add_all(versions, more_versions);
-  smartlist_free(more_versions);
-
-  /* Check to make sure everything looks like a version. */
-  if (warn) {
-    SMARTLIST_FOREACH_BEGIN(versions, const char *, v) {
-      tor_version_t ver;
-      if (tor_version_parse(v, &ver) < 0) {
-        log_warn(LD_DIRSERV, "Recommended version %s does not look valid. "
-                 " (I'll include it anyway, since you told me to.)",
-                 escaped(v));
-      }
-    } SMARTLIST_FOREACH_END(v);
-  }
-
-  sort_version_list(versions, 1);
-  result = smartlist_join_strings(versions,",",0,NULL);
-  SMARTLIST_FOREACH(versions,char *,s,tor_free(s));
-  smartlist_free(versions);
-  return result;
-}
-
 /** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid,
  * not hibernating, having observed bw greater 0, and not too old. Else
  * return 0.
@@ -1093,7 +1071,7 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
   if (!node->is_running || !node->is_valid || ri->is_hibernating) {
     return 0;
   }
-  /* Only require bandwith capacity in non-test networks, or
+  /* Only require bandwidth capacity in non-test networks, or
    * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
   if (!ri->bandwidthcapacity) {
     if (get_options()->TestingTorNetwork) {
@@ -1163,8 +1141,10 @@ directory_fetches_dir_info_later(const or_options_t *options)
   return options->UseBridges != 0;
 }
 
-/** Return true iff we want to fetch and keep certificates for authorities
+/** Return true iff we want to serve certificates for authorities
  * that we don't acknowledge as authorities ourself.
+ * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
+ * and keep these certificates.
  */
 int
 directory_caches_unknown_auth_certs(const or_options_t *options)
@@ -1172,11 +1152,14 @@ directory_caches_unknown_auth_certs(const or_options_t *options)
   return dir_server_mode(options) || options->BridgeRelay;
 }
 
-/** Return 1 if we want to keep descriptors, networkstatuses, etc around.
+/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
  * Else return 0.
  * Check options->DirPort_set and directory_permits_begindir_requests()
  * to see if we are willing to serve these directory documents to others via
  * the DirPort and begindir-over-ORPort, respectively.
+ *
+ * To check if we should fetch documents, use we_want_to_fetch_flavor and
+ * we_want_to_fetch_unknown_auth_certs instead of this function.
  */
 int
 directory_caches_dir_info(const or_options_t *options)
@@ -1191,7 +1174,7 @@ directory_caches_dir_info(const or_options_t *options)
     should_refuse_unknown_exits(options);
 }
 
-/** Return 1 if we want to allow remote people to ask us directory
+/** Return 1 if we want to allow remote clients to ask us directory
  * requests via the "begin_dir" interface, which doesn't require
  * having any separate port open. */
 int
@@ -1240,8 +1223,8 @@ new_cached_dir(char *s, time_t published)
   d->dir = s;
   d->dir_len = strlen(s);
   d->published = published;
-  if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len,
-                        ZLIB_METHOD)) {
+  if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len),
+                   d->dir, d->dir_len, ZLIB_METHOD)) {
     log_warn(LD_BUG, "Error compressing directory");
   }
   return d;
@@ -1252,7 +1235,7 @@ static void
 clear_cached_dir(cached_dir_t *d)
 {
   tor_free(d->dir);
-  tor_free(d->dir_z);
+  tor_free(d->dir_compressed);
   memset(d, 0, sizeof(cached_dir_t));
 }
 
@@ -1275,6 +1258,7 @@ void
 dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
                                            const char *flavor_name,
                                            const common_digests_t *digests,
+                                           const uint8_t *sha3_as_signed,
                                            time_t published)
 {
   cached_dir_t *new_networkstatus;
@@ -1284,6 +1268,8 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
 
   new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
   memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
+  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
+         DIGEST256_LEN);
   old_networkstatus = strmap_set(cached_consensuses, flavor_name,
                                  new_networkstatus);
   if (old_networkstatus)
@@ -1416,7 +1402,7 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
    * tests aren't instant. If we haven't been running long enough,
    * trust the relay. */
 
-  if (stats_n_seconds_working >
+  if (get_uptime() >
       get_options()->MinUptimeHidServDirectoryV2 * 1.1)
     uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
                  real_uptime(router, now));
@@ -1460,6 +1446,24 @@ router_counts_toward_thresholds(const node_t *node, time_t now,
     (have_mbw || !require_mbw);
 }
 
+/** Look through the routerlist, and using the measured bandwidth cache count
+ * how many measured bandwidths we know.  This is used to decide whether we
+ * ever trust advertised bandwidths for purposes of assigning flags. */
+void
+dirserv_count_measured_bws(const smartlist_t *routers)
+{
+  /* Initialize this first */
+  routers_with_measured_bw = 0;
+
+  /* Iterate over the routerlist and count measured bandwidths */
+  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
+    /* Check if we know a measured bandwidth for this one */
+    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
+      ++routers_with_measured_bw;
+    }
+  } SMARTLIST_FOREACH_END(ri);
+}
+
 /** Look through the routerlist, the Mean Time Between Failure history, and
  * the Weighted Fractional Uptime history, and use them to set thresholds for
  * the Stable, Fast, and Guard flags.  Update the fields stable_uptime,
@@ -1467,7 +1471,7 @@ router_counts_toward_thresholds(const node_t *node, time_t now,
  * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits.
  *
  * Also, set the is_exit flag of each router appropriately. */
-static void
+void
 dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
 {
   int n_active, n_active_nonexit, n_familiar;
@@ -1518,15 +1522,21 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
         node->ri &&
         node->ri->purpose != ROUTER_PURPOSE_BRIDGE)
       continue;
+
+    routerinfo_t *ri = node->ri;
+    if (ri) {
+      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
+                       exit_policy_is_general_exit(ri->exit_policy));
+    }
+
     if (router_counts_toward_thresholds(node, now, omit_as_sybil,
                                         require_mbw)) {
-      routerinfo_t *ri = node->ri;
       const char *id = node->identity;
       uint32_t bw_kb;
+
       /* resolve spurious clang shallow analysis null pointer errors */
       tor_assert(ri);
-      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
-                       exit_policy_is_general_exit(ri->exit_policy));
+
       uptimes[n_active] = (uint32_t)real_uptime(ri, now);
       mtbfs[n_active] = rep_hist_get_stability(id, now);
       tks  [n_active] = rep_hist_get_weighted_time_known(id, now);
@@ -1692,7 +1702,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
 }
 
 /** Clear and free the measured bandwidth cache */
-STATIC void
+void
 dirserv_clear_measured_bw_cache(void)
 {
   if (mbw_cache) {
@@ -1724,18 +1734,10 @@ dirserv_expire_measured_bw_cache(time_t now)
   }
 }
 
-/** Get the current size of the measured bandwidth cache */
-STATIC int
-dirserv_get_measured_bw_cache_size(void)
-{
-  if (mbw_cache) return digestmap_size(mbw_cache);
-  else return 0;
-}
-
 /** Query the cache by identity digest, return value indicates whether
  * we found it. The bw_out and as_of_out pointers receive the cached
  * bandwidth value and the time it was cached if not NULL. */
-STATIC int
+int
 dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
                                    time_t *as_of_out)
 {
@@ -1756,61 +1758,18 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
 }
 
 /** Predicate wrapper for dirserv_query_measured_bw_cache() */
-STATIC int
+int
 dirserv_has_measured_bw(const char *node_id)
 {
   return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
 }
 
-/** Get the best estimate of a router's bandwidth for dirauth purposes,
- * preferring measured to advertised values if available. */
-
-static uint32_t
-dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri)
-{
-  uint32_t bw_kb = 0;
-  /*
-   * Yeah, measured bandwidths in measured_bw_line_t are (implicitly
-   * signed) longs and the ones router_get_advertised_bandwidth() returns
-   * are uint32_t.
-   */
-  long mbw_kb = 0;
-
-  if (ri) {
-    /*
-   * * First try to see if we have a measured bandwidth; don't bother with
-     * as_of_out here, on the theory that a stale measured bandwidth is still
-     * better to trust than an advertised one.
-     */
-    if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
-                                        &mbw_kb, NULL)) {
-      /* Got one! */
-      bw_kb = (uint32_t)mbw_kb;
-    } else {
-      /* If not, fall back to advertised */
-      bw_kb = router_get_advertised_bandwidth(ri) / 1000;
-    }
-  }
-
-  return bw_kb;
-}
-
-/** Look through the routerlist, and using the measured bandwidth cache count
- * how many measured bandwidths we know.  This is used to decide whether we
- * ever trust advertised bandwidths for purposes of assigning flags. */
-static void
-dirserv_count_measured_bws(const smartlist_t *routers)
+/** Get the current size of the measured bandwidth cache */
+int
+dirserv_get_measured_bw_cache_size(void)
 {
-  /* Initialize this first */
-  routers_with_measured_bw = 0;
-
-  /* Iterate over the routerlist and count measured bandwidths */
-  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
-    /* Check if we know a measured bandwidth for this one */
-    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
-      ++routers_with_measured_bw;
-    }
-  } SMARTLIST_FOREACH_END(ri);
+  if (mbw_cache) return digestmap_size(mbw_cache);
+  else return 0;
 }
 
 /** Return the bandwidth we believe for assigning flags; prefer measured
@@ -1873,44 +1832,31 @@ dirserv_get_flag_thresholds_line(void)
   return result;
 }
 
-/** Given a platform string as in a routerinfo_t (possibly null), return a
- * newly allocated version string for a networkstatus document, or NULL if the
- * platform doesn't give a Tor version. */
-static char *
-version_from_platform(const char *platform)
-{
-  if (platform && !strcmpstart(platform, "Tor ")) {
-    const char *eos = find_whitespace(platform+4);
-    if (eos && !strcmpstart(eos, " (r")) {
-      /* XXXX Unify this logic with the other version extraction
-       * logic in routerparse.c. */
-      eos = find_whitespace(eos+1);
-    }
-    if (eos) {
-      return tor_strndup(platform, eos-platform);
-    }
-  }
-  return NULL;
-}
-
 /** Helper: write the router-status information in <b>rs</b> into a newly
  * allocated character buffer.  Use the same format as in network-status
  * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
+ *
+ * consensus_method is the current consensus method when format is
+ * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
+ * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
+ *
  * Return 0 on success, -1 on failure.
  *
  * The format argument has one of the following values:
  *   NS_V2 - Output an entry suitable for a V2 NS opinion document
  *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
+ *        for consensus_method.
  *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
- *        consensus entry.
+ *        consensus entry for consensus_method.
  *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
  *        it contains additional information for the vote.
- *   NS_CONTROL_PORT - Output a NS document for the control port
+ *   NS_CONTROL_PORT - Output a NS document for the control port.
  */
 char *
 routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                           const char *protocols,
                           routerstatus_format_type_t format,
+                          int consensus_method,
                           const vote_routerstatus_t *vrs)
 {
   char *summary;
@@ -1941,8 +1887,10 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
    * networkstatus_type_t values, with an additional control port value
    * added -MP */
 
-  /* V3 microdesc consensuses don't have "a" lines. */
-  if (format == NS_V3_CONSENSUS_MICRODESC)
+  /* V3 microdesc consensuses only have "a" lines in later consensus methods
+   */
+  if (format == NS_V3_CONSENSUS_MICRODESC &&
+      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
     goto done;
 
   /* Possible "a" line. At most one for now. */
@@ -1951,7 +1899,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                            fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
   }
 
-  if (format == NS_V3_CONSENSUS)
+  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
     goto done;
 
   smartlist_add_asprintf(chunks,
@@ -2043,7 +1991,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                              vrs->status.guardfraction_percentage);
     }
 
-    smartlist_add(chunks, tor_strdup("\n"));
+    smartlist_add_strdup(chunks, "\n");
 
     if (desc) {
       summary = policy_summarize(desc->exit_policy, AF_INET);
@@ -2053,7 +2001,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
 
     if (format == NS_V3_VOTE && vrs) {
       if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
-        smartlist_add(chunks, tor_strdup("id ed25519 none\n"));
+        smartlist_add_strdup(chunks, "id ed25519 none\n");
       } else {
         char ed_b64[BASE64_DIGEST256_LEN+1];
         digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
@@ -2072,153 +2020,9 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
   return result;
 }
 
-/** Helper for sorting: compares two routerinfos first by address, and then by
- * descending order of "usefulness".  (An authority is more useful than a
- * non-authority; a running router is more useful than a non-running router;
- * and a router with more bandwidth is more useful than one with less.)
- **/
-static int
-compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
-{
-  routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b;
-  int first_is_auth, second_is_auth;
-  uint32_t bw_kb_first, bw_kb_second;
-  const node_t *node_first, *node_second;
-  int first_is_running, second_is_running;
-
-  /* we return -1 if first should appear before second... that is,
-   * if first is a better router. */
-  if (first->addr < second->addr)
-    return -1;
-  else if (first->addr > second->addr)
-    return 1;
-
-  /* Potentially, this next bit could cause k n lg n memeq calls.  But in
-   * reality, we will almost never get here, since addresses will usually be
-   * different. */
-
-  first_is_auth =
-    router_digest_is_trusted_dir(first->cache_info.identity_digest);
-  second_is_auth =
-    router_digest_is_trusted_dir(second->cache_info.identity_digest);
-
-  if (first_is_auth && !second_is_auth)
-    return -1;
-  else if (!first_is_auth && second_is_auth)
-    return 1;
-
-  node_first = node_get_by_id(first->cache_info.identity_digest);
-  node_second = node_get_by_id(second->cache_info.identity_digest);
-  first_is_running = node_first && node_first->is_running;
-  second_is_running = node_second && node_second->is_running;
-
-  if (first_is_running && !second_is_running)
-    return -1;
-  else if (!first_is_running && second_is_running)
-    return 1;
-
-  bw_kb_first = dirserv_get_bandwidth_for_router_kb(first);
-  bw_kb_second = dirserv_get_bandwidth_for_router_kb(second);
-
-  if (bw_kb_first > bw_kb_second)
-     return -1;
-  else if (bw_kb_first < bw_kb_second)
-    return 1;
-
-  /* They're equal! Compare by identity digest, so there's a
-   * deterministic order and we avoid flapping. */
-  return fast_memcmp(first->cache_info.identity_digest,
-                     second->cache_info.identity_digest,
-                     DIGEST_LEN);
-}
-
-/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t
- * whose keys are the identity digests of those routers that we're going to
- * exclude for Sybil-like appearance. */
-static digestmap_t *
-get_possible_sybil_list(const smartlist_t *routers)
-{
-  const or_options_t *options = get_options();
-  digestmap_t *omit_as_sybil;
-  smartlist_t *routers_by_ip = smartlist_new();
-  uint32_t last_addr;
-  int addr_count;
-  /* Allow at most this number of Tor servers on a single IP address, ... */
-  int max_with_same_addr = options->AuthDirMaxServersPerAddr;
-  /* ... unless it's a directory authority, in which case allow more. */
-  int max_with_same_addr_on_authority = options->AuthDirMaxServersPerAuthAddr;
-  if (max_with_same_addr <= 0)
-    max_with_same_addr = INT_MAX;
-  if (max_with_same_addr_on_authority <= 0)
-    max_with_same_addr_on_authority = INT_MAX;
-
-  smartlist_add_all(routers_by_ip, routers);
-  smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_);
-  omit_as_sybil = digestmap_new();
-
-  last_addr = 0;
-  addr_count = 0;
-  SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) {
-      if (last_addr != ri->addr) {
-        last_addr = ri->addr;
-        addr_count = 1;
-      } else if (++addr_count > max_with_same_addr) {
-        if (!router_addr_is_trusted_dir(ri->addr) ||
-            addr_count > max_with_same_addr_on_authority)
-          digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
-      }
-  } SMARTLIST_FOREACH_END(ri);
-
-  smartlist_free(routers_by_ip);
-  return omit_as_sybil;
-}
-
-/** If there are entries in <b>routers</b> with exactly the same ed25519 keys,
- * remove the older one.  If they are exactly the same age, remove the one
- * with the greater descriptor digest. May alter the order of the list. */
-static void
-routers_make_ed_keys_unique(smartlist_t *routers)
-{
-  routerinfo_t *ri2;
-  digest256map_t *by_ed_key = digest256map_new();
-
-  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
-    ri->omit_from_vote = 0;
-    if (ri->cache_info.signing_key_cert == NULL)
-      continue; /* No ed key */
-    const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey;
-    if ((ri2 = digest256map_get(by_ed_key, pk))) {
-      /* Duplicate; must omit one.  Set the omit_from_vote flag in whichever
-       * one has the earlier published_on. */
-      const time_t ri_pub = ri->cache_info.published_on;
-      const time_t ri2_pub = ri2->cache_info.published_on;
-      if (ri2_pub < ri_pub ||
-          (ri2_pub == ri_pub &&
-           fast_memcmp(ri->cache_info.signed_descriptor_digest,
-                     ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) {
-        digest256map_set(by_ed_key, pk, ri);
-        ri2->omit_from_vote = 1;
-      } else {
-        ri->omit_from_vote = 1;
-      }
-    } else {
-      /* Add to map */
-      digest256map_set(by_ed_key, pk, ri);
-    }
-  } SMARTLIST_FOREACH_END(ri);
-
-  digest256map_free(by_ed_key, NULL);
-
-  /* Now remove every router where the omit_from_vote flag got set. */
-  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
-    if (ri->omit_from_vote) {
-      SMARTLIST_DEL_CURRENT(routers, ri);
-    }
-  } SMARTLIST_FOREACH_END(ri);
-}
-
 /** Extract status information from <b>ri</b> and from other authority
- * functions and store it in <b>rs</b>>.
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
  *
  * We assume that ri-\>is_running has already been set, e.g. by
  *   dirserv_set_router_is_running(ri, now);
@@ -2249,6 +2053,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
   rs->is_valid = node->is_valid;
 
   if (node->is_fast && node->is_stable &&
+      ri->supports_tunnelled_dir_requests &&
       ((options->AuthDirGuardBWGuarantee &&
         routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
        routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
@@ -2284,6 +2089,9 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
        OR port and it's reachable so copy it to the routerstatus.  */
     tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
     rs->ipv6_orport = ri->ipv6_orport;
+  } else {
+    tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
+    rs->ipv6_orport = 0;
   }
 
   if (options->TestingTorNetwork) {
@@ -2324,25 +2132,6 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs)
   }
 }
 
-/** Routerstatus <b>rs</b> is part of a group of routers that are on
- * too narrow an IP-space. Clear out its flags: we don't want people
- * using it.
- *
- * Leave its BadExit flag alone though, since if we think it's a bad exit,
- * we want to vote that way in case all the other authorities are voting
- * Running and Exit.
- */
-static void
-clear_status_flags_on_sybil(routerstatus_t *rs)
-{
-  rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast =
-    rs->is_flagged_running = rs->is_named = rs->is_valid =
-    rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0;
-  /* FFFF we might want some mechanism to check later on if we
-   * missed zeroing any flags: it's easy to add a new flag but
-   * forget to add it to this clause. */
-}
-
 /** The guardfraction of the guard with identity fingerprint <b>guard_id</b>
  *  is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for
  *  this guard in <b>vote_routerstatuses</b>, and if we do, register the
@@ -2659,17 +2448,38 @@ dirserv_read_guardfraction_file(const char *fname,
 
 /**
  * Helper function to parse out a line in the measured bandwidth file
- * into a measured_bw_line_t output structure. Returns -1 on failure
- * or 0 on success.
+ * into a measured_bw_line_t output structure.
+ *
+ * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete
+ * bw line, return -1 and warn, since we are after the headers and we should
+ * only parse bw lines. Return 0 otherwise.
+ *
+ * If <b>line_is_after_headers</b> is false then it means that we are not past
+ * the header block yet. If we encounter an incomplete bw line, return -1 but
+ * don't warn since there could be additional header lines coming. If we
+ * encounter a proper bw line, return 0 (and we got past the headers).
  */
 STATIC int
-measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line)
+measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
+                       int line_is_after_headers)
 {
   char *line = tor_strdup(orig_line);
   char *cp = line;
   int got_bw = 0;
   int got_node_id = 0;
   char *strtok_state; /* lame sauce d'jour */
+
+  if (strlen(line) == 0) {
+    log_warn(LD_DIRSERV, "Empty line in bandwidth file");
+    tor_free(line);
+    return -1;
+  }
+
+  /* Remove end of line character, so that is not part of the token */
+  if (line[strlen(line) - 1] == '\n') {
+    line[strlen(line) - 1] = '\0';
+  }
+
   cp = tor_strtok_r(cp, " \t", &strtok_state);
 
   if (!cp) {
@@ -2687,7 +2497,13 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line)
   }
 
   do {
-    if (strcmpstart(cp, "bw=") == 0) {
+    // If the line contains vote=0, ignore it.
+    if (strcmpstart(cp, "vote=0") == 0) {
+      log_debug(LD_DIRSERV, "Ignoring bandwidth file line that contains "
+                "vote=0: %s",escaped(orig_line));
+      tor_free(line);
+      return -1;
+    } else if (strcmpstart(cp, "bw=") == 0) {
       int parse_ok = 0;
       char *endptr;
       if (got_bw) {
@@ -2698,7 +2514,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line)
       }
       cp+=strlen("bw=");
 
-      out->bw_kb = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr);
+      out->bw_kb = tor_parse_long(cp, 10, 0, LONG_MAX, &parse_ok, &endptr);
       if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) {
         log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s",
                  escaped(orig_line));
@@ -2731,6 +2547,13 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line)
   if (got_bw && got_node_id) {
     tor_free(line);
     return 0;
+  } else if (line_is_after_headers == 0) {
+    /* There could be additional header lines, therefore do not give warnings
+     * but returns -1 since it's not a complete bw line. */
+    log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s",
+             escaped(orig_line));
+    tor_free(line);
+    return -1;
   } else {
     log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
              escaped(orig_line));
@@ -2779,6 +2602,11 @@ dirserv_read_measured_bandwidths(const char *from_file,
   int applied_lines = 0;
   time_t file_time, now;
   int ok;
+   /* This flag will be 1 only when the first successful bw measurement line
+   * has been encountered, so that measured_bw_line_parse don't give warnings
+   * if there are additional header lines, as introduced in Bandwidth List spec
+   * version 1.1.0 */
+  int line_is_after_headers = 0;
 
   /* Initialise line, so that we can't possibly run off the end. */
   memset(line, 0, sizeof(line));
@@ -2826,7 +2654,11 @@ dirserv_read_measured_bandwidths(const char *from_file,
   while (!feof(fp)) {
     measured_bw_line_t parsed_line;
     if (fgets(line, sizeof(line), fp) && strlen(line)) {
-      if (measured_bw_line_parse(&parsed_line, line) != -1) {
+      if (measured_bw_line_parse(&parsed_line, line,
+                                 line_is_after_headers) != -1) {
+        /* This condition will be true when the first complete valid bw line
+         * has been encountered, which means the end of the header lines. */
+        line_is_after_headers = 1;
         /* Also cache the line for dirserv_get_bandwidth_for_router() */
         dirserv_cache_measured_bw(&parsed_line, file_time);
         if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
@@ -2845,346 +2677,67 @@ dirserv_read_measured_bandwidths(const char *from_file,
   return 0;
 }
 
-/** Return a new networkstatus_t* containing our current opinion. (For v3
- * authorities) */
-networkstatus_t *
-dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
-                                        authority_cert_t *cert)
-{
-  const or_options_t *options = get_options();
-  networkstatus_t *v3_out = NULL;
-  uint32_t addr;
-  char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
-  const char *contact;
-  smartlist_t *routers, *routerstatuses;
-  char identity_digest[DIGEST_LEN];
-  char signing_key_digest[DIGEST_LEN];
-  int listbadexits = options->AuthDirListBadExits;
-  routerlist_t *rl = router_get_routerlist();
-  time_t now = time(NULL);
-  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  networkstatus_voter_info_t *voter = NULL;
-  vote_timing_t timing;
-  digestmap_t *omit_as_sybil = NULL;
-  const int vote_on_reachability = running_long_enough_to_decide_unreachable();
-  smartlist_t *microdescriptors = NULL;
-
-  tor_assert(private_key);
-  tor_assert(cert);
-
-  if (crypto_pk_get_digest(private_key, signing_key_digest)<0) {
-    log_err(LD_BUG, "Error computing signing key digest");
-    return NULL;
-  }
-  if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) {
-    log_err(LD_BUG, "Error computing identity key digest");
-    return NULL;
-  }
-  if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) {
-    log_warn(LD_NET, "Couldn't resolve my hostname");
-    return NULL;
-  }
-  if (!hostname || !strchr(hostname, '.')) {
-    tor_free(hostname);
-    hostname = tor_dup_ip(addr);
-  }
-
-  if (options->VersioningAuthoritativeDir) {
-    client_versions =
-      format_recommended_version_list(options->RecommendedClientVersions, 0);
-    server_versions =
-      format_recommended_version_list(options->RecommendedServerVersions, 0);
-  }
-
-  contact = get_options()->ContactInfo;
-  if (!contact)
-    contact = "(none)";
-
-  /*
-   * Do this so dirserv_compute_performance_thresholds() and
-   * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
-   */
-  if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL);
-  } else {
-    /*
-     * No bandwidths file; clear the measured bandwidth cache in case we had
-     * one last time around.
-     */
-    if (dirserv_get_measured_bw_cache_size() > 0) {
-      dirserv_clear_measured_bw_cache();
-    }
-  }
-
-  /* precompute this part, since we need it to decide what "stable"
-   * means. */
-  SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, {
-    dirserv_set_router_is_running(ri, now);
-  });
-
-  routers = smartlist_new();
-  smartlist_add_all(routers, rl->routers);
-  routers_make_ed_keys_unique(routers);
-  /* After this point, don't use rl->routers; use 'routers' instead. */
-  routers_sort_by_identity(routers);
-  omit_as_sybil = get_possible_sybil_list(routers);
-
-  DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) {
-    (void) ignore;
-    rep_hist_make_router_pessimal(sybil_id, now);
-  } DIGESTMAP_FOREACH_END;
-
-  /* Count how many have measured bandwidths so we know how to assign flags;
-   * this must come before dirserv_compute_performance_thresholds() */
-  dirserv_count_measured_bws(routers);
-
-  dirserv_compute_performance_thresholds(omit_as_sybil);
-
-  routerstatuses = smartlist_new();
-  microdescriptors = smartlist_new();
-
-  SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
-    if (ri->cache_info.published_on >= cutoff) {
-      routerstatus_t *rs;
-      vote_routerstatus_t *vrs;
-      node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest);
-      if (!node)
-        continue;
-
-      vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      rs = &vrs->status;
-      set_routerstatus_from_routerinfo(rs, node, ri, now,
-                                       listbadexits);
-
-      if (ri->cache_info.signing_key_cert) {
-        memcpy(vrs->ed25519_id,
-               ri->cache_info.signing_key_cert->signing_key.pubkey,
-               ED25519_PUBKEY_LEN);
-      }
-
-      if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest))
-        clear_status_flags_on_sybil(rs);
-
-      if (!vote_on_reachability)
-        rs->is_flagged_running = 0;
-
-      vrs->version = version_from_platform(ri->platform);
-      if (ri->protocol_list) {
-        vrs->protocols = tor_strdup(ri->protocol_list);
-      } else {
-        vrs->protocols = tor_strdup(
-                              protover_compute_for_old_tor(vrs->version));
-      }
-      vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now,
-                                                        microdescriptors);
-
-      smartlist_add(routerstatuses, vrs);
-    }
-  } SMARTLIST_FOREACH_END(ri);
-
-  {
-    smartlist_t *added =
-      microdescs_add_list_to_cache(get_microdesc_cache(),
-                                   microdescriptors, SAVED_NOWHERE, 0);
-    smartlist_free(added);
-    smartlist_free(microdescriptors);
-  }
-
-  smartlist_free(routers);
-  digestmap_free(omit_as_sybil, NULL);
-
-  /* Apply guardfraction information to routerstatuses. */
-  if (options->GuardfractionFile) {
-    dirserv_read_guardfraction_file(options->GuardfractionFile,
-                                    routerstatuses);
-  }
-
-  /* This pass through applies the measured bw lines to the routerstatuses */
-  if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses);
-  } else {
-    /*
-     * No bandwidths file; clear the measured bandwidth cache in case we had
-     * one last time around.
-     */
-    if (dirserv_get_measured_bw_cache_size() > 0) {
-      dirserv_clear_measured_bw_cache();
-    }
-  }
-
-  v3_out = tor_malloc_zero(sizeof(networkstatus_t));
-
-  v3_out->type = NS_TYPE_VOTE;
-  dirvote_get_preferred_voting_intervals(&timing);
-  v3_out->published = now;
-  {
-    char tbuf[ISO_TIME_LEN+1];
-    networkstatus_t *current_consensus =
-      networkstatus_get_live_consensus(now);
-    long last_consensus_interval; /* only used to pick a valid_after */
-    if (current_consensus)
-      last_consensus_interval = current_consensus->fresh_until -
-        current_consensus->valid_after;
-    else
-      last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
-    v3_out->valid_after =
-      dirvote_get_start_of_next_interval(now, (int)last_consensus_interval,
-                                      options->TestingV3AuthVotingStartOffset);
-    format_iso_time(tbuf, v3_out->valid_after);
-    log_notice(LD_DIR,"Choosing valid-after time in vote as %s: "
-               "consensus_set=%d, last_interval=%d",
-               tbuf, current_consensus?1:0, (int)last_consensus_interval);
-  }
-  v3_out->fresh_until = v3_out->valid_after + timing.vote_interval;
-  v3_out->valid_until = v3_out->valid_after +
-    (timing.vote_interval * timing.n_intervals_valid);
-  v3_out->vote_seconds = timing.vote_delay;
-  v3_out->dist_seconds = timing.dist_delay;
-  tor_assert(v3_out->vote_seconds > 0);
-  tor_assert(v3_out->dist_seconds > 0);
-  tor_assert(timing.n_intervals_valid > 0);
-
-  v3_out->client_versions = client_versions;
-  v3_out->server_versions = server_versions;
-
-  /* These are hardwired, to avoid disaster. */
-  v3_out->recommended_relay_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
-  v3_out->recommended_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
-  v3_out->required_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 LinkAuth=1 Microdesc=1-2 Relay=2");
-  v3_out->required_relay_protocols =
-    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=3-4 LinkAuth=1 Microdesc=1 Relay=1-2");
-
-  /* We are not allowed to vote to require anything we don't have. */
-  tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
-  tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL));
-
-  /* We should not recommend anything we don't have. */
-  tor_assert_nonfatal(protover_all_supported(
-                         v3_out->recommended_relay_protocols, NULL));
-  tor_assert_nonfatal(protover_all_supported(
-                         v3_out->recommended_client_protocols, NULL));
-
-  v3_out->package_lines = smartlist_new();
-  {
-    config_line_t *cl;
-    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
-      if (validate_recommended_package_line(cl->value))
-        smartlist_add(v3_out->package_lines, tor_strdup(cl->value));
-    }
-  }
-
-  v3_out->known_flags = smartlist_new();
-  smartlist_split_string(v3_out->known_flags,
-                "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
-                0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  if (vote_on_reachability)
-    smartlist_add(v3_out->known_flags, tor_strdup("Running"));
-  if (listbadexits)
-    smartlist_add(v3_out->known_flags, tor_strdup("BadExit"));
-  smartlist_sort_strings(v3_out->known_flags);
-
-  if (options->ConsensusParams) {
-    v3_out->net_params = smartlist_new();
-    smartlist_split_string(v3_out->net_params,
-                           options->ConsensusParams, NULL, 0, 0);
-    smartlist_sort_strings(v3_out->net_params);
-  }
-
-  voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
-  voter->nickname = tor_strdup(options->Nickname);
-  memcpy(voter->identity_digest, identity_digest, DIGEST_LEN);
-  voter->sigs = smartlist_new();
-  voter->address = hostname;
-  voter->addr = addr;
-  voter->dir_port = router_get_advertised_dir_port(options, 0);
-  voter->or_port = router_get_advertised_or_port(options);
-  voter->contact = tor_strdup(contact);
-  if (options->V3AuthUseLegacyKey) {
-    authority_cert_t *c = get_my_v3_legacy_cert();
-    if (c) {
-      if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) {
-        log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key");
-        memset(voter->legacy_id_digest, 0, DIGEST_LEN);
-      }
-    }
-  }
-
-  v3_out->voters = smartlist_new();
-  smartlist_add(v3_out->voters, voter);
-  v3_out->cert = authority_cert_dup(cert);
-  v3_out->routerstatus_list = routerstatuses;
-  /* Note: networkstatus_digest is unset; it won't get set until we actually
-   * format the vote. */
-
-  return v3_out;
-}
-
 /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
  * pointers, adds copies of digests to fps_out, and doesn't use the
  * /tor/server/ prefix.  For a /d/ request, adds descriptor digests; for other
  * requests, adds identity digests.
  */
 int
-dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
-                                    const char **msg, int for_unencrypted_conn,
-                                    int is_extrainfo)
+dirserv_get_routerdesc_spool(smartlist_t *spool_out,
+                             const char *key,
+                             dir_spool_source_t source,
+                             int conn_is_encrypted,
+                             const char **msg_out)
 {
-  int by_id = 1;
-  *msg = NULL;
+  *msg_out = NULL;
 
   if (!strcmp(key, "all")) {
-    routerlist_t *rl = router_get_routerlist();
-    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
-                      smartlist_add(fps_out,
-                      tor_memdup(r->cache_info.identity_digest, DIGEST_LEN)));
-    /* Treat "all" requests as if they were unencrypted */
-    for_unencrypted_conn = 1;
+    const routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) {
+      spooled_resource_t *spooled;
+      spooled = spooled_resource_new(source,
+                              (const uint8_t *)r->cache_info.identity_digest,
+                              DIGEST_LEN);
+      /* Treat "all" requests as if they were unencrypted */
+      conn_is_encrypted = 0;
+      smartlist_add(spool_out, spooled);
+    } SMARTLIST_FOREACH_END(r);
   } else if (!strcmp(key, "authority")) {
     const routerinfo_t *ri = router_get_my_routerinfo();
     if (ri)
-      smartlist_add(fps_out,
-                    tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN));
+      smartlist_add(spool_out,
+                    spooled_resource_new(source,
+                             (const uint8_t *)ri->cache_info.identity_digest,
+                             DIGEST_LEN));
   } else if (!strcmpstart(key, "d/")) {
-    by_id = 0;
     key += strlen("d/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
   } else if (!strcmpstart(key, "fp/")) {
     key += strlen("fp/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
   } else {
-    *msg = "Key not recognized";
+    *msg_out = "Not found";
     return -1;
   }
 
-  if (for_unencrypted_conn) {
+  if (! conn_is_encrypted) {
     /* Remove anything that insists it not be sent unencrypted. */
-    SMARTLIST_FOREACH_BEGIN(fps_out, char *, cp) {
-        const signed_descriptor_t *sd;
-        if (by_id)
-          sd = get_signed_descriptor_by_fp(cp,is_extrainfo,0);
-        else if (is_extrainfo)
-          sd = extrainfo_get_by_descriptor_digest(cp);
-        else
-          sd = router_get_by_descriptor_digest(cp);
-        if (sd && !sd->send_unencrypted) {
-          tor_free(cp);
-          SMARTLIST_DEL_CURRENT(fps_out, cp);
-        }
-    } SMARTLIST_FOREACH_END(cp);
+    SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) {
+      const uint8_t *body = NULL;
+      size_t bodylen = 0;
+      int r = spooled_resource_lookup_body(spooled, conn_is_encrypted,
+                                           &body, &bodylen, NULL);
+      if (r < 0 || body == NULL || bodylen == 0) {
+        SMARTLIST_DEL_CURRENT(spool_out, spooled);
+        spooled_resource_free(spooled);
+      }
+    } SMARTLIST_FOREACH_END(spooled);
   }
 
-  if (!smartlist_len(fps_out)) {
-    *msg = "Servers unavailable";
+  if (!smartlist_len(spool_out)) {
+    *msg_out = "Servers unavailable";
     return -1;
   }
   return 0;
@@ -3280,7 +2833,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
 void
 dirserv_orconn_tls_done(const tor_addr_t *addr,
                         uint16_t or_port,
-                        const char *digest_rcvd)
+                        const char *digest_rcvd,
+                        const ed25519_public_key_t *ed_id_rcvd)
 {
   node_t *node = NULL;
   tor_addr_port_t orport;
@@ -3292,8 +2846,26 @@ dirserv_orconn_tls_done(const tor_addr_t *addr,
   node = node_get_mutable_by_id(digest_rcvd);
   if (node == NULL || node->ri == NULL)
     return;
+
   ri = node->ri;
 
+  if (get_options()->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      ri->cache_info.signing_key_cert) {
+    /* We allow the node to have an ed25519 key if we haven't been told one in
+     * the routerinfo, but if we *HAVE* been told one in the routerinfo, it
+     * needs to match. */
+    const ed25519_public_key_t *expected_id =
+      &ri->cache_info.signing_key_cert->signing_key;
+    tor_assert(!ed25519_public_key_is_zero(expected_id));
+    if (! ed_id_rcvd || ! ed25519_pubkey_eq(ed_id_rcvd, expected_id)) {
+      log_info(LD_DIRSERV, "Router at %s:%d with RSA ID %s "
+               "did not present expected Ed25519 ID.",
+               fmt_addr(addr), or_port, hex_str(digest_rcvd, DIGEST_LEN));
+      return; /* Don't mark it as reachable. */
+    }
+  }
+
   tor_addr_copy(&orport.addr, addr);
   orport.port = or_port;
   if (router_has_orport(ri, &orport)) {
@@ -3301,7 +2873,7 @@ dirserv_orconn_tls_done(const tor_addr_t *addr,
     if (!authdir_mode_bridge(get_options()) ||
         ri->purpose == ROUTER_PURPOSE_BRIDGE) {
       char addrstr[TOR_ADDR_BUF_LEN];
-      /* This is a bridge or we're not a bridge authorititative --
+      /* This is a bridge or we're not a bridge authority --
          mark it as reachable.  */
       log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.",
                router_describe(ri),
@@ -3349,21 +2921,32 @@ dirserv_should_launch_reachability_test(const routerinfo_t *ri,
 void
 dirserv_single_reachability_test(time_t now, routerinfo_t *router)
 {
+  const or_options_t *options = get_options();
   channel_t *chan = NULL;
-  node_t *node = NULL;
+  const node_t *node = NULL;
   tor_addr_t router_addr;
+  const ed25519_public_key_t *ed_id_key;
   (void) now;
 
   tor_assert(router);
-  node = node_get_mutable_by_id(router->cache_info.identity_digest);
+  node = node_get_by_id(router->cache_info.identity_digest);
   tor_assert(node);
 
+  if (options->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node, 1) &&
+      router->cache_info.signing_key_cert) {
+    ed_id_key = &router->cache_info.signing_key_cert->signing_key;
+  } else {
+    ed_id_key = NULL;
+  }
+
   /* IPv4. */
   log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
             router->nickname, fmt_addr32(router->addr), router->or_port);
   tor_addr_from_ipv4h(&router_addr, router->addr);
   chan = channel_tls_connect(&router_addr, router->or_port,
-                             router->cache_info.identity_digest);
+                             router->cache_info.identity_digest,
+                             ed_id_key);
   if (chan) command_setup_channel(chan);
 
   /* Possible IPv6. */
@@ -3375,7 +2958,8 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
               tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
               router->ipv6_orport);
     chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
-                               router->cache_info.identity_digest);
+                               router->cache_info.identity_digest,
+                               ed_id_key);
     if (chan) command_setup_channel(chan);
   }
 }
@@ -3418,410 +3002,502 @@ dirserv_test_reachability(time_t now)
   ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */
 }
 
-/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
- * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
- * flavor name if we want a flavored v3 status, return a pointer to the
- * appropriate cached dir object, or NULL if there isn't one available. */
-static cached_dir_t *
-lookup_cached_dir_by_fp(const char *fp)
+/* ==========
+ * Spooling code.
+ * ========== */
+
+spooled_resource_t *
+spooled_resource_new(dir_spool_source_t source,
+                     const uint8_t *digest, size_t digestlen)
 {
-  cached_dir_t *d = NULL;
-  if (tor_digest_is_zero(fp) && cached_consensuses) {
-    d = strmap_get(cached_consensuses, "ns");
-  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses &&
-           (d = strmap_get(cached_consensuses, fp))) {
-    /* this here interface is a nasty hack XXXX */;
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = source;
+  switch (source) {
+    case DIR_SPOOL_NETWORKSTATUS:
+      spooled->spool_eagerly = 0;
+      break;
+    case DIR_SPOOL_SERVER_BY_DIGEST:
+    case DIR_SPOOL_SERVER_BY_FP:
+    case DIR_SPOOL_EXTRA_BY_DIGEST:
+    case DIR_SPOOL_EXTRA_BY_FP:
+    case DIR_SPOOL_MICRODESC:
+    default:
+      spooled->spool_eagerly = 1;
+      break;
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+      tor_assert_unreached();
+      break;
   }
-  return d;
+  tor_assert(digestlen <= sizeof(spooled->digest));
+  if (digest)
+    memcpy(spooled->digest, digest, digestlen);
+  return spooled;
 }
 
-/** Remove from <b>fps</b> every networkstatus key where both
- * a) we have a networkstatus document and
- * b) it is not newer than <b>cutoff</b>.
+/**
+ * Create a new spooled_resource_t to spool the contents of <b>entry</b> to
+ * the user.  Return the spooled object on success, or NULL on failure (which
+ * is probably caused by a failure to map the body of the item from disk).
  *
- * Return 1 if any items were present at all; else return 0.
+ * Adds a reference to entry's reference counter.
  */
-int
-dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff)
-{
-  int found_any = 0;
-  SMARTLIST_FOREACH_BEGIN(fps, char *, digest) {
-    cached_dir_t *d = lookup_cached_dir_by_fp(digest);
-    if (!d)
-      continue;
-    found_any = 1;
-    if (d->published <= cutoff) {
-      tor_free(digest);
-      SMARTLIST_DEL_CURRENT(fps, digest);
-    }
-  } SMARTLIST_FOREACH_END(digest);
-
-  return found_any;
+spooled_resource_t *
+spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
+{
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY;
+  spooled->spool_eagerly = 0;
+  consensus_cache_entry_incref(entry);
+  spooled->consensus_cache_entry = entry;
+
+  int r = consensus_cache_entry_get_body(entry,
+                                         &spooled->cce_body,
+                                         &spooled->cce_len);
+  if (r == 0) {
+    return spooled;
+  } else {
+    spooled_resource_free(spooled);
+    return NULL;
+  }
 }
 
-/** Return the cache-info for identity fingerprint <b>fp</b>, or
- * its extra-info document if <b>extrainfo</b> is true. Return
- * NULL if not found or if the descriptor is older than
- * <b>publish_cutoff</b>. */
-static const signed_descriptor_t *
-get_signed_descriptor_by_fp(const char *fp, int extrainfo,
-                            time_t publish_cutoff)
+/** Release all storage held by <b>spooled</b>. */
+void
+spooled_resource_free_(spooled_resource_t *spooled)
 {
-  if (router_digest_is_me(fp)) {
-    if (extrainfo)
-      return &(router_get_my_extrainfo()->cache_info);
-    else
-      return &(router_get_my_routerinfo()->cache_info);
-  } else {
-    const routerinfo_t *ri = router_get_by_id_digest(fp);
-    if (ri &&
-        ri->cache_info.published_on > publish_cutoff) {
-      if (extrainfo)
-        return extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-      else
-        return &ri->cache_info;
-    }
+  if (spooled == NULL)
+    return;
+
+  if (spooled->cached_dir_ref) {
+    cached_dir_decref(spooled->cached_dir_ref);
   }
-  return NULL;
-}
 
-/** Return true iff we have any of the documents (extrainfo or routerdesc)
- * specified by the fingerprints in <b>fps</b> and <b>spool_src</b>.  Used to
- * decide whether to send a 404.  */
-int
-dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src)
-{
-  time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH;
-  SMARTLIST_FOREACH_BEGIN(fps, const char *, fp) {
-      switch (spool_src)
-      {
-        case DIR_SPOOL_EXTRA_BY_DIGEST:
-          if (extrainfo_get_by_descriptor_digest(fp)) return 1;
-          break;
-        case DIR_SPOOL_SERVER_BY_DIGEST:
-          if (router_get_by_descriptor_digest(fp)) return 1;
-          break;
-        case DIR_SPOOL_EXTRA_BY_FP:
-        case DIR_SPOOL_SERVER_BY_FP:
-          if (get_signed_descriptor_by_fp(fp,
-                spool_src == DIR_SPOOL_EXTRA_BY_FP, publish_cutoff))
-            return 1;
-          break;
-      }
-  } SMARTLIST_FOREACH_END(fp);
-  return 0;
+  if (spooled->consensus_cache_entry) {
+    consensus_cache_entry_decref(spooled->consensus_cache_entry);
+  }
+
+  tor_free(spooled);
 }
 
-/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of
- * a microdescriptor we have. */
-int
-dirserv_have_any_microdesc(const smartlist_t *fps)
+/** When spooling data from a cached_dir_t object, we always add
+ * at least this much. */
+#define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192
+
+/** Return an compression ratio for compressing objects from <b>source</b>.
+ */
+static double
+estimate_compression_ratio(dir_spool_source_t source)
 {
-  microdesc_cache_t *cache = get_microdesc_cache();
-  SMARTLIST_FOREACH(fps, const char *, fp,
-                    if (microdesc_cache_lookup_by_digest256(cache, fp))
-                      return 1);
-  return 0;
+  /* We should put in better estimates here, depending on the number of
+     objects and their type */
+  (void) source;
+  return 0.5;
 }
 
-/** Return an approximate estimate of the number of bytes that will
- * be needed to transmit the server descriptors (if is_serverdescs --
- * they can be either d/ or fp/ queries) or networkstatus objects (if
- * !is_serverdescs) listed in <b>fps</b>.  If <b>compressed</b> is set,
- * we guess how large the data will be after compression.
+/** Return an estimated number of bytes needed for transmitting the
+ * resource in <b>spooled</b> on <b>conn</b>
  *
- * The return value is an estimate; it might be larger or smaller.
- **/
-size_t
-dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
-                           int compressed)
-{
-  size_t result;
-  tor_assert(fps);
-  if (is_serverdescs) {
-    int n = smartlist_len(fps);
-    const routerinfo_t *me = router_get_my_routerinfo();
-    result = (me?me->cache_info.signed_descriptor_len:2048) * n;
-    if (compressed)
-      result /= 2; /* observed compressibility is between 35 and 55%. */
+ * As a convenient side-effect, set *<b>published_out</b> to the resource's
+ * publication time.
+ */
+static size_t
+spooled_resource_estimate_size(const spooled_resource_t *spooled,
+                               dir_connection_t *conn,
+                               int compressed,
+                               time_t *published_out)
+{
+  if (spooled->spool_eagerly) {
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen,
+                                         published_out);
+    if (r == -1 || body == NULL || bodylen == 0)
+      return 0;
+    if (compressed) {
+      double ratio = estimate_compression_ratio(spooled->spool_source);
+      bodylen = (size_t)(bodylen * ratio);
+    }
+    return bodylen;
   } else {
-    result = 0;
-    SMARTLIST_FOREACH(fps, const char *, digest, {
-        cached_dir_t *dir = lookup_cached_dir_by_fp(digest);
-        if (dir)
-          result += compressed ? dir->dir_z_len : dir->dir_len;
-      });
+    cached_dir_t *cached;
+    if (spooled->consensus_cache_entry) {
+      if (published_out) {
+        consensus_cache_entry_get_valid_after(
+            spooled->consensus_cache_entry, published_out);
+      }
+
+      return spooled->cce_len;
+    }
+    if (spooled->cached_dir_ref) {
+      cached = spooled->cached_dir_ref;
+    } else {
+      cached = spooled_resource_lookup_cached_dir(spooled,
+                                                  published_out);
+    }
+    if (cached == NULL) {
+      return 0;
+    }
+    size_t result = compressed ? cached->dir_compressed_len : cached->dir_len;
+    return result;
   }
-  return result;
 }
 
-/** Given a list of microdescriptor hashes, guess how many bytes will be
- * needed to transmit them, and return the guess. */
-size_t
-dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed)
-{
-  size_t result = smartlist_len(fps) * microdesc_average_size(NULL);
-  if (compressed)
-    result /= 2;
-  return result;
-}
+/** Return code for spooled_resource_flush_some */
+typedef enum {
+  SRFS_ERR = -1,
+  SRFS_MORE = 0,
+  SRFS_DONE
+} spooled_resource_flush_status_t;
 
-/** When we're spooling data onto our outbuf, add more whenever we dip
- * below this threshold. */
-#define DIRSERV_BUFFER_MIN 16384
+/** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>.
+ * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from
+ * this spooled resource, or SRFS_DONE if we are done flushing this spooled
+ * resource.
+ */
+static spooled_resource_flush_status_t
+spooled_resource_flush_some(spooled_resource_t *spooled,
+                            dir_connection_t *conn)
+{
+  if (spooled->spool_eagerly) {
+    /* Spool_eagerly resources are sent all-at-once. */
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen, NULL);
+    if (r == -1 || body == NULL || bodylen == 0) {
+      /* Absent objects count as "done". */
+      return SRFS_DONE;
+    }
+    if (conn->compress_state) {
+      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
+    } else {
+      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
+    }
+    return SRFS_DONE;
+  } else {
+    cached_dir_t *cached = spooled->cached_dir_ref;
+    consensus_cache_entry_t *cce = spooled->consensus_cache_entry;
+    if (cached == NULL && cce == NULL) {
+      /* The cached_dir_t hasn't been materialized yet. So let's look it up. */
+      cached = spooled->cached_dir_ref =
+        spooled_resource_lookup_cached_dir(spooled, NULL);
+      if (!cached) {
+        /* Absent objects count as done. */
+        return SRFS_DONE;
+      }
+      ++cached->refcnt;
+      tor_assert_nonfatal(spooled->cached_dir_offset == 0);
+    }
 
-/** Spooling helper: called when we have no more data to spool to <b>conn</b>.
- * Flushes any remaining data to be (un)compressed, and changes the spool
- * source to NONE.  Returns 0 on success, negative on failure. */
-static int
-connection_dirserv_finish_spooling(dir_connection_t *conn)
-{
-  if (conn->zlib_state) {
-    connection_write_to_buf_zlib("", 0, conn, 1);
-    tor_zlib_free(conn->zlib_state);
-    conn->zlib_state = NULL;
+    if (BUG(!cached && !cce))
+      return SRFS_DONE;
+
+    int64_t total_len;
+    const char *ptr;
+    if (cached) {
+      total_len = cached->dir_compressed_len;
+      ptr = cached->dir_compressed;
+    } else {
+      total_len = spooled->cce_len;
+      ptr = (const char *)spooled->cce_body;
+    }
+    /* How many bytes left to flush? */
+    int64_t remaining;
+    remaining = total_len - spooled->cached_dir_offset;
+    if (BUG(remaining < 0))
+      return SRFS_ERR;
+    ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
+    if (conn->compress_state) {
+      connection_buf_add_compress(
+              ptr + spooled->cached_dir_offset,
+              bytes, conn, 0);
+    } else {
+      connection_buf_add(ptr + spooled->cached_dir_offset,
+                              bytes, TO_CONN(conn));
+    }
+    spooled->cached_dir_offset += bytes;
+    if (spooled->cached_dir_offset >= (off_t)total_len) {
+      return SRFS_DONE;
+    } else {
+      return SRFS_MORE;
+    }
   }
-  conn->dir_spool_src = DIR_SPOOL_NONE;
-  return 0;
 }
 
-/** Spooling helper: called when we're sending a bunch of server descriptors,
- * and the outbuf has become too empty. Pulls some entries from
- * fingerprint_stack, and writes the corresponding servers onto outbuf.  If we
- * run out of entries, flushes the zlib state and sets the spool source to
- * NONE.  Returns 0 on success, negative on failure.
+/** Helper: find the cached_dir_t for a spooled_resource_t, for
+ * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided,
+ * to the published time of the cached_dir_t.
+ *
+ * DOES NOT increase the reference count on the result.  Callers must do that
+ * themselves if they mean to hang on to it.
  */
+static cached_dir_t *
+spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled,
+                                   time_t *published_out)
+{
+  tor_assert(spooled->spool_eagerly == 0);
+  cached_dir_t *d = lookup_cached_dir_by_fp(spooled->digest);
+  if (d != NULL) {
+    if (published_out)
+      *published_out = d->published;
+  }
+  return d;
+}
+
+/** Helper: Look up the body for an eagerly-served spooled_resource.  If
+ * <b>conn_is_encrypted</b> is false, don't look up any resource that
+ * shouldn't be sent over an unencrypted connection.  On success, set
+ * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer
+ * to the resource's body, size, and publication date, and return 0.
+ * On failure return -1. */
 static int
-connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn)
+spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                             int conn_is_encrypted,
+                             const uint8_t **body_out,
+                             size_t *size_out,
+                             time_t *published_out)
 {
-  int by_fp = (conn->dir_spool_src == DIR_SPOOL_SERVER_BY_FP ||
-               conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP);
-  int extra = (conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP ||
-               conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_DIGEST);
-  time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH;
+  tor_assert(spooled->spool_eagerly == 1);
 
-  const or_options_t *options = get_options();
+  const signed_descriptor_t *sd = NULL;
 
-  while (smartlist_len(conn->fingerprint_stack) &&
-         connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    const char *body;
-    char *fp = smartlist_pop_last(conn->fingerprint_stack);
-    const signed_descriptor_t *sd = NULL;
-    if (by_fp) {
-      sd = get_signed_descriptor_by_fp(fp, extra, publish_cutoff);
-    } else {
-      sd = extra ? extrainfo_get_by_descriptor_digest(fp)
-        : router_get_by_descriptor_digest(fp);
+  switch (spooled->spool_source) {
+    case DIR_SPOOL_EXTRA_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 1);
+      break;
     }
-    tor_free(fp);
-    if (!sd)
-      continue;
-    if (!connection_dir_is_encrypted(conn) && !sd->send_unencrypted) {
-      /* we did this check once before (so we could have an accurate size
-       * estimate and maybe send a 404 if somebody asked for only bridges on a
-       * connection), but we need to do it again in case a previously
-       * unknown bridge descriptor has shown up between then and now. */
-      continue;
+    case DIR_SPOOL_SERVER_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 0);
+      break;
     }
-
-    /** If we are the bridge authority and the descriptor is a bridge
-     * descriptor, remember that we served this descriptor for desc stats. */
-    if (options->BridgeAuthoritativeDir && by_fp) {
-      const routerinfo_t *router =
-          router_get_by_id_digest(sd->identity_digest);
-      /* router can be NULL here when the bridge auth is asked for its own
-       * descriptor. */
-      if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
-        rep_hist_note_desc_served(sd->identity_digest);
+    case DIR_SPOOL_SERVER_BY_DIGEST: {
+      sd = router_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
     }
-    body = signed_descriptor_get_body(sd);
-    if (conn->zlib_state) {
-      int last = ! smartlist_len(conn->fingerprint_stack);
-      connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn,
-                                   last);
-      if (last) {
-        tor_zlib_free(conn->zlib_state);
-        conn->zlib_state = NULL;
+    case DIR_SPOOL_EXTRA_BY_DIGEST: {
+      sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_MICRODESC: {
+      microdesc_t *md = microdesc_cache_lookup_by_digest256(
+                                  get_microdesc_cache(),
+                                  (const char *)spooled->digest);
+      if (! md || ! md->body) {
+        return -1;
       }
-    } else {
-      connection_write_to_buf(body,
-                              sd->signed_descriptor_len,
-                              TO_CONN(conn));
+      *body_out = (const uint8_t *)md->body;
+      *size_out = md->bodylen;
+      if (published_out)
+        *published_out = TIME_MAX;
+      return 0;
     }
+    case DIR_SPOOL_NETWORKSTATUS:
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+    default:
+      /* LCOV_EXCL_START */
+      tor_assert_nonfatal_unreached();
+      return -1;
+      /* LCOV_EXCL_STOP */
   }
 
-  if (!smartlist_len(conn->fingerprint_stack)) {
-    /* We just wrote the last one; finish up. */
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib("", 0, conn, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
-    conn->dir_spool_src = DIR_SPOOL_NONE;
-    smartlist_free(conn->fingerprint_stack);
-    conn->fingerprint_stack = NULL;
+  /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */
+
+  if (sd == NULL) {
+    return -1;
   }
+  if (sd->send_unencrypted == 0 && ! conn_is_encrypted) {
+    /* we did this check once before (so we could have an accurate size
+     * estimate and maybe send a 404 if somebody asked for only bridges on
+     * a connection), but we need to do it again in case a previously
+     * unknown bridge descriptor has shown up between then and now. */
+    return -1;
+  }
+  *body_out = (const uint8_t *) signed_descriptor_get_body(sd);
+  *size_out = sd->signed_descriptor_len;
+  if (published_out)
+    *published_out = sd->published_on;
   return 0;
 }
 
-/** Spooling helper: called when we're sending a bunch of microdescriptors,
- * and the outbuf has become too empty. Pulls some entries from
- * fingerprint_stack, and writes the corresponding microdescs onto outbuf.  If
- * we run out of entries, flushes the zlib state and sets the spool source to
- * NONE.  Returns 0 on success, negative on failure.
- */
-static int
-connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn)
-{
-  microdesc_cache_t *cache = get_microdesc_cache();
-  while (smartlist_len(conn->fingerprint_stack) &&
-         connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    char *fp256 = smartlist_pop_last(conn->fingerprint_stack);
-    microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256);
-    tor_free(fp256);
-    if (!md || !md->body)
-      continue;
-    if (conn->zlib_state) {
-      int last = !smartlist_len(conn->fingerprint_stack);
-      connection_write_to_buf_zlib(md->body, md->bodylen, conn, last);
-      if (last) {
-        tor_zlib_free(conn->zlib_state);
-        conn->zlib_state = NULL;
-      }
-    } else {
-      connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn));
-    }
-  }
-  if (!smartlist_len(conn->fingerprint_stack)) {
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib("", 0, conn, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
-    conn->dir_spool_src = DIR_SPOOL_NONE;
-    smartlist_free(conn->fingerprint_stack);
-    conn->fingerprint_stack = NULL;
+/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
+ * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
+ * flavor name if we want a flavored v3 status, return a pointer to the
+ * appropriate cached dir object, or NULL if there isn't one available. */
+static cached_dir_t *
+lookup_cached_dir_by_fp(const uint8_t *fp)
+{
+  cached_dir_t *d = NULL;
+  if (tor_digest_is_zero((const char *)fp) && cached_consensuses) {
+    d = strmap_get(cached_consensuses, "ns");
+  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) {
+    /* this here interface is a nasty hack: we're shoving a flavor into
+     * a digest field. */
+    d = strmap_get(cached_consensuses, (const char *)fp);
   }
-  return 0;
+  return d;
 }
 
-/** Spooling helper: Called when we're sending a directory or networkstatus,
- * and the outbuf has become too empty.  Pulls some bytes from
- * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and
- * puts them on the outbuf.  If we run out of entries, flushes the zlib state
- * and sets the spool source to NONE.  Returns 0 on success, negative on
- * failure. */
-static int
-connection_dirserv_add_dir_bytes_to_outbuf(dir_connection_t *conn)
-{
-  ssize_t bytes;
-  int64_t remaining;
-
-  bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn));
-  tor_assert(bytes > 0);
-  tor_assert(conn->cached_dir);
-  if (bytes < 8192)
-    bytes = 8192;
-  remaining = conn->cached_dir->dir_z_len - conn->cached_dir_offset;
-  if (bytes > remaining)
-    bytes = (ssize_t) remaining;
-
-  if (conn->zlib_state) {
-    connection_write_to_buf_zlib(
-                             conn->cached_dir->dir_z + conn->cached_dir_offset,
-                             bytes, conn, bytes == remaining);
-  } else {
-    connection_write_to_buf(conn->cached_dir->dir_z + conn->cached_dir_offset,
-                            bytes, TO_CONN(conn));
+/** Try to guess the number of bytes that will be needed to send the
+ * spooled objects for <b>conn</b>'s outgoing spool.  In the process,
+ * remove every element of the spool that refers to an absent object, or
+ * which was published earlier than <b>cutoff</b>.  Set *<b>size_out</b>
+ * to the number of bytes, and *<b>n_expired_out</b> to the number of
+ * objects removed for being too old. */
+void
+dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                            time_t cutoff,
+                                            int compression,
+                                            size_t *size_out,
+                                            int *n_expired_out)
+{
+  if (BUG(!conn))
+    return;
+
+  smartlist_t *spool = conn->spool;
+  if (!spool) {
+    if (size_out)
+      *size_out = 0;
+    if (n_expired_out)
+      *n_expired_out = 0;
+    return;
   }
-  conn->cached_dir_offset += bytes;
-  if (conn->cached_dir_offset == (int)conn->cached_dir->dir_z_len) {
-    /* We just wrote the last one; finish up. */
-    connection_dirserv_finish_spooling(conn);
-    cached_dir_decref(conn->cached_dir);
-    conn->cached_dir = NULL;
+  int n_expired = 0;
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) {
+    time_t published = TIME_MAX;
+    size_t sz = spooled_resource_estimate_size(spooled, conn,
+                                               compression, &published);
+    if (published < cutoff) {
+      ++n_expired;
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else if (sz == 0) {
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else {
+      total += sz;
+    }
+  } SMARTLIST_FOREACH_END(spooled);
+
+  if (size_out) {
+    *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total;
   }
-  return 0;
+  if (n_expired_out)
+    *n_expired_out = n_expired;
 }
 
-/** Spooling helper: Called when we're spooling networkstatus objects on
- * <b>conn</b>, and the outbuf has become too empty.  If the current
- * networkstatus object (in <b>conn</b>-\>cached_dir) has more data, pull data
- * from there.  Otherwise, pop the next fingerprint from fingerprint_stack,
- * and start spooling the next networkstatus.  (A digest of all 0 bytes is
- * treated as a request for the current consensus.) If we run out of entries,
- * flushes the zlib state and sets the spool source to NONE.  Returns 0 on
- * success, negative on failure. */
+/** Helper: used to sort a connection's spool. */
 static int
-connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn)
-{
-
-  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    if (conn->cached_dir) {
-      int uncompressing = (conn->zlib_state != NULL);
-      int r = connection_dirserv_add_dir_bytes_to_outbuf(conn);
-      if (conn->dir_spool_src == DIR_SPOOL_NONE) {
-        /* add_dir_bytes thinks we're done with the cached_dir.  But we
-         * may have more cached_dirs! */
-        conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
-        /* This bit is tricky.  If we were uncompressing the last
-         * networkstatus, we may need to make a new zlib object to
-         * uncompress the next one. */
-        if (uncompressing && ! conn->zlib_state &&
-            conn->fingerprint_stack &&
-            smartlist_len(conn->fingerprint_stack)) {
-          conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-        }
-      }
-      if (r) return r;
-    } else if (conn->fingerprint_stack &&
-               smartlist_len(conn->fingerprint_stack)) {
-      /* Add another networkstatus; start serving it. */
-      char *fp = smartlist_pop_last(conn->fingerprint_stack);
-      cached_dir_t *d = lookup_cached_dir_by_fp(fp);
-      tor_free(fp);
-      if (d) {
-        ++d->refcnt;
-        conn->cached_dir = d;
-        conn->cached_dir_offset = 0;
-      }
-    } else {
-      connection_dirserv_finish_spooling(conn);
-      smartlist_free(conn->fingerprint_stack);
-      conn->fingerprint_stack = NULL;
-      return 0;
+dirserv_spool_sort_comparison_(const void **a_, const void **b_)
+{
+  const spooled_resource_t *a = *a_;
+  const spooled_resource_t *b = *b_;
+  return fast_memcmp(a->digest, b->digest, sizeof(a->digest));
+}
+
+/** Sort all the entries in <b>conn</b> by digest. */
+void
+dirserv_spool_sort(dir_connection_t *conn)
+{
+  if (conn->spool == NULL)
+    return;
+  smartlist_sort(conn->spool, dirserv_spool_sort_comparison_);
+}
+
+/** Return the cache-info for identity fingerprint <b>fp</b>, or
+ * its extra-info document if <b>extrainfo</b> is true. Return
+ * NULL if not found or if the descriptor is older than
+ * <b>publish_cutoff</b>. */
+static const signed_descriptor_t *
+get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo)
+{
+  if (router_digest_is_me((const char *)fp)) {
+    if (extrainfo)
+      return &(router_get_my_extrainfo()->cache_info);
+    else
+      return &(router_get_my_routerinfo()->cache_info);
+  } else {
+    const routerinfo_t *ri = router_get_by_id_digest((const char *)fp);
+    if (ri) {
+      if (extrainfo)
+        return extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+      else
+        return &ri->cache_info;
     }
   }
-  return 0;
+  return NULL;
 }
 
-/** Called whenever we have flushed some directory data in state
- * SERVER_WRITING. */
+/** When we're spooling data onto our outbuf, add more whenever we dip
+ * below this threshold. */
+#define DIRSERV_BUFFER_MIN 16384
+
+/**
+ * Called whenever we have flushed some directory data in state
+ * SERVER_WRITING, or whenever we want to fill the buffer with initial
+ * directory data (so that subsequent writes will occur, and trigger this
+ * function again.)
+ *
+ * Return 0 on success, and -1 on failure.
+ */
 int
 connection_dirserv_flushed_some(dir_connection_t *conn)
 {
   tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING);
-
-  if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN)
+  if (conn->spool == NULL)
     return 0;
 
-  switch (conn->dir_spool_src) {
-    case DIR_SPOOL_EXTRA_BY_DIGEST:
-    case DIR_SPOOL_EXTRA_BY_FP:
-    case DIR_SPOOL_SERVER_BY_DIGEST:
-    case DIR_SPOOL_SERVER_BY_FP:
-      return connection_dirserv_add_servers_to_outbuf(conn);
-    case DIR_SPOOL_MICRODESC:
-      return connection_dirserv_add_microdescs_to_outbuf(conn);
-    case DIR_SPOOL_CACHED_DIR:
-      return connection_dirserv_add_dir_bytes_to_outbuf(conn);
-    case DIR_SPOOL_NETWORKSTATUS:
-      return connection_dirserv_add_networkstatus_bytes_to_outbuf(conn);
-    case DIR_SPOOL_NONE:
-    default:
+  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN &&
+         smartlist_len(conn->spool)) {
+    spooled_resource_t *spooled =
+      smartlist_get(conn->spool, smartlist_len(conn->spool)-1);
+    spooled_resource_flush_status_t status;
+    status = spooled_resource_flush_some(spooled, conn);
+    if (status == SRFS_ERR) {
+      return -1;
+    } else if (status == SRFS_MORE) {
       return 0;
+    }
+    tor_assert(status == SRFS_DONE);
+
+    /* If we're here, we're done flushing this resource. */
+    tor_assert(smartlist_pop_last(conn->spool) == spooled);
+    spooled_resource_free(spooled);
+  }
+
+  if (smartlist_len(conn->spool) > 0) {
+    /* We're still spooling something. */
+    return 0;
+  }
+
+  /* If we get here, we're done. */
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
+  if (conn->compress_state) {
+    /* Flush the compression state: there could be more bytes pending in there,
+     * and we don't want to omit bytes. */
+    connection_buf_add_compress("", 0, conn, 1);
+    tor_compress_free(conn->compress_state);
+    conn->compress_state = NULL;
   }
+  return 0;
+}
+
+/** Remove every element from <b>conn</b>'s outgoing spool, and delete
+ * the spool. */
+void
+dir_conn_clear_spool(dir_connection_t *conn)
+{
+  if (!conn || ! conn->spool)
+    return;
+  SMARTLIST_FOREACH(conn->spool, spooled_resource_t *, s,
+                    spooled_resource_free(s));
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
 }
 
 /** Return true iff <b>line</b> is a valid RecommendedPackages line.
diff --git a/src/or/dirserv.h b/src/or/dirserv.h
index 624cd7e0b7..b243439cc2 100644
--- a/src/or/dirserv.h
+++ b/src/or/dirserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,6 +32,61 @@
 /** Maximum allowable length of a version line in a networkstatus. */
 #define MAX_V_LINE_LEN 128
 
+/** Ways to convert a spoolable_resource_t to a bunch of bytes. */
+typedef enum dir_spool_source_t {
+    DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP,
+    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
+    DIR_SPOOL_MICRODESC,
+    DIR_SPOOL_NETWORKSTATUS,
+    DIR_SPOOL_CONSENSUS_CACHE_ENTRY,
+} dir_spool_source_t;
+#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
+
+/** Object to remember the identity of an object that we are spooling,
+ * or about to spool, in response to a directory request.
+ *
+ * (Why do we spool?  Because some directory responses are very large,
+ * and we don't want to just shove the complete answer into the output
+ * buffer: that would take a ridiculous amount of RAM.)
+ *
+ * If the spooled resource is relatively small (like microdescriptors,
+ * descriptors, etc), we look them up by ID as needed, and add the whole
+ * thing onto the output buffer at once.  If the spooled reseource is
+ * big (like networkstatus documents), we reference-count it, and add it
+ * a few K at a time.
+ */
+typedef struct spooled_resource_t {
+  /**
+   * If true, we add the entire object to the outbuf.  If false,
+   * we spool the object a few K at a time.
+   */
+  unsigned spool_eagerly : 1;
+  /**
+   * Tells us what kind of object to get, and how to look it up.
+   */
+  dir_spool_source_bitfield_t spool_source : 7;
+  /**
+   * Tells us the specific object to spool.
+   */
+  uint8_t digest[DIGEST256_LEN];
+  /**
+   * A large object that we're spooling. Holds a reference count.  Only
+   * used when spool_eagerly is false.
+   */
+  struct cached_dir_t *cached_dir_ref;
+  /**
+   * A different kind of large object that we might be spooling. Also
+   * reference-counted.  Also only used when spool_eagerly is false.
+   */
+  struct consensus_cache_entry_t *consensus_cache_entry;
+  const uint8_t *cce_body;
+  size_t cce_len;
+  /**
+   * The current offset into cached_dir or cce_body. Only used when
+   * spool_eagerly is false */
+  off_t cached_dir_offset;
+} spooled_resource_t;
+
 int connection_dirserv_flushed_some(dir_connection_t *conn);
 
 int dirserv_add_own_fingerprint(crypto_pk_t *pk);
@@ -63,17 +118,19 @@ cached_dir_t *dirserv_get_consensus(const char *flavor_name);
 void dirserv_set_cached_consensus_networkstatus(const char *consensus,
                                               const char *flavor_name,
                                               const common_digests_t *digests,
+                                              const uint8_t *sha3_as_signed,
                                               time_t published);
 void dirserv_clear_old_networkstatuses(time_t cutoff);
-int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
-                                        const char **msg,
-                                        int for_unencrypted_conn,
-                                        int is_extrainfo);
+int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
+                                 dir_spool_source_t source,
+                                 int conn_is_encrypted,
+                                 const char **msg_out);
 int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
                             const char **msg);
 void dirserv_orconn_tls_done(const tor_addr_t *addr,
                              uint16_t or_port,
-                             const char *digest_rcvd);
+                             const char *digest_rcvd,
+                             const ed25519_public_key_t *ed_id_rcvd);
 int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
                                             const routerinfo_t *ri_old);
 void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
@@ -88,24 +145,27 @@ void dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                       uint32_t authstatus);
 
 int dirserv_would_reject_router(const routerstatus_t *rs);
-int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff);
-int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src);
-int dirserv_have_any_microdesc(const smartlist_t *fps);
-size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
-                                  int compressed);
-size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed);
-
 char *routerstatus_format_entry(
                               const routerstatus_t *rs,
                               const char *version,
                               const char *protocols,
                               routerstatus_format_type_t format,
+                              int consensus_method,
                               const vote_routerstatus_t *vrs);
 void dirserv_free_all(void);
 void cached_dir_decref(cached_dir_t *d);
 cached_dir_t *new_cached_dir(char *s, time_t published);
 char *format_recommended_version_list(const config_line_t *line, int warn);
 int validate_recommended_package_line(const char *line);
+int dirserv_query_measured_bw_cache_kb(const char *node_id,
+                                       long *bw_out,
+                                       time_t *as_of_out);
+void dirserv_clear_measured_bw_cache(void);
+int dirserv_has_measured_bw(const char *node_id);
+int dirserv_get_measured_bw_cache_size(void);
+void dirserv_count_measured_bws(const smartlist_t *routers);
+int running_long_enough_to_decide_unreachable(void);
+void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
 
 #ifdef DIRSERV_PRIVATE
 
@@ -114,25 +174,20 @@ STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
 /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */
 #define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */
 
-STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line);
+STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line,
+                                  int line_is_after_headers);
 
 STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line,
                            smartlist_t *routerstatuses);
 
 STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
                                time_t as_of);
-STATIC void dirserv_clear_measured_bw_cache(void);
 STATIC void dirserv_expire_measured_bw_cache(time_t now);
-STATIC int dirserv_get_measured_bw_cache_size(void);
-STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id,
-                                              long *bw_out,
-                                              time_t *as_of_out);
-STATIC int dirserv_has_measured_bw(const char *node_id);
 
 STATIC int
 dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
                                       smartlist_t *vote_routerstatuses);
-#endif
+#endif /* defined(DIRSERV_PRIVATE) */
 
 int dirserv_read_measured_bandwidths(const char *from_file,
                                      smartlist_t *routerstatuses);
@@ -140,4 +195,20 @@ int dirserv_read_measured_bandwidths(const char *from_file,
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);
 
-#endif
+spooled_resource_t *spooled_resource_new(dir_spool_source_t source,
+                                         const uint8_t *digest,
+                                         size_t digestlen);
+spooled_resource_t *spooled_resource_new_from_cache_entry(
+                                      struct consensus_cache_entry_t *entry);
+void spooled_resource_free_(spooled_resource_t *spooled);
+#define spooled_resource_free(sp) \
+  FREE_AND_NULL(spooled_resource_t, spooled_resource_free_, (sp))
+void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                                 time_t cutoff,
+                                                 int compression,
+                                                 size_t *size_out,
+                                                 int *n_expired_out);
+void dirserv_spool_sort(dir_connection_t *conn);
+void dir_conn_clear_spool(dir_connection_t *conn);
+
+#endif /* !defined(TOR_DIRSERV_H) */
diff --git a/src/or/dns.c b/src/or/dns.c
index 33628373d5..ba734ed900 100644
--- a/src/or/dns.c
+++ b/src/or/dns.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,7 @@
  *      dns_seems_to_be_broken().
  *    <li>When a client has asked the relay, in a RELAY_BEGIN cell, to connect
  *      to a given server by hostname.  This happens via dns_resolve().
- *    <li>When a client has asked the rela, in a RELAY_RESOLVE cell, to look
+ *    <li>When a client has asked the relay, in a RELAY_RESOLVE cell, to look
  *      up a given server's IP address(es) by hostname. This also happens via
  *      dns_resolve().
  *   </ol>
@@ -56,6 +56,7 @@
 #include "connection.h"
 #include "connection_edge.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "dns.h"
 #include "main.h"
 #include "policies.h"
@@ -102,7 +103,7 @@ static void assert_cache_ok_(void);
 #define assert_cache_ok() assert_cache_ok_()
 #else
 #define assert_cache_ok() STMT_NIL
-#endif
+#endif /* defined(DEBUG_DNS_CACHE) */
 static void assert_resolve_ok(cached_resolve_t *resolve);
 
 /** Hash table of cached_resolve objects. */
@@ -160,8 +161,9 @@ evdns_log_cb(int warn, const char *msg)
   }
   if (!strcmpstart(msg, "Nameserver ") && (cp=strstr(msg, " has failed: "))) {
     char *ns = tor_strndup(msg+11, cp-(msg+11));
-    const char *err = strchr(cp, ':')+2;
-    tor_assert(err);
+    const char *colon = strchr(cp, ':');
+    tor_assert(colon);
+    const char *err = colon+2;
     /* Don't warn about a single failed nameserver; we'll warn with 'all
      * nameservers have failed' if we're completely out of nameservers;
      * otherwise, the situation is tolerable. */
@@ -181,6 +183,18 @@ evdns_log_cb(int warn, const char *msg)
   } else if (!strcmp(msg, "All nameservers have failed")) {
     control_event_server_status(LOG_WARN, "NAMESERVER_ALL_DOWN");
     all_down = 1;
+  } else if (!strcmpstart(msg, "Address mismatch on received DNS")) {
+    static ratelim_t mismatch_limit = RATELIM_INIT(3600);
+    const char *src = strstr(msg, " Apparent source");
+    if (!src || get_options()->SafeLogging) {
+      src = "";
+    }
+    log_fn_ratelim(&mismatch_limit, severity, LD_EXIT,
+                   "eventdns: Received a DNS packet from "
+                   "an IP address to which we did not send a request. This "
+                   "could be a DNS spoofing attempt, or some kind of "
+                   "misconfiguration.%s", src);
+    return;
   }
   tor_log(severity, LD_EXIT, "eventdns: %s", msg);
 }
@@ -365,7 +379,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires)
   resolve->expire = expires;
   smartlist_pqueue_add(cached_resolve_pqueue,
                        compare_cached_resolves_by_expiry_,
-                       STRUCT_OFFSET(cached_resolve_t, minheap_idx),
+                       offsetof(cached_resolve_t, minheap_idx),
                        resolve);
 }
 
@@ -412,7 +426,7 @@ purge_expired_resolves(time_t now)
       break;
     smartlist_pqueue_pop(cached_resolve_pqueue,
                          compare_cached_resolves_by_expiry_,
-                         STRUCT_OFFSET(cached_resolve_t, minheap_idx));
+                         offsetof(cached_resolve_t, minheap_idx));
 
     if (resolve->state == CACHE_STATE_PENDING) {
       log_debug(LD_EXIT,
@@ -444,7 +458,7 @@ purge_expired_resolves(time_t now)
         if (!pendconn->base_.marked_for_close) {
           connection_edge_end(pendconn, END_STREAM_REASON_TIMEOUT);
           circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
-          connection_free(TO_CONN(pendconn));
+          connection_free_(TO_CONN(pendconn));
         }
         tor_free(pend);
       }
@@ -657,7 +671,7 @@ dns_resolve(edge_connection_t *exitconn)
         /* If we made the connection pending, then we freed it already in
          * dns_cancel_pending_resolve().  If we marked it for close, it'll
          * get freed from the main loop.  Otherwise, can free it now. */
-        connection_free(TO_CONN(exitconn));
+        connection_free_(TO_CONN(exitconn));
       }
       break;
     default:
@@ -948,14 +962,14 @@ assert_connection_edge_not_dns_pending(edge_connection_t *conn)
   for (pend = resolve->pending_connections; pend; pend = pend->next) {
     tor_assert(pend->conn != conn);
   }
-#else
+#else /* !(1) */
   cached_resolve_t **resolve;
   HT_FOREACH(resolve, cache_map, &cache_root) {
     for (pend = (*resolve)->pending_connections; pend; pend = pend->next) {
       tor_assert(pend->conn != conn);
     }
   }
-#endif
+#endif /* 1 */
 }
 
 /** Log an error and abort if any connection waiting for a DNS resolve is
@@ -1088,7 +1102,7 @@ dns_cancel_pending_resolve,(const char *address))
     if (circ)
       circuit_detach_stream(circ, pendconn);
     if (!pendconn->base_.marked_for_close)
-      connection_free(TO_CONN(pendconn));
+      connection_free_(TO_CONN(pendconn));
     resolve->pending_connections = pend->next;
     tor_free(pend);
   }
@@ -1217,7 +1231,7 @@ inform_pending_connections(cached_resolve_t *resolve)
         /* This detach must happen after we send the resolved cell. */
         circuit_detach_stream(circuit_get_by_edge_conn(pendconn), pendconn);
       }
-      connection_free(TO_CONN(pendconn));
+      connection_free_(TO_CONN(pendconn));
     } else {
       circuit_t *circ;
       if (pendconn->base_.purpose == EXIT_PURPOSE_CONNECT) {
@@ -1246,7 +1260,7 @@ inform_pending_connections(cached_resolve_t *resolve)
         circ = circuit_get_by_edge_conn(pendconn);
         tor_assert(circ);
         circuit_detach_stream(circ, pendconn);
-        connection_free(TO_CONN(pendconn));
+        connection_free_(TO_CONN(pendconn));
       }
     }
     resolve->pending_connections = pend->next;
@@ -1383,7 +1397,7 @@ configure_nameservers(int force)
       evdns_base_load_hosts(the_evdns_base,
           sandbox_intern_string("/etc/hosts"));
     }
-#endif
+#endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
     log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
     if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
         sandbox_intern_string(conf_fname)))) {
@@ -1421,7 +1435,7 @@ configure_nameservers(int force)
     tor_free(resolv_conf_fname);
     resolv_conf_mtime = 0;
   }
-#endif
+#endif /* defined(_WIN32) */
 
 #define SET(k,v)  evdns_base_set_option(the_evdns_base, (k), (v))
 
@@ -1568,10 +1582,11 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses,
                 escaped_safe_str(hostname));
       tor_free(escaped_address);
     } else if (count) {
-      log_warn(LD_EXIT, "eventdns returned only non-IPv4 answers for %s.",
+      log_info(LD_EXIT, "eventdns returned only unrecognized answer types "
+               " for %s.",
                escaped_safe_str(string_address));
     } else {
-      log_warn(LD_BUG, "eventdns returned no addresses or error for %s!",
+      log_info(LD_EXIT, "eventdns returned no addresses or error for %s.",
                escaped_safe_str(string_address));
     }
   }
@@ -1655,7 +1670,7 @@ launch_resolve,(cached_resolve_t *resolve))
   tor_addr_t a;
   int r;
 
-  if (get_options()->DisableNetwork)
+  if (net_is_disabled())
     return -1;
 
   /* What? Nameservers not configured?  Sounds like a bug. */
@@ -1762,7 +1777,7 @@ wildcard_increment_answer(const char *id)
         "invalid addresses. Apparently they are hijacking DNS failures. "
         "I'll try to correct for this by treating future occurrences of "
         "\"%s\" as 'not found'.", id, *ip, id);
-      smartlist_add(dns_wildcard_list, tor_strdup(id));
+      smartlist_add_strdup(dns_wildcard_list, id);
     }
     if (!dns_wildcard_notice_given)
       control_event_server_status(LOG_NOTICE, "DNS_HIJACKED");
@@ -1786,7 +1801,7 @@ add_wildcarded_test_address(const char *address)
   n_test_addrs = get_options()->ServerDNSTestAddresses ?
     smartlist_len(get_options()->ServerDNSTestAddresses) : 0;
 
-  smartlist_add(dns_wildcarded_test_address_list, tor_strdup(address));
+  smartlist_add_strdup(dns_wildcarded_test_address_list, address);
   n = smartlist_len(dns_wildcarded_test_address_list);
   if (n > n_test_addrs/2) {
     tor_log(dns_wildcarded_test_address_notice_given ? LOG_INFO : LOG_NOTICE,
@@ -1890,7 +1905,7 @@ launch_test_addresses(evutil_socket_t fd, short event, void *args)
   (void)event;
   (void)args;
 
-  if (options->DisableNetwork)
+  if (net_is_disabled())
     return;
 
   log_info(LD_EXIT, "Launching checks to see whether our nameservers like to "
@@ -1947,7 +1962,7 @@ dns_launch_wildcard_checks(void)
       launch_wildcard_check(8, 16, ipv6, ".com");
       launch_wildcard_check(8, 16, ipv6, ".org");
       launch_wildcard_check(8, 16, ipv6, ".net");
-  }
+    }
   }
 }
 
@@ -2040,7 +2055,7 @@ assert_resolve_ok(cached_resolve_t *resolve)
       tor_assert(!resolve->hostname);
     else
       tor_assert(!resolve->result_ipv4.addr_ipv4);
-#endif
+#endif /* 0 */
     /*XXXXX ADD MORE */
   }
 }
@@ -2090,7 +2105,7 @@ assert_cache_ok_(void)
 
   smartlist_pqueue_assert_ok(cached_resolve_pqueue,
                              compare_cached_resolves_by_expiry_,
-                             STRUCT_OFFSET(cached_resolve_t, minheap_idx));
+                             offsetof(cached_resolve_t, minheap_idx));
 
   SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res,
     {
@@ -2104,10 +2119,10 @@ assert_cache_ok_(void)
     });
 }
 
-#endif
+#endif /* defined(DEBUG_DNS_CACHE) */
 
-cached_resolve_t
-*dns_get_cache_entry(cached_resolve_t *query)
+cached_resolve_t *
+dns_get_cache_entry(cached_resolve_t *query)
 {
   return HT_FIND(cache_map, &cache_root, query);
 }
diff --git a/src/or/dns.h b/src/or/dns.h
index 951a2a3467..28d9f947b4 100644
--- a/src/or/dns.h
+++ b/src/or/dns.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -64,7 +64,7 @@ set_exitconn_info_from_resolve,(edge_connection_t *exitconn,
 MOCK_DECL(STATIC int,
 launch_resolve,(cached_resolve_t *resolve));
 
-#endif
+#endif /* defined(DNS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_DNS_H) */
 
diff --git a/src/or/dns_structs.h b/src/or/dns_structs.h
index bc6067213d..e22f23ac15 100644
--- a/src/or/dns_structs.h
+++ b/src/or/dns_structs.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -98,5 +98,5 @@ typedef struct cached_resolve_t {
   int minheap_idx;
 } cached_resolve_t;
 
-#endif
+#endif /* !defined(TOR_DNS_STRUCTS_H) */
 
diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c
index f5a4f2ac0f..7e344deeab 100644
--- a/src/or/dnsserv.c
+++ b/src/or/dnsserv.c
@@ -1,12 +1,24 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file dnsserv.c
- * \brief Implements client-side DNS proxy server code.  Note:
- * this is the DNS Server code, not the Server DNS code.  Confused?  This code
- * runs on client-side, and acts as a DNS server.  The code in dns.c, on the
- * other hand, runs on Tor servers, and acts as a DNS client.
+ * \brief Implements client-side DNS proxy server code.
+ *
+ * When a user enables the DNSPort configuration option to have their local
+ * Tor client handle DNS requests, this module handles it.  It functions as a
+ * "DNS Server" on the client side, which client applications use.
+ *
+ * Inbound DNS requests are represented as entry_connection_t here (since
+ * that's how Tor represents client-side streams), which are kept associated
+ * with an evdns_server_request structure as exposed by Libevent's
+ * evdns code.
+ *
+ * Upon receiving a DNS request, libevent calls our evdns_server_callback()
+ * function here, which causes this module to create an entry_connection_t
+ * request as appropriate.  Later, when that request is answered,
+ * connection_edge.c calls dnsserv_resolved() so we can finish up and tell the
+ * DNS client.
  **/
 
 #include "or.h"
@@ -160,7 +172,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_)
   if (connection_add(ENTRY_TO_CONN(entry_conn)) < 0) {
     log_warn(LD_APP, "Couldn't register dummy connection for DNS request");
     evdns_server_request_respond(req, DNS_ERR_SERVERFAILED);
-    connection_free(ENTRY_TO_CONN(entry_conn));
+    connection_free_(ENTRY_TO_CONN(entry_conn));
     return;
   }
 
@@ -196,6 +208,7 @@ dnsserv_launch_request(const char *name, int reverse,
 
   /* Make a new dummy AP connection, and attach the request to it. */
   entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  entry_conn->entry_cfg.dns_request = 1;
   conn = ENTRY_TO_EDGE_CONN(entry_conn);
   CONNECTION_AP_EXPECT_NONPENDING(entry_conn);
   conn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
@@ -214,10 +227,10 @@ dnsserv_launch_request(const char *name, int reverse,
     TO_CONN(conn)->port = control_conn->base_.port;
     TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
   }
-#else
+#else /* !(defined(AF_UNIX)) */
   TO_CONN(conn)->port = control_conn->base_.port;
   TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
-#endif
+#endif /* defined(AF_UNIX) */
 
   if (reverse)
     entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR;
@@ -237,7 +250,7 @@ dnsserv_launch_request(const char *name, int reverse,
 
   if (connection_add(TO_CONN(conn))<0) {
     log_warn(LD_APP, "Couldn't register dummy connection for RESOLVE request");
-    connection_free(TO_CONN(conn));
+    connection_free_(TO_CONN(conn));
     return -1;
   }
 
@@ -272,7 +285,7 @@ dnsserv_reject_request(entry_connection_t *conn)
 }
 
 /** Look up the original name that corresponds to 'addr' in req.  We use this
- * to preserve case in order to facilitate people using 0x20-hacks to avoid
+ * to preserve case in order to facilitate clients using 0x20-hacks to avoid
  * DNS poisoning. */
 static const char *
 evdns_get_orig_address(const struct evdns_server_request *req,
diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h
index ad0e248c83..2af366eee5 100644
--- a/src/or/dnsserv.h
+++ b/src/or/dnsserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,5 +23,5 @@ void dnsserv_reject_request(entry_connection_t *conn);
 int dnsserv_launch_request(const char *name, int is_reverse,
                            control_connection_t *control_conn);
 
-#endif
+#endif /* !defined(TOR_DNSSERV_H) */
 
diff --git a/src/or/dos.c b/src/or/dos.c
index 4d1797eece..ee731accea 100644
--- a/src/or/dos.c
+++ b/src/or/dos.c
@@ -11,10 +11,12 @@
 #include "or.h"
 #include "channel.h"
 #include "config.h"
+#include "crypto_rand.h"
 #include "geoip.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
+#include "relay.h"
 #include "router.h"
 
 #include "dos.h"
@@ -622,10 +624,12 @@ dos_log_heartbeat(void)
   char *conn_msg = NULL;
   char *cc_msg = NULL;
   char *single_hop_client_msg = NULL;
+  char *circ_stats_msg = NULL;
 
-  if (!dos_is_enabled()) {
-    goto end;
-  }
+  /* Stats number coming from relay.c append_cell_to_circuit_queue(). */
+  tor_asprintf(&circ_stats_msg,
+               " %" PRIu64 " circuits killed with too many cells.",
+               stats_n_circ_max_cell_reached);
 
   if (dos_cc_enabled) {
     tor_asprintf(&cc_msg,
@@ -647,7 +651,8 @@ dos_log_heartbeat(void)
   }
 
   log_notice(LD_HEARTBEAT,
-             "DoS mitigation since startup:%s%s%s",
+             "DoS mitigation since startup:%s%s%s%s",
+             circ_stats_msg,
              (cc_msg != NULL) ? cc_msg : " [cc not enabled]",
              (conn_msg != NULL) ? conn_msg : " [conn not enabled]",
              (single_hop_client_msg != NULL) ? single_hop_client_msg : "");
@@ -655,8 +660,7 @@ dos_log_heartbeat(void)
   tor_free(conn_msg);
   tor_free(cc_msg);
   tor_free(single_hop_client_msg);
-
- end:
+  tor_free(circ_stats_msg);
   return;
 }
 
diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c
index 265b6dcda1..27d760f1a8 100644
--- a/src/or/entrynodes.c
+++ b/src/or/entrynodes.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,19 +10,121 @@
  *
  * Entry nodes can be guards (for general use) or bridges (for censorship
  * circumvention).
+ *
+ * In general, we use entry guards to prevent traffic-sampling attacks:
+ * if we chose every circuit independently, an adversary controlling
+ * some fraction of paths on the network would observe a sample of every
+ * user's traffic. Using guards gives users a chance of not being
+ * profiled.
+ *
+ * The current entry guard selection code is designed to try to avoid
+ * _ever_ trying every guard on the network, to try to stick to guards
+ * that we've used before, to handle hostile/broken networks, and
+ * to behave sanely when the network goes up and down.
+ *
+ * Our algorithm works as follows: First, we maintain a SAMPLE of guards
+ * we've seen in the networkstatus consensus.  We maintain this sample
+ * over time, and store it persistently; it is chosen without reference
+ * to our configuration or firewall rules.  Guards remain in the sample
+ * as they enter and leave the consensus.  We expand this sample as
+ * needed, up to a maximum size.
+ *
+ * As a subset of the sample, we maintain a FILTERED SET of the guards
+ * that we would be willing to use if we could connect to them.  The
+ * filter removes all the guards that we're excluding because they're
+ * bridges (or not bridges), because we have restrictive firewall rules,
+ * because of ExcludeNodes, because we of path bias restrictions,
+ * because they're absent from the network at present, and so on.
+ *
+ * As a subset of the filtered set, we keep a REACHABLE FILTERED SET
+ * (also called a "usable filtered set") of those guards that we call
+ * "reachable" or "maybe reachable".  A guard is reachable if we've
+ * connected to it more recently than we've failed.  A guard is "maybe
+ * reachable" if we have never tried to connect to it, or if we
+ * failed to connect to it so long ago that we no longer think our
+ * failure means it's down.
+ *
+ * As a persistent ordered list whose elements are taken from the
+ * sampled set, we track a CONFIRMED GUARDS LIST.  A guard becomes
+ * confirmed when we successfully build a circuit through it, and decide
+ * to use that circuit.  We order the guards on this list by the order
+ * in which they became confirmed.
+ *
+ * And as a final group, we have an ordered list of PRIMARY GUARDS,
+ * whose elements are taken from the filtered set. We prefer
+ * confirmed guards to non-confirmed guards for this list, and place
+ * other restrictions on it.  The primary guards are the ones that we
+ * connect to "when nothing is wrong" -- circuits through them can be used
+ * immediately.
+ *
+ * To build circuits, we take a primary guard if possible -- or a
+ * reachable filtered confirmed guard if no primary guard is possible --
+ * or a random reachable filtered guard otherwise.  If the guard is
+ * primary, we can use the circuit immediately on success.  Otherwise,
+ * the guard is now "pending" -- we won't use its circuit unless all
+ * of the circuits we're trying to build through better guards have
+ * definitely failed.
+ *
+ * While we're building circuits, we track a little "guard state" for
+ * each circuit. We use this to keep track of whether the circuit is
+ * one that we can use as soon as it's done, or whether it's one that
+ * we should keep around to see if we can do better.  In the latter case,
+ * a periodic call to entry_guards_upgrade_waiting_circuits() will
+ * eventually upgrade it.
  **/
+/* DOCDOC -- expand this.
+ *
+ * Information invariants:
+ *
+ * [x] whenever a guard becomes unreachable, clear its usable_filtered flag.
+ *
+ * [x] Whenever a guard becomes reachable or maybe-reachable, if its filtered
+ * flag is set, set its usable_filtered flag.
+ *
+ * [x] Whenever we get a new consensus, call update_from_consensus(). (LATER.)
+ *
+ * [x] Whenever the configuration changes in a relevant way, update the
+ * filtered/usable flags. (LATER.)
+ *
+ * [x] Whenever we add a guard to the sample, make sure its filtered/usable
+ * flags are set as possible.
+ *
+ * [x] Whenever we remove a guard from the sample, remove it from the primary
+ * and confirmed lists.
+ *
+ * [x] When we make a guard confirmed, update the primary list.
+ *
+ * [x] When we make a guard filtered or unfiltered, update the primary list.
+ *
+ * [x] When we are about to pick a guard, make sure that the primary list is
+ * full.
+ *
+ * [x] Before calling sample_reachable_filtered_entry_guards(), make sure
+ * that the filtered, primary, and confirmed flags are up-to-date.
+ *
+ * [x] Call entry_guard_consider_retry every time we are about to check
+ * is_usable_filtered or is_reachable, and every time we set
+ * is_filtered to 1.
+ *
+ * [x] Call entry_guards_changed_for_guard_selection() whenever we update
+ * a persistent field.
+ */
 
 #define ENTRYNODES_PRIVATE
 
 #include "or.h"
+#include "channel.h"
+#include "bridges.h"
 #include "circpathbias.h"
 #include "circuitbuild.h"
+#include "circuitlist.h"
+#include "circuituse.h"
 #include "circuitstats.h"
 #include "config.h"
 #include "confparse.h"
 #include "connection.h"
-#include "connection_or.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "directory.h"
 #include "entrynodes.h"
 #include "main.h"
@@ -37,2509 +139,3532 @@
 #include "transports.h"
 #include "statefile.h"
 
-/** Information about a configured bridge. Currently this just matches the
- * ones in the torrc file, but one day we may be able to learn about new
- * bridges on our own, and remember them in the state file. */
-typedef struct {
-  /** Address of the bridge. */
-  tor_addr_t addr;
-  /** TLS port for the bridge. */
-  uint16_t port;
-  /** Boolean: We are re-parsing our bridge list, and we are going to remove
-   * this one if we don't find it in the list of configured bridges. */
-  unsigned marked_for_removal : 1;
-  /** Expected identity digest, or all zero bytes if we don't know what the
-   * digest should be. */
-  char identity[DIGEST_LEN];
-
-  /** Name of pluggable transport protocol taken from its config line. */
-  char *transport_name;
-
-  /** When should we next try to fetch a descriptor for this bridge? */
-  download_status_t fetch_status;
-
-  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
-      transports are used for this bridge. */
-  smartlist_t *socks_args;
-} bridge_info_t;
-
-/** A list of our chosen entry guards. */
-static smartlist_t *entry_guards = NULL;
-/** A value of 1 means that the entry_guards list has changed
+/** A list of existing guard selection contexts. */
+static smartlist_t *guard_contexts = NULL;
+/** The currently enabled guard selection context. */
+static guard_selection_t *curr_guard_context = NULL;
+
+/** A value of 1 means that at least one context has changed,
  * and those changes need to be flushed to disk. */
 static int entry_guards_dirty = 0;
 
-static void bridge_free(bridge_info_t *bridge);
-static const node_t *choose_random_entry_impl(cpath_build_state_t *state,
-                                              int for_directory,
-                                              dirinfo_type_t dirtype,
-                                              int *n_options_out);
-static int num_bridges_usable(void);
-
-/* Default number of entry guards in the case where the NumEntryGuards
- * consensus parameter is not set */
-#define DEFAULT_N_GUARDS 1
-/* Minimum and maximum number of entry guards (in case the NumEntryGuards
- * consensus parameter is set). */
-#define MIN_N_GUARDS 1
-#define MAX_N_GUARDS 10
-
-/** Return the list of entry guards, creating it if necessary. */
-const smartlist_t *
-get_entry_guards(void)
-{
-  if (! entry_guards)
-    entry_guards = smartlist_new();
-  return entry_guards;
-}
-
-/** Check whether the entry guard <b>e</b> is usable, given the directory
- * authorities' opinion about the router (stored in <b>ri</b>) and the user's
- * configuration (in <b>options</b>). Set <b>e</b>->bad_since
- * accordingly. Return true iff the entry guard's status changes.
- *
- * If it's not usable, set *<b>reason</b> to a static string explaining why.
- */
-static int
-entry_guard_set_status(entry_guard_t *e, const node_t *node,
-                       time_t now, const or_options_t *options,
-                       const char **reason)
+static void entry_guard_set_filtered_flags(const or_options_t *options,
+                                           guard_selection_t *gs,
+                                           entry_guard_t *guard);
+static void pathbias_check_use_success_count(entry_guard_t *guard);
+static void pathbias_check_close_success_count(entry_guard_t *guard);
+static int node_is_possible_guard(const node_t *node);
+static int node_passes_guard_filter(const or_options_t *options,
+                                    const node_t *node);
+static entry_guard_t *entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport);
+static entry_guard_t *get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                              const tor_addr_port_t *addrport);
+static int entry_guard_obeys_restriction(const entry_guard_t *guard,
+                                         const entry_guard_restriction_t *rst);
+
+/** Return 0 if we should apply guardfraction information found in the
+ *  consensus. A specific consensus can be specified with the
+ *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
+int
+should_apply_guardfraction(const networkstatus_t *ns)
 {
-  char buf[HEX_DIGEST_LEN+1];
-  int changed = 0;
+  /* We need to check the corresponding torrc option and the consensus
+   * parameter if we need to. */
+  const or_options_t *options = get_options();
 
-  *reason = NULL;
+  /* If UseGuardFraction is 'auto' then check the same-named consensus
+   * parameter. If the consensus parameter is not present, default to
+   * "off". */
+  if (options->UseGuardFraction == -1) {
+    return networkstatus_get_param(ns, "UseGuardFraction",
+                                   0, /* default to "off" */
+                                   0, 1);
+  }
 
-  /* Do we want to mark this guard as bad? */
+  return options->UseGuardFraction;
+}
+
+/** Return true iff we know a preferred descriptor for <b>guard</b> */
+static int
+guard_has_descriptor(const entry_guard_t *guard)
+{
+  const node_t *node = node_get_by_id(guard->identity);
   if (!node)
-    *reason = "unlisted";
-  else if (!node->is_running)
-    *reason = "down";
-  else if (options->UseBridges && (!node->ri ||
-                                   node->ri->purpose != ROUTER_PURPOSE_BRIDGE))
-    *reason = "not a bridge";
-  else if (options->UseBridges && !node_is_a_configured_bridge(node))
-    *reason = "not a configured bridge";
-  else if (!options->UseBridges && !node->is_possible_guard &&
-           !routerset_contains_node(options->EntryNodes,node))
-    *reason = "not recommended as a guard";
-  else if (routerset_contains_node(options->ExcludeNodes, node))
-    *reason = "excluded";
-  /* We only care about OR connection connectivity for entry guards. */
-  else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
-    *reason = "unreachable by config";
-  else if (e->path_bias_disabled)
-    *reason = "path-biased";
-
-  if (*reason && ! e->bad_since) {
-    /* Router is newly bad. */
-    base16_encode(buf, sizeof(buf), e->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard %s (%s) is %s: marking as unusable.",
-             e->nickname, buf, *reason);
-
-    e->bad_since = now;
-    control_event_guard(e->nickname, e->identity, "BAD");
-    changed = 1;
-  } else if (!*reason && e->bad_since) {
-    /* There's nothing wrong with the router any more. */
-    base16_encode(buf, sizeof(buf), e->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard %s (%s) is no longer unusable: "
-             "marking as ok.", e->nickname, buf);
-
-    e->bad_since = 0;
-    control_event_guard(e->nickname, e->identity, "GOOD");
-    changed = 1;
+    return 0;
+  return node_has_preferred_descriptor(node, 1);
+}
+
+/**
+ * Try to determine the correct type for a selection named "name",
+ * if <b>type</b> is GS_TYPE_INFER.
+ */
+STATIC guard_selection_type_t
+guard_selection_infer_type(guard_selection_type_t type,
+                           const char *name)
+{
+  if (type == GS_TYPE_INFER) {
+    if (!strcmp(name, "bridges"))
+      type = GS_TYPE_BRIDGE;
+    else if (!strcmp(name, "restricted"))
+      type = GS_TYPE_RESTRICTED;
+    else
+      type = GS_TYPE_NORMAL;
   }
+  return type;
+}
 
-  if (node) {
-    int is_dir = node_is_dir(node);
-    if (options->UseBridges && node_is_a_configured_bridge(node))
-      is_dir = 1;
-    if (e->is_dir_cache != is_dir) {
-      e->is_dir_cache = is_dir;
-      changed = 1;
-    }
+/**
+ * Allocate and return a new guard_selection_t, with the name <b>name</b>.
+ */
+STATIC guard_selection_t *
+guard_selection_new(const char *name,
+                    guard_selection_type_t type)
+{
+  guard_selection_t *gs;
+
+  type = guard_selection_infer_type(type, name);
+
+  gs = tor_malloc_zero(sizeof(*gs));
+  gs->name = tor_strdup(name);
+  gs->type = type;
+  gs->sampled_entry_guards = smartlist_new();
+  gs->confirmed_entry_guards = smartlist_new();
+  gs->primary_entry_guards = smartlist_new();
+
+  return gs;
+}
+
+/**
+ * Return the guard selection called <b>name</b>. If there is none, and
+ * <b>create_if_absent</b> is true, then create and return it.  If there
+ * is none, and <b>create_if_absent</b> is false, then return NULL.
+ */
+STATIC guard_selection_t *
+get_guard_selection_by_name(const char *name,
+                            guard_selection_type_t type,
+                            int create_if_absent)
+{
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
   }
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    if (!strcmp(gs->name, name))
+      return gs;
+  } SMARTLIST_FOREACH_END(gs);
+
+  if (! create_if_absent)
+    return NULL;
 
-  return changed;
+  log_debug(LD_GUARD, "Creating a guard selection called %s", name);
+  guard_selection_t *new_selection = guard_selection_new(name, type);
+  smartlist_add(guard_contexts, new_selection);
+
+  return new_selection;
 }
 
-/** Return true iff enough time has passed since we last tried to connect
- * to the unreachable guard <b>e</b> that we're willing to try again. */
-STATIC int
-entry_is_time_to_retry(const entry_guard_t *e, time_t now)
+/**
+ * Allocate the first guard context that we're planning to use,
+ * and make it the current context.
+ */
+static void
+create_initial_guard_context(void)
 {
-  struct guard_retry_period_s {
-    time_t period_duration;
-    time_t interval_during_period;
-  };
+  tor_assert(! curr_guard_context);
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
+  }
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(
+                             get_options(),
+                             networkstatus_get_live_consensus(approx_time()),
+                             NULL,
+                             &type);
+  tor_assert(name); // "name" can only be NULL if we had an old name.
+  tor_assert(type != GS_TYPE_INFER);
+  log_notice(LD_GUARD, "Starting with guard context \"%s\"", name);
+  curr_guard_context = get_guard_selection_by_name(name, type, 1);
+}
+
+/** Get current default guard_selection_t, creating it if necessary */
+guard_selection_t *
+get_guard_selection_info(void)
+{
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+  }
 
-  struct guard_retry_period_s periods[] = {
-     {    6*60*60,    60*60 }, /* For first 6 hrs., retry hourly; */
-     { 3*24*60*60,  4*60*60 }, /* Then retry every 4 hrs. until the
-                                  3-day mark; */
-     { 7*24*60*60, 18*60*60 }, /* After 3 days, retry every 18 hours until
-                                  1 week mark. */
-     {   TIME_MAX, 36*60*60 }  /* After 1 week, retry every 36 hours. */
-  };
+  return curr_guard_context;
+}
 
-  time_t ith_deadline_for_retry;
-  time_t unreachable_for;
-  unsigned i;
+/** Return a statically allocated human-readable description of <b>guard</b>
+ */
+const char *
+entry_guard_describe(const entry_guard_t *guard)
+{
+  static char buf[256];
+  tor_snprintf(buf, sizeof(buf),
+               "%s ($%s)",
+               strlen(guard->nickname) ? guard->nickname : "[bridge]",
+               hex_str(guard->identity, DIGEST_LEN));
+  return buf;
+}
 
-  if (e->last_attempted < e->unreachable_since)
-    return 1;
+/** Return <b>guard</b>'s 20-byte RSA identity digest */
+const char *
+entry_guard_get_rsa_id_digest(const entry_guard_t *guard)
+{
+  return guard->identity;
+}
 
-  unreachable_for = now - e->unreachable_since;
+/** Return the pathbias state associated with <b>guard</b>. */
+guard_pathbias_t *
+entry_guard_get_pathbias_state(entry_guard_t *guard)
+{
+  return &guard->pb;
+}
 
-  for (i = 0; i < ARRAY_LENGTH(periods); i++) {
-    if (unreachable_for <= periods[i].period_duration) {
-      ith_deadline_for_retry = e->last_attempted +
-                               periods[i].interval_during_period;
+HANDLE_IMPL(entry_guard, entry_guard_t, ATTR_UNUSED STATIC)
 
-      return (now > ith_deadline_for_retry);
-    }
-  }
-  return 0;
+/** Return an interval betweeen 'now' and 'max_backdate' seconds in the past,
+ * chosen uniformly at random.  We use this before recording persistent
+ * dates, so that we aren't leaking exactly when we recorded it.
+ */
+MOCK_IMPL(STATIC time_t,
+randomize_time,(time_t now, time_t max_backdate))
+{
+  tor_assert(max_backdate > 0);
+
+  time_t earliest = now - max_backdate;
+  time_t latest = now;
+  if (earliest <= 0)
+    earliest = 1;
+  if (latest <= earliest)
+    latest = earliest + 1;
+
+  return crypto_rand_time_range(earliest, latest);
 }
 
-/** Return the node corresponding to <b>e</b>, if <b>e</b> is
- * working well enough that we are willing to use it as an entry
- * right now. (Else return NULL.) In particular, it must be
- * - Listed as either up or never yet contacted;
- * - Present in the routerlist;
- * - Listed as 'stable' or 'fast' by the current dirserver consensus,
- *   if demanded by <b>need_uptime</b> or <b>need_capacity</b>
- *   (unless it's a configured EntryNode);
- * - Allowed by our current ReachableORAddresses config option; and
- * - Currently thought to be reachable by us (unless <b>assume_reachable</b>
- *   is true).
- *
- * If the answer is no, set *<b>msg</b> to an explanation of why.
+/**
+ * @name parameters for networkstatus algorithm
  *
- * If need_descriptor is true, only return the node if we currently have
- * a descriptor (routerinfo or microdesc) for it.
+ * These parameters are taken from the consensus; some are overrideable in
+ * the torrc.
  */
-STATIC const node_t *
-entry_is_live(const entry_guard_t *e, entry_is_live_flags_t flags,
-              const char **msg)
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this fraction
+ * of the guards on the network.
+ */
+STATIC double
+get_max_sample_threshold(void)
 {
-  const node_t *node;
-  const or_options_t *options = get_options();
-  int need_uptime = (flags & ENTRY_NEED_UPTIME) != 0;
-  int need_capacity = (flags & ENTRY_NEED_CAPACITY) != 0;
-  const int assume_reachable = (flags & ENTRY_ASSUME_REACHABLE) != 0;
-  const int need_descriptor = (flags & ENTRY_NEED_DESCRIPTOR) != 0;
-
-  tor_assert(msg);
+  int32_t pct =
+    networkstatus_get_param(NULL, "guard-max-sample-threshold-percent",
+                            DFLT_MAX_SAMPLE_THRESHOLD_PERCENT,
+                            1, 100);
+  return pct / 100.0;
+}
+/**
+ * We never let our sampled guard set grow larger than this number.
+ */
+STATIC int
+get_max_sample_size_absolute(void)
+{
+  return (int) networkstatus_get_param(NULL, "guard-max-sample-size",
+                                       DFLT_MAX_SAMPLE_SIZE,
+                                       1, INT32_MAX);
+}
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+STATIC int
+get_min_filtered_sample_size(void)
+{
+  return networkstatus_get_param(NULL, "guard-min-filtered-sample-size",
+                                 DFLT_MIN_FILTERED_SAMPLE_SIZE,
+                                 1, INT32_MAX);
+}
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+STATIC int
+get_remove_unlisted_guards_after_days(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-remove-unlisted-guards-after-days",
+                                 DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS,
+                                 1, 365*10);
+}
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+STATIC int
+get_guard_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL,
+                                 "guard-lifetime-days",
+                                 DFLT_GUARD_LIFETIME_DAYS, 1, 365*10);
+  return days * 86400;
+}
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+STATIC int
+get_guard_confirmed_min_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL, "guard-confirmed-min-lifetime-days",
+                                 DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS,
+                                 1, 365*10);
+  return days * 86400;
+}
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+STATIC int
+get_n_primary_guards(void)
+{
+  /* If the user has explicitly configured the number of primary guards, do
+   * what the user wishes to do */
+  const int configured_primaries = get_options()->NumPrimaryGuards;
+  if (configured_primaries) {
+    return configured_primaries;
+  }
 
-  if (e->path_bias_disabled) {
-    *msg = "path-biased";
-    return NULL;
+  /* otherwise check for consensus parameter and if that's not set either, just
+   * use the default value. */
+  return networkstatus_get_param(NULL,
+                                 "guard-n-primary-guards",
+                                 DFLT_N_PRIMARY_GUARDS, 1, INT32_MAX);
+}
+/**
+ * Return the number of the live primary guards we should look at when
+ * making a circuit.
+ */
+STATIC int
+get_n_primary_guards_to_use(guard_usage_t usage)
+{
+  int configured;
+  const char *param_name;
+  int param_default;
+
+  /* If the user has explicitly configured the amount of guards, use
+     that. Otherwise, fall back to the default value. */
+  if (usage == GUARD_USAGE_DIRGUARD) {
+    configured = get_options()->NumDirectoryGuards;
+    param_name = "guard-n-primary-dir-guards-to-use";
+    param_default = DFLT_N_PRIMARY_DIR_GUARDS_TO_USE;
+  } else {
+    configured = get_options()->NumEntryGuards;
+    param_name = "guard-n-primary-guards-to-use";
+    param_default = DFLT_N_PRIMARY_GUARDS_TO_USE;
   }
-  if (e->bad_since) {
-    *msg = "bad";
-    return NULL;
+  if (configured >= 1) {
+    return configured;
   }
-  /* no good if it's unreachable, unless assume_unreachable or can_retry. */
-  if (!assume_reachable && !e->can_retry &&
-      e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) {
-    *msg = "unreachable";
-    return NULL;
+  return networkstatus_get_param(NULL,
+                                 param_name, param_default, 1, INT32_MAX);
+}
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+STATIC int
+get_internet_likely_down_interval(void)
+{
+  return networkstatus_get_param(NULL, "guard-internet-likely-down-interval",
+                                 DFLT_INTERNET_LIKELY_DOWN_INTERVAL,
+                                 1, INT32_MAX);
+}
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+STATIC int
+get_nonprimary_guard_connect_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-connect-timeout",
+                                 DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+STATIC int
+get_nonprimary_guard_idle_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-idle-timeout",
+                                 DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+STATIC double
+get_meaningful_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-meaningful-restriction-percent",
+                                        DFLT_MEANINGFUL_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+STATIC double
+get_extreme_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-extreme-restriction-percent",
+                                        DFLT_EXTREME_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+
+/* Mark <b>guard</b> as maybe reachable again. */
+static void
+mark_guard_maybe_reachable(entry_guard_t *guard)
+{
+  if (guard->is_reachable != GUARD_REACHABLE_NO) {
+    return;
   }
-  node = node_get_by_id(e->identity);
-  if (!node) {
-    *msg = "no node info";
-    return NULL;
+
+  /* Note that we do not clear failing_since: this guard is now only
+   * _maybe-reachable_. */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+}
+
+/**
+ * Called when the network comes up after having seemed to be down for
+ * a while: Mark the primary guards as maybe-reachable so that we'll
+ * try them again.
+ */
+STATIC void
+mark_primary_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/* Called when we exhaust all guards in our sampled set: Marks all guards as
+   maybe-reachable so that we 'll try them again. */
+static void
+mark_all_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**@}*/
+
+/**
+ * Given our options and our list of nodes, return the name of the
+ * guard selection that we should use.  Return NULL for "use the
+ * same selection you were using before.
+ */
+STATIC const char *
+choose_guard_selection(const or_options_t *options,
+                       const networkstatus_t *live_ns,
+                       const guard_selection_t *old_selection,
+                       guard_selection_type_t *type_out)
+{
+  tor_assert(options);
+  tor_assert(type_out);
+
+  if (options->UseBridges) {
+    *type_out = GS_TYPE_BRIDGE;
+    return "bridges";
   }
-  if (need_descriptor && !node_has_descriptor(node)) {
-    *msg = "no descriptor";
-    return NULL;
+
+  if (! live_ns) {
+    /* without a networkstatus, we can't tell any more than that. */
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
   }
-  if (get_options()->UseBridges) {
-    if (node_get_purpose(node) != ROUTER_PURPOSE_BRIDGE) {
-      *msg = "not a bridge";
-      return NULL;
-    }
-    if (!node_is_a_configured_bridge(node)) {
-      *msg = "not a configured bridge";
-      return NULL;
-    }
-  } else { /* !get_options()->UseBridges */
-    if (node_get_purpose(node) != ROUTER_PURPOSE_GENERAL) {
-      *msg = "not general-purpose";
-      return NULL;
+
+  const smartlist_t *nodes = nodelist_get_list();
+  int n_guards = 0, n_passing_filter = 0;
+  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+    if (node_is_possible_guard(node)) {
+      ++n_guards;
+      if (node_passes_guard_filter(options, node)) {
+        ++n_passing_filter;
+      }
     }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* We use separate 'high' and 'low' thresholds here to prevent flapping
+   * back and forth */
+  const int meaningful_threshold_high =
+    (int)(n_guards * get_meaningful_restriction_threshold() * 1.05);
+  const int meaningful_threshold_mid =
+    (int)(n_guards * get_meaningful_restriction_threshold());
+  const int meaningful_threshold_low =
+    (int)(n_guards * get_meaningful_restriction_threshold() * .95);
+  const int extreme_threshold =
+    (int)(n_guards * get_extreme_restriction_threshold());
+
+  /*
+    If we have no previous selection, then we're "restricted" iff we are
+    below the meaningful restriction threshold.  That's easy enough.
+
+    But if we _do_ have a previous selection, we make it a little
+    "sticky": we only move from "restricted" to "default" when we find
+    that we're above the threshold plus 5%, and we only move from
+    "default" to "restricted" when we're below the threshold minus 5%.
+    That should prevent us from flapping back and forth if we happen to
+    be hovering very close to the default.
+
+    The extreme threshold is for warning only.
+  */
+
+  static int have_warned_extreme_threshold = 0;
+  if (n_guards &&
+      n_passing_filter < extreme_threshold &&
+      ! have_warned_extreme_threshold) {
+    have_warned_extreme_threshold = 1;
+    const double exclude_frac =
+      (n_guards - n_passing_filter) / (double)n_guards;
+    log_warn(LD_GUARD, "Your configuration excludes %d%% of all possible "
+             "guards. That's likely to make you stand out from the "
+             "rest of the world.", (int)(exclude_frac * 100));
   }
-  if (routerset_contains_node(options->EntryNodes, node)) {
-    /* they asked for it, they get it */
-    need_uptime = need_capacity = 0;
-  }
-  if (node_is_unreliable(node, need_uptime, need_capacity, 0)) {
-    *msg = "not fast/stable";
-    return NULL;
+
+  /* Easy case: no previous selection. Just check if we are in restricted or
+     normal guard selection. */
+  if (old_selection == NULL) {
+    if (n_passing_filter >= meaningful_threshold_mid) {
+      *type_out = GS_TYPE_NORMAL;
+      return "default";
+    } else {
+      *type_out = GS_TYPE_RESTRICTED;
+      return "restricted";
+    }
   }
-  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0)) {
-    *msg = "unreachable by config";
-    return NULL;
+
+  /* Trickier case: we do have a previous guard selection context. */
+  tor_assert(old_selection);
+
+  /* Use high and low thresholds to decide guard selection, and if we fall in
+     the middle then keep the current guard selection context. */
+  if (n_passing_filter >= meaningful_threshold_high) {
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
+  } else if (n_passing_filter < meaningful_threshold_low) {
+    *type_out = GS_TYPE_RESTRICTED;
+    return "restricted";
+  } else {
+    /* we are in the middle: maintain previous guard selection */
+    *type_out = old_selection->type;
+    return old_selection->name;
   }
-  return node;
 }
 
-/** Return the number of entry guards that we think are usable. */
+/**
+ * Check whether we should switch from our current guard selection to a
+ * different one.  If so, switch and return 1.  Return 0 otherwise.
+ *
+ * On a 1 return, the caller should mark all currently live circuits unusable
+ * for new streams, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
 int
-num_live_entry_guards(int for_directory)
+update_guard_selection_choice(const or_options_t *options)
 {
-  int n = 0;
-  const char *msg;
-  /* Set the entry node attributes we are interested in. */
-  entry_is_live_flags_t entry_flags = ENTRY_NEED_CAPACITY;
-  if (!for_directory) {
-    entry_flags |= ENTRY_NEED_DESCRIPTOR;
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+    return 1;
   }
 
-  if (! entry_guards)
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-      if (for_directory && !entry->is_dir_cache)
-        continue;
-      if (entry_is_live(entry, entry_flags, &msg))
-        ++n;
-  } SMARTLIST_FOREACH_END(entry);
-  return n;
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *new_name = choose_guard_selection(
+                             options,
+                             networkstatus_get_live_consensus(approx_time()),
+                             curr_guard_context,
+                             &type);
+  tor_assert(new_name);
+  tor_assert(type != GS_TYPE_INFER);
+
+  const char *cur_name = curr_guard_context->name;
+  if (! strcmp(cur_name, new_name)) {
+    log_debug(LD_GUARD,
+              "Staying with guard context \"%s\" (no change)", new_name);
+    return 0; // No change
+  }
+
+  log_notice(LD_GUARD, "Switching to guard context \"%s\" (was using \"%s\")",
+             new_name, cur_name);
+  guard_selection_t *new_guard_context;
+  new_guard_context = get_guard_selection_by_name(new_name, type, 1);
+  tor_assert(new_guard_context);
+  tor_assert(new_guard_context != curr_guard_context);
+  curr_guard_context = new_guard_context;
+
+  return 1;
 }
 
-/** If <b>digest</b> matches the identity of any node in the
- * entry_guards list, return that node. Else return NULL. */
-entry_guard_t *
-entry_guard_get_by_id_digest(const char *digest)
+/**
+ * Return true iff <b>node</b> has all the flags needed for us to consider it
+ * a possible guard when sampling guards.
+ */
+static int
+node_is_possible_guard(const node_t *node)
 {
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry,
-                    if (tor_memeq(digest, entry->identity, DIGEST_LEN))
-                      return entry;
-                   );
+  /* The "GUARDS" set is all nodes in the nodelist for which this predicate
+   * holds. */
+
+  tor_assert(node);
+  return (node->is_possible_guard &&
+          node->is_stable &&
+          node->is_fast &&
+          node->is_valid &&
+          node_is_dir(node) &&
+          !router_digest_is_me(node->identity));
+}
+
+/**
+ * Return the sampled guard with the RSA identity digest <b>rsa_id</b>, or
+ * NULL if we don't have one. */
+STATIC entry_guard_t *
+get_sampled_guard_with_id(guard_selection_t *gs,
+                          const uint8_t *rsa_id)
+{
+  tor_assert(gs);
+  tor_assert(rsa_id);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (tor_memeq(guard->identity, rsa_id, DIGEST_LEN))
+      return guard;
+  } SMARTLIST_FOREACH_END(guard);
   return NULL;
 }
 
-/** Dump a description of our list of entry guards to the log at level
- * <b>severity</b>. */
-static void
-log_entry_guards(int severity)
-{
-  smartlist_t *elements = smartlist_new();
-  char *s;
-
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e)
-    {
-      const char *msg = NULL;
-      if (entry_is_live(e, ENTRY_NEED_CAPACITY, &msg))
-        smartlist_add_asprintf(elements, "%s [%s] (up %s)",
-                     e->nickname,
-                     hex_str(e->identity, DIGEST_LEN),
-                     e->made_contact ? "made-contact" : "never-contacted");
-      else
-        smartlist_add_asprintf(elements, "%s [%s] (%s, %s)",
-                     e->nickname,
-                     hex_str(e->identity, DIGEST_LEN),
-                     msg,
-                     e->made_contact ? "made-contact" : "never-contacted");
-    }
-  SMARTLIST_FOREACH_END(e);
+/** If <b>gs</b> contains a sampled entry guard matching <b>bridge</b>,
+ * return that guard. Otherwise return NULL. */
+static entry_guard_t *
+get_sampled_guard_for_bridge(guard_selection_t *gs,
+                             const bridge_info_t *bridge)
+{
+  const uint8_t *id = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+  entry_guard_t *guard;
+  if (BUG(!addrport))
+    return NULL; // LCOV_EXCL_LINE
+  guard = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (! guard || (id && tor_memneq(id, guard->identity, DIGEST_LEN)))
+    return NULL;
+  else
+    return guard;
+}
 
-  s = smartlist_join_strings(elements, ",", 0, NULL);
-  SMARTLIST_FOREACH(elements, char*, cp, tor_free(cp));
-  smartlist_free(elements);
-  log_fn(severity,LD_CIRC,"%s",s);
-  tor_free(s);
+/** If we know a bridge_info_t matching <b>guard</b>, return that
+ * bridge.  Otherwise return NULL. */
+static bridge_info_t *
+get_bridge_info_for_guard(const entry_guard_t *guard)
+{
+  const uint8_t *identity = NULL;
+  if (! tor_digest_is_zero(guard->identity)) {
+    identity = (const uint8_t *)guard->identity;
+  }
+  if (BUG(guard->bridge_addr == NULL))
+    return NULL;
+
+  return get_configured_bridge_by_exact_addr_port_digest(
+                                                 &guard->bridge_addr->addr,
+                                                 guard->bridge_addr->port,
+                                                 (const char*)identity);
 }
 
-/** Called when one or more guards that we would previously have used for some
- * purpose are no longer in use because a higher-priority guard has become
- * usable again. */
-static void
-control_event_guard_deferred(void)
-{
-  /* XXXX We don't actually have a good way to figure out _how many_ entries
-   * are live for some purpose.  We need an entry_is_even_slightly_live()
-   * function for this to work right.  NumEntryGuards isn't reliable: if we
-   * need guards with weird properties, we can have more than that number
-   * live.
-   **/
-#if 0
-  int n = 0;
-  const char *msg;
-  const or_options_t *options = get_options();
-  if (!entry_guards)
-    return;
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry,
-    {
-      if (entry_is_live(entry, 0, 1, 0, &msg)) {
-        if (n++ == options->NumEntryGuards) {
-          control_event_guard(entry->nickname, entry->identity, "DEFERRED");
-          return;
-        }
-      }
-    });
-#endif
+/**
+ * Return true iff we have a sampled guard with the RSA identity digest
+ * <b>rsa_id</b>. */
+static inline int
+have_sampled_guard_with_id(guard_selection_t *gs, const uint8_t *rsa_id)
+{
+  return get_sampled_guard_with_id(gs, rsa_id) != NULL;
 }
 
-/** Largest amount that we'll backdate chosen_on_date */
-#define CHOSEN_ON_DATE_SLOP (30*86400)
+/**
+ * Allocate a new entry_guard_t object for <b>node</b>, add it to the
+ * sampled entry guards in <b>gs</b>, and return it. <b>node</b> must
+ * not currently be a sampled guard in <b>gs</b>.
+ */
+STATIC entry_guard_t *
+entry_guard_add_to_sample(guard_selection_t *gs,
+                          const node_t *node)
+{
+  log_info(LD_GUARD, "Adding %s to the entry guard sample set.",
+           node_describe(node));
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(have_sampled_guard_with_id(gs, (const uint8_t*)node->identity)))
+    return NULL; // LCOV_EXCL_LINE
 
-/** Add a new (preferably stable and fast) router to our
- * entry_guards list. Return a pointer to the router if we succeed,
- * or NULL if we can't find any more suitable entries.
- *
- * If <b>chosen</b> is defined, use that one, and if it's not
- * already in our entry_guards list, put it at the *beginning*.
- * Else, put the one we pick at the end of the list. */
-STATIC const node_t *
-add_an_entry_guard(const node_t *chosen, int reset_status, int prepend,
-                   int for_discovery, int for_directory)
+  return entry_guard_add_to_sample_impl(gs,
+                                        (const uint8_t*)node->identity,
+                                        node_get_nickname(node),
+                                        NULL);
+}
+
+/**
+ * Backend: adds a new sampled guard to <b>gs</b>, with given identity,
+ * nickname, and ORPort.  rsa_id_digest and bridge_addrport are optional, but
+ * we need one of them. nickname is optional. The caller is responsible for
+ * maintaining the size limit of the SAMPLED_GUARDS set.
+ */
+static entry_guard_t *
+entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport)
+{
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  tor_assert(gs);
+
+  // XXXX #20827 take ed25519 identity here too.
+
+  /* Make sure we can actually identify the guard. */
+  if (BUG(!rsa_id_digest && !bridge_addrport))
+    return NULL; // LCOV_EXCL_LINE
+
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+
+  /* persistent fields */
+  guard->is_persistent = (rsa_id_digest != NULL);
+  guard->selection_name = tor_strdup(gs->name);
+  if (rsa_id_digest)
+    memcpy(guard->identity, rsa_id_digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  guard->sampled_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+  tor_free(guard->sampled_by_version);
+  guard->sampled_by_version = tor_strdup(VERSION);
+  guard->currently_listed = 1;
+  guard->confirmed_idx = -1;
+
+  /* non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (bridge_addrport)
+    guard->bridge_addr = tor_memdup(bridge_addrport, sizeof(*bridge_addrport));
+
+  smartlist_add(gs->sampled_entry_guards, guard);
+  guard->in_selection = gs;
+  entry_guard_set_filtered_flags(get_options(), gs, guard);
+  entry_guards_changed_for_guard_selection(gs);
+  return guard;
+}
+
+/**
+ * Add an entry guard to the "bridges" guard selection sample, with
+ * information taken from <b>bridge</b>. Return that entry guard.
+ */
+static entry_guard_t *
+entry_guard_add_bridge_to_sample(guard_selection_t *gs,
+                                 const bridge_info_t *bridge)
 {
-  const node_t *node;
-  entry_guard_t *entry;
-
-  if (chosen) {
-    node = chosen;
-    entry = entry_guard_get_by_id_digest(node->identity);
-    if (entry) {
-      if (reset_status) {
-        entry->bad_since = 0;
-        entry->can_retry = 1;
-      }
-      entry->is_dir_cache = node_is_dir(node);
-      if (get_options()->UseBridges && node_is_a_configured_bridge(node))
-        entry->is_dir_cache = 1;
+  const uint8_t *id_digest = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
 
-      return NULL;
-    }
-  } else if (!for_directory) {
-    node = choose_good_entry_server(CIRCUIT_PURPOSE_C_GENERAL, NULL);
-    if (!node)
-      return NULL;
-  } else {
-    const routerstatus_t *rs;
-    rs = router_pick_directory_server(MICRODESC_DIRINFO|V3_DIRINFO,
-                                      PDS_FOR_GUARD);
-    if (!rs)
-      return NULL;
-    node = node_get_by_id(rs->identity_digest);
-    if (!node)
-      return NULL;
-  }
-  if (node->using_as_guard)
+  tor_assert(addrport);
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(get_sampled_guard_for_bridge(gs, bridge)))
+    return NULL; // LCOV_EXCL_LINE
+
+  return entry_guard_add_to_sample_impl(gs, id_digest, NULL, addrport);
+}
+
+/**
+ * Return the entry_guard_t in <b>gs</b> whose address is <b>addrport</b>,
+ * or NULL if none exists.
+*/
+static entry_guard_t *
+get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                 const tor_addr_port_t *addrport)
+{
+  if (! gs)
     return NULL;
-  if (entry_guard_get_by_id_digest(node->identity) != NULL) {
-    log_info(LD_CIRC, "I was about to add a duplicate entry guard.");
-    /* This can happen if we choose a guard, then the node goes away, then
-     * comes back. */
-    ((node_t*) node)->using_as_guard = 1;
+  if (BUG(!addrport))
     return NULL;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    if (g->bridge_addr && tor_addr_port_eq(addrport, g->bridge_addr))
+      return g;
+  } SMARTLIST_FOREACH_END(g);
+  return NULL;
+}
+
+/** Update the guard subsystem's knowledge of the identity of the bridge
+ * at <b>addrport</b>.  Idempotent.
+ */
+void
+entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                    const uint8_t *rsa_id_digest)
+{
+  guard_selection_t *gs = get_guard_selection_by_name("bridges",
+                                                      GS_TYPE_BRIDGE,
+                                                      0);
+  if (!gs)
+    return;
+
+  entry_guard_t *g = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (!g)
+    return;
+
+  int make_persistent = 0;
+
+  if (tor_digest_is_zero(g->identity)) {
+    memcpy(g->identity, rsa_id_digest, DIGEST_LEN);
+    make_persistent = 1;
+  } else if (tor_memeq(g->identity, rsa_id_digest, DIGEST_LEN)) {
+    /* Nothing to see here; we learned something we already knew. */
+    if (BUG(! g->is_persistent))
+      make_persistent = 1;
+  } else {
+    char old_id[HEX_DIGEST_LEN+1];
+    base16_encode(old_id, sizeof(old_id), g->identity, sizeof(g->identity));
+    log_warn(LD_BUG, "We 'learned' an identity %s for a bridge at %s:%d, but "
+             "we already knew a different one (%s). Ignoring the new info as "
+             "possibly bogus.",
+             hex_str((const char *)rsa_id_digest, DIGEST_LEN),
+             fmt_and_decorate_addr(&addrport->addr), addrport->port,
+             old_id);
+    return; // redundant, but let's be clear: we're not making this persistent.
   }
-  entry = tor_malloc_zero(sizeof(entry_guard_t));
-  log_info(LD_CIRC, "Chose %s as new entry guard.",
-           node_describe(node));
-  strlcpy(entry->nickname, node_get_nickname(node), sizeof(entry->nickname));
-  memcpy(entry->identity, node->identity, DIGEST_LEN);
-  entry->is_dir_cache = node_is_dir(node);
-  if (get_options()->UseBridges && node_is_a_configured_bridge(node))
-    entry->is_dir_cache = 1;
-
-  /* Choose expiry time smudged over the past month. The goal here
-   * is to a) spread out when Tor clients rotate their guards, so they
-   * don't all select them on the same day, and b) avoid leaving a
-   * precise timestamp in the state file about when we first picked
-   * this guard. For details, see the Jan 2010 or-dev thread. */
-  time_t now = time(NULL);
-  entry->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now);
-  entry->chosen_by_version = tor_strdup(VERSION);
-
-  /* Are we picking this guard because all of our current guards are
-   * down so we need another one (for_discovery is 1), or because we
-   * decided we need more variety in our guard list (for_discovery is 0)?
-   *
-   * Currently we hack this behavior into place by setting "made_contact"
-   * for guards of the latter variety, so we'll be willing to use any of
-   * them right off the bat.
-   */
-  if (!for_discovery)
-    entry->made_contact = 1;
 
-  ((node_t*)node)->using_as_guard = 1;
-  if (prepend)
-    smartlist_insert(entry_guards, 0, entry);
-  else
-    smartlist_add(entry_guards, entry);
-  control_event_guard(entry->nickname, entry->identity, "NEW");
-  control_event_guard_deferred();
-  log_entry_guards(LOG_INFO);
-  return node;
+  if (make_persistent) {
+    g->is_persistent = 1;
+    entry_guards_changed_for_guard_selection(gs);
+  }
 }
 
-/** Choose how many entry guards or directory guards we'll use. If
- * <b>for_directory</b> is true, we return how many directory guards to
- * use; else we return how many entry guards to use. */
+/**
+ * Return the number of sampled guards in <b>gs</b> that are "filtered"
+ * (that is, we're willing to connect to them) and that are "usable"
+ * (that is, either "reachable" or "maybe reachable").
+ *
+ * If a restriction is provided in <b>rst</b>, do not count any guards that
+ * violate it.
+ */
 STATIC int
-decide_num_guards(const or_options_t *options, int for_directory)
+num_reachable_filtered_guards(const guard_selection_t *gs,
+                              const entry_guard_restriction_t *rst)
 {
-  if (for_directory) {
-    int answer;
-    if (options->NumDirectoryGuards != 0)
-      return options->NumDirectoryGuards;
-    answer = networkstatus_get_param(NULL, "NumDirectoryGuards", 0, 0, 10);
-    if (answer) /* non-zero means use the consensus value */
-      return answer;
-  }
-
-  if (options->NumEntryGuards)
-    return options->NumEntryGuards;
+  int n_reachable_filtered_guards = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_usable_filtered_guard)
+      ++n_reachable_filtered_guards;
+  } SMARTLIST_FOREACH_END(guard);
+  return n_reachable_filtered_guards;
+}
 
-  /* Use the value from the consensus, or 3 if no guidance. */
-  return networkstatus_get_param(NULL, "NumEntryGuards", DEFAULT_N_GUARDS,
-                                 MIN_N_GUARDS, MAX_N_GUARDS);
+/** Return the actual maximum size for the sample in <b>gs</b>,
+ * given that we know about <b>n_guards</b> total. */
+static int
+get_max_sample_size(guard_selection_t *gs,
+                    int n_guards)
+{
+  const int using_bridges = (gs->type == GS_TYPE_BRIDGE);
+  const int min_sample = get_min_filtered_sample_size();
+
+  /* If we are in bridge mode, expand our sample set as needed without worrying
+   * about max size. We should respect the user's wishes to use many bridges if
+   * that's what they have specified in their configuration file. */
+  if (using_bridges)
+    return INT_MAX;
+
+  const int max_sample_by_pct = (int)(n_guards * get_max_sample_threshold());
+  const int max_sample_absolute = get_max_sample_size_absolute();
+  const int max_sample = MIN(max_sample_by_pct, max_sample_absolute);
+  if (max_sample < min_sample)
+    return min_sample;
+  else
+    return max_sample;
 }
 
-/** If the use of entry guards is configured, choose more entry guards
- * until we have enough in the list. */
-static void
-pick_entry_guards(const or_options_t *options, int for_directory)
-{
-  int changed = 0;
-  const int num_needed = decide_num_guards(options, for_directory);
+/**
+ * Return a smartlist of the all the guards that are not currently
+ * members of the sample (GUARDS - SAMPLED_GUARDS).  The elements of
+ * this list are node_t pointers in the non-bridge case, and
+ * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out/b>
+ * to the number of guards that we found in GUARDS, including those
+ * that were already sampled.
+ */
+static smartlist_t *
+get_eligible_guards(const or_options_t *options,
+                    guard_selection_t *gs,
+                    int *n_guards_out)
+{
+  /* Construct eligible_guards as GUARDS - SAMPLED_GUARDS */
+  smartlist_t *eligible_guards = smartlist_new();
+  int n_guards = 0; // total size of "GUARDS"
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const smartlist_t *bridges = bridge_list_get();
+    SMARTLIST_FOREACH_BEGIN(bridges, bridge_info_t *, bridge) {
+      ++n_guards;
+      if (NULL != get_sampled_guard_for_bridge(gs, bridge)) {
+        continue;
+      }
+      smartlist_add(eligible_guards, bridge);
+    } SMARTLIST_FOREACH_END(bridge);
+  } else {
+    const smartlist_t *nodes = nodelist_get_list();
+    const int n_sampled = smartlist_len(gs->sampled_entry_guards);
+
+    /* Build a bloom filter of our current guards: let's keep this O(N). */
+    digestset_t *sampled_guard_ids = digestset_new(n_sampled);
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, const entry_guard_t *,
+                            guard) {
+      digestset_add(sampled_guard_ids, guard->identity);
+    } SMARTLIST_FOREACH_END(guard);
+
+    SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+      if (! node_is_possible_guard(node))
+        continue;
+      if (gs->type == GS_TYPE_RESTRICTED) {
+        /* In restricted mode, we apply the filter BEFORE sampling, so
+         * that we are sampling from the nodes that we might actually
+         * select. If we sampled first, we might wind up with a sample
+         * that didn't include any EntryNodes at all. */
+        if (! node_passes_guard_filter(options, node))
+          continue;
+      }
+      ++n_guards;
+      if (digestset_contains(sampled_guard_ids, node->identity))
+        continue;
+      smartlist_add(eligible_guards, (node_t*)node);
+    } SMARTLIST_FOREACH_END(node);
 
-  tor_assert(entry_guards);
+    /* Now we can free that bloom filter. */
+    digestset_free(sampled_guard_ids);
+  }
 
-  while (num_live_entry_guards(for_directory) < num_needed) {
-    if (!add_an_entry_guard(NULL, 0, 0, 0, for_directory))
-      break;
-    changed = 1;
+  *n_guards_out = n_guards;
+  return eligible_guards;
+}
+
+/** Helper: given a smartlist of either bridge_info_t (if gs->type is
+ * GS_TYPE_BRIDGE) or node_t (otherwise), pick one that can be a guard,
+ * add it as a guard, remove it from the list, and return a new
+ * entry_guard_t.  Return NULL on failure. */
+static entry_guard_t *
+select_and_add_guard_item_for_sample(guard_selection_t *gs,
+                                     smartlist_t *eligible_guards)
+{
+  entry_guard_t *added_guard;
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = smartlist_choose(eligible_guards);
+    if (BUG(!bridge))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, bridge);
+    added_guard = entry_guard_add_bridge_to_sample(gs, bridge);
+  } else {
+    const node_t *node =
+      node_sl_choose_by_bandwidth(eligible_guards, WEIGHT_FOR_GUARD);
+    if (BUG(!node))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, node);
+    added_guard = entry_guard_add_to_sample(gs, node);
   }
-  if (changed)
-    entry_guards_changed();
-}
 
-/** How long (in seconds) do we allow an entry guard to be nonfunctional,
- * unlisted, excluded, or otherwise nonusable before we give up on it? */
-#define ENTRY_GUARD_REMOVE_AFTER (30*24*60*60)
+  return added_guard;
+}
 
-/** Release all storage held by <b>e</b>. */
-static void
-entry_guard_free(entry_guard_t *e)
+/**
+ * Return true iff we need a consensus to update our guards, but we don't
+ * have one. (We can return 0 here either if the consensus is _not_ missing,
+ * or if we don't need a consensus because we're using bridges.)
+ */
+static int
+live_consensus_is_missing(const guard_selection_t *gs)
 {
-  if (!e)
-    return;
-  tor_free(e->chosen_by_version);
-  tor_free(e);
+  tor_assert(gs);
+  if (gs->type == GS_TYPE_BRIDGE) {
+    /* We don't update bridges from the consensus; they aren't there. */
+    return 0;
+  }
+  return networkstatus_get_live_consensus(approx_time()) == NULL;
 }
 
 /**
- * Return the minimum lifetime of working entry guard, in seconds,
- * as given in the consensus networkstatus.  (Plus CHOSEN_ON_DATE_SLOP,
- * so that we can do the chosen_on_date randomization while achieving the
- * desired minimum lifetime.)
+ * Add new guards to the sampled guards in <b>gs</b> until there are
+ * enough usable filtered guards, but never grow the sample beyond its
+ * maximum size.  Return the last guard added, or NULL if none were
+ * added.
  */
-static int32_t
-guards_get_lifetime(void)
+STATIC entry_guard_t *
+entry_guards_expand_sample(guard_selection_t *gs)
 {
+  tor_assert(gs);
   const or_options_t *options = get_options();
-#define DFLT_GUARD_LIFETIME (86400 * 60)   /* Two months. */
-#define MIN_GUARD_LIFETIME  (86400 * 30)   /* One months. */
-#define MAX_GUARD_LIFETIME  (86400 * 1826) /* Five years. */
 
-  if (options->GuardLifetime >= 1) {
-    return CLAMP(MIN_GUARD_LIFETIME,
-                 options->GuardLifetime,
-                 MAX_GUARD_LIFETIME) + CHOSEN_ON_DATE_SLOP;
+  if (live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not expanding the sample guard set; we have "
+             "no live consensus.");
+    return NULL;
   }
 
-  return networkstatus_get_param(NULL, "GuardLifetime",
-                                 DFLT_GUARD_LIFETIME,
-                                 MIN_GUARD_LIFETIME,
-                                 MAX_GUARD_LIFETIME) + CHOSEN_ON_DATE_SLOP;
-}
-
-/** Remove any entry guard which was selected by an unknown version of Tor,
- * or which was selected by a version of Tor that's known to select
- * entry guards badly, or which was selected more 2 months ago. */
-/* XXXX The "obsolete guards" and "chosen long ago guards" things should
- * probably be different functions. */
-static int
-remove_obsolete_entry_guards(time_t now)
-{
-  int changed = 0, i;
-  int32_t guard_lifetime = guards_get_lifetime();
-
-  for (i = 0; i < smartlist_len(entry_guards); ++i) {
-    entry_guard_t *entry = smartlist_get(entry_guards, i);
-    const char *ver = entry->chosen_by_version;
-    const char *msg = NULL;
-    tor_version_t v;
-    int version_is_bad = 0, date_is_bad = 0;
-    if (!ver) {
-      msg = "does not say what version of Tor it was selected by";
-      version_is_bad = 1;
-    } else if (tor_version_parse(ver, &v)) {
-      msg = "does not seem to be from any recognized version of Tor";
-      version_is_bad = 1;
-    }
-    if (!version_is_bad && entry->chosen_on_date + guard_lifetime < now) {
-      /* It's been too long since the date listed in our state file. */
-      msg = "was selected several months ago";
-      date_is_bad = 1;
+  int n_sampled = smartlist_len(gs->sampled_entry_guards);
+  entry_guard_t *added_guard = NULL;
+  int n_usable_filtered_guards = num_reachable_filtered_guards(gs, NULL);
+  int n_guards = 0;
+  smartlist_t *eligible_guards = get_eligible_guards(options, gs, &n_guards);
+
+  const int max_sample = get_max_sample_size(gs, n_guards);
+  const int min_filtered_sample = get_min_filtered_sample_size();
+
+  log_info(LD_GUARD, "Expanding the sample guard set. We have %d guards "
+           "in the sample, and %d eligible guards to extend it with.",
+           n_sampled, smartlist_len(eligible_guards));
+
+  while (n_usable_filtered_guards < min_filtered_sample) {
+    /* Has our sample grown too large to expand? */
+    if (n_sampled >= max_sample) {
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just hit the maximum sample threshold of %d",
+               max_sample);
+      goto done;
     }
 
-    if (version_is_bad || date_is_bad) { /* we need to drop it */
-      char dbuf[HEX_DIGEST_LEN+1];
-      tor_assert(msg);
-      base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-      log_fn(version_is_bad ? LOG_NOTICE : LOG_INFO, LD_CIRC,
-             "Entry guard '%s' (%s) %s. (Version=%s.) Replacing it.",
-             entry->nickname, dbuf, msg, ver?escaped(ver):"none");
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, i--);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
+    /* Did we run out of guards? */
+    if (smartlist_len(eligible_guards) == 0) {
+      /* LCOV_EXCL_START
+         As long as MAX_SAMPLE_THRESHOLD makes can't be adjusted to
+         allow all guards to be sampled, this can't be reached.
+       */
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just ran out of eligible guards");
+      goto done;
+      /* LCOV_EXCL_STOP */
     }
+
+    /* Otherwise we can add at least one new guard. */
+    added_guard = select_and_add_guard_item_for_sample(gs, eligible_guards);
+    if (!added_guard)
+      goto done; // LCOV_EXCL_LINE -- only fails on BUG.
+
+    ++n_sampled;
+
+    if (added_guard->is_usable_filtered_guard)
+      ++n_usable_filtered_guards;
   }
 
-  return changed ? 1 : 0;
+ done:
+  smartlist_free(eligible_guards);
+  return added_guard;
 }
 
-/** Remove all entry guards that have been down or unlisted for so
- * long that we don't think they'll come up again. Return 1 if we
- * removed any, or 0 if we did nothing. */
-static int
-remove_dead_entry_guards(time_t now)
-{
-  char dbuf[HEX_DIGEST_LEN+1];
-  char tbuf[ISO_TIME_LEN+1];
-  int i;
-  int changed = 0;
-
-  for (i = 0; i < smartlist_len(entry_guards); ) {
-    entry_guard_t *entry = smartlist_get(entry_guards, i);
-    if (entry->bad_since &&
-        ! entry->path_bias_disabled &&
-        entry->bad_since + ENTRY_GUARD_REMOVE_AFTER < now) {
-
-      base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-      format_local_iso_time(tbuf, entry->bad_since);
-      log_info(LD_CIRC, "Entry guard '%s' (%s) has been down or unlisted "
-               "since %s local time; removing.",
-               entry->nickname, dbuf, tbuf);
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, i);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    } else
-      ++i;
-  }
-  return changed ? 1 : 0;
-}
-
-/** Remove all currently listed entry guards. So new ones will be chosen. */
-void
-remove_all_entry_guards(void)
+/**
+ * Helper: <b>guard</b> has just been removed from the sampled guards:
+ * also remove it from primary and confirmed. */
+static void
+remove_guard_from_confirmed_and_primary_lists(guard_selection_t *gs,
+                                              entry_guard_t *guard)
 {
-  char dbuf[HEX_DIGEST_LEN+1];
+  if (guard->is_primary) {
+    guard->is_primary = 0;
+    smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+  } else {
+    if (BUG(smartlist_contains(gs->primary_entry_guards, guard))) {
+      smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+    }
+  }
 
-  while (smartlist_len(entry_guards)) {
-    entry_guard_t *entry = smartlist_get(entry_guards, 0);
-    base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard '%s' (%s) has been dropped.",
-             entry->nickname, dbuf);
-    control_event_guard(entry->nickname, entry->identity, "DROPPED");
-    entry_guard_free(entry);
-    smartlist_del(entry_guards, 0);
+  if (guard->confirmed_idx >= 0) {
+    smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+    guard->confirmed_idx = -1;
+    guard->confirmed_on_date = 0;
+  } else {
+    if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard))) {
+      // LCOV_EXCL_START
+      smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+      // LCOV_EXCL_STOP
+    }
   }
-  log_entry_guards(LOG_INFO);
-  entry_guards_changed();
 }
 
-/** A new directory or router-status has arrived; update the down/listed
- * status of the entry guards.
- *
- * An entry is 'down' if the directory lists it as nonrunning.
- * An entry is 'unlisted' if the directory doesn't include it.
- *
- * Don't call this on startup; only on a fresh download. Otherwise we'll
- * think that things are unlisted.
- */
-void
-entry_guards_compute_status(const or_options_t *options, time_t now)
+/** Return true iff <b>guard</b> is currently "listed" -- that is, it
+ * appears in the consensus, or as a configured bridge (as
+ * appropriate) */
+MOCK_IMPL(STATIC int,
+entry_guard_is_listed,(guard_selection_t *gs, const entry_guard_t *guard))
 {
-  int changed = 0;
-  digestmap_t *reasons;
+  if (gs->type == GS_TYPE_BRIDGE) {
+    return NULL != get_bridge_info_for_guard(guard);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
 
-  if (! entry_guards)
+    return node && node_is_possible_guard(node);
+  }
+}
+
+/**
+ * Update the status of all sampled guards based on the arrival of a
+ * new consensus networkstatus document.  This will include marking
+ * some guards as listed or unlisted, and removing expired guards. */
+STATIC void
+sampled_guards_update_from_consensus(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  const int REMOVE_UNLISTED_GUARDS_AFTER =
+    (get_remove_unlisted_guards_after_days() * 86400);
+  const int unlisted_since_slop = REMOVE_UNLISTED_GUARDS_AFTER / 5;
+
+  // It's important to use only a live consensus here; we don't want to
+  // make changes based on anything expired or old.
+  if (live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not updating the sample guard set; we have "
+             "no live consensus.");
     return;
+  }
+  log_info(LD_GUARD, "Updating sampled guard status based on received "
+           "consensus.");
+
+  int n_changes = 0;
+
+  /* First: Update listed/unlisted. */
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* XXXX #20827 check ed ID too */
+    const int is_listed = entry_guard_is_listed(gs, guard);
+
+    if (is_listed && ! guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 1;
+      guard->unlisted_since_date = 0;
+      log_info(LD_GUARD, "Sampled guard %s is now listed again.",
+               entry_guard_describe(guard));
+    } else if (!is_listed && guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 0;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_info(LD_GUARD, "Sampled guard %s is now unlisted.",
+               entry_guard_describe(guard));
+    } else if (is_listed && guard->currently_listed) {
+      log_debug(LD_GUARD, "Sampled guard %s is still listed.",
+               entry_guard_describe(guard));
+    } else {
+      tor_assert(! is_listed && ! guard->currently_listed);
+      log_debug(LD_GUARD, "Sampled guard %s is still unlisted.",
+                entry_guard_describe(guard));
+    }
 
-  if (options->EntryNodes) /* reshuffle the entry guard list if needed */
-    entry_nodes_should_be_added();
-
-  reasons = digestmap_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry)
-    {
-      const node_t *r = node_get_by_id(entry->identity);
-      const char *reason = NULL;
-      if (entry_guard_set_status(entry, r, now, options, &reason))
-        changed = 1;
-
-      if (entry->bad_since)
-        tor_assert(reason);
-      if (reason)
-        digestmap_set(reasons, entry->identity, (char*)reason);
+    /* Clean up unlisted_since_date, just in case. */
+    if (guard->currently_listed && guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = 0;
+      log_warn(LD_BUG, "Sampled guard %s was listed, but with "
+               "unlisted_since_date set. Fixing.",
+               entry_guard_describe(guard));
+    } else if (!guard->currently_listed && ! guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_warn(LD_BUG, "Sampled guard %s was unlisted, but with "
+               "unlisted_since_date unset. Fixing.",
+               entry_guard_describe(guard));
     }
-  SMARTLIST_FOREACH_END(entry);
-
-  if (remove_dead_entry_guards(now))
-    changed = 1;
-  if (remove_obsolete_entry_guards(now))
-    changed = 1;
-
-  if (changed) {
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-      const char *reason = digestmap_get(reasons, entry->identity);
-      const char *live_msg = "";
-      const node_t *r = entry_is_live(entry, ENTRY_NEED_CAPACITY, &live_msg);
-      log_info(LD_CIRC, "Summary: Entry %s [%s] is %s, %s%s%s, and %s%s.",
-               entry->nickname,
-               hex_str(entry->identity, DIGEST_LEN),
-               entry->unreachable_since ? "unreachable" : "reachable",
-               entry->bad_since ? "unusable" : "usable",
-               reason ? ", ": "",
-               reason ? reason : "",
-               r ? "live" : "not live / ",
-               r ? "" : live_msg);
-    } SMARTLIST_FOREACH_END(entry);
-    log_info(LD_CIRC, "    (%d/%d entry guards are usable/new)",
-             num_live_entry_guards(0), smartlist_len(entry_guards));
-    log_entry_guards(LOG_INFO);
-    entry_guards_changed();
+  } SMARTLIST_FOREACH_END(guard);
+
+  const time_t remove_if_unlisted_since =
+    approx_time() - REMOVE_UNLISTED_GUARDS_AFTER;
+  const time_t maybe_remove_if_sampled_before =
+    approx_time() - get_guard_lifetime();
+  const time_t remove_if_confirmed_before =
+    approx_time() - get_guard_confirmed_min_lifetime();
+
+  /* Then: remove the ones that have been junk for too long */
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    int rmv = 0;
+
+    if (guard->currently_listed == 0 &&
+        guard->unlisted_since_date < remove_if_unlisted_since) {
+      /*
+        "We have a live consensus, and {IS_LISTED} is false, and
+         {FIRST_UNLISTED_AT} is over {REMOVE_UNLISTED_GUARDS_AFTER}
+         days in the past."
+      */
+      log_info(LD_GUARD, "Removing sampled guard %s: it has been unlisted "
+               "for over %d days", entry_guard_describe(guard),
+               get_remove_unlisted_guards_after_days());
+      rmv = 1;
+    } else if (guard->sampled_on_date < maybe_remove_if_sampled_before) {
+      /* We have a live consensus, and {ADDED_ON_DATE} is over
+        {GUARD_LIFETIME} ago, *and* {CONFIRMED_ON_DATE} is either
+        "never", or over {GUARD_CONFIRMED_MIN_LIFETIME} ago.
+      */
+      if (guard->confirmed_on_date == 0) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, but never confirmed.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400);
+      } else if (guard->confirmed_on_date < remove_if_confirmed_before) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, and confirmed over %d days ago.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400,
+                 get_guard_confirmed_min_lifetime() / 86400);
+      }
+    }
+
+    if (rmv) {
+      ++n_changes;
+      SMARTLIST_DEL_CURRENT(gs->sampled_entry_guards, guard);
+      remove_guard_from_confirmed_and_primary_lists(gs, guard);
+      entry_guard_free(guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (n_changes) {
+    gs->primary_guards_up_to_date = 0;
+    entry_guards_update_filtered_sets(gs);
+    /* We don't need to rebuild the confirmed list right here -- we may have
+     * removed confirmed guards above, but we can't have added any new
+     * confirmed guards.
+     */
+    entry_guards_changed_for_guard_selection(gs);
   }
+}
+
+/**
+ * Return true iff <b>node</b> is a Tor relay that we are configured to
+ * be able to connect to. */
+static int
+node_passes_guard_filter(const or_options_t *options,
+                         const node_t *node)
+{
+  /* NOTE: Make sure that this function stays in sync with
+   * options_transition_affects_entry_guards */
+  if (routerset_contains_node(options->ExcludeNodes, node))
+    return 0;
+
+  if (options->EntryNodes &&
+      !routerset_contains_node(options->EntryNodes, node))
+    return 0;
+
+  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
+    return 0;
 
-  digestmap_free(reasons, NULL);
+  if (node_is_a_configured_bridge(node))
+    return 0;
+
+  return 1;
 }
 
-/** Called when a connection to an OR with the identity digest <b>digest</b>
- * is established (<b>succeeded</b>==1) or has failed (<b>succeeded</b>==0).
- * If the OR is an entry, change that entry's up/down status.
- * Return 0 normally, or -1 if we want to tear down the new connection.
- *
- * If <b>mark_relay_status</b>, also call router_set_status() on this
- * relay.
- */
-/* XXX We could change succeeded and mark_relay_status into 'int flags'.
- * Too many boolean arguments is a recipe for confusion.
- */
-int
-entry_guard_register_connect_status(const char *digest, int succeeded,
-                                    int mark_relay_status, time_t now)
+/** Helper: Return true iff <b>bridge</b> passes our configuration
+ * filter-- if it is a relay that we are configured to be able to
+ * connect to. */
+static int
+bridge_passes_guard_filter(const or_options_t *options,
+                           const bridge_info_t *bridge)
 {
-  int changed = 0;
-  int refuse_conn = 0;
-  int first_contact = 0;
-  entry_guard_t *entry = NULL;
-  int idx = -1;
-  char buf[HEX_DIGEST_LEN+1];
+  tor_assert(bridge);
+  if (!bridge)
+    return 0;
 
-  if (! entry_guards)
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge))
     return 0;
 
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-    tor_assert(e);
-    if (tor_memeq(e->identity, digest, DIGEST_LEN)) {
-      entry = e;
-      idx = e_sl_idx;
-      break;
-    }
-  } SMARTLIST_FOREACH_END(e);
+  /* Ignore entrynodes */
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
 
-  if (!entry)
+  if (!fascist_firewall_allows_address_addr(&addrport->addr,
+                                            addrport->port,
+                                            FIREWALL_OR_CONNECTION,
+                                            0, 0))
     return 0;
 
-  base16_encode(buf, sizeof(buf), entry->identity, DIGEST_LEN);
-
-  if (succeeded) {
-    if (entry->unreachable_since) {
-      log_info(LD_CIRC, "Entry guard '%s' (%s) is now reachable again. Good.",
-               entry->nickname, buf);
-      entry->can_retry = 0;
-      entry->unreachable_since = 0;
-      entry->last_attempted = now;
-      control_event_guard(entry->nickname, entry->identity, "UP");
-      changed = 1;
-    }
-    if (!entry->made_contact) {
-      entry->made_contact = 1;
-      first_contact = changed = 1;
-    }
-  } else { /* ! succeeded */
-    if (!entry->made_contact) {
-      /* We've never connected to this one. */
-      log_info(LD_CIRC,
-               "Connection to never-contacted entry guard '%s' (%s) failed. "
-               "Removing from the list. %d/%d entry guards usable/new.",
-               entry->nickname, buf,
-               num_live_entry_guards(0)-1, smartlist_len(entry_guards)-1);
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, idx);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    } else if (!entry->unreachable_since) {
-      log_info(LD_CIRC, "Unable to connect to entry guard '%s' (%s). "
-               "Marking as unreachable.", entry->nickname, buf);
-      entry->unreachable_since = entry->last_attempted = now;
-      control_event_guard(entry->nickname, entry->identity, "DOWN");
-      changed = 1;
-      entry->can_retry = 0; /* We gave it an early chance; no good. */
-    } else {
-      char tbuf[ISO_TIME_LEN+1];
-      format_iso_time(tbuf, entry->unreachable_since);
-      log_debug(LD_CIRC, "Failed to connect to unreachable entry guard "
-                "'%s' (%s).  It has been unreachable since %s.",
-                entry->nickname, buf, tbuf);
-      entry->last_attempted = now;
-      entry->can_retry = 0; /* We gave it an early chance; no good. */
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> is a Tor relay that we are configured to
+ * be able to connect to, and we haven't disabled it for omission from
+ * the consensus or path bias issues. */
+static int
+entry_guard_passes_filter(const or_options_t *options, guard_selection_t *gs,
+                          entry_guard_t *guard)
+{
+  if (guard->currently_listed == 0)
+    return 0;
+  if (guard->pb.path_bias_disabled)
+    return 0;
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = get_bridge_info_for_guard(guard);
+    if (bridge == NULL)
+      return 0;
+    return bridge_passes_guard_filter(options, bridge);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node == NULL) {
+      // This can happen when currently_listed is true, and we're not updating
+      // it because we don't have a live consensus.
+      return 0;
     }
+
+    return node_passes_guard_filter(options, node);
   }
+}
 
-  /* if the caller asked us to, also update the is_running flags for this
-   * relay */
-  if (mark_relay_status)
-    router_set_status(digest, succeeded);
-
-  if (first_contact) {
-    /* We've just added a new long-term entry guard. Perhaps the network just
-     * came back? We should give our earlier entries another try too,
-     * and close this connection so we don't use it before we've given
-     * the others a shot. */
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-        if (e == entry)
-          break;
-        if (e->made_contact) {
-          const char *msg;
-          const node_t *r = entry_is_live(e,
-                     ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE,
-                     &msg);
-          if (r && e->unreachable_since) {
-            refuse_conn = 1;
-            e->can_retry = 1;
-          }
-        }
-    } SMARTLIST_FOREACH_END(e);
-    if (refuse_conn) {
-      log_info(LD_CIRC,
-               "Connected to new entry guard '%s' (%s). Marking earlier "
-               "entry guards up. %d/%d entry guards usable/new.",
-               entry->nickname, buf,
-               num_live_entry_guards(0), smartlist_len(entry_guards));
-      log_entry_guards(LOG_INFO);
-      changed = 1;
+/** Return true iff <b>guard</b> is in the same family as <b>node</b>.
+ */
+static int
+guard_in_node_family(const entry_guard_t *guard, const node_t *node)
+{
+  const node_t *guard_node = node_get_by_id(guard->identity);
+  if (guard_node) {
+    return nodes_in_same_family(guard_node, node);
+  } else {
+    /* If we don't have a node_t for the guard node, we might have
+     * a bridge_info_t for it. So let's check to see whether the bridge
+     * address matches has any family issues.
+     *
+     * (Strictly speaking, I believe this check is unnecessary, since we only
+     * use it to avoid the exit's family when building circuits, and we don't
+     * build multihop circuits until we have a routerinfo_t for the
+     * bridge... at which point, we'll also have a node_t for the
+     * bridge. Nonetheless, it seems wise to include it, in case our
+     * assumptions change down the road.  -nickm.)
+     */
+    if (get_options()->EnforceDistinctSubnets && guard->bridge_addr) {
+      tor_addr_t node_addr;
+      node_get_addr(node, &node_addr);
+      if (addrs_in_same_network_family(&node_addr,
+                                       &guard->bridge_addr->addr)) {
+        return 1;
+      }
     }
+    return 0;
   }
+}
 
-  if (changed)
-    entry_guards_changed();
-  return refuse_conn ? -1 : 0;
+/* Allocate and return a new exit guard restriction (where <b>exit_id</b> is of
+ * size DIGEST_LEN) */
+STATIC entry_guard_restriction_t *
+guard_create_exit_restriction(const uint8_t *exit_id)
+{
+  entry_guard_restriction_t *rst = NULL;
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_EXIT_NODE;
+  memcpy(rst->exclude_id, exit_id, DIGEST_LEN);
+  return rst;
 }
 
-/** When we try to choose an entry guard, should we parse and add
- * config's EntryNodes first? */
-static int should_add_entry_nodes = 0;
+/** If we have fewer than this many possible usable guards, don't set
+ * MD-availability-based restrictions: we might blacklist all of them. */
+#define MIN_GUARDS_FOR_MD_RESTRICTION 10
 
-/** Called when the value of EntryNodes changes in our configuration. */
-void
-entry_nodes_should_be_added(void)
+/** Return true if we should set md dirserver restrictions. We might not want
+ *  to set those if our guard options are too restricted, since we don't want
+ *  to blacklist all of them. */
+static int
+should_set_md_dirserver_restriction(void)
 {
-  log_info(LD_CIRC, "EntryNodes config option set. Putting configured "
-           "relays at the front of the entry guard list.");
-  should_add_entry_nodes = 1;
+  const guard_selection_t *gs = get_guard_selection_info();
+  int num_usable_guards = num_reachable_filtered_guards(gs, NULL);
+
+  /* Don't set restriction if too few reachable filtered guards. */
+  if (num_usable_guards < MIN_GUARDS_FOR_MD_RESTRICTION) {
+    log_info(LD_GUARD, "Not setting md restriction: only %d"
+             " usable guards.", num_usable_guards);
+    return 0;
+  }
+
+  /* We have enough usable guards: set MD restriction */
+  return 1;
 }
 
-/** Update the using_as_guard fields of all the nodes. We do this after we
- * remove entry guards from the list: This is the only function that clears
- * the using_as_guard field. */
-static void
-update_node_guard_status(void)
+/** Allocate and return an outdated md guard restriction. Return NULL if no
+ *  such restriction is needed. */
+STATIC entry_guard_restriction_t *
+guard_create_dirserver_md_restriction(void)
 {
-  smartlist_t *nodes = nodelist_get_list();
-  SMARTLIST_FOREACH(nodes, node_t *, node, node->using_as_guard = 0);
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-    node_t *node = node_get_mutable_by_id(entry->identity);
-    if (node)
-      node->using_as_guard = 1;
-  } SMARTLIST_FOREACH_END(entry);
+  entry_guard_restriction_t *rst = NULL;
+
+  if (!should_set_md_dirserver_restriction()) {
+    log_debug(LD_GUARD, "Not setting md restriction: too few "
+              "filtered guards.");
+    return NULL;
+  }
+
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_OUTDATED_MD_DIRSERVER;
+
+  return rst;
 }
 
-/** Adjust the entry guards list so that it only contains entries from
- * EntryNodes, adding new entries from EntryNodes to the list as needed. */
-STATIC void
-entry_guards_set_from_config(const or_options_t *options)
+/* Return True if <b>guard</b> obeys the exit restriction <b>rst</b>. */
+static int
+guard_obeys_exit_restriction(const entry_guard_t *guard,
+                             const entry_guard_restriction_t *rst)
 {
-  smartlist_t *entry_nodes, *worse_entry_nodes, *entry_fps;
-  smartlist_t *old_entry_guards_on_list, *old_entry_guards_not_on_list;
-  const int numentryguards = decide_num_guards(options, 0);
-  tor_assert(entry_guards);
+  tor_assert(rst->type == RST_EXIT_NODE);
 
-  should_add_entry_nodes = 0;
+  // Exclude the exit ID and all of its family.
+  const node_t *node = node_get_by_id((const char*)rst->exclude_id);
+  if (node && guard_in_node_family(guard, node))
+    return 0;
 
-  if (!options->EntryNodes) {
-    /* It's possible that a controller set EntryNodes, thus making
-     * should_add_entry_nodes set, then cleared it again, all before the
-     * call to choose_random_entry() that triggered us. If so, just return.
-     */
-    return;
+  return tor_memneq(guard->identity, rst->exclude_id, DIGEST_LEN);
+}
+
+/** Return True if <b>guard</b> should be used as a dirserver for fetching
+ *  microdescriptors. */
+static int
+guard_obeys_md_dirserver_restriction(const entry_guard_t *guard)
+{
+  /* If this guard is an outdated dirserver, don't use it. */
+  if (microdesc_relay_is_outdated_dirserver(guard->identity)) {
+    log_info(LD_GENERAL, "Skipping %s dirserver: outdated",
+             hex_str(guard->identity, DIGEST_LEN));
+    return 0;
   }
 
-  {
-    char *string = routerset_to_string(options->EntryNodes);
-    log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string);
-    tor_free(string);
+  log_debug(LD_GENERAL, "%s dirserver obeys md restrictions",
+            hex_str(guard->identity, DIGEST_LEN));
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> obeys the restrictions defined in <b>rst</b>.
+ * (If <b>rst</b> is NULL, there are no restrictions.)
+ */
+static int
+entry_guard_obeys_restriction(const entry_guard_t *guard,
+                              const entry_guard_restriction_t *rst)
+{
+  tor_assert(guard);
+  if (! rst)
+    return 1; // No restriction?  No problem.
+
+  if (rst->type == RST_EXIT_NODE) {
+    return guard_obeys_exit_restriction(guard, rst);
+  } else if (rst->type == RST_OUTDATED_MD_DIRSERVER) {
+    return guard_obeys_md_dirserver_restriction(guard);
   }
 
-  entry_nodes = smartlist_new();
-  worse_entry_nodes = smartlist_new();
-  entry_fps = smartlist_new();
-  old_entry_guards_on_list = smartlist_new();
-  old_entry_guards_not_on_list = smartlist_new();
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
 
-  /* Split entry guards into those on the list and those not. */
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flags on <b>guard</b>. */
+void
+entry_guard_set_filtered_flags(const or_options_t *options,
+                               guard_selection_t *gs,
+                               entry_guard_t *guard)
+{
+  unsigned was_filtered = guard->is_filtered_guard;
+  guard->is_filtered_guard = 0;
+  guard->is_usable_filtered_guard = 0;
 
-  routerset_get_all_nodes(entry_nodes, options->EntryNodes,
-                          options->ExcludeNodes, 0);
-  SMARTLIST_FOREACH(entry_nodes, const node_t *,node,
-                    smartlist_add(entry_fps, (void*)node->identity));
+  if (entry_guard_passes_filter(options, gs, guard)) {
+    guard->is_filtered_guard = 1;
 
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, {
-    if (smartlist_contains_digest(entry_fps, e->identity))
-      smartlist_add(old_entry_guards_on_list, e);
-    else
-      smartlist_add(old_entry_guards_not_on_list, e);
-  });
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      guard->is_usable_filtered_guard = 1;
+
+    entry_guard_consider_retry(guard);
+  }
+  log_debug(LD_GUARD, "Updated sampled guard %s: filtered=%d; "
+            "reachable_filtered=%d.", entry_guard_describe(guard),
+            guard->is_filtered_guard, guard->is_usable_filtered_guard);
 
-  /* Remove all currently configured guard nodes, excluded nodes, unreachable
-   * nodes, or non-Guard nodes from entry_nodes. */
-  SMARTLIST_FOREACH_BEGIN(entry_nodes, const node_t *, node) {
-    if (entry_guard_get_by_id_digest(node->identity)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
+  if (!bool_eq(was_filtered, guard->is_filtered_guard)) {
+    /* This guard might now be primary or nonprimary. */
+    gs->primary_guards_up_to_date = 0;
+  }
+}
+
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flag on every guard in <b>gs</b>. */
+STATIC void
+entry_guards_update_filtered_sets(guard_selection_t *gs)
+{
+  const or_options_t *options = get_options();
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_set_filtered_flags(options, gs, guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**
+ * Return a random guard from the reachable filtered sample guards
+ * in <b>gs</b>, subject to the exclusion rules listed in <b>flags</b>.
+ * Return NULL if no such guard can be found.
+ *
+ * Make sure that the sample is big enough, and that all the filter flags
+ * are set correctly, before calling this function.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it.
+ **/
+STATIC entry_guard_t *
+sample_reachable_filtered_entry_guards(guard_selection_t *gs,
+                                       const entry_guard_restriction_t *rst,
+                                       unsigned flags)
+{
+  tor_assert(gs);
+  entry_guard_t *result = NULL;
+  const unsigned exclude_confirmed = flags & SAMPLE_EXCLUDE_CONFIRMED;
+  const unsigned exclude_primary = flags & SAMPLE_EXCLUDE_PRIMARY;
+  const unsigned exclude_pending = flags & SAMPLE_EXCLUDE_PENDING;
+  const unsigned no_update_primary = flags & SAMPLE_NO_UPDATE_PRIMARY;
+  const unsigned need_descriptor = flags & SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  const int n_reachable_filtered = num_reachable_filtered_guards(gs, rst);
+
+  log_info(LD_GUARD, "Trying to sample a reachable guard: We know of %d "
+           "in the USABLE_FILTERED set.", n_reachable_filtered);
+
+  const int min_filtered_sample = get_min_filtered_sample_size();
+  if (n_reachable_filtered < min_filtered_sample) {
+    log_info(LD_GUARD, "  (That isn't enough. Trying to expand the sample.)");
+    entry_guards_expand_sample(gs);
+  }
+
+  if (exclude_primary && !gs->primary_guards_up_to_date && !no_update_primary)
+    entry_guards_update_primary(gs);
+
+  /* Build the set of reachable filtered guards. */
+  smartlist_t *reachable_filtered_sample = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);// redundant, but cheap.
+    if (! entry_guard_obeys_restriction(guard, rst))
       continue;
-    } else if (routerset_contains_node(options->ExcludeNodes, node)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
+    if (! guard->is_usable_filtered_guard)
       continue;
-    } else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
-                                             0)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
+    if (exclude_confirmed && guard->confirmed_idx >= 0)
       continue;
-    } else if (! node->is_possible_guard) {
-      smartlist_add(worse_entry_nodes, (node_t*)node);
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
-    }
-  } SMARTLIST_FOREACH_END(node);
+    if (exclude_primary && guard->is_primary)
+      continue;
+    if (exclude_pending && guard->is_pending)
+      continue;
+    if (need_descriptor && !guard_has_descriptor(guard))
+      continue;
+    smartlist_add(reachable_filtered_sample, guard);
+  } SMARTLIST_FOREACH_END(guard);
 
-  /* Now build the new entry_guards list. */
-  smartlist_clear(entry_guards);
-  /* First, the previously configured guards that are in EntryNodes. */
-  smartlist_add_all(entry_guards, old_entry_guards_on_list);
-  /* Next, scramble the rest of EntryNodes, putting the guards first. */
-  smartlist_shuffle(entry_nodes);
-  smartlist_shuffle(worse_entry_nodes);
-  smartlist_add_all(entry_nodes, worse_entry_nodes);
-
-  /* Next, the rest of EntryNodes */
-  SMARTLIST_FOREACH_BEGIN(entry_nodes, const node_t *, node) {
-    add_an_entry_guard(node, 0, 0, 1, 0);
-    if (smartlist_len(entry_guards) > numentryguards * 10)
-      break;
-  } SMARTLIST_FOREACH_END(node);
-  log_notice(LD_GENERAL, "%d entries in guards", smartlist_len(entry_guards));
-  /* Finally, free the remaining previously configured guards that are not in
-   * EntryNodes. */
-  SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e,
-                    entry_guard_free(e));
+  log_info(LD_GUARD, "  (After filters [%x], we have %d guards to consider.)",
+           flags, smartlist_len(reachable_filtered_sample));
 
-  update_node_guard_status();
+  if (smartlist_len(reachable_filtered_sample)) {
+    result = smartlist_choose(reachable_filtered_sample);
+    log_info(LD_GUARD, "  (Selected %s.)",
+             result ? entry_guard_describe(result) : "<null>");
+  }
+  smartlist_free(reachable_filtered_sample);
 
-  smartlist_free(entry_nodes);
-  smartlist_free(worse_entry_nodes);
-  smartlist_free(entry_fps);
-  smartlist_free(old_entry_guards_on_list);
-  smartlist_free(old_entry_guards_not_on_list);
-  entry_guards_changed();
+  return result;
 }
 
-/** Return 0 if we're fine adding arbitrary routers out of the
- * directory to our entry guard list, or return 1 if we have a
- * list already and we must stick to it.
+/**
+ * Helper: compare two entry_guard_t by their confirmed_idx values.
+ * Used to sort the confirmed list.
  */
-int
-entry_list_is_constrained(const or_options_t *options)
+static int
+compare_guards_by_confirmed_idx(const void **a_, const void **b_)
 {
-  if (options->EntryNodes)
+  const entry_guard_t *a = *a_, *b = *b_;
+  if (a->confirmed_idx < b->confirmed_idx)
+    return -1;
+  else if (a->confirmed_idx > b->confirmed_idx)
     return 1;
-  if (options->UseBridges)
-    return 1;
-  return 0;
+  else
+    return 0;
 }
 
-/** Pick a live (up and listed) entry guard from entry_guards. If
- * <b>state</b> is non-NULL, this is for a specific circuit --
- * make sure not to pick this circuit's exit or any node in the
- * exit's family. If <b>state</b> is NULL, we're looking for a random
- * guard (likely a bridge).  If <b>dirinfo</b> is not NO_DIRINFO (zero),
- * then only select from nodes that know how to answer directory questions
- * of that type. */
-const node_t *
-choose_random_entry(cpath_build_state_t *state)
+/**
+ * Find the confirmed guards from among the sampled guards in <b>gs</b>,
+ * and put them in confirmed_entry_guards in the correct
+ * order. Recalculate their indices.
+ */
+STATIC void
+entry_guards_update_confirmed(guard_selection_t *gs)
 {
-  return choose_random_entry_impl(state, 0, NO_DIRINFO, NULL);
+  smartlist_clear(gs->confirmed_entry_guards);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx >= 0)
+      smartlist_add(gs->confirmed_entry_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_confirmed_idx);
+
+  int any_changed = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx != guard_sl_idx) {
+      any_changed = 1;
+      guard->confirmed_idx = guard_sl_idx;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  gs->next_confirmed_idx = smartlist_len(gs->confirmed_entry_guards);
+
+  if (any_changed) {
+    entry_guards_changed_for_guard_selection(gs);
+  }
 }
 
-/** Pick a live (up and listed) directory guard from entry_guards for
- * downloading information of type <b>type</b>. */
-const node_t *
-choose_random_dirguard(dirinfo_type_t type)
+/**
+ * Mark <b>guard</b> as a confirmed guard -- that is, one that we have
+ * connected to, and intend to use again.
+ */
+STATIC void
+make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard)
 {
-  return choose_random_entry_impl(NULL, 1, type, NULL);
+  if (BUG(guard->confirmed_on_date && guard->confirmed_idx >= 0))
+    return; // LCOV_EXCL_LINE
+
+  if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard)))
+    return; // LCOV_EXCL_LINE
+
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  guard->confirmed_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+
+  log_info(LD_GUARD, "Marking %s as a confirmed guard (index %d)",
+           entry_guard_describe(guard),
+           gs->next_confirmed_idx);
+
+  guard->confirmed_idx = gs->next_confirmed_idx++;
+  smartlist_add(gs->confirmed_entry_guards, guard);
+
+  // This confirmed guard might kick something else out of the primary
+  // guards.
+  gs->primary_guards_up_to_date = 0;
+
+  entry_guards_changed_for_guard_selection(gs);
 }
 
-/** Filter <b>all_entry_guards</b> for usable entry guards and put them
- * in <b>live_entry_guards</b>. We filter based on whether the node is
- * currently alive, and on whether it satisfies the restrictions
- * imposed by the other arguments of this function.
- *
- * We don't place more guards than NumEntryGuards in <b>live_entry_guards</b>.
- *
- * If <b>chosen_exit</b> is set, it contains the exit node of this
- * circuit. Make sure to not use it or its family as an entry guard.
- *
- * If <b>need_uptime</b> is set, we are looking for a stable entry guard.
- * if <b>need_capacity</b> is set, we are looking for a fast entry guard.
- *
- * The rest of the arguments are the same as in choose_random_entry_impl().
- *
- * Return 1 if we should choose a guard right away. Return 0 if we
- * should try to add more nodes to our list before deciding on a
- * guard.
+/**
+ * Recalculate the list of primary guards (the ones we'd prefer to use) from
+ * the filtered sample and the confirmed list.
  */
-STATIC int
-populate_live_entry_guards(smartlist_t *live_entry_guards,
-                           const smartlist_t *all_entry_guards,
-                           const node_t *chosen_exit,
-                           dirinfo_type_t dirinfo_type,
-                           int for_directory,
-                           int need_uptime, int need_capacity)
+STATIC void
+entry_guards_update_primary(guard_selection_t *gs)
 {
-  const or_options_t *options = get_options();
-  const node_t *node = NULL;
-  const int num_needed = decide_num_guards(options, for_directory);
-  smartlist_t *exit_family = smartlist_new();
-  int retval = 0;
-  entry_is_live_flags_t entry_flags = 0;
+  tor_assert(gs);
 
-  (void) dirinfo_type;
+  // prevent recursion. Recursion is potentially very bad here.
+  static int running = 0;
+  tor_assert(!running);
+  running = 1;
 
-  { /* Set the flags we want our entry node to have */
-    if (need_uptime) {
-      entry_flags |= ENTRY_NEED_UPTIME;
-    }
-    if (need_capacity) {
-      entry_flags |= ENTRY_NEED_CAPACITY;
-    }
-    if (!for_directory) {
-      entry_flags |= ENTRY_NEED_DESCRIPTOR;
-    }
-  }
+  const int N_PRIMARY_GUARDS = get_n_primary_guards();
+
+  smartlist_t *new_primary_guards = smartlist_new();
+  smartlist_t *old_primary_guards = smartlist_new();
+  smartlist_add_all(old_primary_guards, gs->primary_entry_guards);
 
-  tor_assert(all_entry_guards);
+  /* Set this flag now, to prevent the calls below from recursing. */
+  gs->primary_guards_up_to_date = 1;
+
+  /* First, can we fill it up with confirmed guards? */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Can we keep any older primary guards? First remove all the ones
+   * that we already kept. */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    if (smartlist_contains(new_primary_guards, guard)) {
+      SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
 
-  if (chosen_exit) {
-    nodelist_add_node_and_family(exit_family, chosen_exit);
+  /* Now add any that are still good. */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+    SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Mark the remaining previous primary guards as non-primary */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    guard->is_primary = 0;
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Finally, fill out the list with sampled guards. */
+  while (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS) {
+    entry_guard_t *guard = sample_reachable_filtered_entry_guards(gs, NULL,
+                                            SAMPLE_EXCLUDE_CONFIRMED|
+                                            SAMPLE_EXCLUDE_PRIMARY|
+                                            SAMPLE_NO_UPDATE_PRIMARY);
+    if (!guard)
+      break;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
   }
 
-  SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) {
-      const char *msg;
-      node = entry_is_live(entry, entry_flags, &msg);
-      if (!node)
-        continue; /* down, no point */
-      if (for_directory) {
-        if (!entry->is_dir_cache)
-          continue; /* We need a directory and didn't get one. */
-      }
-      if (node == chosen_exit)
-        continue; /* don't pick the same node for entry and exit */
-      if (smartlist_contains(exit_family, node))
-        continue; /* avoid relays that are family members of our exit */
-      smartlist_add(live_entry_guards, (void*)node);
-      if (!entry->made_contact) {
-        /* Always start with the first not-yet-contacted entry
-         * guard. Otherwise we might add several new ones, pick
-         * the second new one, and now we've expanded our entry
-         * guard list without needing to. */
-        retval = 1;
-        goto done;
-      }
-      if (smartlist_len(live_entry_guards) >= num_needed) {
-        retval = 1;
-        goto done; /* We picked enough entry guards. Done! */
+#if 1
+  /* Debugging. */
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, guard, {
+    tor_assert_nonfatal(
+                   bool_eq(guard->is_primary,
+                           smartlist_contains(new_primary_guards, guard)));
+  });
+#endif /* 1 */
+
+  int any_change = 0;
+  if (smartlist_len(gs->primary_entry_guards) !=
+      smartlist_len(new_primary_guards)) {
+    any_change = 1;
+  } else {
+    SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, g) {
+      if (g != smartlist_get(new_primary_guards, g_sl_idx)) {
+        any_change = 1;
       }
-  } SMARTLIST_FOREACH_END(entry);
+    } SMARTLIST_FOREACH_END(g);
+  }
 
- done:
-  smartlist_free(exit_family);
+  if (any_change) {
+    log_info(LD_GUARD, "Primary entry guards have changed. "
+             "New primary guard list is: ");
+    int n = smartlist_len(new_primary_guards);
+    SMARTLIST_FOREACH_BEGIN(new_primary_guards, entry_guard_t *, g) {
+      log_info(LD_GUARD, "  %d/%d: %s%s%s",
+               g_sl_idx+1, n, entry_guard_describe(g),
+               g->confirmed_idx >= 0 ? " (confirmed)" : "",
+               g->is_filtered_guard ? "" : " (excluded by filter)");
+    } SMARTLIST_FOREACH_END(g);
+  }
 
-  return retval;
+  smartlist_free(old_primary_guards);
+  smartlist_free(gs->primary_entry_guards);
+  gs->primary_entry_guards = new_primary_guards;
+  gs->primary_guards_up_to_date = 1;
+  running = 0;
 }
 
-/** Pick a node to be used as the entry guard of a circuit.
- *
- * If <b>state</b> is set, it contains the information we know about
- * the upcoming circuit.
- *
- * If <b>for_directory</b> is set, we are looking for a directory guard.
- *
- * <b>dirinfo_type</b> contains the kind of directory information we
- * are looking for in our node, or NO_DIRINFO (zero) if we are not
- * looking for any particular directory information (when set to
- * NO_DIRINFO, the <b>dirinfo_type</b> filter is ignored).
- *
- * If <b>n_options_out</b> is set, we set it to the number of
- * candidate guard nodes we had before picking a specific guard node.
- *
- * On success, return the node that should be used as the entry guard
- * of the circuit.  Return NULL if no such node could be found.
- *
- * Helper for choose_random{entry,dirguard}.
-*/
-static const node_t *
-choose_random_entry_impl(cpath_build_state_t *state, int for_directory,
-                         dirinfo_type_t dirinfo_type, int *n_options_out)
+/**
+ * Return the number of seconds after the last attempt at which we should
+ * retry a guard that has been failing since <b>failing_since</b>.
+ */
+static int
+get_retry_schedule(time_t failing_since, time_t now,
+                   int is_primary)
 {
-  const or_options_t *options = get_options();
-  smartlist_t *live_entry_guards = smartlist_new();
-  const node_t *chosen_exit =
-    state?build_state_get_exit_node(state) : NULL;
-  const node_t *node = NULL;
-  int need_uptime = state ? state->need_uptime : 0;
-  int need_capacity = state ? state->need_capacity : 0;
-  int preferred_min = 0;
-  const int num_needed = decide_num_guards(options, for_directory);
-  int retval = 0;
-
-  if (n_options_out)
-    *n_options_out = 0;
-
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-
-  if (should_add_entry_nodes)
-    entry_guards_set_from_config(options);
-
-  if (!entry_list_is_constrained(options) &&
-      smartlist_len(entry_guards) < num_needed)
-    pick_entry_guards(options, for_directory);
-
- retry:
-  smartlist_clear(live_entry_guards);
-
-  /* Populate the list of live entry guards so that we pick one of
-     them. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      entry_guards,
-                                      chosen_exit,
-                                      dirinfo_type,
-                                      for_directory,
-                                      need_uptime, need_capacity);
-
-  if (retval == 1) { /* We should choose a guard right now. */
-    goto choose_and_finish;
-  }
-
-  if (entry_list_is_constrained(options)) {
-    /* If we prefer the entry nodes we've got, and we have at least
-     * one choice, that's great. Use it. */
-    preferred_min = 1;
-  } else {
-    /* Try to have at least 2 choices available. This way we don't
-     * get stuck with a single live-but-crummy entry and just keep
-     * using it.
-     * (We might get 2 live-but-crummy entry guards, but so be it.) */
-    preferred_min = 2;
-  }
-
-  if (smartlist_len(live_entry_guards) < preferred_min) {
-    if (!entry_list_is_constrained(options)) {
-      /* still no? try adding a new entry then */
-      /* XXX if guard doesn't imply fast and stable, then we need
-       * to tell add_an_entry_guard below what we want, or it might
-       * be a long time til we get it. -RD */
-      node = add_an_entry_guard(NULL, 0, 0, 1, for_directory);
-      if (node) {
-        entry_guards_changed();
-        /* XXX we start over here in case the new node we added shares
-         * a family with our exit node. There's a chance that we'll just
-         * load up on entry guards here, if the network we're using is
-         * one big family. Perhaps we should teach add_an_entry_guard()
-         * to understand nodes-to-avoid-if-possible? -RD */
-        goto retry;
-      }
-    }
-    if (!node && need_uptime) {
-      need_uptime = 0; /* try without that requirement */
-      goto retry;
-    }
-    if (!node && need_capacity) {
-      /* still no? last attempt, try without requiring capacity */
-      need_capacity = 0;
-      goto retry;
-    }
+  const unsigned SIX_HOURS = 6 * 3600;
+  const unsigned FOUR_DAYS = 4 * 86400;
+  const unsigned SEVEN_DAYS = 7 * 86400;
 
-    /* live_entry_guards may be empty below. Oh well, we tried. */
+  time_t tdiff;
+  if (now > failing_since) {
+    tdiff = now - failing_since;
+  } else {
+    tdiff = 0;
   }
 
- choose_and_finish:
-  if (entry_list_is_constrained(options)) {
-    /* We need to weight by bandwidth, because our bridges or entryguards
-     * were not already selected proportional to their bandwidth. */
-    node = node_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD);
-  } else {
-    /* We choose uniformly at random here, because choose_good_entry_server()
-     * already weights its choices by bandwidth, so we don't want to
-     * *double*-weight our guard selection. */
-    node = smartlist_choose(live_entry_guards);
+  const struct {
+    time_t maximum; int primary_delay; int nonprimary_delay;
+  } delays[] = {
+    { SIX_HOURS,    10*60,  1*60*60 },
+    { FOUR_DAYS,    90*60,  4*60*60 },
+    { SEVEN_DAYS, 4*60*60, 18*60*60 },
+    { TIME_MAX,   9*60*60, 36*60*60 }
+  };
+
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(delays); ++i) {
+    if (tdiff <= delays[i].maximum) {
+      return is_primary ? delays[i].primary_delay : delays[i].nonprimary_delay;
+    }
   }
-  if (n_options_out)
-    *n_options_out = smartlist_len(live_entry_guards);
-  smartlist_free(live_entry_guards);
-  return node;
+  /* LCOV_EXCL_START -- can't reach, since delays ends with TIME_MAX. */
+  tor_assert_nonfatal_unreached();
+  return 36*60*60;
+  /* LCOV_EXCL_STOP */
 }
 
-/** Parse <b>state</b> and learn about the entry guards it describes.
- * If <b>set</b> is true, and there are no errors, replace the global
- * entry_list with what we find.
- * On success, return 0. On failure, alloc into *<b>msg</b> a string
- * describing the error, and return -1.
+/**
+ * If <b>guard</b> is unreachable, consider whether enough time has passed
+ * to consider it maybe-reachable again.
  */
-int
-entry_guards_parse_state(or_state_t *state, int set, char **msg)
+STATIC void
+entry_guard_consider_retry(entry_guard_t *guard)
 {
-  entry_guard_t *node = NULL;
-  smartlist_t *new_entry_guards = smartlist_new();
-  config_line_t *line;
-  time_t now = time(NULL);
-  const char *state_version = state->TorVersion;
-  digestmap_t *added_by = digestmap_new();
-
-  *msg = NULL;
-  for (line = state->EntryGuards; line; line = line->next) {
-    if (!strcasecmp(line->key, "EntryGuard")) {
-      smartlist_t *args = smartlist_new();
-      node = tor_malloc_zero(sizeof(entry_guard_t));
-      /* all entry guards on disk have been contacted */
-      node->made_contact = 1;
-      smartlist_add(new_entry_guards, node);
-      smartlist_split_string(args, line->value, " ",
-                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(args)<2) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Too few arguments to EntryGuard");
-      } else if (!is_legal_nickname(smartlist_get(args,0))) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Bad nickname for EntryGuard");
-      } else {
-        strlcpy(node->nickname, smartlist_get(args,0), MAX_NICKNAME_LEN+1);
-        if (base16_decode(node->identity, DIGEST_LEN, smartlist_get(args,1),
-                          strlen(smartlist_get(args,1))) != DIGEST_LEN) {
-          *msg = tor_strdup("Unable to parse entry nodes: "
-                            "Bad hex digest for EntryGuard");
-        }
-      }
-      if (smartlist_len(args) >= 3) {
-        const char *is_cache = smartlist_get(args, 2);
-        if (!strcasecmp(is_cache, "DirCache")) {
-          node->is_dir_cache = 1;
-        } else if (!strcasecmp(is_cache, "NoDirCache")) {
-          node->is_dir_cache = 0;
-        } else {
-          log_warn(LD_CONFIG, "Bogus third argument to EntryGuard line: %s",
-                   escaped(is_cache));
-        }
-      }
-      SMARTLIST_FOREACH(args, char*, cp, tor_free(cp));
-      smartlist_free(args);
-      if (*msg)
-        break;
-    } else if (!strcasecmp(line->key, "EntryGuardDownSince") ||
-               !strcasecmp(line->key, "EntryGuardUnlistedSince")) {
-      time_t when;
-      time_t last_try = 0;
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardDownSince/UnlistedSince without EntryGuard");
-        break;
-      }
-      if (parse_iso_time_(line->value, &when, 0)<0) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Bad time in EntryGuardDownSince/UnlistedSince");
-        break;
-      }
-      if (when > now) {
-        /* It's a bad idea to believe info in the future: you can wind
-         * up with timeouts that aren't allowed to happen for years. */
-        continue;
-      }
-      if (strlen(line->value) >= ISO_TIME_LEN+ISO_TIME_LEN+1) {
-        /* ignore failure */
-        (void) parse_iso_time(line->value+ISO_TIME_LEN+1, &last_try);
-      }
-      if (!strcasecmp(line->key, "EntryGuardDownSince")) {
-        node->unreachable_since = when;
-        node->last_attempted = last_try;
-      } else {
-        node->bad_since = when;
-      }
-    } else if (!strcasecmp(line->key, "EntryGuardAddedBy")) {
-      char d[DIGEST_LEN];
-      /* format is digest version date */
-      if (strlen(line->value) < HEX_DIGEST_LEN+1+1+1+ISO_TIME_LEN) {
-        log_warn(LD_BUG, "EntryGuardAddedBy line is not long enough.");
-        continue;
-      }
-      if (base16_decode(d, sizeof(d),
-                        line->value, HEX_DIGEST_LEN) != sizeof(d) ||
-                        line->value[HEX_DIGEST_LEN] != ' ') {
-        log_warn(LD_BUG, "EntryGuardAddedBy line %s does not begin with "
-                 "hex digest", escaped(line->value));
+  if (guard->is_reachable != GUARD_REACHABLE_NO)
+    return; /* No retry needed. */
+
+  const time_t now = approx_time();
+  const int delay =
+    get_retry_schedule(guard->failing_since, now, guard->is_primary);
+  const time_t last_attempt = guard->last_tried_to_connect;
+
+  if (BUG(last_attempt == 0) ||
+      now >= last_attempt + delay) {
+    /* We should mark this retriable. */
+    char tbuf[ISO_TIME_LEN+1];
+    format_local_iso_time(tbuf, last_attempt);
+    log_info(LD_GUARD, "Marked %s%sguard %s for possible retry, since we "
+             "haven't tried to use it since %s.",
+             guard->is_primary?"primary ":"",
+             guard->confirmed_idx>=0?"confirmed ":"",
+             entry_guard_describe(guard),
+             tbuf);
+
+    guard->is_reachable = GUARD_REACHABLE_MAYBE;
+    if (guard->is_filtered_guard)
+      guard->is_usable_filtered_guard = 1;
+  }
+}
+
+/** Tell the entry guards subsystem that we have confirmed that as of
+ * just now, we're on the internet. */
+void
+entry_guards_note_internet_connectivity(guard_selection_t *gs)
+{
+  gs->last_time_on_internet = approx_time();
+}
+
+/**
+ * Get a guard for use with a circuit.  Prefer to pick a running primary
+ * guard; then a non-pending running filtered confirmed guard; then a
+ * non-pending runnable filtered guard.  Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate.  Set <b>state_out</b> to the new guard-state
+ * of the circuit.
+ */
+STATIC entry_guard_t *
+select_entry_guard_for_circuit(guard_selection_t *gs,
+                               guard_usage_t usage,
+                               const entry_guard_restriction_t *rst,
+                               unsigned *state_out)
+{
+  const int need_descriptor = (usage == GUARD_USAGE_TRAFFIC);
+  tor_assert(gs);
+  tor_assert(state_out);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  int num_entry_guards = get_n_primary_guards_to_use(usage);
+  smartlist_t *usable_primary_guards = smartlist_new();
+
+  /* "If any entry in PRIMARY_GUARDS has {is_reachable} status of
+      <maybe> or <yes>, return the first such guard." */
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_reachable != GUARD_REACHABLE_NO) {
+      if (need_descriptor && !guard_has_descriptor(guard)) {
         continue;
       }
-      digestmap_set(added_by, d, tor_strdup(line->value+HEX_DIGEST_LEN+1));
-    } else if (!strcasecmp(line->key, "EntryGuardPathUseBias")) {
-      const or_options_t *options = get_options();
-      double use_cnt, success_cnt;
-
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardPathUseBias without EntryGuard");
+      *state_out = GUARD_CIRC_STATE_USABLE_ON_COMPLETION;
+      guard->last_tried_to_connect = approx_time();
+      smartlist_add(usable_primary_guards, guard);
+      if (smartlist_len(usable_primary_guards) >= num_entry_guards)
         break;
-      }
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (smartlist_len(usable_primary_guards)) {
+    entry_guard_t *guard = smartlist_choose(usable_primary_guards);
+    smartlist_free(usable_primary_guards);
+    log_info(LD_GUARD, "Selected primary guard %s for circuit.",
+             entry_guard_describe(guard));
+    return guard;
+  }
+  smartlist_free(usable_primary_guards);
+
+  /* "Otherwise, if the ordered intersection of {CONFIRMED_GUARDS}
+      and {USABLE_FILTERED_GUARDS} is nonempty, return the first
+      entry in that intersection that has {is_pending} set to
+      false." */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->is_primary)
+      continue; /* we already considered this one. */
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    entry_guard_consider_retry(guard);
+    if (guard->is_usable_filtered_guard && ! guard->is_pending) {
+      if (need_descriptor && !guard_has_descriptor(guard))
+        continue; /* not a bug */
+      guard->is_pending = 1;
+      guard->last_tried_to_connect = approx_time();
+      *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+      log_info(LD_GUARD, "No primary guards available. Selected confirmed "
+               "guard %s for circuit. Will try other guards before using "
+               "this circuit.",
+               entry_guard_describe(guard));
+      return guard;
+    }
+  } SMARTLIST_FOREACH_END(guard);
 
-      if (tor_sscanf(line->value, "%lf %lf",
-                     &use_cnt, &success_cnt) != 2) {
-        log_info(LD_GENERAL, "Malformed path use bias line for node %s",
-                 node->nickname);
-        continue;
-      }
+  /* "Otherwise, if there is no such entry, select a member at
+      random from {USABLE_FILTERED_GUARDS}." */
+  {
+    entry_guard_t *guard;
+    unsigned flags = 0;
+    if (need_descriptor)
+      flags |= SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+    guard = sample_reachable_filtered_entry_guards(gs,
+                                                   rst,
+                                                   SAMPLE_EXCLUDE_CONFIRMED |
+                                                   SAMPLE_EXCLUDE_PRIMARY |
+                                                   SAMPLE_EXCLUDE_PENDING |
+                                                   flags);
+    if (guard == NULL) {
+      log_info(LD_GUARD, "Absolutely no sampled guards were available. "
+               "Marking all guards for retry and starting from top again.");
+      mark_all_guards_maybe_reachable(gs);
+      return NULL;
+    }
+    guard->is_pending = 1;
+    guard->last_tried_to_connect = approx_time();
+    *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+    log_info(LD_GUARD, "No primary or confirmed guards available. Selected "
+             "random guard %s for circuit. Will try other guards before "
+             "using this circuit.",
+             entry_guard_describe(guard));
+    return guard;
+  }
+}
 
-      if (use_cnt < success_cnt) {
-        int severity = LOG_INFO;
-        /* If this state file was written by a Tor that would have
-         * already fixed it, then the overcounting bug is still there.. */
-        if (tor_version_as_new_as(state_version, "0.2.4.13-alpha")) {
-          severity = LOG_NOTICE;
-        }
-        log_fn(severity, LD_BUG,
-                   "State file contains unexpectedly high usage success "
-                   "counts %lf/%lf for Guard %s ($%s)",
-                   success_cnt, use_cnt,
-                   node->nickname, hex_str(node->identity, DIGEST_LEN));
-        success_cnt = use_cnt;
-      }
+/**
+ * Note that we failed to connect to or build circuits through <b>guard</b>.
+ * Use with a guard returned by select_entry_guard_for_circuit().
+ */
+STATIC void
+entry_guards_note_guard_failure(guard_selection_t *gs,
+                                entry_guard_t *guard)
+{
+  tor_assert(gs);
 
-      node->use_attempts = use_cnt;
-      node->use_successes = success_cnt;
-
-      log_info(LD_GENERAL, "Read %f/%f path use bias for node %s",
-               node->use_successes, node->use_attempts, node->nickname);
-
-      /* Note: We rely on the < comparison here to allow us to set a 0
-       * rate and disable the feature entirely. If refactoring, don't
-       * change to <= */
-      if (pathbias_get_use_success_count(node)/node->use_attempts
-            < pathbias_get_extreme_use_rate(options) &&
-          pathbias_get_dropguards(options)) {
-        node->path_bias_disabled = 1;
-        log_info(LD_GENERAL,
-                 "Path use bias is too high (%f/%f); disabling node %s",
-                 node->circ_successes, node->circ_attempts, node->nickname);
-      }
-    } else if (!strcasecmp(line->key, "EntryGuardPathBias")) {
-      const or_options_t *options = get_options();
-      double hop_cnt, success_cnt, timeouts, collapsed, successful_closed,
-               unusable;
-
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardPathBias without EntryGuard");
-        break;
-      }
+  guard->is_reachable = GUARD_REACHABLE_NO;
+  guard->is_usable_filtered_guard = 0;
 
-      /* First try 3 params, then 2. */
-      /* In the long run: circuit_success ~= successful_circuit_close +
-       *                                     collapsed_circuits +
-       *                                     unusable_circuits */
-      if (tor_sscanf(line->value, "%lf %lf %lf %lf %lf %lf",
-                  &hop_cnt, &success_cnt, &successful_closed,
-                  &collapsed, &unusable, &timeouts) != 6) {
-        int old_success, old_hops;
-        if (tor_sscanf(line->value, "%u %u", &old_success, &old_hops) != 2) {
-          continue;
-        }
-        log_info(LD_GENERAL, "Reading old-style EntryGuardPathBias %s",
-                 escaped(line->value));
-
-        success_cnt = old_success;
-        successful_closed = old_success;
-        hop_cnt = old_hops;
-        timeouts = 0;
-        collapsed = 0;
-        unusable = 0;
-      }
+  guard->is_pending = 0;
+  if (guard->failing_since == 0)
+    guard->failing_since = approx_time();
 
-      if (hop_cnt < success_cnt) {
-        int severity = LOG_INFO;
-        /* If this state file was written by a Tor that would have
-         * already fixed it, then the overcounting bug is still there.. */
-        if (tor_version_as_new_as(state_version, "0.2.4.13-alpha")) {
-          severity = LOG_NOTICE;
-        }
-        log_fn(severity, LD_BUG,
-                "State file contains unexpectedly high success counts "
-                "%lf/%lf for Guard %s ($%s)",
-                success_cnt, hop_cnt,
-                node->nickname, hex_str(node->identity, DIGEST_LEN));
-        success_cnt = hop_cnt;
-      }
+  log_info(LD_GUARD, "Recorded failure for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
+}
 
-      node->circ_attempts = hop_cnt;
-      node->circ_successes = success_cnt;
-
-      node->successful_circuits_closed = successful_closed;
-      node->timeouts = timeouts;
-      node->collapsed_circuits = collapsed;
-      node->unusable_circuits = unusable;
-
-      log_info(LD_GENERAL, "Read %f/%f path bias for node %s",
-               node->circ_successes, node->circ_attempts, node->nickname);
-      /* Note: We rely on the < comparison here to allow us to set a 0
-       * rate and disable the feature entirely. If refactoring, don't
-       * change to <= */
-      if (pathbias_get_close_success_count(node)/node->circ_attempts
-            < pathbias_get_extreme_rate(options) &&
-          pathbias_get_dropguards(options)) {
-        node->path_bias_disabled = 1;
-        log_info(LD_GENERAL,
-                 "Path bias is too high (%f/%f); disabling node %s",
-                 node->circ_successes, node->circ_attempts, node->nickname);
-      }
+/**
+ * Note that we successfully connected to, and built a circuit through
+ * <b>guard</b>. Given the old guard-state of the circuit in <b>old_state</b>,
+ * return the new guard-state of the circuit.
+ *
+ * Be aware: the circuit is only usable when its guard-state becomes
+ * GUARD_CIRC_STATE_COMPLETE.
+ **/
+STATIC unsigned
+entry_guards_note_guard_success(guard_selection_t *gs,
+                                entry_guard_t *guard,
+                                unsigned old_state)
+{
+  tor_assert(gs);
+
+  /* Save this, since we're about to overwrite it. */
+  const time_t last_time_on_internet = gs->last_time_on_internet;
+  gs->last_time_on_internet = approx_time();
+
+  guard->is_reachable = GUARD_REACHABLE_YES;
+  guard->failing_since = 0;
+  guard->is_pending = 0;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+
+  if (guard->confirmed_idx < 0) {
+    make_guard_confirmed(gs, guard);
+    if (!gs->primary_guards_up_to_date)
+      entry_guards_update_primary(gs);
+  }
 
-    } else {
-      log_warn(LD_BUG, "Unexpected key %s", line->key);
-    }
+  unsigned new_state;
+  switch (old_state) {
+    case GUARD_CIRC_STATE_COMPLETE:
+    case GUARD_CIRC_STATE_USABLE_ON_COMPLETION:
+      new_state = GUARD_CIRC_STATE_COMPLETE;
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      /* Fall through. */
+    case GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD:
+      if (guard->is_primary) {
+        /* XXXX #20832 -- I don't actually like this logic. It seems to make
+         * us a little more susceptible to evil-ISP attacks.  The mitigations
+         * I'm thinking of, however, aren't local to this point, so I'll leave
+         * it alone. */
+        /* This guard may have become primary by virtue of being confirmed.
+         * If so, the circuit for it is now complete.
+         */
+        new_state = GUARD_CIRC_STATE_COMPLETE;
+      } else {
+        new_state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+      }
+      break;
   }
 
-  SMARTLIST_FOREACH_BEGIN(new_entry_guards, entry_guard_t *, e) {
-     char *sp;
-     char *val = digestmap_get(added_by, e->identity);
-     if (val && (sp = strchr(val, ' '))) {
-       time_t when;
-       *sp++ = '\0';
-       if (parse_iso_time(sp, &when)<0) {
-         log_warn(LD_BUG, "Can't read time %s in EntryGuardAddedBy", sp);
-       } else {
-         e->chosen_by_version = tor_strdup(val);
-         e->chosen_on_date = when;
-       }
-     } else {
-       if (state_version) {
-         e->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now);
-         e->chosen_by_version = tor_strdup(state_version);
-       }
-     }
-     if (e->path_bias_disabled && !e->bad_since)
-       e->bad_since = time(NULL);
+  if (! guard->is_primary) {
+    if (last_time_on_internet + get_internet_likely_down_interval()
+        < approx_time()) {
+      mark_primary_guards_maybe_reachable(gs);
     }
-  SMARTLIST_FOREACH_END(e);
+  }
 
-  if (*msg || !set) {
-    SMARTLIST_FOREACH(new_entry_guards, entry_guard_t *, e,
-                      entry_guard_free(e));
-    smartlist_free(new_entry_guards);
-  } else { /* !err && set */
-    if (entry_guards) {
-      SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e,
-                        entry_guard_free(e));
-      smartlist_free(entry_guards);
-    }
-    entry_guards = new_entry_guards;
-    entry_guards_dirty = 0;
-    /* XXX hand new_entry_guards to this func, and move it up a
-     * few lines, so we don't have to re-dirty it */
-    if (remove_obsolete_entry_guards(now))
-      entry_guards_dirty = 1;
+  log_info(LD_GUARD, "Recorded success for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
 
-    update_node_guard_status();
-  }
-  digestmap_free(added_by, tor_free_);
-  return *msg ? -1 : 0;
+  return new_state;
 }
 
-/** How long will we let a change in our guard nodes stay un-saved
- * when we are trying to avoid disk writes? */
-#define SLOW_GUARD_STATE_FLUSH_TIME 600
-/** How long will we let a change in our guard nodes stay un-saved
- * when we are not trying to avoid disk writes? */
-#define FAST_GUARD_STATE_FLUSH_TIME 30
-
-/** Our list of entry guards has changed, or some element of one
- * of our entry guards has changed. Write the changes to disk within
- * the next few minutes.
+/**
+ * Helper: Return true iff <b>a</b> has higher priority than <b>b</b>.
  */
-void
-entry_guards_changed(void)
+STATIC int
+entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b)
 {
-  time_t when;
-  entry_guards_dirty = 1;
+  tor_assert(a && b);
+  if (a == b)
+    return 0;
 
-  if (get_options()->AvoidDiskWrites)
-    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
-  else
-    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
+  /* Confirmed is always better than unconfirmed; lower index better
+     than higher */
+  if (a->confirmed_idx < 0) {
+    if (b->confirmed_idx >= 0)
+      return 0;
+  } else {
+    if (b->confirmed_idx < 0)
+      return 1;
 
-  /* or_state_save() will call entry_guards_update_state(). */
-  or_state_mark_dirty(get_or_state(), when);
+    /* Lower confirmed_idx is better than higher. */
+    return (a->confirmed_idx < b->confirmed_idx);
+  }
+
+  /* If we reach this point, both are unconfirmed. If one is pending, it
+   * has higher priority. */
+  if (a->is_pending) {
+    if (! b->is_pending)
+      return 1;
+
+    /* Both are pending: earlier last_tried_connect wins. */
+    return a->last_tried_to_connect < b->last_tried_to_connect;
+  } else {
+    if (b->is_pending)
+      return 0;
+
+    /* Neither is pending: priorities are equal. */
+    return 0;
+  }
 }
 
-/** If the entry guard info has not changed, do nothing and return.
- * Otherwise, free the EntryGuards piece of <b>state</b> and create
- * a new one out of the global entry_guards list, and then mark
- * <b>state</b> dirty so it will get saved to disk.
+/** Release all storage held in <b>restriction</b> */
+STATIC void
+entry_guard_restriction_free_(entry_guard_restriction_t *rst)
+{
+  tor_free(rst);
+}
+
+/**
+ * Release all storage held in <b>state</b>.
  */
 void
-entry_guards_update_state(or_state_t *state)
+circuit_guard_state_free_(circuit_guard_state_t *state)
 {
-  config_line_t **next, *line;
-  if (! entry_guards_dirty)
+  if (!state)
     return;
+  entry_guard_restriction_free(state->restrictions);
+  entry_guard_handle_free(state->guard);
+  tor_free(state);
+}
 
-  config_free_lines(state->EntryGuards);
-  next = &state->EntryGuards;
-  *next = NULL;
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-      char dbuf[HEX_DIGEST_LEN+1];
-      if (!e->made_contact)
-        continue; /* don't write this one to disk */
-      *next = line = tor_malloc_zero(sizeof(config_line_t));
-      line->key = tor_strdup("EntryGuard");
-      base16_encode(dbuf, sizeof(dbuf), e->identity, DIGEST_LEN);
-      tor_asprintf(&line->value, "%s %s %sDirCache", e->nickname, dbuf,
-                   e->is_dir_cache ? "" : "No");
-      next = &(line->next);
-      if (e->unreachable_since) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardDownSince");
-        line->value = tor_malloc(ISO_TIME_LEN+1+ISO_TIME_LEN+1);
-        format_iso_time(line->value, e->unreachable_since);
-        if (e->last_attempted) {
-          line->value[ISO_TIME_LEN] = ' ';
-          format_iso_time(line->value+ISO_TIME_LEN+1, e->last_attempted);
-        }
-        next = &(line->next);
-      }
-      if (e->bad_since) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardUnlistedSince");
-        line->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(line->value, e->bad_since);
-        next = &(line->next);
-      }
-      if (e->chosen_on_date && e->chosen_by_version &&
-          !strchr(e->chosen_by_version, ' ')) {
-        char d[HEX_DIGEST_LEN+1];
-        char t[ISO_TIME_LEN+1];
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardAddedBy");
-        base16_encode(d, sizeof(d), e->identity, DIGEST_LEN);
-        format_iso_time(t, e->chosen_on_date);
-        tor_asprintf(&line->value, "%s %s %s",
-                     d, e->chosen_by_version, t);
-        next = &(line->next);
-      }
-      if (e->circ_attempts > 0) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardPathBias");
-        /* In the long run: circuit_success ~= successful_circuit_close +
-         *                                     collapsed_circuits +
-         *                                     unusable_circuits */
-        tor_asprintf(&line->value, "%f %f %f %f %f %f",
-                     e->circ_attempts, e->circ_successes,
-                     pathbias_get_close_success_count(e),
-                     e->collapsed_circuits,
-                     e->unusable_circuits, e->timeouts);
-        next = &(line->next);
-      }
-      if (e->use_attempts > 0) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardPathUseBias");
-
-        tor_asprintf(&line->value, "%f %f",
-                     e->use_attempts,
-                     pathbias_get_use_success_count(e));
-        next = &(line->next);
-      }
+/** Allocate and return a new circuit_guard_state_t to track the result
+ * of using <b>guard</b> for a given operation. */
+MOCK_IMPL(STATIC circuit_guard_state_t *,
+circuit_guard_state_new,(entry_guard_t *guard, unsigned state,
+                         entry_guard_restriction_t *rst))
+{
+  circuit_guard_state_t *result;
 
-  } SMARTLIST_FOREACH_END(e);
-  if (!get_options()->AvoidDiskWrites)
-    or_state_mark_dirty(get_or_state(), 0);
-  entry_guards_dirty = 0;
+  result = tor_malloc_zero(sizeof(circuit_guard_state_t));
+  result->guard = entry_guard_handle_new(guard);
+  result->state = state;
+  result->state_set_at = approx_time();
+  result->restrictions = rst;
+
+  return result;
 }
 
-/** If <b>question</b> is the string "entry-guards", then dump
- * to *<b>answer</b> a newly allocated string describing all of
- * the nodes in the global entry_guards list. See control-spec.txt
- * for details.
- * For backward compatibility, we also handle the string "helper-nodes".
- * */
+/**
+ * Pick a suitable entry guard for a circuit in, and place that guard
+ * in *<b>chosen_node_out</b>. Set *<b>guard_state_out</b> to an opaque
+ * state object that will record whether the circuit is ready to be used
+ * or not. Return 0 on success; on failure, return -1.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it, and remember that restriction in <b>guard_state_out</b> for
+ * later use. (Takes ownership of the <b>rst</b> object.)
+ */
 int
-getinfo_helper_entry_guards(control_connection_t *conn,
-                            const char *question, char **answer,
-                            const char **errmsg)
-{
-  (void) conn;
-  (void) errmsg;
+entry_guard_pick_for_circuit(guard_selection_t *gs,
+                             guard_usage_t usage,
+                             entry_guard_restriction_t *rst,
+                             const node_t **chosen_node_out,
+                             circuit_guard_state_t **guard_state_out)
+{
+  tor_assert(gs);
+  tor_assert(chosen_node_out);
+  tor_assert(guard_state_out);
+  *chosen_node_out = NULL;
+  *guard_state_out = NULL;
+
+  unsigned state = 0;
+  entry_guard_t *guard =
+    select_entry_guard_for_circuit(gs, usage, rst, &state);
+  if (! guard)
+    goto fail;
+  if (BUG(state == 0))
+    goto fail;
+  const node_t *node = node_get_by_id(guard->identity);
+  // XXXX #20827 check Ed ID.
+  if (! node)
+    goto fail;
+  if (BUG(usage != GUARD_USAGE_DIRGUARD &&
+          !node_has_preferred_descriptor(node, 1)))
+    goto fail;
+
+  *chosen_node_out = node;
+  *guard_state_out = circuit_guard_state_new(guard, state, rst);
 
-  if (!strcmp(question,"entry-guards") ||
-      !strcmp(question,"helper-nodes")) {
-    smartlist_t *sl = smartlist_new();
-    char tbuf[ISO_TIME_LEN+1];
-    char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
-    if (!entry_guards)
-      entry_guards = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-        const char *status = NULL;
-        time_t when = 0;
-        const node_t *node;
-
-        if (!e->made_contact) {
-          status = "never-connected";
-        } else if (e->bad_since) {
-          when = e->bad_since;
-          status = "unusable";
-        } else if (e->unreachable_since) {
-          when = e->unreachable_since;
-          status = "down";
-        } else {
-          status = "up";
-        }
-
-        node = node_get_by_id(e->identity);
-        if (node) {
-          node_get_verbose_nickname(node, nbuf);
-        } else {
-          nbuf[0] = '$';
-          base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
-          /* e->nickname field is not very reliable if we don't know about
-           * this router any longer; don't include it. */
-        }
-
-        if (when) {
-          format_iso_time(tbuf, when);
-          smartlist_add_asprintf(sl, "%s %s %s\n", nbuf, status, tbuf);
-        } else {
-          smartlist_add_asprintf(sl, "%s %s\n", nbuf, status);
-        }
-    } SMARTLIST_FOREACH_END(e);
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  }
   return 0;
+ fail:
+  entry_guard_restriction_free(rst);
+  return -1;
 }
 
-/** Return 0 if we should apply guardfraction information found in the
- *  consensus. A specific consensus can be specified with the
- *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
-int
-should_apply_guardfraction(const networkstatus_t *ns)
+/**
+ * Called by the circuit building module when a circuit has succeeded: informs
+ * the guards code that the guard in *<b>guard_state_p</b> is working, and
+ * advances the state of the guard module.  On a GUARD_USABLE_NEVER return
+ * value, the circuit is broken and should not be used.  On a GUARD_USABLE_NOW
+ * return value, the circuit is ready to use.  On a GUARD_MAYBE_USABLE_LATER
+ * return value, the circuit should not be used until we find out whether
+ * preferred guards will work for us.
+ */
+guard_usable_t
+entry_guard_succeeded(circuit_guard_state_t **guard_state_p)
 {
-  /* We need to check the corresponding torrc option and the consensus
-   * parameter if we need to. */
-  const or_options_t *options = get_options();
+  if (BUG(*guard_state_p == NULL))
+    return GUARD_USABLE_NEVER;
 
-  /* If UseGuardFraction is 'auto' then check the same-named consensus
-   * parameter. If the consensus parameter is not present, default to
-   * "off". */
-  if (options->UseGuardFraction == -1) {
-    return networkstatus_get_param(ns, "UseGuardFraction",
-                                   0, /* default to "off" */
-                                   0, 1);
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return GUARD_USABLE_NEVER;
+
+  unsigned newstate =
+    entry_guards_note_guard_success(guard->in_selection, guard,
+                                    (*guard_state_p)->state);
+
+  (*guard_state_p)->state = newstate;
+  (*guard_state_p)->state_set_at = approx_time();
+
+  if (newstate == GUARD_CIRC_STATE_COMPLETE) {
+    return GUARD_USABLE_NOW;
+  } else {
+    return GUARD_MAYBE_USABLE_LATER;
   }
+}
 
-  return options->UseGuardFraction;
+/** Cancel the selection of *<b>guard_state_p</b> without declaring
+ * success or failure. It is safe to call this function if success or
+ * failure _has_ already been declared. */
+void
+entry_guard_cancel(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return;
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard)
+    return;
+
+  /* XXXX prop271 -- last_tried_to_connect_at will be erroneous here, but this
+   * function will only get called in "bug" cases anyway. */
+  guard->is_pending = 0;
+  circuit_guard_state_free(*guard_state_p);
+  *guard_state_p = NULL;
 }
 
-/* Given the original bandwidth of a guard and its guardfraction,
- * calculate how much bandwidth the guard should have as a guard and
- * as a non-guard.
- *
- * Quoting from proposal236:
- *
- *   Let Wpf denote the weight from the 'bandwidth-weights' line a
- *   client would apply to N for position p if it had the guard
- *   flag, Wpn the weight if it did not have the guard flag, and B the
- *   measured bandwidth of N in the consensus.  Then instead of choosing
- *   N for position p proportionally to Wpf*B or Wpn*B, clients should
- *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
- *
- * This function fills the <b>guardfraction_bw</b> structure. It sets
- * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
+/**
+ * Called by the circuit building module when a circuit has failed:
+ * informs the guards code that the guard in *<b>guard_state_p</b> is
+ * not working, and advances the state of the guard module.
  */
 void
-guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
-                                  int orig_bandwidth,
-                                  uint32_t guardfraction_percentage)
+entry_guard_failed(circuit_guard_state_t **guard_state_p)
 {
-  double guardfraction_fraction;
-
-  /* Turn the percentage into a fraction. */
-  tor_assert(guardfraction_percentage <= 100);
-  guardfraction_fraction = guardfraction_percentage / 100.0;
+  if (BUG(*guard_state_p == NULL))
+    return;
 
-  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
-  tor_assert(guard_bw <= INT_MAX);
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return;
 
-  guardfraction_bw->guard_bw = (int) guard_bw;
+  entry_guards_note_guard_failure(guard->in_selection, guard);
 
-  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
+  (*guard_state_p)->state = GUARD_CIRC_STATE_DEAD;
+  (*guard_state_p)->state_set_at = approx_time();
 }
 
-/** A list of configured bridges. Whenever we actually get a descriptor
- * for one, we add it as an entry guard.  Note that the order of bridges
- * in this list does not necessarily correspond to the order of bridges
- * in the torrc. */
-static smartlist_t *bridge_list = NULL;
-
-/** Mark every entry of the bridge list to be removed on our next call to
- * sweep_bridge_list unless it has first been un-marked. */
+/**
+ * Run the entry_guard_failed() function on every circuit that is
+ * pending on <b>chan</b>.
+ */
 void
-mark_bridge_list(void)
+entry_guard_chan_failed(channel_t *chan)
 {
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
-                    b->marked_for_removal = 1);
-}
+  if (!chan)
+    return;
 
-/** Remove every entry of the bridge list that was marked with
- * mark_bridge_list if it has not subsequently been un-marked. */
-void
-sweep_bridge_list(void)
-{
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-    if (b->marked_for_removal) {
-      SMARTLIST_DEL_CURRENT(bridge_list, b);
-      bridge_free(b);
+  smartlist_t *pending = smartlist_new();
+  circuit_get_all_pending_on_channel(pending, chan);
+  SMARTLIST_FOREACH_BEGIN(pending, circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    if (origin_circ->guard_state) {
+      /* We might have no guard state if we didn't use a guard on this
+       * circuit (eg it's for a fallback directory). */
+      entry_guard_failed(&origin_circ->guard_state);
     }
-  } SMARTLIST_FOREACH_END(b);
+  } SMARTLIST_FOREACH_END(circ);
+  smartlist_free(pending);
 }
 
-/** Initialize the bridge list to empty, creating it if needed. */
-static void
-clear_bridge_list(void)
+/**
+ * Return true iff every primary guard in <b>gs</b> is believed to
+ * be unreachable.
+ */
+STATIC int
+entry_guards_all_primary_guards_are_down(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      return 0;
+  } SMARTLIST_FOREACH_END(guard);
+  return 1;
+}
+
+/** Wrapper for entry_guard_has_higher_priority that compares the
+ * guard-priorities of a pair of circuits. Return 1 if <b>a</b> has higher
+ * priority than <b>b</b>.
+ *
+ * If a restriction is provided in <b>rst</b>, then do not consider
+ * <b>a</b> to have higher priority if it violates the restriction.
+ */
+static int
+circ_state_has_higher_priority(origin_circuit_t *a,
+                               const entry_guard_restriction_t *rst,
+                               origin_circuit_t *b)
 {
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
-  smartlist_clear(bridge_list);
-}
+  circuit_guard_state_t *state_a = origin_circuit_get_guard_state(a);
+  circuit_guard_state_t *state_b = origin_circuit_get_guard_state(b);
 
-/** Free the bridge <b>bridge</b>. */
-static void
-bridge_free(bridge_info_t *bridge)
-{
-  if (!bridge)
-    return;
+  tor_assert(state_a);
+  tor_assert(state_b);
 
-  tor_free(bridge->transport_name);
-  if (bridge->socks_args) {
-    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
-    smartlist_free(bridge->socks_args);
-  }
+  entry_guard_t *guard_a = entry_guard_handle_get(state_a->guard);
+  entry_guard_t *guard_b = entry_guard_handle_get(state_b->guard);
 
-  tor_free(bridge);
+  if (! guard_a) {
+    /* Unknown guard -- never higher priority. */
+    return 0;
+  } else if (! guard_b) {
+    /* Known guard -- higher priority than any unknown guard. */
+    return 1;
+  } else  if (! entry_guard_obeys_restriction(guard_a, rst)) {
+    /* Restriction violated; guard_a cannot have higher priority. */
+    return 0;
+  } else {
+    /* Both known -- compare.*/
+    return entry_guard_has_higher_priority(guard_a, guard_b);
+  }
 }
 
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches any of the
- * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
- * NULL. */
-static bridge_info_t *
-get_configured_bridge_by_orports_digest(const char *digest,
-                                        const smartlist_t *orports)
-{
-  if (!bridge_list)
-    return NULL;
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if (tor_digest_is_zero(bridge->identity)) {
-        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
-          {
-            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
-                bridge->port == ap->port)
-              return bridge;
-          }
-        SMARTLIST_FOREACH_END(ap);
+/**
+ * Look at all of the origin_circuit_t * objects in <b>all_circuits_in</b>,
+ * and see if any of them that were previously not ready to use for
+ * guard-related reasons are now ready to use. Place those circuits
+ * in <b>newly_complete_out</b>, and mark them COMPLETE.
+ *
+ * Return 1 if we upgraded any circuits, and 0 otherwise.
+ */
+int
+entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                      const smartlist_t *all_circuits_in,
+                                      smartlist_t *newly_complete_out)
+{
+  tor_assert(gs);
+  tor_assert(all_circuits_in);
+  tor_assert(newly_complete_out);
+
+  if (! entry_guards_all_primary_guards_are_down(gs)) {
+    /* We only upgrade a waiting circuit if the primary guards are all
+     * down. */
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but not all primary guards were definitely down.");
+    return 0;
+  }
+
+  int n_waiting = 0;
+  int n_complete = 0;
+  int n_complete_blocking = 0;
+  origin_circuit_t *best_waiting_circuit = NULL;
+  smartlist_t *all_circuits = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(all_circuits_in, origin_circuit_t *, circ) {
+    // We filter out circuits that aren't ours, or which we can't
+    // reason about.
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (state == NULL)
+      continue;
+    entry_guard_t *guard = entry_guard_handle_get(state->guard);
+    if (!guard || guard->in_selection != gs)
+      continue;
+
+    smartlist_add(all_circuits, circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+
+    if (state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD) {
+      ++n_waiting;
+      if (! best_waiting_circuit ||
+          circ_state_has_higher_priority(circ, NULL, best_waiting_circuit)) {
+        best_waiting_circuit = circ;
       }
-      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
     }
-  SMARTLIST_FOREACH_END(bridge);
-  return NULL;
-}
+  } SMARTLIST_FOREACH_END(circ);
 
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
- * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
- * address/port matches only. */
-static bridge_info_t *
-get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
-                                          uint16_t port,
-                                          const char *digest)
-{
-  if (!bridge_list)
-    return NULL;
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
-          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
-          bridge->port == port)
-        return bridge;
-      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
+  if (! best_waiting_circuit) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but didn't find any.");
+    goto no_change;
+  }
+
+  /* We'll need to keep track of what restrictions were used when picking this
+   * circuit, so that we don't allow any circuit without those restrictions to
+   * block it. */
+  const entry_guard_restriction_t *rst_on_best_waiting =
+    origin_circuit_get_guard_state(best_waiting_circuit)->restrictions;
+
+  /* First look at the complete circuits: Do any block this circuit? */
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    /* "C2 "blocks" C1 if:
+        * C2 obeys all the restrictions that C1 had to obey, AND
+        * C2 has higher priority than C1, AND
+        * Either C2 is <complete>, or C2 is <waiting_for_better_guard>,
+          or C2 has been <usable_if_no_better_guard> for no more than
+          {NONPRIMARY_GUARD_CONNECT_TIMEOUT} seconds."
+    */
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if BUG((state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_COMPLETE)
+      continue;
+    ++n_complete;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_complete_blocking;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_complete_blocking) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d complete and %d guard-stalled. At least one complete "
+              "circuit had higher priority, so not upgrading.",
+              n_complete, n_waiting);
+    goto no_change;
+  }
+
+  /* " * If any circuit C1 is <waiting_for_better_guard>, AND:
+          * All primary guards have reachable status of <no>.
+          * There is no circuit C2 that "blocks" C1.
+         Then, upgrade C1 to <complete>.""
+  */
+  int n_blockers_found = 0;
+  const time_t state_set_at_cutoff =
+    approx_time() - get_nonprimary_guard_connect_timeout();
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD)
+      continue;
+    if (state->state_set_at <= state_set_at_cutoff)
+      continue;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_blockers_found;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_blockers_found) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d guard-stalled, but %d pending circuit(s) had higher "
+              "guard priority, so not upgrading.",
+              n_waiting, n_blockers_found);
+    goto no_change;
+  }
+
+  /* Okay. We have a best waiting circuit, and we aren't waiting for
+     anything better.  Add all circuits with that priority to the
+     list, and call them COMPLETE. */
+  int n_succeeded = 0;
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (circ != best_waiting_circuit && rst_on_best_waiting) {
+      /* Can't upgrade other circ with same priority as best; might
+         be blocked. */
+      continue;
     }
-  SMARTLIST_FOREACH_END(bridge);
-  return NULL;
-}
+    if (state->state != GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD)
+      continue;
+    if (circ_state_has_higher_priority(best_waiting_circuit, NULL, circ))
+      continue;
 
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
- * return 1.  Else return 0. If <b>digest</b> is NULL, check for
- * address/port matches only. */
-int
-addr_is_a_configured_bridge(const tor_addr_t *addr,
-                                uint16_t port,
-                                const char *digest)
-{
-  tor_assert(addr);
-  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
-}
+    state->state = GUARD_CIRC_STATE_COMPLETE;
+    state->state_set_at = approx_time();
+    smartlist_add(newly_complete_out, circ);
+    ++n_succeeded;
+  } SMARTLIST_FOREACH_END(circ);
 
-/** If we have a bridge configured whose digest matches
- * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
- * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
- * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
-int
-extend_info_is_a_configured_bridge(const extend_info_t *ei)
-{
-  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
-  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
-}
+  log_info(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+           "%d guard-stalled, %d complete. %d of the guard-stalled "
+           "circuit(s) had high enough priority to upgrade.",
+           n_waiting, n_complete, n_succeeded);
 
-/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
- * it up via router descriptor <b>ri</b>. */
-static bridge_info_t *
-get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
-{
-  bridge_info_t *bi = NULL;
-  smartlist_t *orports = router_get_all_orports(ri);
-  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
-                                               orports);
-  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
-  smartlist_free(orports);
-  return bi;
+  tor_assert_nonfatal(n_succeeded >= 1);
+  smartlist_free(all_circuits);
+  return 1;
+
+ no_change:
+  smartlist_free(all_circuits);
+  return 0;
 }
 
-/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
+/**
+ * Return true iff the circuit whose state is <b>guard_state</b> should
+ * expire.
+ */
 int
-routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
+entry_guard_state_should_expire(circuit_guard_state_t *guard_state)
 {
-  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
+  if (guard_state == NULL)
+    return 0;
+  const time_t expire_if_waiting_since =
+    approx_time() - get_nonprimary_guard_idle_timeout();
+  return (guard_state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD
+          && guard_state->state_set_at < expire_if_waiting_since);
 }
 
-/** Return 1 if <b>node</b> is one of our configured bridges, else 0. */
+/**
+ * Update all derived pieces of the guard selection state in <b>gs</b>.
+ * Return true iff we should stop using all previously generated circuits.
+ */
 int
-node_is_a_configured_bridge(const node_t *node)
+entry_guards_update_all(guard_selection_t *gs)
 {
-  int retval = 0;
-  smartlist_t *orports = node_get_all_orports(node);
-  retval = get_configured_bridge_by_orports_digest(node->identity,
-                                                   orports) != NULL;
-  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
-  smartlist_free(orports);
-  return retval;
+  sampled_guards_update_from_consensus(gs);
+  entry_guards_update_filtered_sets(gs);
+  entry_guards_update_confirmed(gs);
+  entry_guards_update_primary(gs);
+  return 0;
 }
 
-/** We made a connection to a router at <b>addr</b>:<b>port</b>
- * without knowing its digest. Its digest turned out to be <b>digest</b>.
- * If it was a bridge, and we still don't know its digest, record it.
+/**
+ * Return a newly allocated string for encoding the persistent parts of
+ * <b>guard</b> to the state file.
  */
-void
-learned_router_identity(const tor_addr_t *addr, uint16_t port,
-                        const char *digest)
-{
-  bridge_info_t *bridge =
-    get_configured_bridge_by_addr_port_digest(addr, port, digest);
-  if (bridge && tor_digest_is_zero(bridge->identity)) {
-    char *transport_info = NULL;
-    const char *transport_name =
-      find_transport_name_by_bridge_addrport(addr, port);
-    if (transport_name)
-      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
+STATIC char *
+entry_guard_encode_for_state(entry_guard_t *guard)
+{
+  /*
+   * The meta-format we use is K=V K=V K=V... where K can be any
+   * characters excepts space and =, and V can be any characters except
+   * space.  The order of entries is not allowed to matter.
+   * Unrecognized K=V entries are persisted; recognized but erroneous
+   * entries are corrected.
+   */
 
-    memcpy(bridge->identity, digest, DIGEST_LEN);
-    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
-               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
-               transport_info ? transport_info : "");
-    tor_free(transport_info);
+  smartlist_t *result = smartlist_new();
+  char tbuf[ISO_TIME_LEN+1];
+
+  tor_assert(guard);
+
+  smartlist_add_asprintf(result, "in=%s", guard->selection_name);
+  smartlist_add_asprintf(result, "rsa_id=%s",
+                         hex_str(guard->identity, DIGEST_LEN));
+  if (guard->bridge_addr) {
+    smartlist_add_asprintf(result, "bridge_addr=%s:%d",
+                           fmt_and_decorate_addr(&guard->bridge_addr->addr),
+                           guard->bridge_addr->port);
+  }
+  if (strlen(guard->nickname) && is_legal_nickname(guard->nickname)) {
+    smartlist_add_asprintf(result, "nickname=%s", guard->nickname);
   }
-}
 
-/** Return true if <b>bridge</b> has the same identity digest as
- *  <b>digest</b>. If <b>digest</b> is NULL, it matches
- *  bridges with unspecified identity digests. */
-static int
-bridge_has_digest(const bridge_info_t *bridge, const char *digest)
-{
-  if (digest)
-    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
-  else
-    return tor_digest_is_zero(bridge->identity);
+  format_iso_time_nospace(tbuf, guard->sampled_on_date);
+  smartlist_add_asprintf(result, "sampled_on=%s", tbuf);
+
+  if (guard->sampled_by_version) {
+    smartlist_add_asprintf(result, "sampled_by=%s",
+                           guard->sampled_by_version);
+  }
+
+  if (guard->unlisted_since_date > 0) {
+    format_iso_time_nospace(tbuf, guard->unlisted_since_date);
+    smartlist_add_asprintf(result, "unlisted_since=%s", tbuf);
+  }
+
+  smartlist_add_asprintf(result, "listed=%d",
+                         (int)guard->currently_listed);
+
+  if (guard->confirmed_idx >= 0) {
+    format_iso_time_nospace(tbuf, guard->confirmed_on_date);
+    smartlist_add_asprintf(result, "confirmed_on=%s", tbuf);
+
+    smartlist_add_asprintf(result, "confirmed_idx=%d", guard->confirmed_idx);
+  }
+
+  const double EPSILON = 1.0e-6;
+
+  /* Make a copy of the pathbias object, since we will want to update
+     some of them */
+  guard_pathbias_t *pb = tor_memdup(&guard->pb, sizeof(*pb));
+  pb->use_successes = pathbias_get_use_success_count(guard);
+  pb->successful_circuits_closed = pathbias_get_close_success_count(guard);
+
+  #define PB_FIELD(field) do {                                          \
+      if (pb->field >= EPSILON) {                                       \
+        smartlist_add_asprintf(result, "pb_" #field "=%f", pb->field);  \
+      }                                                                 \
+    } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+  tor_free(pb);
+#undef PB_FIELD
+
+  if (guard->extra_state_fields)
+    smartlist_add_strdup(result, guard->extra_state_fields);
+
+  char *joined = smartlist_join_strings(result, " ", 0, NULL);
+  SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+  smartlist_free(result);
+
+  return joined;
 }
 
-/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
- * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
- * existing bridge with the same address and port, and warn the user as
- * appropriate.
+/**
+ * Given a string generated by entry_guard_encode_for_state(), parse it
+ * (if possible) and return an entry_guard_t object for it.  Return NULL
+ * on complete failure.
  */
-static void
-bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
-                         const char *digest, const char *transport_name)
-{
-  /* Iterate the already-registered bridge list:
+STATIC entry_guard_t *
+entry_guard_parse_from_state(const char *s)
+{
+  /* Unrecognized entries get put in here. */
+  smartlist_t *extra = smartlist_new();
+
+  /* These fields get parsed from the string. */
+  char *in = NULL;
+  char *rsa_id = NULL;
+  char *nickname = NULL;
+  char *sampled_on = NULL;
+  char *sampled_by = NULL;
+  char *unlisted_since = NULL;
+  char *listed  = NULL;
+  char *confirmed_on = NULL;
+  char *confirmed_idx = NULL;
+  char *bridge_addr = NULL;
+
+  // pathbias
+  char *pb_use_attempts = NULL;
+  char *pb_use_successes = NULL;
+  char *pb_circ_attempts = NULL;
+  char *pb_circ_successes = NULL;
+  char *pb_successful_circuits_closed = NULL;
+  char *pb_collapsed_circuits = NULL;
+  char *pb_unusable_circuits = NULL;
+  char *pb_timeouts = NULL;
+
+  /* Split up the entries.  Put the ones we know about in strings and the
+   * rest in "extra". */
+  {
+    smartlist_t *entries = smartlist_new();
+
+    strmap_t *vals = strmap_new(); // Maps keyword to location
+#define FIELD(f) \
+    strmap_set(vals, #f, &f);
+    FIELD(in);
+    FIELD(rsa_id);
+    FIELD(nickname);
+    FIELD(sampled_on);
+    FIELD(sampled_by);
+    FIELD(unlisted_since);
+    FIELD(listed);
+    FIELD(confirmed_on);
+    FIELD(confirmed_idx);
+    FIELD(bridge_addr);
+    FIELD(pb_use_attempts);
+    FIELD(pb_use_successes);
+    FIELD(pb_circ_attempts);
+    FIELD(pb_circ_successes);
+    FIELD(pb_successful_circuits_closed);
+    FIELD(pb_collapsed_circuits);
+    FIELD(pb_unusable_circuits);
+    FIELD(pb_timeouts);
+#undef FIELD
+
+    smartlist_split_string(entries, s, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
+      const char *eq = strchr(entry, '=');
+      if (!eq) {
+        smartlist_add(extra, entry);
+        continue;
+      }
+      char *key = tor_strndup(entry, eq-entry);
+      char **target = strmap_get(vals, key);
+      if (target == NULL || *target != NULL) {
+        /* unrecognized or already set */
+        smartlist_add(extra, entry);
+        tor_free(key);
+        continue;
+      }
 
-     If you find a bridge with the same adress and port, mark it for
-     removal. It doesn't make sense to have two active bridges with
-     the same IP:PORT. If the bridge in question has a different
-     digest or transport than <b>digest</b>/<b>transport_name</b>,
-     it's probably a misconfiguration and we should warn the user.
-  */
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
-    if (bridge->marked_for_removal)
-      continue;
+      *target = tor_strdup(eq+1);
+      tor_free(key);
+      tor_free(entry);
+    } SMARTLIST_FOREACH_END(entry);
 
-    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
-
-      bridge->marked_for_removal = 1;
-
-      if (!bridge_has_digest(bridge, digest) ||
-          strcmp_opt(bridge->transport_name, transport_name)) {
-        /* warn the user */
-        char *bridge_description_new, *bridge_description_old;
-        tor_asprintf(&bridge_description_new, "%s:%s:%s",
-                     fmt_addrport(addr, port),
-                     digest ? hex_str(digest, DIGEST_LEN) : "",
-                     transport_name ? transport_name : "");
-        tor_asprintf(&bridge_description_old, "%s:%s:%s",
-                     fmt_addrport(&bridge->addr, bridge->port),
-                     tor_digest_is_zero(bridge->identity) ?
-                     "" : hex_str(bridge->identity,DIGEST_LEN),
-                     bridge->transport_name ? bridge->transport_name : "");
-
-        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
-                 " with the already registered bridge '%s'. We will discard"
-                 " the old bridge and keep '%s'. If this is not what you"
-                 " wanted, please change your configuration file accordingly.",
-                 bridge_description_new, bridge_description_old,
-                 bridge_description_new);
-
-        tor_free(bridge_description_new);
-        tor_free(bridge_description_old);
-      }
-    }
-  } SMARTLIST_FOREACH_END(bridge);
-}
+    smartlist_free(entries);
+    strmap_free(vals, NULL);
+  }
 
-/** Return True if we have a bridge that uses a transport with name
- *  <b>transport_name</b>. */
-MOCK_IMPL(int,
-transport_is_needed, (const char *transport_name))
-{
-  if (!bridge_list)
-    return 0;
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+  guard->is_persistent = 1;
 
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (bridge->transport_name &&
-        !strcmp(bridge->transport_name, transport_name))
-      return 1;
-  } SMARTLIST_FOREACH_END(bridge);
+  if (in == NULL) {
+    log_warn(LD_CIRC, "Guard missing 'in' field");
+    goto err;
+  }
 
-  return 0;
-}
+  guard->selection_name = in;
+  in = NULL;
 
-/** Register the bridge information in <b>bridge_line</b> to the
- *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
-void
-bridge_add_from_config(bridge_line_t *bridge_line)
-{
-  bridge_info_t *b;
+  if (rsa_id == NULL) {
+    log_warn(LD_CIRC, "Guard missing RSA ID field");
+    goto err;
+  }
 
-  { /* Log the bridge we are about to register: */
-    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
-              fmt_addrport(&bridge_line->addr, bridge_line->port),
-              bridge_line->transport_name ?
-              bridge_line->transport_name : "no transport",
-              tor_digest_is_zero(bridge_line->digest) ?
-              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
+  /* Process the identity and nickname. */
+  if (base16_decode(guard->identity, sizeof(guard->identity),
+                    rsa_id, strlen(rsa_id)) != DIGEST_LEN) {
+    log_warn(LD_CIRC, "Unable to decode guard identity %s", escaped(rsa_id));
+    goto err;
+  }
 
-    if (bridge_line->socks_args) { /* print socks arguments */
-      int i = 0;
+  if (nickname) {
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  } else {
+    guard->nickname[0]='$';
+    base16_encode(guard->nickname+1, sizeof(guard->nickname)-1,
+                  guard->identity, DIGEST_LEN);
+  }
 
-      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
+  if (bridge_addr) {
+    tor_addr_port_t res;
+    memset(&res, 0, sizeof(res));
+    int r = tor_addr_port_parse(LOG_WARN, bridge_addr,
+                                &res.addr, &res.port, -1);
+    if (r == 0)
+      guard->bridge_addr = tor_memdup(&res, sizeof(res));
+    /* On error, we already warned. */
+  }
 
-      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
-                smartlist_len(bridge_line->socks_args));
-      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
-                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
+  /* Process the various time fields. */
+
+#define HANDLE_TIME(field) do {                                 \
+    if (field) {                                                \
+      int r = parse_iso_time_nospace(field, &field ## _time);   \
+      if (r < 0) {                                              \
+        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
+                 #field, escaped(field));                       \
+        field##_time = -1;                                      \
+      }                                                         \
+    }                                                           \
+  } while (0)
+
+  time_t sampled_on_time = 0;
+  time_t unlisted_since_time = 0;
+  time_t confirmed_on_time = 0;
+
+  HANDLE_TIME(sampled_on);
+  HANDLE_TIME(unlisted_since);
+  HANDLE_TIME(confirmed_on);
+
+  if (sampled_on_time <= 0)
+    sampled_on_time = approx_time();
+  if (unlisted_since_time < 0)
+    unlisted_since_time = 0;
+  if (confirmed_on_time < 0)
+    confirmed_on_time = 0;
+
+  #undef HANDLE_TIME
+
+  guard->sampled_on_date = sampled_on_time;
+  guard->unlisted_since_date = unlisted_since_time;
+  guard->confirmed_on_date = confirmed_on_time;
+
+  /* Take sampled_by_version verbatim. */
+  guard->sampled_by_version = sampled_by;
+  sampled_by = NULL; /* prevent free */
+
+  /* Listed is a boolean */
+  if (listed && strcmp(listed, "0"))
+    guard->currently_listed = 1;
+
+  /* The index is a nonnegative integer. */
+  guard->confirmed_idx = -1;
+  if (confirmed_idx) {
+    int ok=1;
+    long idx = tor_parse_long(confirmed_idx, 10, 0, INT_MAX, &ok, NULL);
+    if (! ok) {
+      log_warn(LD_GUARD, "Guard has invalid confirmed_idx %s",
+               escaped(confirmed_idx));
+    } else {
+      guard->confirmed_idx = (int)idx;
     }
   }
 
-  bridge_resolve_conflicts(&bridge_line->addr,
-                           bridge_line->port,
-                           bridge_line->digest,
-                           bridge_line->transport_name);
+  /* Anything we didn't recognize gets crammed together */
+  if (smartlist_len(extra) > 0) {
+    guard->extra_state_fields = smartlist_join_strings(extra, " ", 0, NULL);
+  }
+
+  /* initialize non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+
+#define PB_FIELD(field)                                                 \
+  do {                                                                  \
+    if (pb_ ## field) {                                                 \
+      int ok = 1;                                                       \
+      double r = tor_parse_double(pb_ ## field, 0.0, 1e9, &ok, NULL);   \
+      if (! ok) {                                                       \
+        log_warn(LD_CIRC, "Guard has invalid pb_%s %s",                 \
+                 #field, pb_ ## field);                                 \
+      } else {                                                          \
+        guard->pb.field = r;                                            \
+      }                                                                 \
+    }                                                                   \
+  } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+#undef PB_FIELD
+
+  pathbias_check_use_success_count(guard);
+  pathbias_check_close_success_count(guard);
+
+  /* We update everything on this guard later, after we've parsed
+   * everything.  */
+
+  goto done;
+
+ err:
+  // only consider it an error if the guard state was totally unparseable.
+  entry_guard_free(guard);
+  guard = NULL;
+
+ done:
+  tor_free(in);
+  tor_free(rsa_id);
+  tor_free(nickname);
+  tor_free(sampled_on);
+  tor_free(sampled_by);
+  tor_free(unlisted_since);
+  tor_free(listed);
+  tor_free(confirmed_on);
+  tor_free(confirmed_idx);
+  tor_free(bridge_addr);
+  tor_free(pb_use_attempts);
+  tor_free(pb_use_successes);
+  tor_free(pb_circ_attempts);
+  tor_free(pb_circ_successes);
+  tor_free(pb_successful_circuits_closed);
+  tor_free(pb_collapsed_circuits);
+  tor_free(pb_unusable_circuits);
+  tor_free(pb_timeouts);
+
+  SMARTLIST_FOREACH(extra, char *, cp, tor_free(cp));
+  smartlist_free(extra);
+
+  return guard;
+}
 
-  b = tor_malloc_zero(sizeof(bridge_info_t));
-  tor_addr_copy(&b->addr, &bridge_line->addr);
-  b->port = bridge_line->port;
-  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
-  if (bridge_line->transport_name)
-    b->transport_name = bridge_line->transport_name;
-  b->fetch_status.schedule = DL_SCHED_BRIDGE;
-  b->fetch_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL;
-  b->socks_args = bridge_line->socks_args;
-  if (!bridge_list)
-    bridge_list = smartlist_new();
+/**
+ * Replace the Guards entries in <b>state</b> with a list of all our sampled
+ * guards.
+ */
+static void
+entry_guards_update_guards_in_state(or_state_t *state)
+{
+  if (!guard_contexts)
+    return;
+  config_line_t *lines = NULL;
+  config_line_t **nextline = &lines;
 
-  tor_free(bridge_line); /* Deallocate bridge_line now. */
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+      if (guard->is_persistent == 0)
+        continue;
+      *nextline = tor_malloc_zero(sizeof(config_line_t));
+      (*nextline)->key = tor_strdup("Guard");
+      (*nextline)->value = entry_guard_encode_for_state(guard);
+      nextline = &(*nextline)->next;
+    } SMARTLIST_FOREACH_END(guard);
+  } SMARTLIST_FOREACH_END(gs);
 
-  smartlist_add(bridge_list, b);
+  config_free_lines(state->Guard);
+  state->Guard = lines;
 }
 
-/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+/**
+ * Replace our sampled guards from the Guards entries in <b>state</b>. Return 0
+ * on success, -1 on failure. (If <b>set</b> is true, replace nothing -- only
+ * check whether replacing would work.)
+ */
 static int
-routerset_contains_bridge(const routerset_t *routerset,
-                          const bridge_info_t *bridge)
+entry_guards_load_guards_from_state(or_state_t *state, int set)
+{
+  const config_line_t *line = state->Guard;
+  int n_errors = 0;
+
+  if (!guard_contexts)
+    guard_contexts = smartlist_new();
+
+  /* Wipe all our existing guard info. (we shouldn't have any, but
+   * let's be safe.) */
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+      if (curr_guard_context == gs)
+        curr_guard_context = NULL;
+      SMARTLIST_DEL_CURRENT(guard_contexts, gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+
+  for ( ; line != NULL; line = line->next) {
+    entry_guard_t *guard = entry_guard_parse_from_state(line->value);
+    if (guard == NULL) {
+      ++n_errors;
+      continue;
+    }
+    tor_assert(guard->selection_name);
+    if (!strcmp(guard->selection_name, "legacy")) {
+      ++n_errors;
+      entry_guard_free(guard);
+      continue;
+    }
+
+    if (set) {
+      guard_selection_t *gs;
+      gs = get_guard_selection_by_name(guard->selection_name,
+                                       GS_TYPE_INFER, 1);
+      tor_assert(gs);
+      smartlist_add(gs->sampled_entry_guards, guard);
+      guard->in_selection = gs;
+    } else {
+      entry_guard_free(guard);
+    }
+  }
+
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      entry_guards_update_all(gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+  return n_errors ? -1 : 0;
+}
+
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the provided guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest_for_guard_selection(guard_selection_t *gs,
+                                                 const char *digest)
 {
-  int result;
-  extend_info_t *extinfo;
-  tor_assert(bridge);
-  if (!routerset)
-    return 0;
+  return get_sampled_guard_with_id(gs, (const uint8_t*)digest);
+}
 
-  extinfo = extend_info_new(
-         NULL, bridge->identity, NULL, NULL, &bridge->addr, bridge->port);
-  result = routerset_contains_extendinfo(routerset, extinfo);
-  extend_info_free(extinfo);
-  return result;
+/** Return the node_t associated with a single entry_guard_t. May
+ * return NULL if the guard is not currently in the consensus. */
+const node_t *
+entry_guard_find_node(const entry_guard_t *guard)
+{
+  tor_assert(guard);
+  return node_get_by_id(guard->identity);
 }
 
-/** If <b>digest</b> is one of our known bridges, return it. */
-static bridge_info_t *
-find_bridge_by_digest(const char *digest)
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the default guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest(const char *digest)
 {
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
-    {
-      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
-    });
-  return NULL;
+  return entry_guard_get_by_id_digest_for_guard_selection(
+      get_guard_selection_info(), digest);
 }
 
-/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
- *  supports a pluggable transport, return its name. Otherwise, return
- *  NULL. */
-const char *
-find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+/** We are about to connect to bridge with identity <b>digest</b> to fetch its
+ *  descriptor. Create a new guard state for this connection and return it. */
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest)
 {
-  if (!bridge_list)
+  circuit_guard_state_t *guard_state = NULL;
+  entry_guard_t *guard = NULL;
+
+  guard = entry_guard_get_by_id_digest_for_guard_selection(
+                                    get_guard_selection_info(), digest);
+  if (!guard) {
     return NULL;
+  }
 
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (tor_addr_eq(&bridge->addr, addr) &&
-        (bridge->port == port))
-      return bridge->transport_name;
-  } SMARTLIST_FOREACH_END(bridge);
+  /* Update the guard last_tried_to_connect time since it's checked by the
+   * guard susbsystem. */
+  guard->last_tried_to_connect = approx_time();
 
-  return NULL;
+  /* Create the guard state */
+  guard_state = circuit_guard_state_new(guard,
+                                        GUARD_CIRC_STATE_USABLE_ON_COMPLETION,
+                                        NULL);
+
+  return guard_state;
 }
 
-/** If <b>addr</b> and <b>port</b> match the address and port of a
- * bridge of ours that uses pluggable transports, place its transport
- * in <b>transport</b>.
- *
- * Return 0 on success (found a transport, or found a bridge with no
- * transport, or found no bridge); return -1 if we should be using a
- * transport, but the transport could not be found.
+/** Release all storage held by <b>e</b>. */
+STATIC void
+entry_guard_free_(entry_guard_t *e)
+{
+  if (!e)
+    return;
+  entry_guard_handles_clear(e);
+  tor_free(e->sampled_by_version);
+  tor_free(e->extra_state_fields);
+  tor_free(e->selection_name);
+  tor_free(e->bridge_addr);
+  tor_free(e);
+}
+
+/** Return 0 if we're fine adding arbitrary routers out of the
+ * directory to our entry guard list, or return 1 if we have a
+ * list already and we must stick to it.
  */
 int
-get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
-                                  const transport_t **transport)
+entry_list_is_constrained(const or_options_t *options)
 {
-  *transport = NULL;
-  if (!bridge_list)
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (tor_addr_eq(&bridge->addr, addr) &&
-        (bridge->port == port)) { /* bridge matched */
-      if (bridge->transport_name) { /* it also uses pluggable transports */
-        *transport = transport_get_by_name(bridge->transport_name);
-        if (*transport == NULL) { /* it uses pluggable transports, but
-                                     the transport could not be found! */
-          return -1;
-        }
-        return 0;
-      } else { /* bridge matched, but it doesn't use transports. */
-        break;
-      }
-    }
-  } SMARTLIST_FOREACH_END(bridge);
-
-  *transport = NULL;
+  // XXXX #21425 look at the current selection.
+  if (options->EntryNodes)
+    return 1;
+  if (options->UseBridges)
+    return 1;
   return 0;
 }
 
-/** Return a smartlist containing all the SOCKS arguments that we
- *  should pass to the SOCKS proxy. */
-const smartlist_t *
-get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+/** Return the number of bridges that have descriptors that are marked with
+ * purpose 'bridge' and are running. If use_maybe_reachable is
+ * true, include bridges that might be reachable in the count.
+ * Otherwise, if it is false, only include bridges that have recently been
+ * found running in the count.
+ *
+ * We use this function to decide if we're ready to start building
+ * circuits through our bridges, or if we need to wait until the
+ * directory "server/authority" requests finish. */
+MOCK_IMPL(int,
+num_bridges_usable,(int use_maybe_reachable))
 {
-  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
-                                                                    port,
-                                                                    NULL);
-  return bridge ? bridge->socks_args : NULL;
+  int n_options = 0;
+
+  if (BUG(!get_options()->UseBridges)) {
+    return 0;
+  }
+  guard_selection_t *gs  = get_guard_selection_info();
+  if (BUG(gs->type != GS_TYPE_BRIDGE)) {
+    return 0;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* Definitely not usable */
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    /* If we want to be really sure the bridges will work, skip maybes */
+    if (!use_maybe_reachable && guard->is_reachable == GUARD_REACHABLE_MAYBE)
+      continue;
+    if (tor_digest_is_zero(guard->identity))
+      continue;
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node && node->ri)
+      ++n_options;
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_options;
 }
 
-/** We need to ask <b>bridge</b> for its server descriptor. */
+/** Check the pathbias use success count of <b>node</b> and disable it if it
+ *  goes over our thresholds. */
 static void
-launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
+pathbias_check_use_success_count(entry_guard_t *node)
 {
   const or_options_t *options = get_options();
-
-  if (connection_get_by_type_addr_port_purpose(
-      CONN_TYPE_DIR, &bridge->addr, bridge->port,
-      DIR_PURPOSE_FETCH_SERVERDESC))
-    return; /* it's already on the way */
-
-  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
-    download_status_mark_impossible(&bridge->fetch_status);
-    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
-             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
-    return;
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.use_attempts > EPSILON &&
+      pathbias_get_use_success_count(node)/node->pb.use_attempts
+      < pathbias_get_extreme_use_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path use bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
   }
+}
 
-  /* Until we get a descriptor for the bridge, we only know one address for
-   * it. */
-  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
-                                            FIREWALL_OR_CONNECTION, 0, 0)) {
-    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
-               "bridge, but that bridge is not reachable through our "
-               "firewall.");
-    return;
+/** Check the pathbias close count of <b>node</b> and disable it if it goes
+ *  over our thresholds. */
+static void
+pathbias_check_close_success_count(entry_guard_t *node)
+{
+  const or_options_t *options = get_options();
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.circ_attempts > EPSILON &&
+      pathbias_get_close_success_count(node)/node->pb.circ_attempts
+      < pathbias_get_extreme_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
   }
-
-  directory_initiate_command(&bridge->addr, bridge->port,
-                             NULL, 0, /*no dirport*/
-                             bridge->identity,
-                             DIR_PURPOSE_FETCH_SERVERDESC,
-                             ROUTER_PURPOSE_BRIDGE,
-                             DIRIND_ONEHOP, "authority.z", NULL, 0, 0);
 }
 
-/** Fetching the bridge descriptor from the bridge authority returned a
- * "not found". Fall back to trying a direct fetch. */
-void
-retry_bridge_descriptor_fetch_directly(const char *digest)
+/** Parse <b>state</b> and learn about the entry guards it describes.
+ * If <b>set</b> is true, and there are no errors, replace the guard
+ * list in the default guard selection context with what we find.
+ * On success, return 0. On failure, alloc into *<b>msg</b> a string
+ * describing the error, and return -1.
+ */
+int
+entry_guards_parse_state(or_state_t *state, int set, char **msg)
 {
-  bridge_info_t *bridge = find_bridge_by_digest(digest);
-  if (!bridge)
-    return; /* not found? oh well. */
+  entry_guards_dirty = 0;
+  int r1 = entry_guards_load_guards_from_state(state, set);
+  entry_guards_dirty = 0;
 
-  launch_direct_bridge_descriptor_fetch(bridge);
+  if (r1 < 0) {
+    if (msg && *msg == NULL) {
+      *msg = tor_strdup("parsing error");
+    }
+    return -1;
+  }
+  return 0;
 }
 
-/** For each bridge in our list for which we don't currently have a
- * descriptor, fetch a new copy of its descriptor -- either directly
- * from the bridge or via a bridge authority. */
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are trying to avoid disk writes? */
+#define SLOW_GUARD_STATE_FLUSH_TIME 600
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are not trying to avoid disk writes? */
+#define FAST_GUARD_STATE_FLUSH_TIME 30
+
+/** Our list of entry guards has changed for a particular guard selection
+ * context, or some element of one of our entry guards has changed for one.
+ * Write the changes to disk within the next few minutes.
+ */
 void
-fetch_bridge_descriptors(const or_options_t *options, time_t now)
+entry_guards_changed_for_guard_selection(guard_selection_t *gs)
 {
-  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
-  int ask_bridge_directly;
-  int can_use_bridge_authority;
+  time_t when;
 
-  if (!bridge_list)
-    return;
+  tor_assert(gs != NULL);
 
-  /* If we still have unconfigured managed proxies, don't go and
-     connect to a bridge. */
-  if (pt_proxies_configuration_pending())
-    return;
+  entry_guards_dirty = 1;
 
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if (!download_status_is_ready(&bridge->fetch_status, now,
-                                    IMPOSSIBLE_TO_DOWNLOAD))
-        continue; /* don't bother, no need to retry yet */
-      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
-        download_status_mark_impossible(&bridge->fetch_status);
-        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
-                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
-        continue;
-      }
+  if (get_options()->AvoidDiskWrites)
+    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
+  else
+    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
 
-      /* schedule another fetch as if this one will fail, in case it does */
-      download_status_failed(&bridge->fetch_status, 0);
-
-      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
-                                 num_bridge_auths;
-      ask_bridge_directly = !can_use_bridge_authority ||
-                            !options->UpdateBridgesFromAuthority;
-      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
-                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
-                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
-
-      if (ask_bridge_directly &&
-          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
-                                                FIREWALL_OR_CONNECTION, 0,
-                                                0)) {
-        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
-                   "firewall policy. %s.",
-                   fmt_addrport(&bridge->addr, bridge->port),
-                   can_use_bridge_authority ?
-                     "Asking bridge authority instead" : "Skipping");
-        if (can_use_bridge_authority)
-          ask_bridge_directly = 0;
-        else
-          continue;
-      }
+  /* or_state_save() will call entry_guards_update_state() and
+     entry_guards_update_guards_in_state()
+  */
+  or_state_mark_dirty(get_or_state(), when);
+}
 
-      if (ask_bridge_directly) {
-        /* we need to ask the bridge itself for its descriptor. */
-        launch_direct_bridge_descriptor_fetch(bridge);
-      } else {
-        /* We have a digest and we want to ask an authority. We could
-         * combine all the requests into one, but that may give more
-         * hints to the bridge authority than we want to give. */
-        char resource[10 + HEX_DIGEST_LEN];
-        memcpy(resource, "fp/", 3);
-        base16_encode(resource+3, HEX_DIGEST_LEN+1,
-                      bridge->identity, DIGEST_LEN);
-        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
-        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
-                 resource);
-        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
-                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
-      }
-    }
-  SMARTLIST_FOREACH_END(bridge);
+/** Our list of entry guards has changed for the default guard selection
+ * context, or some element of one of our entry guards has changed. Write
+ * the changes to disk within the next few minutes.
+ */
+void
+entry_guards_changed(void)
+{
+  entry_guards_changed_for_guard_selection(get_guard_selection_info());
 }
 
-/** If our <b>bridge</b> is configured to be a different address than
- * the bridge gives in <b>node</b>, rewrite the routerinfo
- * we received to use the address we meant to use. Now we handle
- * multihomed bridges better.
+/** If the entry guard info has not changed, do nothing and return.
+ * Otherwise, free the EntryGuards piece of <b>state</b> and create
+ * a new one out of the global entry_guards list, and then mark
+ * <b>state</b> dirty so it will get saved to disk.
  */
-static void
-rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
+void
+entry_guards_update_state(or_state_t *state)
 {
-  /* XXXX move this function. */
-  /* XXXX overridden addresses should really live in the node_t, so that the
-   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
-   *   do that safely if we know that no function that connects to an OR
-   *   does so through an address from any source other than node_get_addr().
-   */
-  tor_addr_t addr;
-  const or_options_t *options = get_options();
+  entry_guards_dirty = 0;
 
-  if (node->ri) {
-    routerinfo_t *ri = node->ri;
-    tor_addr_from_ipv4h(&addr, ri->addr);
+  // Handles all guard info.
+  entry_guards_update_guards_in_state(state);
 
-    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
-         bridge->port == ri->or_port) ||
-        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
-         bridge->port == ri->ipv6_orport)) {
-      /* they match, so no need to do anything */
-    } else {
-      if (tor_addr_family(&bridge->addr) == AF_INET) {
-        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
-        ri->or_port = bridge->port;
-        log_info(LD_DIR,
-                 "Adjusted bridge routerinfo for '%s' to match configured "
-                 "address %s:%d.",
-                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
-      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
-        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
-        ri->ipv6_orport = bridge->port;
-        log_info(LD_DIR,
-                 "Adjusted bridge routerinfo for '%s' to match configured "
-                 "address %s.",
-                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
-      } else {
-        log_err(LD_BUG, "Address family not supported: %d.",
-                tor_addr_family(&bridge->addr));
-        return;
-      }
-    }
+  entry_guards_dirty = 0;
 
-    if (options->ClientPreferIPv6ORPort == -1) {
-      /* Mark which address to use based on which bridge_t we got. */
-      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
-                              !tor_addr_is_null(&node->ri->ipv6_addr));
-    } else {
-      /* Mark which address to use based on user preference */
-      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
-                              !tor_addr_is_null(&node->ri->ipv6_addr));
-    }
+  if (!get_options()->AvoidDiskWrites)
+    or_state_mark_dirty(get_or_state(), 0);
+  entry_guards_dirty = 0;
+}
 
-    /* XXXipv6 we lack support for falling back to another address for
-       the same relay, warn the user */
-    if (!tor_addr_is_null(&ri->ipv6_addr)) {
-      tor_addr_port_t ap;
-      node_get_pref_orport(node, &ap);
-      log_notice(LD_CONFIG,
-                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
-                 "Will prefer using its %s address (%s) based on %s.",
-                 ri->nickname,
-                 node->ipv6_preferred ? "IPv6" : "IPv4",
-                 fmt_addrport(&ap.addr, ap.port),
-                 options->ClientPreferIPv6ORPort == -1 ?
-                 "the configured Bridge address" :
-                 "ClientPreferIPv6ORPort");
-    }
+/** Return true iff the circuit's guard can succeed that is can be used. */
+int
+entry_guard_could_succeed(const circuit_guard_state_t *guard_state)
+{
+  if (!guard_state) {
+    return 0;
   }
-  if (node->rs) {
-    routerstatus_t *rs = node->rs;
-    tor_addr_from_ipv4h(&addr, rs->addr);
 
-    if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
-        bridge->port == rs->or_port) {
-      /* they match, so no need to do anything */
-    } else {
-      rs->addr = tor_addr_to_ipv4h(&bridge->addr);
-      rs->or_port = bridge->port;
-      log_info(LD_DIR,
-               "Adjusted bridge routerstatus for '%s' to match "
-               "configured address %s.",
-               rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
-    }
+  entry_guard_t *guard = entry_guard_handle_get(guard_state->guard);
+  if (!guard || BUG(guard->in_selection == NULL)) {
+    return 0;
   }
+
+  return 1;
 }
 
-/** We just learned a descriptor for a bridge. See if that
- * digest is in our entry guard list, and add it if not. */
-void
-learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
-{
-  tor_assert(ri);
-  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
-  if (get_options()->UseBridges) {
-    int first = num_bridges_usable() <= 1;
-    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
-    time_t now = time(NULL);
-    router_set_status(ri->cache_info.identity_digest, 1);
-
-    if (bridge) { /* if we actually want to use this one */
-      node_t *node;
-      /* it's here; schedule its re-fetch for a long time from now. */
-      if (!from_cache)
-        download_status_reset(&bridge->fetch_status);
-
-      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
-      tor_assert(node);
-      rewrite_node_address_for_bridge(bridge, node);
-      if (tor_digest_is_zero(bridge->identity)) {
-        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
-        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
-                   hex_str(bridge->identity, DIGEST_LEN),
-                   fmt_and_decorate_addr(&bridge->addr),
-                   (int) bridge->port);
-      }
-      add_an_entry_guard(node, 1, 1, 0, 0);
-
-      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
-                 from_cache ? "cached" : "fresh", router_describe(ri));
-      /* set entry->made_contact so if it goes down we don't drop it from
-       * our entry node list */
-      entry_guard_register_connect_status(ri->cache_info.identity_digest,
-                                          1, 0, now);
-      if (first) {
-        routerlist_retry_directory_downloads(now);
-      }
-    }
+/**
+ * Format a single entry guard in the format expected by the controller.
+ * Return a newly allocated string.
+ */
+STATIC char *
+getinfo_helper_format_single_entry_guard(const entry_guard_t *e)
+{
+  const char *status = NULL;
+  time_t when = 0;
+  const node_t *node;
+  char tbuf[ISO_TIME_LEN+1];
+  char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  /* This is going to be a bit tricky, since the status
+   * codes weren't really intended for prop271 guards.
+   *
+   * XXXX use a more appropriate format for exporting this information
+   */
+  if (e->confirmed_idx < 0) {
+    status = "never-connected";
+  } else if (! e->currently_listed) {
+    when = e->unlisted_since_date;
+    status = "unusable";
+  } else if (! e->is_filtered_guard) {
+    status = "unusable";
+  } else if (e->is_reachable == GUARD_REACHABLE_NO) {
+    when = e->failing_since;
+    status = "down";
+  } else {
+    status = "up";
+  }
+
+  node = entry_guard_find_node(e);
+  if (node) {
+    node_get_verbose_nickname(node, nbuf);
+  } else {
+    nbuf[0] = '$';
+    base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
+    /* e->nickname field is not very reliable if we don't know about
+     * this router any longer; don't include it. */
   }
+
+  char *result = NULL;
+  if (when) {
+    format_iso_time(tbuf, when);
+    tor_asprintf(&result, "%s %s %s\n", nbuf, status, tbuf);
+  } else {
+    tor_asprintf(&result, "%s %s\n", nbuf, status);
+  }
+  return result;
 }
 
-/** Return the number of bridges that have descriptors that
- * are marked with purpose 'bridge' and are running.
+/** If <b>question</b> is the string "entry-guards", then dump
+ * to *<b>answer</b> a newly allocated string describing all of
+ * the nodes in the global entry_guards list. See control-spec.txt
+ * for details.
+ * For backward compatibility, we also handle the string "helper-nodes".
  *
- * We use this function to decide if we're ready to start building
- * circuits through our bridges, or if we need to wait until the
- * directory "server/authority" requests finish. */
+ * XXX this should be totally redesigned after prop 271 too, and that's
+ * going to take some control spec work.
+ * */
 int
-any_bridge_descriptors_known(void)
+getinfo_helper_entry_guards(control_connection_t *conn,
+                            const char *question, char **answer,
+                            const char **errmsg)
 {
-  tor_assert(get_options()->UseBridges);
-  return choose_random_entry(NULL) != NULL;
+  guard_selection_t *gs = get_guard_selection_info();
+
+  tor_assert(gs != NULL);
+
+  (void) conn;
+  (void) errmsg;
+
+  if (!strcmp(question,"entry-guards") ||
+      !strcmp(question,"helper-nodes")) {
+    const smartlist_t *guards;
+    guards = gs->sampled_entry_guards;
+
+    smartlist_t *sl = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(guards, const entry_guard_t *, e) {
+      char *cp = getinfo_helper_format_single_entry_guard(e);
+      smartlist_add(sl, cp);
+    } SMARTLIST_FOREACH_END(e);
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  }
+  return 0;
 }
 
-/** Return the number of bridges that have descriptors that are marked with
- * purpose 'bridge' and are running.
+/* Given the original bandwidth of a guard and its guardfraction,
+ * calculate how much bandwidth the guard should have as a guard and
+ * as a non-guard.
+ *
+ * Quoting from proposal236:
+ *
+ *   Let Wpf denote the weight from the 'bandwidth-weights' line a
+ *   client would apply to N for position p if it had the guard
+ *   flag, Wpn the weight if it did not have the guard flag, and B the
+ *   measured bandwidth of N in the consensus.  Then instead of choosing
+ *   N for position p proportionally to Wpf*B or Wpn*B, clients should
+ *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+ *
+ * This function fills the <b>guardfraction_bw</b> structure. It sets
+ * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
  */
-static int
-num_bridges_usable(void)
+void
+guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
+                                  int orig_bandwidth,
+                                  uint32_t guardfraction_percentage)
 {
-  int n_options = 0;
-  tor_assert(get_options()->UseBridges);
-  (void) choose_random_entry_impl(NULL, 0, 0, &n_options);
-  return n_options;
+  double guardfraction_fraction;
+
+  /* Turn the percentage into a fraction. */
+  tor_assert(guardfraction_percentage <= 100);
+  guardfraction_fraction = guardfraction_percentage / 100.0;
+
+  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
+  tor_assert(guard_bw <= INT_MAX);
+
+  guardfraction_bw->guard_bw = (int) guard_bw;
+
+  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
 }
 
-/** Return a smartlist containing all bridge identity digests */
-MOCK_IMPL(smartlist_t *,
-list_bridge_identities, (void))
+/** Helper: Update the status of all entry guards, in whatever algorithm
+ * is used. Return true if we should stop using all previously generated
+ * circuits, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
+int
+guards_update_all(void)
 {
-  smartlist_t *result = NULL;
-  char *digest_tmp;
+  int mark_circuits = 0;
+  if (update_guard_selection_choice(get_options()))
+    mark_circuits = 1;
 
-  if (get_options()->UseBridges && bridge_list) {
-    result = smartlist_new();
+  tor_assert(curr_guard_context);
 
-    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-      digest_tmp = tor_malloc(DIGEST_LEN);
-      memcpy(digest_tmp, b->identity, DIGEST_LEN);
-      smartlist_add(result, digest_tmp);
-    } SMARTLIST_FOREACH_END(b);
-  }
+  if (entry_guards_update_all(curr_guard_context))
+    mark_circuits = 1;
 
-  return result;
+  return mark_circuits;
+}
+
+/** Helper: pick a guard for a circuit, with whatever algorithm is
+    used. */
+const node_t *
+guards_choose_guard(cpath_build_state_t *state,
+                    uint8_t purpose,
+                    circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  const uint8_t *exit_id = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* Only apply restrictions if we have a specific exit node in mind, and only
+   * if we are not doing vanguard circuits: we don't want to apply guard
+   * restrictions to vanguard circuits. */
+  if (state && !circuit_should_use_vanguards(purpose) &&
+      (exit_id = build_state_get_exit_rsa_id(state))) {
+    /* We're building to a targeted exit node, so that node can't be
+     * chosen as our guard for this circuit.  Remember that fact in a
+     * restriction. */
+    rst = guard_create_exit_restriction(exit_id);
+    tor_assert(rst);
+  }
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_TRAFFIC,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
 }
 
-/** Get the download status for a bridge descriptor given its identity */
-MOCK_IMPL(download_status_t *,
-get_bridge_dl_status_by_id, (const char *digest))
+/** Remove all currently listed entry guards for a given guard selection
+ * context.  This frees and replaces <b>gs</b>, so don't use <b>gs</b>
+ * after calling this function. */
+void
+remove_all_entry_guards_for_guard_selection(guard_selection_t *gs)
 {
-  download_status_t *dl = NULL;
+  // This function shouldn't exist. XXXX
+  tor_assert(gs != NULL);
+  char *old_name = tor_strdup(gs->name);
+  guard_selection_type_t old_type = gs->type;
 
-  if (digest && get_options()->UseBridges && bridge_list) {
-    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
-        dl = &(b->fetch_status);
-        break;
-      }
-    } SMARTLIST_FOREACH_END(b);
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, entry, {
+    control_event_guard(entry->nickname, entry->identity, "DROPPED");
+  });
+
+  if (gs == curr_guard_context) {
+    curr_guard_context = NULL;
   }
 
-  return dl;
+  smartlist_remove(guard_contexts, gs);
+  guard_selection_free(gs);
+
+  gs = get_guard_selection_by_name(old_name, old_type, 1);
+  entry_guards_changed_for_guard_selection(gs);
+  tor_free(old_name);
 }
 
-/** Return 1 if we have at least one descriptor for an entry guard
- * (bridge or member of EntryNodes) and all descriptors we know are
- * down. Else return 0. If <b>act</b> is 1, then mark the down guards
- * up; else just observe and report. */
-static int
-entries_retry_helper(const or_options_t *options, int act)
+/** Remove all currently listed entry guards, so new ones will be chosen.
+ *
+ * XXXX This function shouldn't exist -- it's meant to support the DROPGUARDS
+ * command, which is deprecated.
+ */
+void
+remove_all_entry_guards(void)
 {
-  const node_t *node;
-  int any_known = 0;
-  int any_running = 0;
-  int need_bridges = options->UseBridges != 0;
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-      node = node_get_by_id(e->identity);
-      if (node && node_has_descriptor(node) &&
-          node_is_bridge(node) == need_bridges &&
-          (!need_bridges || (!e->bad_since &&
-                             node_is_a_configured_bridge(node)))) {
-        any_known = 1;
-        if (node->is_running)
-          any_running = 1; /* some entry is both known and running */
-        else if (act) {
-          /* Mark all current connections to this OR as unhealthy, since
-           * otherwise there could be one that started 30 seconds
-           * ago, and in 30 seconds it will time out, causing us to mark
-           * the node down and undermine the retry attempt. We mark even
-           * the established conns, since if the network just came back
-           * we'll want to attach circuits to fresh conns. */
-          connection_or_set_bad_connections(node->identity, 1);
-
-          /* mark this entry node for retry */
-          router_set_status(node->identity, 1);
-          e->can_retry = 1;
-          e->bad_since = 0;
-        }
-      }
-  } SMARTLIST_FOREACH_END(e);
-  log_debug(LD_DIR, "%d: any_known %d, any_running %d",
-            act, any_known, any_running);
-  return any_known && !any_running;
+  remove_all_entry_guards_for_guard_selection(get_guard_selection_info());
+}
+
+/** Helper: pick a directory guard, with whatever algorithm is used. */
+const node_t *
+guards_choose_dirguard(uint8_t dir_purpose,
+                       circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* If we are fetching microdescs, don't query outdated dirservers. */
+  if (dir_purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    rst = guard_create_dirserver_md_restriction();
+  }
+
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_DIRGUARD,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
 }
 
-/** Do we know any descriptors for our bridges / entrynodes, and are
- * all the ones we have descriptors for down? */
+/**
+ * If we're running with a constrained guard set, then maybe mark our guards
+ * usable.  Return 1 if we do; 0 if we don't.
+ */
 int
-entries_known_but_down(const or_options_t *options)
+guards_retry_optimistic(const or_options_t *options)
 {
-  tor_assert(entry_list_is_constrained(options));
-  return entries_retry_helper(options, 0);
+  if (! entry_list_is_constrained(options))
+    return 0;
+
+  mark_primary_guards_maybe_reachable(get_guard_selection_info());
+
+  return 1;
 }
 
-/** Mark all down known bridges / entrynodes up. */
-void
-entries_retry_all(const or_options_t *options)
+/**
+ * Check if we are missing any crucial dirinfo for the guard subsystem to
+ * work. Return NULL if everything went well, otherwise return a newly
+ * allocated string with an informative error message. In the latter case, use
+ * the genreal descriptor information <b>using_mds</b>, <b>num_present</b> and
+ * <b>num_usable</b> to improve the error message. */
+char *
+guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs,
+                                        int using_mds,
+                                        int num_present, int num_usable)
+{
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  char *ret_str = NULL;
+  int n_missing_descriptors = 0;
+  int n_considered = 0;
+  int num_primary_to_check;
+
+  /* We want to check for the descriptor of at least the first two primary
+   * guards in our list, since these are the guards that we typically use for
+   * circuits. */
+  num_primary_to_check = get_n_primary_guards_to_use(GUARD_USAGE_TRAFFIC);
+  num_primary_to_check++;
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    n_considered++;
+    if (!guard_has_descriptor(guard))
+      n_missing_descriptors++;
+    if (n_considered >= num_primary_to_check)
+      break;
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* If we are not missing any descriptors, return NULL. */
+  if (!n_missing_descriptors) {
+    return NULL;
+  }
+
+  /* otherwise return a helpful error string */
+  tor_asprintf(&ret_str, "We're missing descriptors for %d/%d of our "
+               "primary entry guards (total %sdescriptors: %d/%d).",
+               n_missing_descriptors, num_primary_to_check,
+               using_mds?"micro":"", num_present, num_usable);
+
+  return ret_str;
+}
+
+/** As guard_selection_have_enough_dir_info_to_build_circuits, but uses
+ * the default guard selection. */
+char *
+entry_guards_get_err_str_if_dir_info_missing(int using_mds,
+                                     int num_present, int num_usable)
 {
-  tor_assert(entry_list_is_constrained(options));
-  entries_retry_helper(options, 1);
+  return guard_selection_get_err_str_if_dir_info_missing(
+                                                 get_guard_selection_info(),
+                                                 using_mds,
+                                                 num_present, num_usable);
 }
 
-/** Return true if at least one of our bridges runs a Tor version that can
- * provide microdescriptors to us. If not, we'll fall back to asking for
- * full descriptors. */
-int
-any_bridge_supports_microdescriptors(void)
+/** Free one guard selection context */
+STATIC void
+guard_selection_free_(guard_selection_t *gs)
 {
-  const node_t *node;
-  if (!get_options()->UseBridges || !entry_guards)
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-    node = node_get_by_id(e->identity);
-    if (node && node->is_running &&
-        node_is_bridge(node) && node_is_a_configured_bridge(node)) {
-      /* This is one of our current bridges, and we know enough about
-       * it to know that it will be able to answer our questions. */
-       return 1;
-    }
-  } SMARTLIST_FOREACH_END(e);
-  return 0;
+  if (!gs) return;
+
+  tor_free(gs->name);
+
+  if (gs->sampled_entry_guards) {
+    SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, e,
+                      entry_guard_free(e));
+    smartlist_free(gs->sampled_entry_guards);
+    gs->sampled_entry_guards = NULL;
+  }
+
+  smartlist_free(gs->confirmed_entry_guards);
+  smartlist_free(gs->primary_entry_guards);
+
+  tor_free(gs);
 }
 
 /** Release all storage held by the list of entry guards and related
@@ -2547,15 +3672,16 @@ any_bridge_supports_microdescriptors(void)
 void
 entry_guards_free_all(void)
 {
-  if (entry_guards) {
-    SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e,
-                      entry_guard_free(e));
-    smartlist_free(entry_guards);
-    entry_guards = NULL;
+  /* Null out the default */
+  curr_guard_context = NULL;
+  /* Free all the guard contexts */
+  if (guard_contexts != NULL) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+    } SMARTLIST_FOREACH_END(gs);
+    smartlist_free(guard_contexts);
+    guard_contexts = NULL;
   }
-  clear_bridge_list();
-  smartlist_free(bridge_list);
-  bridge_list = NULL;
   circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
 }
 
diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h
index 1021e67d43..e8c91da41b 100644
--- a/src/or/entrynodes.h
+++ b/src/or/entrynodes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,25 +12,27 @@
 #ifndef TOR_ENTRYNODES_H
 #define TOR_ENTRYNODES_H
 
-#if 1
-/* XXXX NM I would prefer that all of this stuff be private to
- * entrynodes.c. */
+#include "handles.h"
 
-/** An entry_guard_t represents our information about a chosen long-term
- * first hop, known as a "helper" node in the literature. We can't just
- * use a node_t, since we want to remember these even when we
- * don't have any directory info. */
-typedef struct entry_guard_t {
-  char nickname[MAX_NICKNAME_LEN+1];
-  char identity[DIGEST_LEN];
-  time_t chosen_on_date; /**< Approximately when was this guard added?
-                          * "0" if we don't know. */
-  char *chosen_by_version; /**< What tor version added this guard? NULL
-                            * if we don't know. */
-  unsigned int made_contact : 1; /**< 0 if we have never connected to this
-                                  * router, 1 if we have. */
-  unsigned int can_retry : 1; /**< Should we retry connecting to this entry,
-                               * in spite of having it marked as unreachable?*/
+/* Forward declare for guard_selection_t; entrynodes.c has the real struct */
+typedef struct guard_selection_s guard_selection_t;
+
+/* Forward declare for entry_guard_t; the real declaration is private. */
+typedef struct entry_guard_t entry_guard_t;
+
+/* Forward declaration for circuit_guard_state_t; the real declaration is
+   private. */
+typedef struct circuit_guard_state_t circuit_guard_state_t;
+
+/* Forward declaration for entry_guard_restriction_t; the real declaration is
+   private. */
+typedef struct entry_guard_restriction_t entry_guard_restriction_t;
+
+/* Information about a guard's pathbias status.
+ * These fields are used in circpathbias.c to try to detect entry
+ * nodes that are failing circuits at a suspicious frequency.
+ */
+typedef struct guard_pathbias_t {
   unsigned int path_bias_noticed : 1; /**< Did we alert the user about path
                                        * bias for this node already? */
   unsigned int path_bias_warned : 1; /**< Did we alert the user about path bias
@@ -43,15 +45,6 @@ typedef struct entry_guard_t {
                                        * use bias for this node already? */
   unsigned int path_bias_use_extreme : 1; /**< Did we alert the user about path
                                        * use bias for this node already? */
-  unsigned int is_dir_cache : 1; /**< Is this node a directory cache? */
-  time_t bad_since; /**< 0 if this guard is currently usable, or the time at
-                      * which it was observed to become (according to the
-                      * directory or the user configuration) unusable. */
-  time_t unreachable_since; /**< 0 if we can connect to this guard, or the
-                             * time at which we first noticed we couldn't
-                             * connect to it. */
-  time_t last_attempted; /**< 0 if we can connect to this guard, or the time
-                          * at which we last failed to connect to it. */
 
   double circ_attempts; /**< Number of circuits this guard has "attempted" */
   double circ_successes; /**< Number of successfully built circuits using
@@ -68,98 +61,560 @@ typedef struct entry_guard_t {
   double use_attempts; /**< Number of circuits we tried to use with streams */
   double use_successes; /**< Number of successfully used circuits using
                                * this guard as first hop. */
-} entry_guard_t;
+} guard_pathbias_t;
+
+#if defined(ENTRYNODES_PRIVATE)
+/**
+ * @name values for entry_guard_t.is_reachable.
+ *
+ * See entry_guard_t.is_reachable for more information.
+ */
+/**@{*/
+#define GUARD_REACHABLE_NO    0
+#define GUARD_REACHABLE_YES   1
+#define GUARD_REACHABLE_MAYBE 2
+/**@}*/
+
+/** An entry_guard_t represents our information about a chosen long-term
+ * first hop, known as a "helper" node in the literature. We can't just
+ * use a node_t, since we want to remember these even when we
+ * don't have any directory info. */
+struct entry_guard_t {
+  HANDLE_ENTRY(entry_guard, entry_guard_t);
+
+  char nickname[MAX_HEX_NICKNAME_LEN+1];
+  char identity[DIGEST_LEN];
+  ed25519_public_key_t ed_id;
+
+  /**
+   * @name new guard selection algorithm fields.
+   *
+   * Only the new (prop271) algorithm uses these.  For a more full
+   * description of the algorithm, see the module documentation for
+   * entrynodes.c
+   */
+  /**@{*/
+
+  /* == Persistent fields, present for all sampled guards. */
+  /** When was this guard added to the sample? */
+  time_t sampled_on_date;
+  /** Since what date has this guard been "unlisted"?  A guard counts as
+   * unlisted if we have a live consensus that does not include it, or
+   * if we have a live consensus that does not include it as a usable
+   * guard.  This field is zero when the guard is listed. */
+  time_t unlisted_since_date; // can be zero
+  /** What version of Tor added this guard to the sample? */
+  char *sampled_by_version;
+  /** Is this guard listed right now? If this is set, then
+   * unlisted_since_date should be set too. */
+  unsigned currently_listed : 1;
+
+  /* == Persistent fields, for confirmed guards only */
+  /** When was this guard confirmed? (That is, when did we first use it
+   * successfully and decide to keep it?) This field is zero if this is not a
+   * confirmed guard. */
+  time_t confirmed_on_date; /* 0 if not confirmed */
+  /**
+   * In what order was this guard confirmed? Guards with lower indices
+   * appear earlier on the confirmed list.  If the confirmed list is compacted,
+   * this field corresponds to the index of this guard on the confirmed list.
+   *
+   * This field is set to -1 if this guard is not confirmed.
+   */
+  int confirmed_idx; /* -1 if not confirmed; otherwise the order that this
+                      * item should occur in the CONFIRMED_GUARDS ordered
+                      * list */
+
+  /**
+   * Which selection does this guard belong to?
+   */
+  char *selection_name;
 
+  /** Bridges only: address of the bridge. */
+  tor_addr_port_t *bridge_addr;
+
+  /* ==== Non-persistent fields. */
+  /* == These are used by sampled guards */
+  /** When did we last decide to try using this guard for a circuit? 0 for
+   * "not since we started up." */
+  time_t last_tried_to_connect;
+  /** How reachable do we consider this guard to be? One of
+   * GUARD_REACHABLE_NO, GUARD_REACHABLE_YES, or GUARD_REACHABLE_MAYBE. */
+  unsigned is_reachable : 2;
+  /** Boolean: true iff this guard is pending. A pending guard is one
+   * that we have an in-progress circuit through, and which we do not plan
+   * to try again until it either succeeds or fails. Primary guards can
+   * never be pending. */
+  unsigned is_pending : 1;
+  /** If true, don't write this guard to disk. (Used for bridges with unknown
+   * identities) */
+  unsigned is_persistent : 1;
+  /** When did we get the earliest connection failure for this guard?
+   * We clear this field on a successful connect.  We do _not_ clear it
+   * when we mark the guard as "MAYBE" reachable.
+   */
+  time_t failing_since;
+
+  /* == Set inclusion flags. */
+  /** If true, this guard is in the filtered set.  The filtered set includes
+   * all sampled guards that our configuration allows us to use. */
+  unsigned is_filtered_guard : 1;
+  /** If true, this guard is in the usable filtered set. The usable filtered
+   * set includes all filtered guards that are not believed to be
+   * unreachable. (That is, those for which is_reachable is not
+   * GUARD_REACHABLE_NO) */
+  unsigned is_usable_filtered_guard : 1;
+  unsigned is_primary:1;
+
+  /** This string holds any fields that we are maintaining because
+   * we saw them in the state, even if we don't understand them. */
+  char *extra_state_fields;
+
+  /** Backpointer to the guard selection that this guard belongs to.
+   * The entry_guard_t must never outlive its guard_selection. */
+  guard_selection_t *in_selection;
+  /**@}*/
+
+  /** Path bias information for this guard. */
+  guard_pathbias_t pb;
+};
+
+/**
+ * Possible rules for a guard selection to follow
+ */
+typedef enum guard_selection_type_t {
+  /** Infer the type of this selection from its name. */
+  GS_TYPE_INFER=0,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * complete list of guards in the consensus. */
+  GS_TYPE_NORMAL=1,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * configured bridges, and allowing it to grow as large as all the configured
+   * bridges */
+  GS_TYPE_BRIDGE,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * set of filtered nodes. */
+  GS_TYPE_RESTRICTED,
+} guard_selection_type_t;
+
+/**
+ * All of the the context for guard selection on a particular client.
+ *
+ * We maintain multiple guard selection contexts for a client, depending
+ * aspects on its current configuration -- whether an extremely
+ * restrictive EntryNodes is used, whether UseBridges is enabled, and so
+ * on.)
+ *
+ * See the module documentation for entrynodes.c for more information
+ * about guard selection algorithms.
+ */
+struct guard_selection_s {
+  /**
+   * The name for this guard-selection object. (Must not contain spaces).
+   */
+  char *name;
+
+  /**
+   * What rules does this guard-selection object follow?
+   */
+  guard_selection_type_t type;
+
+  /**
+   * A value of 1 means that primary_entry_guards is up-to-date with respect to
+   * the consensus and status info that we currently have; 0 means we need to
+   * recalculate it before using primary_entry_guards or the is_primary flag on
+   * any guard.
+   */
+  int primary_guards_up_to_date;
+
+  /**
+   * A list of the sampled entry guards, as entry_guard_t structures.
+   * Not in any particular order.  When we 'sample' a guard, we are
+   * noting it as a possible guard to pick in the future. The use of
+   * sampling here prevents us from being forced by an attacker to try
+   * every guard on the network. This list is persistent.
+   */
+  smartlist_t *sampled_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * have successfully contacted and decided to use. Every member of
+   * this list is a member of sampled_entry_guards. Every member should
+   * have confirmed_on_date set, and have confirmed_idx greater than
+   * any earlier member of the list.
+   *
+   * This list is persistent. It is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *confirmed_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * are willing to use the most happily.  These guards may or may not
+   * yet be confirmed yet.  If we can use one of these guards, we are
+   * probably not on a network that is trying to restrict our guard
+   * choices.
+   *
+   * This list is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *primary_entry_guards;
+
+  /** When did we last successfully build a circuit or use a circuit? */
+  time_t last_time_on_internet;
+
+  /** What confirmed_idx value should the next-added member of
+   * confirmed_entry_guards receive? */
+  int next_confirmed_idx;
+
+};
+
+struct entry_guard_handle_t;
+
+/** Types of restrictions we impose when picking guard nodes */
+typedef enum guard_restriction_type_t {
+  /* Don't pick the same guard node as our exit node (or its family) */
+  RST_EXIT_NODE = 0,
+  /* Don't pick dirguards that have previously shown to be outdated */
+  RST_OUTDATED_MD_DIRSERVER = 1
+} guard_restriction_type_t;
+
+/**
+ * A restriction to remember which entry guards are off-limits for a given
+ * circuit.
+ *
+ * Note: This mechanism is NOT for recording which guards are never to be
+ * used: only which guards cannot be used on <em>one particular circuit</em>.
+ */
+struct entry_guard_restriction_t {
+  /* What type of restriction are we imposing? */
+  guard_restriction_type_t type;
+
+  /* In case of restriction type RST_EXIT_NODE, the guard's RSA identity
+   * digest must not equal this; and it must not be in the same family as any
+   * node with this digest. */
+  uint8_t exclude_id[DIGEST_LEN];
+};
+
+/**
+ * Per-circuit state to track whether we'll be able to use the circuit.
+ */
+struct circuit_guard_state_t {
+  /** Handle to the entry guard object for this circuit. */
+  struct entry_guard_handle_t *guard;
+  /** The time at which <b>state</b> last changed. */
+  time_t state_set_at;
+  /** One of GUARD_CIRC_STATE_* */
+  uint8_t state;
+
+  /**
+   * A set of restrictions that were placed on this guard when we selected it
+   * for this particular circuit.  We need to remember the restrictions here,
+   * since any guard that breaks these restrictions will not block this
+   * circuit from becoming COMPLETE.
+   */
+  entry_guard_restriction_t *restrictions;
+};
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+/* Common entry points for old and new guard code */
+int guards_update_all(void);
+const node_t *guards_choose_guard(cpath_build_state_t *state,
+                                  uint8_t purpose,
+                                  circuit_guard_state_t **guard_state_out);
+const node_t *guards_choose_dirguard(uint8_t dir_purpose,
+                                     circuit_guard_state_t **guard_state_out);
+
+#if 1
+/* XXXX NM I would prefer that all of this stuff be private to
+ * entrynodes.c. */
+entry_guard_t *entry_guard_get_by_id_digest_for_guard_selection(
+    guard_selection_t *gs, const char *digest);
 entry_guard_t *entry_guard_get_by_id_digest(const char *digest);
+
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest);
+
+void entry_guards_changed_for_guard_selection(guard_selection_t *gs);
 void entry_guards_changed(void);
-const smartlist_t *get_entry_guards(void);
+guard_selection_t * get_guard_selection_info(void);
+int num_live_entry_guards_for_guard_selection(
+    guard_selection_t *gs,
+    int for_directory);
 int num_live_entry_guards(int for_directory);
+#endif /* 1 */
 
-#endif
+const node_t *entry_guard_find_node(const entry_guard_t *guard);
+const char *entry_guard_get_rsa_id_digest(const entry_guard_t *guard);
+const char *entry_guard_describe(const entry_guard_t *guard);
+guard_pathbias_t *entry_guard_get_pathbias_state(entry_guard_t *guard);
 
-#ifdef ENTRYNODES_PRIVATE
-STATIC const node_t *add_an_entry_guard(const node_t *chosen,
-                                        int reset_status, int prepend,
-                                        int for_discovery, int for_directory);
-
-STATIC int populate_live_entry_guards(smartlist_t *live_entry_guards,
-                                      const smartlist_t *all_entry_guards,
-                                      const node_t *chosen_exit,
-                                      dirinfo_type_t dirinfo_type,
-                                      int for_directory,
-                                      int need_uptime, int need_capacity);
-STATIC int decide_num_guards(const or_options_t *options, int for_directory);
-
-STATIC void entry_guards_set_from_config(const or_options_t *options);
-
-/** Flags to be passed to entry_is_live() to indicate what kind of
- * entry nodes we are looking for. */
+/** Enum to specify how we're going to use a given guard, when we're picking
+ * one for immediate use. */
+typedef enum {
+  GUARD_USAGE_TRAFFIC = 0,
+  GUARD_USAGE_DIRGUARD = 1
+} guard_usage_t;
+
+#define circuit_guard_state_free(val) \
+  FREE_AND_NULL(circuit_guard_state_t, circuit_guard_state_free_, (val))
+
+void circuit_guard_state_free_(circuit_guard_state_t *state);
+int entry_guard_pick_for_circuit(guard_selection_t *gs,
+                                 guard_usage_t usage,
+                                 entry_guard_restriction_t *rst,
+                                 const node_t **chosen_node_out,
+                                 circuit_guard_state_t **guard_state_out);
+
+/* We just connected to an entry guard. What should we do with the circuit? */
 typedef enum {
-  ENTRY_NEED_UPTIME = 1<<0,
-  ENTRY_NEED_CAPACITY = 1<<1,
-  ENTRY_ASSUME_REACHABLE = 1<<2,
-  ENTRY_NEED_DESCRIPTOR = 1<<3,
-} entry_is_live_flags_t;
+  GUARD_USABLE_NEVER = -1, /* Never use the circuit */
+  GUARD_MAYBE_USABLE_LATER = 0, /* Keep it. We might use it in the future */
+  GUARD_USABLE_NOW = 1, /* Use it right now */
+} guard_usable_t;
+
+guard_usable_t entry_guard_succeeded(circuit_guard_state_t **guard_state_p);
+void entry_guard_failed(circuit_guard_state_t **guard_state_p);
+void entry_guard_cancel(circuit_guard_state_t **guard_state_p);
+void entry_guard_chan_failed(channel_t *chan);
+int entry_guards_update_all(guard_selection_t *gs);
+int entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                          const smartlist_t *all_circuits,
+                                          smartlist_t *newly_complete_out);
+int entry_guard_state_should_expire(circuit_guard_state_t *guard_state);
+void entry_guards_note_internet_connectivity(guard_selection_t *gs);
+
+int update_guard_selection_choice(const or_options_t *options);
+
+int entry_guard_could_succeed(const circuit_guard_state_t *guard_state);
+
+MOCK_DECL(int,num_bridges_usable,(int use_maybe_reachable));
+
+#ifdef ENTRYNODES_PRIVATE
+/**
+ * @name Default values for the parameters for the new (prop271) entry guard
+ * algorithm.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this percentage
+ * of the guards on the network.
+ */
+#define DFLT_MAX_SAMPLE_THRESHOLD_PERCENT 20
+/**
+ * We never let our sampled guard set grow larger than this number of
+ * guards.
+ */
+#define DFLT_MAX_SAMPLE_SIZE 60
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+#define DFLT_MIN_FILTERED_SAMPLE_SIZE 20
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+#define DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS 20
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+#define DFLT_GUARD_LIFETIME_DAYS 120
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+#define DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS 60
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+#define DFLT_N_PRIMARY_GUARDS 3
+/**
+ * Of the live guards on the primary guard list, how many do we consider when
+ * choosing a guard to use?
+ */
+#define DFLT_N_PRIMARY_GUARDS_TO_USE 1
+/**
+ * As DFLT_N_PRIMARY_GUARDS, but for choosing which directory guard to use.
+ */
+#define DFLT_N_PRIMARY_DIR_GUARDS_TO_USE 3
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+#define DFLT_INTERNET_LIKELY_DOWN_INTERVAL (10*60)
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+#define DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT 15
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+#define DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT (10*60)
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+#define DFLT_MEANINGFUL_RESTRICTION_PERCENT 20
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+#define DFLT_EXTREME_RESTRICTION_PERCENT 1
+/**@}*/
+
+STATIC double get_max_sample_threshold(void);
+STATIC int get_max_sample_size_absolute(void);
+STATIC int get_min_filtered_sample_size(void);
+STATIC int get_remove_unlisted_guards_after_days(void);
+STATIC int get_guard_lifetime(void);
+STATIC int get_guard_confirmed_min_lifetime(void);
+STATIC int get_n_primary_guards(void);
+STATIC int get_n_primary_guards_to_use(guard_usage_t usage);
+STATIC int get_internet_likely_down_interval(void);
+STATIC int get_nonprimary_guard_connect_timeout(void);
+STATIC int get_nonprimary_guard_idle_timeout(void);
+STATIC double get_meaningful_restriction_threshold(void);
+STATIC double get_extreme_restriction_threshold(void);
+
+HANDLE_DECL(entry_guard, entry_guard_t, STATIC)
+#define entry_guard_handle_free(h)    \
+  FREE_AND_NULL(entry_guard_handle_t, entry_guard_handle_free_, (h))
 
-STATIC const node_t *entry_is_live(const entry_guard_t *e,
-                                   entry_is_live_flags_t flags,
-                                   const char **msg);
+STATIC guard_selection_type_t guard_selection_infer_type(
+                           guard_selection_type_t type_in,
+                           const char *name);
+STATIC guard_selection_t *guard_selection_new(const char *name,
+                                              guard_selection_type_t type);
+STATIC guard_selection_t *get_guard_selection_by_name(
+          const char *name, guard_selection_type_t type, int create_if_absent);
+STATIC void guard_selection_free_(guard_selection_t *gs);
+#define guard_selection_free(gs) \
+  FREE_AND_NULL(guard_selection_t, guard_selection_free_, (gs))
+MOCK_DECL(STATIC int, entry_guard_is_listed,
+          (guard_selection_t *gs, const entry_guard_t *guard));
+STATIC const char *choose_guard_selection(const or_options_t *options,
+                                        const networkstatus_t *ns,
+                                        const guard_selection_t *old_selection,
+                                        guard_selection_type_t *type_out);
+STATIC entry_guard_t *get_sampled_guard_with_id(guard_selection_t *gs,
+                                                const uint8_t *rsa_id);
 
-STATIC int entry_is_time_to_retry(const entry_guard_t *e, time_t now);
+MOCK_DECL(STATIC time_t, randomize_time, (time_t now, time_t max_backdate));
 
-#endif
+MOCK_DECL(STATIC circuit_guard_state_t *,
+          circuit_guard_state_new,(entry_guard_t *guard, unsigned state,
+                                   entry_guard_restriction_t *rst));
 
+STATIC entry_guard_t *entry_guard_add_to_sample(guard_selection_t *gs,
+                                                const node_t *node);
+STATIC entry_guard_t *entry_guards_expand_sample(guard_selection_t *gs);
+STATIC char *entry_guard_encode_for_state(entry_guard_t *guard);
+STATIC entry_guard_t *entry_guard_parse_from_state(const char *s);
+#define entry_guard_free(e) \
+  FREE_AND_NULL(entry_guard_t, entry_guard_free_, (e))
+STATIC void entry_guard_free_(entry_guard_t *e);
+STATIC void entry_guards_update_filtered_sets(guard_selection_t *gs);
+STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
+/**
+ * @name Flags for sample_reachable_filtered_entry_guards()
+ */
+/**@{*/
+#define SAMPLE_EXCLUDE_CONFIRMED   (1u<<0)
+#define SAMPLE_EXCLUDE_PRIMARY     (1u<<1)
+#define SAMPLE_EXCLUDE_PENDING     (1u<<2)
+#define SAMPLE_NO_UPDATE_PRIMARY   (1u<<3)
+#define SAMPLE_EXCLUDE_NO_DESCRIPTOR (1u<<4)
+/**@}*/
+STATIC entry_guard_t *sample_reachable_filtered_entry_guards(
+                                    guard_selection_t *gs,
+                                    const entry_guard_restriction_t *rst,
+                                    unsigned flags);
+STATIC void entry_guard_consider_retry(entry_guard_t *guard);
+STATIC void make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard);
+STATIC void entry_guards_update_confirmed(guard_selection_t *gs);
+STATIC void entry_guards_update_primary(guard_selection_t *gs);
+STATIC int num_reachable_filtered_guards(const guard_selection_t *gs,
+                                         const entry_guard_restriction_t *rst);
+STATIC void sampled_guards_update_from_consensus(guard_selection_t *gs);
+/**
+ * @name Possible guard-states for a circuit.
+ */
+/**@{*/
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic as soon as it is successfully
+ * opened. */
+#define GUARD_CIRC_STATE_USABLE_ON_COMPLETION 1
+/** State for an non-open circuit that we shouldn't use for actual
+ * traffic, when it completes, unless other circuits to preferable
+ * guards fail. */
+#define GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD 2
+/** State for an open circuit that we shouldn't use for actual traffic
+ * unless other circuits to preferable guards fail. */
+#define GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD 3
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic. */
+#define GUARD_CIRC_STATE_COMPLETE 4
+/** State for a circuit that is unusable, and will not become usable. */
+#define GUARD_CIRC_STATE_DEAD 5
+/**@}*/
+STATIC void entry_guards_note_guard_failure(guard_selection_t *gs,
+                                            entry_guard_t *guard);
+STATIC entry_guard_t *select_entry_guard_for_circuit(guard_selection_t *gs,
+                                          guard_usage_t usage,
+                                          const entry_guard_restriction_t *rst,
+                                          unsigned *state_out);
+STATIC void mark_primary_guards_maybe_reachable(guard_selection_t *gs);
+STATIC unsigned entry_guards_note_guard_success(guard_selection_t *gs,
+                                                entry_guard_t *guard,
+                                                unsigned old_state);
+STATIC int entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b);
+STATIC char *getinfo_helper_format_single_entry_guard(const entry_guard_t *e);
+
+STATIC entry_guard_restriction_t *guard_create_exit_restriction(
+                                                      const uint8_t *exit_id);
+
+STATIC entry_guard_restriction_t *guard_create_dirserver_md_restriction(void);
+
+STATIC void entry_guard_restriction_free_(entry_guard_restriction_t *rst);
+#define entry_guard_restriction_free(rst)  \
+  FREE_AND_NULL(entry_guard_restriction_t, \
+                entry_guard_restriction_free_, (rst))
+
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+void remove_all_entry_guards_for_guard_selection(guard_selection_t *gs);
 void remove_all_entry_guards(void);
 
-void entry_guards_compute_status(const or_options_t *options, time_t now);
-int entry_guard_register_connect_status(const char *digest, int succeeded,
-                                        int mark_relay_status, time_t now);
-void entry_nodes_should_be_added(void);
+struct bridge_info_t;
+void entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                         const uint8_t *rsa_id_digest);
+
 int entry_list_is_constrained(const or_options_t *options);
-const node_t *choose_random_entry(cpath_build_state_t *state);
-const node_t *choose_random_dirguard(dirinfo_type_t t);
+int guards_retry_optimistic(const or_options_t *options);
+int entry_guards_parse_state_for_guard_selection(
+    guard_selection_t *gs, or_state_t *state, int set, char **msg);
 int entry_guards_parse_state(or_state_t *state, int set, char **msg);
 void entry_guards_update_state(or_state_t *state);
 int getinfo_helper_entry_guards(control_connection_t *conn,
                                 const char *question, char **answer,
                                 const char **errmsg);
 
-void mark_bridge_list(void);
-void sweep_bridge_list(void);
-
-int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
-                                const char *digest);
-int extend_info_is_a_configured_bridge(const extend_info_t *ei);
-int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
-int node_is_a_configured_bridge(const node_t *node);
-void learned_router_identity(const tor_addr_t *addr, uint16_t port,
-                             const char *digest);
-struct bridge_line_t;
-void bridge_add_from_config(struct bridge_line_t *bridge_line);
-void retry_bridge_descriptor_fetch_directly(const char *digest);
-void fetch_bridge_descriptors(const or_options_t *options, time_t now);
-void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
-int any_bridge_descriptors_known(void);
 int entries_known_but_down(const or_options_t *options);
 void entries_retry_all(const or_options_t *options);
 
-int any_bridge_supports_microdescriptors(void);
-const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
-                                                     uint16_t port);
-
-int any_bridges_dont_support_microdescriptors(void);
+char *entry_guards_get_err_str_if_dir_info_missing(int using_mds,
+                                           int num_present, int num_usable);
+char *guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs,
+                                              int using_mds,
+                                              int num_present, int num_usable);
 
 void entry_guards_free_all(void);
 
-const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
-                                                   uint16_t port);
-struct transport_t;
-int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
-                                      const struct transport_t **transport);
-
-MOCK_DECL(int, transport_is_needed, (const char *transport_name));
-int validate_pluggable_transports_config(void);
-
 double pathbias_get_close_success_count(entry_guard_t *guard);
 double pathbias_get_use_success_count(entry_guard_t *guard);
 
@@ -179,9 +634,5 @@ guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
                                   int orig_bandwidth,
                                   uint32_t guardfraction_percentage);
 
-MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
-MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
-          (const char *digest));
-
-#endif
+#endif /* !defined(TOR_ENTRYNODES_H) */
 
diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c
index 676adfd8bf..b842442caf 100644
--- a/src/or/ext_orport.c
+++ b/src/or/ext_orport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,18 +20,21 @@
 #include "or.h"
 #include "connection.h"
 #include "connection_or.h"
-#include "ext_orport.h"
 #include "control.h"
 #include "config.h"
-#include "util.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "ext_orport.h"
 #include "main.h"
+#include "proto_ext_or.h"
+#include "util.h"
 
 /** Allocate and return a structure capable of holding an Extended
  *  ORPort message of body length <b>len</b>. */
 ext_or_cmd_t *
 ext_or_cmd_new(uint16_t len)
 {
-  size_t size = STRUCT_OFFSET(ext_or_cmd_t, body) + len;
+  size_t size = offsetof(ext_or_cmd_t, body) + len;
   ext_or_cmd_t *cmd = tor_malloc(size);
   cmd->len = len;
   return cmd;
@@ -39,7 +42,7 @@ ext_or_cmd_new(uint16_t len)
 
 /** Deallocate the Extended ORPort message in <b>cmd</b>. */
 void
-ext_or_cmd_free(ext_or_cmd_t *cmd)
+ext_or_cmd_free_(ext_or_cmd_t *cmd)
 {
   tor_free(cmd);
 }
@@ -69,10 +72,10 @@ connection_write_ext_or_command(connection_t *conn,
     return -1;
   set_uint16(header, htons(command));
   set_uint16(header+2, htons(bodylen));
-  connection_write_to_buf(header, 4, conn);
+  connection_buf_add(header, 4, conn);
   if (bodylen) {
     tor_assert(body);
-    connection_write_to_buf(body, bodylen, conn);
+    connection_buf_add(body, bodylen, conn);
   }
   return 0;
 }
@@ -170,7 +173,7 @@ connection_ext_or_auth_neg_auth_type(connection_t *conn)
   if (connection_get_inbuf_len(conn) < 1)
     return 0;
 
-  if (connection_fetch_from_buf(authtype, 1, conn) < 0)
+  if (connection_buf_get_bytes(authtype, 1, conn) < 0)
     return -1;
 
   log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]);
@@ -310,7 +313,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn)
   if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN)
     return 0;
 
-  if (connection_fetch_from_buf(client_nonce,
+  if (connection_buf_get_bytes(client_nonce,
                                 EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0)
     return -1;
 
@@ -325,7 +328,7 @@ connection_ext_or_auth_handle_client_nonce(connection_t *conn)
                             &reply, &reply_len) < 0)
     return -1;
 
-  connection_write_to_buf(reply, reply_len, conn);
+  connection_buf_add(reply, reply_len, conn);
 
   memwipe(reply, 0, reply_len);
   tor_free(reply);
@@ -347,9 +350,9 @@ static void
 connection_ext_or_auth_send_result(connection_t *conn, int success)
 {
   if (success)
-    connection_write_to_buf("\x01", 1, conn);
+    connection_buf_add("\x01", 1, conn);
   else
-    connection_write_to_buf("\x00", 1, conn);
+    connection_buf_add("\x00", 1, conn);
 }
 
 /** Receive the client's hash from <b>conn</b>, validate that it's
@@ -367,7 +370,7 @@ connection_ext_or_auth_handle_client_hash(connection_t *conn)
   if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN)
     return 0;
 
-  if (connection_fetch_from_buf(provided_client_hash,
+  if (connection_buf_get_bytes(provided_client_hash,
                                 EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0)
     return -1;
 
@@ -459,6 +462,11 @@ connection_ext_or_handle_cmd_useraddr(connection_t *conn,
   tor_free(addr_str);
   if (res<0)
     return -1;
+  if (port == 0) {
+    log_warn(LD_GENERAL, "Server transport proxy gave us an empty port "
+             "in ExtORPort UserAddr command.");
+    // return -1; // enable this if nothing breaks after a while.
+  }
 
   res = tor_addr_parse(&addr, address_part);
   tor_free(address_part);
@@ -637,7 +645,7 @@ connection_ext_or_start_auth(or_connection_t *or_conn)
   log_debug(LD_GENERAL,
            "ExtORPort authentication: Sending supported authentication types");
 
-  connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn);
+  connection_buf_add((const char *)authtypes, sizeof(authtypes), conn);
   conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE;
 
   return 0;
diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h
index 33d954e8d0..09acbc407e 100644
--- a/src/or/ext_orport.h
+++ b/src/or/ext_orport.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef EXT_ORPORT_H
@@ -10,7 +10,11 @@
 int connection_ext_or_start_auth(or_connection_t *or_conn);
 
 ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
-void ext_or_cmd_free(ext_or_cmd_t *cmd);
+
+#define ext_or_cmd_free(cmd)                            \
+  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+
+void ext_or_cmd_free_(ext_or_cmd_t *cmd);
 void connection_or_set_ext_or_identifier(or_connection_t *conn);
 void connection_or_remove_from_ext_or_id_map(or_connection_t *conn);
 void connection_or_clear_ext_or_id_map(void);
@@ -36,7 +40,7 @@ STATIC int handle_client_auth_nonce(const char *client_nonce,
 extern uint8_t *ext_or_auth_cookie;
 extern int ext_or_auth_cookie_is_set;
 #endif
-#endif
+#endif /* defined(EXT_ORPORT_PRIVATE) */
 
-#endif
+#endif /* !defined(EXT_ORPORT_H) */
 
diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c
index eeeb0f1de3..c938e76678 100644
--- a/src/or/fp_pair.c
+++ b/src/or/fp_pair.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -196,7 +196,7 @@ fp_pair_map_remove(fp_pair_map_t *map, const fp_pair_t *key)
  */
 
 void
-fp_pair_map_free(fp_pair_map_t *map, void (*free_val)(void*))
+fp_pair_map_free_(fp_pair_map_t *map, void (*free_val)(void*))
 {
   fp_pair_map_entry_t **ent, **next, *this;
 
diff --git a/src/or/fp_pair.h b/src/or/fp_pair.h
index b1466581d2..4498a16101 100644
--- a/src/or/fp_pair.h
+++ b/src/or/fp_pair.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,7 +26,12 @@ void * fp_pair_map_get(const fp_pair_map_t *map, const fp_pair_t *key);
 void * fp_pair_map_get_by_digests(const fp_pair_map_t *map,
                                   const char *first, const char *second);
 void * fp_pair_map_remove(fp_pair_map_t *map, const fp_pair_t *key);
-void fp_pair_map_free(fp_pair_map_t *map, void (*free_val)(void*));
+void fp_pair_map_free_(fp_pair_map_t *map, void (*free_val)(void*));
+#define fp_pair_map_free(map, free_val) do {            \
+    fp_pair_map_free_((map), (free_val));               \
+    (map) = NULL;                                       \
+  } while (0)
+
 int fp_pair_map_isempty(const fp_pair_map_t *map);
 int fp_pair_map_size(const fp_pair_map_t *map);
 fp_pair_map_iter_t * fp_pair_map_iter_init(fp_pair_map_t *map);
@@ -41,5 +46,5 @@ void fp_pair_map_assert_ok(const fp_pair_map_t *map);
 
 #undef DECLARE_MAP_FNS
 
-#endif
+#endif /* !defined(_TOR_FP_PAIR_H) */
 
diff --git a/src/or/geoip.c b/src/or/geoip.c
index a39366ed13..d59043a7f6 100644
--- a/src/or/geoip.c
+++ b/src/or/geoip.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -30,6 +30,7 @@
 #define GEOIP_PRIVATE
 #include "or.h"
 #include "ht.h"
+#include "buffers.h"
 #include "config.h"
 #include "control.h"
 #include "dnsserv.h"
@@ -149,7 +150,7 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high,
     idx = ((uintptr_t)idxplus1_)-1;
   }
   {
-    geoip_country_t *c = smartlist_get(geoip_countries, idx);
+    geoip_country_t *c = smartlist_get(geoip_countries, (int)idx);
     tor_assert(!strcasecmp(c->countrycode, country));
   }
 
@@ -541,6 +542,9 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
 HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
              clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
 
+#define clientmap_entry_free(ent) \
+  FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
+
 /** Return the size of a client map entry. */
 static inline size_t
 clientmap_entry_size(const clientmap_entry_t *ent)
@@ -552,7 +556,7 @@ clientmap_entry_size(const clientmap_entry_t *ent)
 
 /** Free all storage held by <b>ent</b>. */
 static void
-clientmap_entry_free(clientmap_entry_t *ent)
+clientmap_entry_free_(clientmap_entry_t *ent)
 {
   if (!ent)
     return;
@@ -624,8 +628,7 @@ geoip_note_client_seen(geoip_client_action_t action,
     /* Only remember statistics if the DoS mitigation subsystem is enabled. If
      * not, only if as entry guard or as bridge. */
     if (!dos_enabled()) {
-      if (!options->EntryStatistics &&
-          (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) {
+      if (!options->EntryStatistics && !should_record_bridge_info(options)) {
         return;
       }
     }
@@ -1017,7 +1020,7 @@ geoip_get_transport_history(void)
 
     /* If it's the first time we see this transport, note it. */
     if (val == 1)
-      smartlist_add(transports_used, tor_strdup(transport_name));
+      smartlist_add_strdup(transports_used, transport_name);
 
     log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
               "I've now seen %d clients.",
@@ -1067,9 +1070,9 @@ static char *
 geoip_get_dirreq_history(dirreq_type_t type)
 {
   char *result = NULL;
+  buf_t *buf = NULL;
   smartlist_t *dirreq_completed = NULL;
   uint32_t complete = 0, timeouts = 0, running = 0;
-  int bufsize = 1024, written;
   dirreq_map_entry_t **ptr, **next;
   struct timeval now;
 
@@ -1102,13 +1105,9 @@ geoip_get_dirreq_history(dirreq_type_t type)
                                               DIR_REQ_GRANULARITY);
   running = round_uint32_to_next_multiple_of(running,
                                              DIR_REQ_GRANULARITY);
-  result = tor_malloc_zero(bufsize);
-  written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u,"
-                         "running=%u", complete, timeouts, running);
-  if (written < 0) {
-    tor_free(result);
-    goto done;
-  }
+  buf = buf_new_with_capacity(1024);
+  buf_add_printf(buf, "complete=%u,timeout=%u,"
+                 "running=%u", complete, timeouts, running);
 
 #define MIN_DIR_REQ_RESPONSES 16
   if (complete >= MIN_DIR_REQ_RESPONSES) {
@@ -1129,7 +1128,7 @@ geoip_get_dirreq_history(dirreq_type_t type)
       dltimes[ent_sl_idx] = bytes_per_second;
     } SMARTLIST_FOREACH_END(ent);
     median_uint32(dltimes, complete); /* sorts as a side effect. */
-    written = tor_snprintf(result + written, bufsize - written,
+    buf_add_printf(buf,
                            ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
                            "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
                            dltimes[0],
@@ -1145,14 +1144,15 @@ geoip_get_dirreq_history(dirreq_type_t type)
                            dltimes[8*complete/10-1],
                            dltimes[9*complete/10-1],
                            dltimes[complete-1]);
-    if (written<0)
-      tor_free(result);
     tor_free(dltimes);
   }
- done:
+
+  result = buf_extract(buf, NULL);
+
   SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
                     tor_free(ent));
   smartlist_free(dirreq_completed);
+  buf_free(buf);
   return result;
 }
 
@@ -1802,6 +1802,15 @@ getinfo_helper_geoip(control_connection_t *control_conn,
     sa_family_t family;
     tor_addr_t addr;
     question += strlen("ip-to-country/");
+
+    if (!strcmp(question, "ipv4-available") ||
+        !strcmp(question, "ipv6-available")) {
+      family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
+      const int available = geoip_is_loaded(family);
+      tor_asprintf(answer, "%d", !! available);
+      return 0;
+    }
+
     family = tor_addr_parse(&addr, question);
     if (family != AF_INET && family != AF_INET6) {
       *errmsg = "Invalid address family";
@@ -1871,5 +1880,8 @@ geoip_free_all(void)
 
   clear_geoip_db();
   tor_free(bridge_stats_extrainfo);
+
+  memset(geoip_digest, 0, sizeof(geoip_digest));
+  memset(geoip6_digest, 0, sizeof(geoip6_digest));
 }
 
diff --git a/src/or/geoip.h b/src/or/geoip.h
index c8ea9f85ea..753bdbf82a 100644
--- a/src/or/geoip.h
+++ b/src/or/geoip.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,7 @@ STATIC int geoip_parse_entry(const char *line, sa_family_t family);
 STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr);
 STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr);
 STATIC void clear_geoip_db(void);
-#endif
+#endif /* defined(GEOIP_PRIVATE) */
 
 /** Entry in a map from IP address to the last time we've seen an incoming
  * connection from that IP address. Used by bridges only to track which
@@ -96,5 +96,5 @@ const char *geoip_get_bridge_stats_extrainfo(time_t);
 char *geoip_get_bridge_stats_controller(time_t);
 char *format_client_stats_heartbeat(time_t now);
 
-#endif
+#endif /* !defined(TOR_GEOIP_H) */
 
diff --git a/src/or/git_revision.c b/src/or/git_revision.c
new file mode 100644
index 0000000000..8f326b8751
--- /dev/null
+++ b/src/or/git_revision.c
@@ -0,0 +1,17 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "git_revision.h"
+
+/** String describing which Tor Git repository version the source was
+ * built from.  This string is generated by a bit of shell kludging in
+ * src/or/include.am, and is usually right.
+ */
+const char tor_git_revision[] =
+#ifndef _MSC_VER
+#include "micro-revision.i"
+#endif
+  "";
+
diff --git a/src/or/git_revision.h b/src/or/git_revision.h
new file mode 100644
index 0000000000..5613cb4335
--- /dev/null
+++ b/src/or/git_revision.h
@@ -0,0 +1,12 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_GIT_REVISION_H
+#define TOR_GIT_REVISION_H
+
+extern const char tor_git_revision[];
+
+#endif /* !defined(TOR_GIT_REVISION_H) */
+
diff --git a/src/or/hibernate.c b/src/or/hibernate.c
index e3c80b5f14..d7d259470f 100644
--- a/src/or/hibernate.c
+++ b/src/or/hibernate.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,9 @@ hibernating, phase 2:
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
+#include "connection_or.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "hibernate.h"
 #include "main.h"
 #include "router.h"
@@ -50,6 +52,10 @@ static time_t hibernate_end_time = 0;
  * we aren't shutting down. */
 static time_t shutdown_time = 0;
 
+/** A timed event that we'll use when it's time to wake up from
+ * hibernation. */
+static mainloop_event_t *wakeup_event = NULL;
+
 /** Possible accounting periods. */
 typedef enum {
   UNIT_MONTH=1, UNIT_WEEK=2, UNIT_DAY=3,
@@ -129,6 +135,8 @@ static time_t start_of_accounting_period_after(time_t now);
 static time_t start_of_accounting_period_containing(time_t now);
 static void accounting_set_wakeup_time(void);
 static void on_hibernate_state_change(hibernate_state_t prev_state);
+static void hibernate_schedule_wakeup_event(time_t now, time_t end_time);
+static void wakeup_event_callback(mainloop_event_t *ev, void *data);
 
 /**
  * Return the human-readable name for the hibernation state <b>state</b>
@@ -296,7 +304,7 @@ accounting_get_end_time,(void))
   return interval_end_time;
 }
 
-/** Called from main.c to tell us that <b>seconds</b> seconds have
+/** Called from connection.c to tell us that <b>seconds</b> seconds have
  * passed, <b>n_read</b> bytes have been read, and <b>n_written</b>
  * bytes have been written. */
 void
@@ -424,8 +432,8 @@ configure_accounting(time_t now)
     if (-0.50 <= delta && delta <= 0.50) {
       /* The start of the period is now a little later or earlier than we
        * remembered.  That's fine; we might lose some bytes we could otherwise
-       * have written, but better to err on the side of obeying people's
-       * accounting settings. */
+       * have written, but better to err on the side of obeying accounting
+       * settings. */
       log_info(LD_ACCT, "Accounting interval moved by %.02f%%; "
                "that's fine.", delta*100);
       interval_end_time = start_of_accounting_period_after(now);
@@ -587,7 +595,10 @@ accounting_set_wakeup_time(void)
     char buf[ISO_TIME_LEN+1];
     format_iso_time(buf, interval_start_time);
 
-    crypto_pk_get_digest(get_server_identity_key(), digest);
+    if (crypto_pk_get_digest(get_server_identity_key(), digest) < 0) {
+      log_err(LD_BUG, "Error getting our key's digest.");
+      tor_assert(0);
+    }
 
     d_env = crypto_digest_new();
     crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN);
@@ -815,8 +826,8 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
     log_notice(LD_GENERAL,"SIGINT received %s; exiting now.",
                hibernate_state == HIBERNATE_STATE_EXITING ?
                "a second time" : "while hibernating");
-    tor_cleanup();
-    exit(0);
+    tor_shutdown_event_loop_and_exit(0);
+    return;
   }
 
   if (new_state == HIBERNATE_STATE_LOWBANDWIDTH &&
@@ -862,7 +873,7 @@ hibernate_end(hibernate_state_t new_state)
 
   hibernate_state = new_state;
   hibernate_end_time = 0; /* no longer hibernating */
-  stats_n_seconds_working = 0; /* reset published uptime */
+  reset_uptime(); /* reset published uptime */
 }
 
 /** A wrapper around hibernate_begin, for when we get SIGINT. */
@@ -872,13 +883,26 @@ hibernate_begin_shutdown(void)
   hibernate_begin(HIBERNATE_STATE_EXITING, time(NULL));
 }
 
-/** Return true iff we are currently hibernating. */
+/**
+ * Return true iff we are currently hibernating -- that is, if we are in
+ * any non-live state.
+ */
 MOCK_IMPL(int,
 we_are_hibernating,(void))
 {
   return hibernate_state != HIBERNATE_STATE_LIVE;
 }
 
+/**
+ * Return true iff we are currently _fully_ hibernating -- that is, if we are
+ * in a state where we expect to handle no network activity at all.
+ */
+MOCK_IMPL(int,
+we_are_fully_hibernating,(void))
+{
+  return hibernate_state == HIBERNATE_STATE_DORMANT;
+}
+
 /** If we aren't currently dormant, close all connections and become
  * dormant. */
 static void
@@ -896,27 +920,30 @@ hibernate_go_dormant(time_t now)
   log_notice(LD_ACCT,"Going dormant. Blowing away remaining connections.");
 
   /* Close all OR/AP/exit conns. Leave dir conns because we still want
-   * to be able to upload server descriptors so people know we're still
+   * to be able to upload server descriptors so clients know we're still
    * running, and download directories so we can detect if we're obsolete.
    * Leave control conns because we still want to be controllable.
    */
   while ((conn = connection_get_by_type(CONN_TYPE_OR)) ||
          (conn = connection_get_by_type(CONN_TYPE_AP)) ||
          (conn = connection_get_by_type(CONN_TYPE_EXIT))) {
-    if (CONN_IS_EDGE(conn))
+    if (CONN_IS_EDGE(conn)) {
       connection_edge_end(TO_EDGE_CONN(conn), END_STREAM_REASON_HIBERNATING);
+    }
     log_info(LD_NET,"Closing conn type %d", conn->type);
-    if (conn->type == CONN_TYPE_AP) /* send socks failure if needed */
+    if (conn->type == CONN_TYPE_AP) {
+      /* send socks failure if needed */
       connection_mark_unattached_ap(TO_ENTRY_CONN(conn),
                                     END_STREAM_REASON_HIBERNATING);
-    else if (conn->type == CONN_TYPE_OR) {
+    } else if (conn->type == CONN_TYPE_OR) {
       if (TO_OR_CONN(conn)->chan) {
-        channel_mark_for_close(TLS_CHAN_TO_BASE(TO_OR_CONN(conn)->chan));
+        connection_or_close_normally(TO_OR_CONN(conn), 0);
       } else {
          connection_mark_for_close(conn);
       }
-    } else
+    } else {
       connection_mark_for_close(conn);
+    }
   }
 
   if (now < interval_wakeup_time)
@@ -928,6 +955,63 @@ hibernate_go_dormant(time_t now)
 
   or_state_mark_dirty(get_or_state(),
                       get_options()->AvoidDiskWrites ? now+600 : 0);
+
+  hibernate_schedule_wakeup_event(now, hibernate_end_time);
+}
+
+/**
+ * Schedule a mainloop event at <b>end_time</b> to wake up from a dormant
+ * state.  We can't rely on this happening from second_elapsed_callback,
+ * since second_elapsed_callback will be shut down when we're dormant.
+ *
+ * (Note that We might immediately go back to sleep after we set the next
+ * wakeup time.)
+ */
+static void
+hibernate_schedule_wakeup_event(time_t now, time_t end_time)
+{
+  struct timeval delay = { 0, 0 };
+
+  if (now >= end_time) {
+    // In these cases we always wait at least a second, to avoid running
+    // the callback in a tight loop.
+    delay.tv_sec = 1;
+  } else {
+    delay.tv_sec = (end_time - now);
+  }
+
+  if (!wakeup_event) {
+    wakeup_event = mainloop_event_postloop_new(wakeup_event_callback, NULL);
+  }
+
+  mainloop_event_schedule(wakeup_event, &delay);
+}
+
+/**
+ * Called at the end of the interval, or at the wakeup time of the current
+ * interval, to exit the dormant state.
+ **/
+static void
+wakeup_event_callback(mainloop_event_t *ev, void *data)
+{
+  (void) ev;
+  (void) data;
+
+  const time_t now = time(NULL);
+  accounting_run_housekeeping(now);
+  consider_hibernation(now);
+  if (hibernate_state != HIBERNATE_STATE_DORMANT) {
+    /* We woke up, so everything's great here */
+    return;
+  }
+
+  /* We're still dormant. */
+  if (now < interval_wakeup_time)
+    hibernate_end_time = interval_wakeup_time;
+  else
+    hibernate_end_time = interval_end_time;
+
+  hibernate_schedule_wakeup_event(now, hibernate_end_time);
 }
 
 /** Called when hibernate_end_time has arrived. */
@@ -977,8 +1061,7 @@ consider_hibernation(time_t now)
     tor_assert(shutdown_time);
     if (shutdown_time <= now) {
       log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
-      tor_cleanup();
-      exit(0);
+      tor_shutdown_event_loop_and_exit(0);
     }
     return; /* if exiting soon, don't worry about bandwidth limits */
   }
@@ -1105,10 +1188,30 @@ getinfo_helper_accounting(control_connection_t *conn,
 static void
 on_hibernate_state_change(hibernate_state_t prev_state)
 {
-  (void)prev_state; /* Should we do something with this? */
   control_event_server_status(LOG_NOTICE,
                               "HIBERNATION_STATUS STATUS=%s",
                               hibernate_state_to_string(hibernate_state));
+
+  /* We are changing hibernation state, this can affect the main loop event
+   * list. Rescan it to update the events state. We do this whatever the new
+   * hibernation state because they can each possibly affect an event. The
+   * initial state means we are booting up so we shouldn't scan here because
+   * at this point the events in the list haven't been initialized. */
+  if (prev_state != HIBERNATE_STATE_INITIAL) {
+    rescan_periodic_events(get_options());
+  }
+
+  reschedule_per_second_timer();
+}
+
+/** Free all resources held by the accounting module */
+void
+accounting_free_all(void)
+{
+  mainloop_event_free(wakeup_event);
+  hibernate_state = HIBERNATE_STATE_INITIAL;
+  hibernate_end_time = 0;
+  shutdown_time = 0;
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -1121,5 +1224,5 @@ hibernate_set_state_for_testing_(hibernate_state_t newstate)
 {
   hibernate_state = newstate;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/or/hibernate.h b/src/or/hibernate.h
index fa9da6de39..453969d052 100644
--- a/src/or/hibernate.h
+++ b/src/or/hibernate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,11 +25,13 @@ void accounting_add_bytes(size_t n_read, size_t n_written, int seconds);
 int accounting_record_bandwidth_usage(time_t now, or_state_t *state);
 void hibernate_begin_shutdown(void);
 MOCK_DECL(int, we_are_hibernating, (void));
+MOCK_DECL(int, we_are_fully_hibernating,(void));
 void consider_hibernation(time_t now);
 int getinfo_helper_accounting(control_connection_t *conn,
                               const char *question, char **answer,
                               const char **errmsg);
 uint64_t get_accounting_max_total(void);
+void accounting_free_all(void);
 
 #ifdef HIBERNATE_PRIVATE
 /** Possible values of hibernate_state */
@@ -53,7 +55,7 @@ typedef enum {
 #ifdef TOR_UNIT_TESTS
 void hibernate_set_state_for_testing_(hibernate_state_t newstate);
 #endif
-#endif
+#endif /* defined(HIBERNATE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_HIBERNATE_H) */
 
diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c
new file mode 100644
index 0000000000..ecc845d17f
--- /dev/null
+++ b/src/or/hs_cache.c
@@ -0,0 +1,977 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.c
+ * \brief Handle hidden service descriptor caches.
+ **/
+
+/* For unit tests.*/
+#define HS_CACHE_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "crypto_util.h"
+#include "hs_ident.h"
+#include "hs_common.h"
+#include "hs_client.h"
+#include "hs_descriptor.h"
+#include "networkstatus.h"
+#include "rendcache.h"
+
+#include "hs_cache.h"
+
+static int cached_client_descriptor_has_expired(time_t now,
+           const hs_cache_client_descriptor_t *cached_desc);
+
+/********************** Directory HS cache ******************/
+
+/* Directory descriptor cache. Map indexed by blinded key. */
+static digest256map_t *hs_cache_v3_dir;
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_dir, desc->key);
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_dir, desc->key, desc);
+}
+
+/* Query our cache and return the entry or NULL if not found. */
+static hs_cache_dir_descriptor_t *
+lookup_v3_desc_as_dir(const uint8_t *key)
+{
+  tor_assert(key);
+  return digest256map_get(hs_cache_v3_dir, key);
+}
+
+#define cache_dir_desc_free(val) \
+  FREE_AND_NULL(hs_cache_dir_descriptor_t, cache_dir_desc_free_, (val))
+
+/* Free a directory descriptor object. */
+static void
+cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_desc_plaintext_data_free(desc->plaintext_data);
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/* Helper function: Use by the free all function using the digest256map
+ * interface to cache entries. */
+static void
+cache_dir_desc_free_void(void *ptr)
+{
+  cache_dir_desc_free_(ptr);
+}
+
+/* Create a new directory cache descriptor object from a encoded descriptor.
+ * On success, return the heap-allocated cache object, otherwise return NULL if
+ * we can't decode the descriptor. */
+static hs_cache_dir_descriptor_t *
+cache_dir_desc_new(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc;
+
+  tor_assert(desc);
+
+  dir_desc = tor_malloc_zero(sizeof(hs_cache_dir_descriptor_t));
+  dir_desc->plaintext_data =
+    tor_malloc_zero(sizeof(hs_desc_plaintext_data_t));
+  dir_desc->encoded_desc = tor_strdup(desc);
+
+  if (hs_desc_decode_plaintext(desc, dir_desc->plaintext_data) < 0) {
+    log_debug(LD_DIR, "Unable to decode descriptor. Rejecting.");
+    goto err;
+  }
+
+  /* The blinded pubkey is the indexed key. */
+  dir_desc->key = dir_desc->plaintext_data->blinded_pubkey.pubkey;
+  dir_desc->created_ts = time(NULL);
+  return dir_desc;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return NULL;
+}
+
+/* Return the size of a cache entry in bytes. */
+static size_t
+cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
+{
+  return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data)
+          + strlen(entry->encoded_desc));
+}
+
+/* Try to store a valid version 3 descriptor in the directory cache. Return 0
+ * on success else a negative value is returned indicating that we have a
+ * newer version in our cache. On error, caller is responsible to free the
+ * given descriptor desc. */
+static int
+cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  hs_cache_dir_descriptor_t *cache_entry;
+
+  tor_assert(desc);
+
+  /* Verify if we have an entry in the cache for that key and if yes, check
+   * if we should replace it? */
+  cache_entry = lookup_v3_desc_as_dir(desc->key);
+  if (cache_entry != NULL) {
+    /* Only replace descriptor if revision-counter is greater than the one
+     * in our cache */
+    if (cache_entry->plaintext_data->revision_counter >=
+        desc->plaintext_data->revision_counter) {
+      log_info(LD_REND, "Descriptor revision counter in our cache is "
+               "greater or equal than the one we received (%d/%d). "
+               "Rejecting!",
+               (int)cache_entry->plaintext_data->revision_counter,
+               (int)desc->plaintext_data->revision_counter);
+      goto err;
+    }
+    /* We now know that the descriptor we just received is a new one so
+     * remove the entry we currently have from our cache so we can then
+     * store the new one. */
+    remove_v3_desc_as_dir(cache_entry);
+    rend_cache_decrement_allocation(cache_get_dir_entry_size(cache_entry));
+    cache_dir_desc_free(cache_entry);
+  }
+  /* Store the descriptor we just got. We are sure here that either we
+   * don't have the entry or we have a newer descriptor and the old one
+   * has been removed from the cache. */
+  store_v3_desc_as_dir(desc);
+
+  /* Update our total cache size with this entry for the OOM. This uses the
+   * old HS protocol cache subsystem for which we are tied with. */
+  rend_cache_increment_allocation(cache_get_dir_entry_size(desc));
+
+  /* XXX: Update HS statistics. We should have specific stats for v3. */
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the query which is the base64 encoded blinded key of a version 3
+ * descriptor, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being the
+ * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
+ * On error, a negative value is returned and desc_out is untouched. */
+static int
+cache_lookup_v3_as_dir(const char *query, const char **desc_out)
+{
+  int found = 0;
+  ed25519_public_key_t blinded_key;
+  const hs_cache_dir_descriptor_t *entry;
+
+  tor_assert(query);
+
+  /* Decode blinded key using the given query value. */
+  if (ed25519_public_from_base64(&blinded_key, query) < 0) {
+    log_info(LD_REND, "Unable to decode the v3 HSDir query %s.",
+             safe_str_client(query));
+    goto err;
+  }
+
+  entry = lookup_v3_desc_as_dir(blinded_key.pubkey);
+  if (entry != NULL) {
+    found = 1;
+    if (desc_out) {
+      *desc_out = entry->encoded_desc;
+    }
+  }
+
+  return found;
+
+ err:
+  return -1;
+}
+
+/* Clean the v3 cache by removing any entry that has expired using the
+ * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
+ * process will use the lifetime found in the plaintext data section. Return
+ * the number of bytes cleaned. */
+STATIC size_t
+cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
+{
+  size_t bytes_removed = 0;
+
+  /* Code flow error if this ever happens. */
+  tor_assert(global_cutoff >= 0);
+
+  if (!hs_cache_v3_dir) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_dir, key,
+                              hs_cache_dir_descriptor_t *, entry) {
+    size_t entry_size;
+    time_t cutoff = global_cutoff;
+    if (!cutoff) {
+      /* Cutoff is the lifetime of the entry found in the descriptor. */
+      cutoff = now - entry->plaintext_data->lifetime_sec;
+    }
+
+    /* If the entry has been created _after_ the cutoff, not expired so
+     * continue to the next entry in our v3 cache. */
+    if (entry->created_ts > cutoff) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_dir_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_dir_desc_free(entry);
+    /* Update our cache entry allocation size for the OOM. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      digest256_to_base64(key_b64, (const char *) key);
+      log_info(LD_REND, "Removing v3 descriptor '%s' from HSDir cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/* Given an encoded descriptor, store it in the directory cache depending on
+ * which version it is. Return a negative value on error. On success, 0 is
+ * returned. */
+int
+hs_cache_store_as_dir(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc = NULL;
+
+  tor_assert(desc);
+
+  /* Create a new cache object. This can fail if the descriptor plaintext data
+   * is unparseable which in this case a log message will be triggered. */
+  dir_desc = cache_dir_desc_new(desc);
+  if (dir_desc == NULL) {
+    goto err;
+  }
+
+  /* Call the right function against the descriptor version. At this point,
+   * we are sure that the descriptor's version is supported else the
+   * decoding would have failed. */
+  switch (dir_desc->plaintext_data->version) {
+  case HS_VERSION_THREE:
+  default:
+    if (cache_store_v3_as_dir(dir_desc) < 0) {
+      goto err;
+    }
+    break;
+  }
+  return 0;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return -1;
+}
+
+/* Using the query, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being
+ * the encoded descriptor. If not found, 0 is returned and desc_out is
+ * untouched. On error, a negative value is returned and desc_out is
+ * untouched. */
+int
+hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                       const char **desc_out)
+{
+  int found;
+
+  tor_assert(query);
+  /* This should never be called with an unsupported version. */
+  tor_assert(hs_desc_is_supported_version(version));
+
+  switch (version) {
+  case HS_VERSION_THREE:
+  default:
+    found = cache_lookup_v3_as_dir(query, desc_out);
+    break;
+  }
+
+  return found;
+}
+
+/* Clean all directory caches using the current time now. */
+void
+hs_cache_clean_as_dir(time_t now)
+{
+  time_t cutoff;
+
+  /* Start with v2 cache cleaning. */
+  cutoff = now - rend_cache_max_entry_lifetime();
+  rend_cache_clean_v2_descs_as_dir(cutoff);
+
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_dir(now, 0);
+}
+
+/********************** Client-side HS cache ******************/
+
+/* Client-side HS descriptor cache. Map indexed by service identity key. */
+static digest256map_t *hs_cache_v3_client;
+
+/* Client-side introduction point state cache. Map indexed by service public
+ * identity key (onion address). It contains hs_cache_client_intro_state_t
+ * objects all related to a specific service. */
+static digest256map_t *hs_cache_client_intro_state;
+
+/* Return the size of a client cache entry in bytes. */
+static size_t
+cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry)
+{
+  return sizeof(*entry) +
+         strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc);
+}
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_client, desc->key.pubkey);
+  /* Update cache size with this entry for the OOM handler. */
+  rend_cache_decrement_allocation(cache_get_client_entry_size(desc));
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_client, desc->key.pubkey, desc);
+  /* Update cache size with this entry for the OOM handler. */
+  rend_cache_increment_allocation(cache_get_client_entry_size(desc));
+}
+
+/* Query our cache and return the entry or NULL if not found or if expired. */
+STATIC hs_cache_client_descriptor_t *
+lookup_v3_desc_as_client(const uint8_t *key)
+{
+  time_t now = approx_time();
+  hs_cache_client_descriptor_t *cached_desc;
+
+  tor_assert(key);
+
+  /* Do the lookup */
+  cached_desc = digest256map_get(hs_cache_v3_client, key);
+  if (!cached_desc) {
+    return NULL;
+  }
+
+  /* Don't return expired entries */
+  if (cached_client_descriptor_has_expired(now, cached_desc)) {
+    return NULL;
+  }
+
+  return cached_desc;
+}
+
+/* Parse the encoded descriptor in <b>desc_str</b> using
+ * <b>service_identity_pk<b> to decrypt it first.
+ *
+ * If everything goes well, allocate and return a new
+ * hs_cache_client_descriptor_t object. In case of error, return NULL. */
+static hs_cache_client_descriptor_t *
+cache_client_desc_new(const char *desc_str,
+                      const ed25519_public_key_t *service_identity_pk)
+{
+  hs_descriptor_t *desc = NULL;
+  hs_cache_client_descriptor_t *client_desc = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(service_identity_pk);
+
+  /* Decode the descriptor we just fetched. */
+  if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) {
+    goto end;
+  }
+  tor_assert(desc);
+
+  /* All is good: make a cache object for this descriptor */
+  client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t));
+  ed25519_pubkey_copy(&client_desc->key, service_identity_pk);
+  /* Set expiration time for this cached descriptor to be the start of the next
+   * time period since that's when clients need to start using the next blinded
+   * pk of the service (and hence will need its next descriptor). */
+  client_desc->expiration_ts = hs_get_start_time_of_next_time_period(0);
+  client_desc->desc = desc;
+  client_desc->encoded_desc = tor_strdup(desc_str);
+
+ end:
+  return client_desc;
+}
+
+#define cache_client_desc_free(val) \
+  FREE_AND_NULL(hs_cache_client_descriptor_t, cache_client_desc_free_, (val))
+
+/** Free memory allocated by <b>desc</b>. */
+static void
+cache_client_desc_free_(hs_cache_client_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_descriptor_free(desc->desc);
+  memwipe(&desc->key, 0, sizeof(desc->key));
+  memwipe(desc->encoded_desc, 0, strlen(desc->encoded_desc));
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/** Helper function: Use by the free all function to clear the client cache */
+static void
+cache_client_desc_free_void(void *ptr)
+{
+  hs_cache_client_descriptor_t *desc = ptr;
+  cache_client_desc_free(desc);
+}
+
+/* Return a newly allocated and initialized hs_cache_intro_state_t object. */
+static hs_cache_intro_state_t *
+cache_intro_state_new(void)
+{
+  hs_cache_intro_state_t *state = tor_malloc_zero(sizeof(*state));
+  state->created_ts = approx_time();
+  return state;
+}
+
+#define cache_intro_state_free(val) \
+  FREE_AND_NULL(hs_cache_intro_state_t, cache_intro_state_free_, (val))
+
+/* Free an hs_cache_intro_state_t object. */
+static void
+cache_intro_state_free_(hs_cache_intro_state_t *state)
+{
+  tor_free(state);
+}
+
+/* Helper function: use by the free all function. */
+static void
+cache_intro_state_free_void(void *state)
+{
+  cache_intro_state_free_(state);
+}
+
+/* Return a newly allocated and initialized hs_cache_client_intro_state_t
+ * object. */
+static hs_cache_client_intro_state_t *
+cache_client_intro_state_new(void)
+{
+  hs_cache_client_intro_state_t *cache = tor_malloc_zero(sizeof(*cache));
+  cache->intro_points = digest256map_new();
+  return cache;
+}
+
+#define cache_client_intro_state_free(val)              \
+  FREE_AND_NULL(hs_cache_client_intro_state_t,          \
+                cache_client_intro_state_free_, (val))
+
+/* Free a cache client intro state object. */
+static void
+cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
+{
+  if (cache == NULL) {
+    return;
+  }
+  digest256map_free(cache->intro_points, cache_intro_state_free_void);
+  tor_free(cache);
+}
+
+/* Helper function: use by the free all function. */
+static void
+cache_client_intro_state_free_void(void *entry)
+{
+  cache_client_intro_state_free_(entry);
+}
+
+/* For the given service identity key service_pk and an introduction
+ * authentication key auth_key, lookup the intro state object. Return 1 if
+ * found and put it in entry if not NULL. Return 0 if not found and entry is
+ * untouched. */
+static int
+cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,
+                                const ed25519_public_key_t *auth_key,
+                                hs_cache_intro_state_t **entry)
+{
+  hs_cache_intro_state_t *state;
+  hs_cache_client_intro_state_t *cache;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  /* Lookup the intro state cache for this service key. */
+  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
+  if (cache == NULL) {
+    goto not_found;
+  }
+
+  /* From the cache we just found for the service, lookup in the introduction
+   * points map for the given authentication key. */
+  state = digest256map_get(cache->intro_points, auth_key->pubkey);
+  if (state == NULL) {
+    goto not_found;
+  }
+  if (entry) {
+    *entry = state;
+  }
+  return 1;
+ not_found:
+  return 0;
+}
+
+/* Note the given failure in state. */
+static void
+cache_client_intro_state_note(hs_cache_intro_state_t *state,
+                              rend_intro_point_failure_t failure)
+{
+  tor_assert(state);
+  switch (failure) {
+  case INTRO_POINT_FAILURE_GENERIC:
+    state->error = 1;
+    break;
+  case INTRO_POINT_FAILURE_TIMEOUT:
+    state->timed_out = 1;
+    break;
+  case INTRO_POINT_FAILURE_UNREACHABLE:
+    state->unreachable_count++;
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+    return;
+  }
+}
+
+/* For the given service identity key service_pk and an introduction
+ * authentication key auth_key, add an entry in the client intro state cache
+ * If no entry exists for the service, it will create one. If state is non
+ * NULL, it will point to the new intro state entry. */
+static void
+cache_client_intro_state_add(const ed25519_public_key_t *service_pk,
+                             const ed25519_public_key_t *auth_key,
+                             hs_cache_intro_state_t **state)
+{
+  hs_cache_intro_state_t *entry, *old_entry;
+  hs_cache_client_intro_state_t *cache;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  /* Lookup the state cache for this service key. */
+  cache = digest256map_get(hs_cache_client_intro_state, service_pk->pubkey);
+  if (cache == NULL) {
+    cache = cache_client_intro_state_new();
+    digest256map_set(hs_cache_client_intro_state, service_pk->pubkey, cache);
+  }
+
+  entry = cache_intro_state_new();
+  old_entry = digest256map_set(cache->intro_points, auth_key->pubkey, entry);
+  /* This should never happened because the code flow is to lookup the entry
+   * before adding it. But, just in case, non fatal assert and free it. */
+  tor_assert_nonfatal(old_entry == NULL);
+  tor_free(old_entry);
+
+  if (state) {
+    *state = entry;
+  }
+}
+
+/* Remove every intro point state entry from cache that has been created
+ * before or at the cutoff. */
+static void
+cache_client_intro_state_clean(time_t cutoff,
+                               hs_cache_client_intro_state_t *cache)
+{
+  tor_assert(cache);
+
+  DIGEST256MAP_FOREACH_MODIFY(cache->intro_points, key,
+                              hs_cache_intro_state_t *, entry) {
+    if (entry->created_ts <= cutoff) {
+      cache_intro_state_free(entry);
+      MAP_DEL_CURRENT(key);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Return true iff no intro points are in this cache. */
+static int
+cache_client_intro_state_is_empty(const hs_cache_client_intro_state_t *cache)
+{
+  return digest256map_isempty(cache->intro_points);
+}
+
+/** Check whether <b>client_desc</b> is useful for us, and store it in the
+ *  client-side HS cache if so. The client_desc is freed if we already have a
+ *  fresher (higher revision counter count) in the cache. */
+static int
+cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
+{
+  hs_cache_client_descriptor_t *cache_entry;
+
+  /* TODO: Heavy code duplication with cache_store_as_dir(). Consider
+   * refactoring and uniting! */
+
+  tor_assert(client_desc);
+
+  /* Check if we already have a descriptor from this HS in cache. If we do,
+   * check if this descriptor is newer than the cached one */
+  cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey);
+  if (cache_entry != NULL) {
+    /* If we have an entry in our cache that has a revision counter greater
+     * than the one we just fetched, discard the one we fetched. */
+    if (cache_entry->desc->plaintext_data.revision_counter >
+        client_desc->desc->plaintext_data.revision_counter) {
+      cache_client_desc_free(client_desc);
+      goto done;
+    }
+    /* Remove old entry. Make space for the new one! */
+    remove_v3_desc_as_client(cache_entry);
+    cache_client_desc_free(cache_entry);
+  }
+
+  /* Store descriptor in cache */
+  store_v3_desc_as_client(client_desc);
+
+ done:
+  return 0;
+}
+
+/* Return true iff the cached client descriptor at <b>cached_desc</b has
+ * expired. */
+static int
+cached_client_descriptor_has_expired(time_t now,
+                               const hs_cache_client_descriptor_t *cached_desc)
+{
+  /* We use the current consensus time to see if we should expire this
+   * descriptor since we use consensus time for all other parts of the protocol
+   * as well (e.g. to build the blinded key and compute time periods). */
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  /* If we don't have a recent consensus, consider this entry expired since we
+   * will want to fetch a new HS desc when we get a live consensus. */
+  if (!ns) {
+    return 1;
+  }
+
+  if (cached_desc->expiration_ts <= ns->valid_after) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/* clean the client cache using now as the current time. Return the total size
+ * of removed bytes from the cache. */
+static size_t
+cache_clean_v3_as_client(time_t now)
+{
+  size_t bytes_removed = 0;
+
+  if (!hs_cache_v3_client) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
+                              hs_cache_client_descriptor_t *, entry) {
+    size_t entry_size;
+
+    /* If the entry has not expired, continue to the next cached entry */
+    if (!cached_client_descriptor_has_expired(now, entry)) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_client_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_client_desc_free(entry);
+    /* Update our OOM. We didn't use the remove() function because we are in
+     * a loop so we have to explicitly decrement. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      digest256_to_base64(key_b64, (const char *) key);
+      log_info(LD_REND, "Removing hidden service v3 descriptor '%s' "
+                        "from client cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
+ *  its HS encoded descriptor if it's stored in our cache, or NULL if not. */
+const char *
+hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (cached_desc) {
+    tor_assert(cached_desc->encoded_desc);
+    return cached_desc->encoded_desc;
+  }
+
+  return NULL;
+}
+
+/** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
+ *  its HS descriptor if it's stored in our cache, or NULL if not. */
+const hs_descriptor_t *
+hs_cache_lookup_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (cached_desc) {
+    tor_assert(cached_desc->desc);
+    return cached_desc->desc;
+  }
+
+  return NULL;
+}
+
+/** Public API: Given an encoded descriptor, store it in the client HS
+ *  cache. Return -1 on error, 0 on success .*/
+int
+hs_cache_store_as_client(const char *desc_str,
+                         const ed25519_public_key_t *identity_pk)
+{
+  hs_cache_client_descriptor_t *client_desc = NULL;
+
+  tor_assert(desc_str);
+  tor_assert(identity_pk);
+
+  /* Create client cache descriptor object */
+  client_desc = cache_client_desc_new(desc_str, identity_pk);
+  if (!client_desc) {
+    log_warn(LD_GENERAL, "Failed to parse received descriptor %s.",
+             escaped(desc_str));
+    goto err;
+  }
+
+  /* Push it to the cache */
+  if (cache_store_as_client(client_desc) < 0) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  cache_client_desc_free(client_desc);
+  return -1;
+}
+
+/* Clean all client caches using the current time now. */
+void
+hs_cache_clean_as_client(time_t now)
+{
+  /* Start with v2 cache cleaning. */
+  rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_client(now);
+}
+
+/* Purge the client descriptor cache. */
+void
+hs_cache_purge_as_client(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_client, key,
+                              hs_cache_client_descriptor_t *, entry) {
+    size_t entry_size = cache_get_client_entry_size(entry);
+    MAP_DEL_CURRENT(key);
+    cache_client_desc_free(entry);
+    /* Update our OOM. We didn't use the remove() function because we are in
+     * a loop so we have to explicitly decrement. */
+    rend_cache_decrement_allocation(entry_size);
+  } DIGEST256MAP_FOREACH_END;
+
+  log_info(LD_REND, "Hidden service client descriptor cache purged.");
+}
+
+/* For a given service identity public key and an introduction authentication
+ * key, note the given failure in the client intro state cache. */
+void
+hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
+                                 const ed25519_public_key_t *auth_key,
+                                 rend_intro_point_failure_t failure)
+{
+  int found;
+  hs_cache_intro_state_t *entry;
+
+  tor_assert(service_pk);
+  tor_assert(auth_key);
+
+  found = cache_client_intro_state_lookup(service_pk, auth_key, &entry);
+  if (!found) {
+    /* Create a new entry and add it to the cache. */
+    cache_client_intro_state_add(service_pk, auth_key, &entry);
+  }
+  /* Note down the entry. */
+  cache_client_intro_state_note(entry, failure);
+}
+
+/* For a given service identity public key and an introduction authentication
+ * key, return true iff it is present in the failure cache. */
+const hs_cache_intro_state_t *
+hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
+                                 const ed25519_public_key_t *auth_key)
+{
+  hs_cache_intro_state_t *state = NULL;
+  cache_client_intro_state_lookup(service_pk, auth_key, &state);
+  return state;
+}
+
+/* Cleanup the client introduction state cache. */
+void
+hs_cache_client_intro_state_clean(time_t now)
+{
+  time_t cutoff = now - HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE;
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
+                              hs_cache_client_intro_state_t *, cache) {
+    /* Cleanup intro points failure. */
+    cache_client_intro_state_clean(cutoff, cache);
+
+    /* Is this cache empty for this service key? If yes, remove it from the
+     * cache. Else keep it. */
+    if (cache_client_intro_state_is_empty(cache)) {
+      cache_client_intro_state_free(cache);
+      MAP_DEL_CURRENT(key);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Purge the client introduction state cache. */
+void
+hs_cache_client_intro_state_purge(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_client_intro_state, key,
+                              hs_cache_client_intro_state_t *, cache) {
+    MAP_DEL_CURRENT(key);
+    cache_client_intro_state_free(cache);
+  } DIGEST256MAP_FOREACH_END;
+
+  log_info(LD_REND, "Hidden service client introduction point state "
+                    "cache purged.");
+}
+
+/**************** Generics *********************************/
+
+/* Do a round of OOM cleanup on all directory caches. Return the amount of
+ * removed bytes. It is possible that the returned value is lower than
+ * min_remove_bytes if the caches get emptied out so the caller should be
+ * aware of this. */
+size_t
+hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t k;
+  size_t bytes_removed = 0;
+
+  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+  tor_assert(min_remove_bytes != 0);
+
+  /* The algorithm is as follow. K is the oldest expected descriptor age.
+   *
+   *   1) Deallocate all entries from v2 cache that are older than K hours.
+   *      1.1) If the amount of remove bytes has been reached, stop.
+   *   2) Deallocate all entries from v3 cache that are older than K hours
+   *      2.1) If the amount of remove bytes has been reached, stop.
+   *   3) Set K = K - RendPostPeriod and repeat process until K is < 0.
+   *
+   * This ends up being O(Kn).
+   */
+
+  /* Set K to the oldest expected age in seconds which is the maximum
+   * lifetime of a cache entry. We'll use the v2 lifetime because it's much
+   * bigger than the v3 thus leading to cleaning older descriptors. */
+  k = rend_cache_max_entry_lifetime();
+
+  do {
+    time_t cutoff;
+
+    /* If K becomes negative, it means we've empty the caches so stop and
+     * return what we were able to cleanup. */
+    if (k < 0) {
+      break;
+    }
+    /* Compute a cutoff value with K and the current time. */
+    cutoff = now - k;
+
+    /* Start by cleaning the v2 cache with that cutoff. */
+    bytes_removed += rend_cache_clean_v2_descs_as_dir(cutoff);
+
+    if (bytes_removed < min_remove_bytes) {
+      /* We haven't remove enough bytes so clean v3 cache. */
+      bytes_removed += cache_clean_v3_as_dir(now, cutoff);
+      /* Decrement K by a post period to shorten the cutoff. */
+      k -= get_options()->RendPostPeriod;
+    }
+  } while (bytes_removed < min_remove_bytes);
+
+  return bytes_removed;
+}
+
+/* Return the maximum size of a v3 HS descriptor. */
+unsigned int
+hs_cache_get_max_descriptor_size(void)
+{
+  return (unsigned) networkstatus_get_param(NULL,
+                                            "HSV3MaxDescriptorSize",
+                                            HS_DESC_MAX_LEN, 1, INT32_MAX);
+}
+
+/* Initialize the hidden service cache subsystem. */
+void
+hs_cache_init(void)
+{
+  /* Calling this twice is very wrong code flow. */
+  tor_assert(!hs_cache_v3_dir);
+  hs_cache_v3_dir = digest256map_new();
+
+  tor_assert(!hs_cache_v3_client);
+  hs_cache_v3_client = digest256map_new();
+
+  tor_assert(!hs_cache_client_intro_state);
+  hs_cache_client_intro_state = digest256map_new();
+}
+
+/* Cleanup the hidden service cache subsystem. */
+void
+hs_cache_free_all(void)
+{
+  digest256map_free(hs_cache_v3_dir, cache_dir_desc_free_void);
+  hs_cache_v3_dir = NULL;
+
+  digest256map_free(hs_cache_v3_client, cache_client_desc_free_void);
+  hs_cache_v3_client = NULL;
+
+  digest256map_free(hs_cache_client_intro_state,
+                    cache_client_intro_state_free_void);
+  hs_cache_client_intro_state = NULL;
+}
+
diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h
new file mode 100644
index 0000000000..0d0085ffdc
--- /dev/null
+++ b/src/or/hs_cache.h
@@ -0,0 +1,128 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.h
+ * \brief Header file for hs_cache.c
+ **/
+
+#ifndef TOR_HS_CACHE_H
+#define TOR_HS_CACHE_H
+
+#include <stdint.h>
+
+#include "crypto_ed25519.h"
+#include "hs_common.h"
+#include "hs_descriptor.h"
+#include "rendcommon.h"
+#include "torcert.h"
+
+/* This is the maximum time an introduction point state object can stay in the
+ * client cache in seconds (2 mins or 120 seconds). */
+#define HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE (2 * 60)
+
+/* Introduction point state. */
+typedef struct hs_cache_intro_state_t {
+  /* When this entry was created and put in the cache. */
+  time_t created_ts;
+
+  /* Did it suffered a generic error? */
+  unsigned int error : 1;
+
+  /* Did it timed out? */
+  unsigned int timed_out : 1;
+
+  /* How many times we tried to reached it and it was unreachable. */
+  uint32_t unreachable_count;
+} hs_cache_intro_state_t;
+
+typedef struct hs_cache_client_intro_state_t {
+  /* Contains hs_cache_intro_state_t object indexed by introduction point
+   * authentication key. */
+  digest256map_t *intro_points;
+} hs_cache_client_intro_state_t;
+
+/* Descriptor representation on the directory side which is a subset of
+ * information that the HSDir can decode and serve it. */
+typedef struct hs_cache_dir_descriptor_t {
+  /* This object is indexed using the blinded pubkey located in the plaintext
+   * data which is populated only once the descriptor has been successfully
+   * decoded and validated. This simply points to that pubkey. */
+  const uint8_t *key;
+
+  /* When does this entry has been created. Used to expire entries. */
+  time_t created_ts;
+
+  /* Descriptor plaintext information. Obviously, we can't decrypt the
+   * encrypted part of the descriptor. */
+  hs_desc_plaintext_data_t *plaintext_data;
+
+  /* Encoded descriptor which is basically in text form. It's a NUL terminated
+   * string thus safe to strlen(). */
+  char *encoded_desc;
+} hs_cache_dir_descriptor_t;
+
+/* Public API */
+
+void hs_cache_init(void);
+void hs_cache_free_all(void);
+void hs_cache_clean_as_dir(time_t now);
+size_t hs_cache_handle_oom(time_t now, size_t min_remove_bytes);
+
+unsigned int hs_cache_get_max_descriptor_size(void);
+
+/* Store and Lookup function. They are version agnostic that is depending on
+ * the requested version of the descriptor, it will be re-routed to the
+ * right function. */
+int hs_cache_store_as_dir(const char *desc);
+int hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                           const char **desc_out);
+
+const hs_descriptor_t *
+hs_cache_lookup_as_client(const ed25519_public_key_t *key);
+const char *
+hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key);
+int hs_cache_store_as_client(const char *desc_str,
+                             const ed25519_public_key_t *identity_pk);
+void hs_cache_clean_as_client(time_t now);
+void hs_cache_purge_as_client(void);
+
+/* Client failure cache. */
+void hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
+                                      const ed25519_public_key_t *auth_key,
+                                      rend_intro_point_failure_t failure);
+const hs_cache_intro_state_t *hs_cache_client_intro_state_find(
+                                       const ed25519_public_key_t *service_pk,
+                                       const ed25519_public_key_t *auth_key);
+void hs_cache_client_intro_state_clean(time_t now);
+void hs_cache_client_intro_state_purge(void);
+
+#ifdef HS_CACHE_PRIVATE
+
+/** Represents a locally cached HS descriptor on a hidden service client. */
+typedef struct hs_cache_client_descriptor_t {
+  /* This object is indexed using the service identity public key */
+  ed25519_public_key_t key;
+
+  /* When will this entry expire? We expire cached client descriptors in the
+   * start of the next time period, since that's when clients need to start
+   * using the next blinded key of the service. */
+  time_t expiration_ts;
+
+  /* The cached descriptor, this object is the owner. It can't be NULL. A
+   * cache object without a valid descriptor is not possible. */
+  hs_descriptor_t *desc;
+
+  /* Encoded descriptor in string form. Can't be NULL. */
+  char *encoded_desc;
+} hs_cache_client_descriptor_t;
+
+STATIC size_t cache_clean_v3_as_dir(time_t now, time_t global_cutoff);
+
+STATIC hs_cache_client_descriptor_t *
+lookup_v3_desc_as_client(const uint8_t *key);
+
+#endif /* defined(HS_CACHE_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CACHE_H) */
+
diff --git a/src/or/hs_cell.c b/src/or/hs_cell.c
new file mode 100644
index 0000000000..03273a44f9
--- /dev/null
+++ b/src/or/hs_cell.c
@@ -0,0 +1,949 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cell.c
+ * \brief Hidden service API for cell creation and handling.
+ **/
+
+#include "or.h"
+#include "config.h"
+#include "crypto_util.h"
+#include "rendservice.h"
+#include "replaycache.h"
+#include "util.h"
+
+#include "hs_cell.h"
+#include "hs_ntor.h"
+
+/* Trunnel. */
+#include "ed25519_cert.h"
+#include "hs/cell_common.h"
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+#include "hs/cell_rendezvous.h"
+
+/* Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
+ * the cell content up to the ENCRYPTED section of length encoded_cell_len.
+ * The encrypted param is the start of the ENCRYPTED section of length
+ * encrypted_len. The mac_key is the key needed for the computation of the MAC
+ * derived from the ntor handshake of length mac_key_len.
+ *
+ * The length mac_out_len must be at least DIGEST256_LEN. */
+static void
+compute_introduce_mac(const uint8_t *encoded_cell, size_t encoded_cell_len,
+                      const uint8_t *encrypted, size_t encrypted_len,
+                      const uint8_t *mac_key, size_t mac_key_len,
+                      uint8_t *mac_out, size_t mac_out_len)
+{
+  size_t offset = 0;
+  size_t mac_msg_len;
+  uint8_t mac_msg[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(encoded_cell);
+  tor_assert(encrypted);
+  tor_assert(mac_key);
+  tor_assert(mac_out);
+  tor_assert(mac_out_len >= DIGEST256_LEN);
+
+  /* Compute the size of the message which is basically the entire cell until
+   * the MAC field of course. */
+  mac_msg_len = encoded_cell_len + (encrypted_len - DIGEST256_LEN);
+  tor_assert(mac_msg_len <= sizeof(mac_msg));
+
+  /* First, put the encoded cell in the msg. */
+  memcpy(mac_msg, encoded_cell, encoded_cell_len);
+  offset += encoded_cell_len;
+  /* Second, put the CLIENT_PK + ENCRYPTED_DATA but ommit the MAC field (which
+   * is junk at this point). */
+  memcpy(mac_msg + offset, encrypted, (encrypted_len - DIGEST256_LEN));
+  offset += (encrypted_len - DIGEST256_LEN);
+  tor_assert(offset == mac_msg_len);
+
+  crypto_mac_sha3_256(mac_out, mac_out_len,
+                      mac_key, mac_key_len,
+                      mac_msg, mac_msg_len);
+  memwipe(mac_msg, 0, sizeof(mac_msg));
+}
+
+/* From a set of keys, subcredential and the ENCRYPTED section of an
+ * INTRODUCE2 cell, return a newly allocated intro cell keys structure.
+ * Finally, the client public key is copied in client_pk. On error, return
+ * NULL. */
+static hs_ntor_intro_cell_keys_t *
+get_introduce2_key_material(const ed25519_public_key_t *auth_key,
+                            const curve25519_keypair_t *enc_key,
+                            const uint8_t *subcredential,
+                            const uint8_t *encrypted_section,
+                            curve25519_public_key_t *client_pk)
+{
+  hs_ntor_intro_cell_keys_t *keys;
+
+  tor_assert(auth_key);
+  tor_assert(enc_key);
+  tor_assert(subcredential);
+  tor_assert(encrypted_section);
+  tor_assert(client_pk);
+
+  keys = tor_malloc_zero(sizeof(*keys));
+
+  /* First bytes of the ENCRYPTED section are the client public key. */
+  memcpy(client_pk->public_key, encrypted_section, CURVE25519_PUBKEY_LEN);
+
+  if (hs_ntor_service_get_introduce1_keys(auth_key, enc_key, client_pk,
+                                          subcredential, keys) < 0) {
+    /* Don't rely on the caller to wipe this on error. */
+    memwipe(client_pk, 0, sizeof(curve25519_public_key_t));
+    tor_free(keys);
+    keys = NULL;
+  }
+  return keys;
+}
+
+/* Using the given encryption key, decrypt the encrypted_section of length
+ * encrypted_section_len of an INTRODUCE2 cell and return a newly allocated
+ * buffer containing the decrypted data. On decryption failure, NULL is
+ * returned. */
+static uint8_t *
+decrypt_introduce2(const uint8_t *enc_key, const uint8_t *encrypted_section,
+                   size_t encrypted_section_len)
+{
+  uint8_t *decrypted = NULL;
+  crypto_cipher_t *cipher = NULL;
+
+  tor_assert(enc_key);
+  tor_assert(encrypted_section);
+
+  /* Decrypt ENCRYPTED section. */
+  cipher = crypto_cipher_new_with_bits((char *) enc_key,
+                                       CURVE25519_PUBKEY_LEN * 8);
+  tor_assert(cipher);
+
+  /* This is symmetric encryption so can't be bigger than the encrypted
+   * section length. */
+  decrypted = tor_malloc_zero(encrypted_section_len);
+  if (crypto_cipher_decrypt(cipher, (char *) decrypted,
+                            (const char *) encrypted_section,
+                            encrypted_section_len) < 0) {
+    tor_free(decrypted);
+    decrypted = NULL;
+    goto done;
+  }
+
+ done:
+  crypto_cipher_free(cipher);
+  return decrypted;
+}
+
+/* Given a pointer to the decrypted data of the ENCRYPTED section of an
+ * INTRODUCE2 cell of length decrypted_len, parse and validate the cell
+ * content. Return a newly allocated cell structure or NULL on error. The
+ * circuit and service object are only used for logging purposes. */
+static trn_cell_introduce_encrypted_t *
+parse_introduce2_encrypted(const uint8_t *decrypted_data,
+                           size_t decrypted_len, const origin_circuit_t *circ,
+                           const hs_service_t *service)
+{
+  trn_cell_introduce_encrypted_t *enc_cell = NULL;
+
+  tor_assert(decrypted_data);
+  tor_assert(circ);
+  tor_assert(service);
+
+  if (trn_cell_introduce_encrypted_parse(&enc_cell, decrypted_data,
+                                         decrypted_len) < 0) {
+    log_info(LD_REND, "Unable to parse the decrypted ENCRYPTED section of "
+                      "the INTRODUCE2 cell on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  if (trn_cell_introduce_encrypted_get_onion_key_type(enc_cell) !=
+      HS_CELL_ONION_KEY_TYPE_NTOR) {
+    log_info(LD_REND, "INTRODUCE2 onion key type is invalid. Got %u but "
+                      "expected %u on circuit %u for service %s",
+             trn_cell_introduce_encrypted_get_onion_key_type(enc_cell),
+             HS_CELL_ONION_KEY_TYPE_NTOR, TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  if (trn_cell_introduce_encrypted_getlen_onion_key(enc_cell) !=
+      CURVE25519_PUBKEY_LEN) {
+    log_info(LD_REND, "INTRODUCE2 onion key length is invalid. Got %u but "
+                      "expected %d on circuit %u for service %s",
+             (unsigned)trn_cell_introduce_encrypted_getlen_onion_key(enc_cell),
+             CURVE25519_PUBKEY_LEN, TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* XXX: Validate NSPEC field as well. */
+
+  return enc_cell;
+ err:
+  trn_cell_introduce_encrypted_free(enc_cell);
+  return NULL;
+}
+
+/* Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
+ * encryption key. The encoded cell is put in cell_out that MUST at least be
+ * of the size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on
+ * success else a negative value and cell_out is untouched. */
+static ssize_t
+build_legacy_establish_intro(const char *circ_nonce, crypto_pk_t *enc_key,
+                             uint8_t *cell_out)
+{
+  ssize_t cell_len;
+
+  tor_assert(circ_nonce);
+  tor_assert(enc_key);
+  tor_assert(cell_out);
+
+  memwipe(cell_out, 0, RELAY_PAYLOAD_SIZE);
+
+  cell_len = rend_service_encode_establish_intro_cell((char*)cell_out,
+                                                      RELAY_PAYLOAD_SIZE,
+                                                      enc_key, circ_nonce);
+  return cell_len;
+}
+
+/* Parse an INTRODUCE2 cell from payload of size payload_len for the given
+ * service and circuit which are used only for logging purposes. The resulting
+ * parsed cell is put in cell_ptr_out.
+ *
+ * This function only parses prop224 INTRODUCE2 cells even when the intro point
+ * is a legacy intro point. That's because intro points don't actually care
+ * about the contents of the introduce cell. Legacy INTRODUCE cells are only
+ * used by the legacy system now.
+ *
+ * Return 0 on success else a negative value and cell_ptr_out is untouched. */
+static int
+parse_introduce2_cell(const hs_service_t *service,
+                      const origin_circuit_t *circ, const uint8_t *payload,
+                      size_t payload_len,
+                      trn_cell_introduce1_t **cell_ptr_out)
+{
+  trn_cell_introduce1_t *cell = NULL;
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(cell_ptr_out);
+
+  /* Parse the cell so we can start cell validation. */
+  if (trn_cell_introduce1_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_PROTOCOL, "Unable to parse INTRODUCE2 cell on circuit %u "
+                          "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  /* Success. */
+  *cell_ptr_out = cell;
+  return 0;
+ err:
+  return -1;
+}
+
+/* Set the onion public key onion_pk in cell, the encrypted section of an
+ * INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
+                                   const uint8_t *onion_pk)
+{
+  tor_assert(cell);
+  tor_assert(onion_pk);
+  /* There is only one possible key type for a non legacy cell. */
+  trn_cell_introduce_encrypted_set_onion_key_type(cell,
+                                                  HS_CELL_ONION_KEY_TYPE_NTOR);
+  trn_cell_introduce_encrypted_set_onion_key_len(cell, CURVE25519_PUBKEY_LEN);
+  trn_cell_introduce_encrypted_setlen_onion_key(cell, CURVE25519_PUBKEY_LEN);
+  memcpy(trn_cell_introduce_encrypted_getarray_onion_key(cell), onion_pk,
+         trn_cell_introduce_encrypted_getlen_onion_key(cell));
+}
+
+/* Set the link specifiers in lspecs in cell, the encrypted section of an
+ * INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
+                                   const smartlist_t *lspecs)
+{
+  tor_assert(cell);
+  tor_assert(lspecs);
+  tor_assert(smartlist_len(lspecs) > 0);
+  tor_assert(smartlist_len(lspecs) <= UINT8_MAX);
+
+  uint8_t lspecs_num = (uint8_t) smartlist_len(lspecs);
+  trn_cell_introduce_encrypted_set_nspec(cell, lspecs_num);
+  /* We aren't duplicating the link specifiers object here which means that
+   * the ownership goes to the trn_cell_introduce_encrypted_t cell and those
+   * object will be freed when the cell is. */
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls,
+                    trn_cell_introduce_encrypted_add_nspecs(cell, ls));
+}
+
+/* Set padding in the enc_cell only if needed that is the total length of both
+ * sections are below the mininum required for an INTRODUCE1 cell. */
+static void
+introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
+                                 trn_cell_introduce_encrypted_t *enc_cell)
+{
+  tor_assert(cell);
+  tor_assert(enc_cell);
+  /* This is the length we expect to have once encoded of the whole cell. */
+  ssize_t full_len = trn_cell_introduce1_encoded_len(cell) +
+                     trn_cell_introduce_encrypted_encoded_len(enc_cell);
+  tor_assert(full_len > 0);
+  if (full_len < HS_CELL_INTRODUCE1_MIN_SIZE) {
+    size_t padding = HS_CELL_INTRODUCE1_MIN_SIZE - full_len;
+    trn_cell_introduce_encrypted_setlen_pad(enc_cell, padding);
+    memset(trn_cell_introduce_encrypted_getarray_pad(enc_cell), 0,
+           trn_cell_introduce_encrypted_getlen_pad(enc_cell));
+  }
+}
+
+/* Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
+ * and the INTRODUCE1 data.
+ *
+ * This can't fail but it is very important that the caller sets every field
+ * in data so the computation of the INTRODUCE1 keys doesn't fail. */
+static void
+introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell,
+                              const trn_cell_introduce_encrypted_t *enc_cell,
+                              const hs_cell_introduce1_data_t *data)
+{
+  size_t offset = 0;
+  ssize_t encrypted_len;
+  ssize_t encoded_cell_len, encoded_enc_cell_len;
+  uint8_t encoded_cell[RELAY_PAYLOAD_SIZE] = {0};
+  uint8_t encoded_enc_cell[RELAY_PAYLOAD_SIZE] = {0};
+  uint8_t *encrypted = NULL;
+  uint8_t mac[DIGEST256_LEN];
+  crypto_cipher_t *cipher = NULL;
+  hs_ntor_intro_cell_keys_t keys;
+
+  tor_assert(cell);
+  tor_assert(enc_cell);
+  tor_assert(data);
+
+  /* Encode the cells up to now of what we have to we can perform the MAC
+   * computation on it. */
+  encoded_cell_len = trn_cell_introduce1_encode(encoded_cell,
+                                                sizeof(encoded_cell), cell);
+  /* We have a much more serious issue if this isn't true. */
+  tor_assert(encoded_cell_len > 0);
+
+  encoded_enc_cell_len =
+    trn_cell_introduce_encrypted_encode(encoded_enc_cell,
+                                        sizeof(encoded_enc_cell), enc_cell);
+  /* We have a much more serious issue if this isn't true. */
+  tor_assert(encoded_enc_cell_len > 0);
+
+  /* Get the key material for the encryption. */
+  if (hs_ntor_client_get_introduce1_keys(data->auth_pk, data->enc_pk,
+                                         data->client_kp,
+                                         data->subcredential, &keys) < 0) {
+    tor_assert_unreached();
+  }
+
+  /* Prepare cipher with the encryption key just computed. */
+  cipher = crypto_cipher_new_with_bits((const char *) keys.enc_key,
+                                       sizeof(keys.enc_key) * 8);
+  tor_assert(cipher);
+
+  /* Compute the length of the ENCRYPTED section which is the CLIENT_PK,
+   * ENCRYPTED_DATA and MAC length. */
+  encrypted_len = sizeof(data->client_kp->pubkey) + encoded_enc_cell_len +
+                  sizeof(mac);
+  tor_assert(encrypted_len < RELAY_PAYLOAD_SIZE);
+  encrypted = tor_malloc_zero(encrypted_len);
+
+  /* Put the CLIENT_PK first. */
+  memcpy(encrypted, data->client_kp->pubkey.public_key,
+         sizeof(data->client_kp->pubkey.public_key));
+  offset += sizeof(data->client_kp->pubkey.public_key);
+  /* Then encrypt and set the ENCRYPTED_DATA. This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted + offset,
+                        (const char *) encoded_enc_cell, encoded_enc_cell_len);
+  crypto_cipher_free(cipher);
+  offset += encoded_enc_cell_len;
+  /* Compute MAC from the above and put it in the buffer. This function will
+   * make the adjustment to the encrypted_len to omit the MAC length. */
+  compute_introduce_mac(encoded_cell, encoded_cell_len,
+                        encrypted, encrypted_len,
+                        keys.mac_key, sizeof(keys.mac_key),
+                        mac, sizeof(mac));
+  memcpy(encrypted + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  tor_assert(offset == (size_t) encrypted_len);
+
+  /* Set the ENCRYPTED section in the cell. */
+  trn_cell_introduce1_setlen_encrypted(cell, encrypted_len);
+  memcpy(trn_cell_introduce1_getarray_encrypted(cell),
+         encrypted, encrypted_len);
+
+  /* Cleanup. */
+  memwipe(&keys, 0, sizeof(keys));
+  memwipe(mac, 0, sizeof(mac));
+  memwipe(encrypted, 0, sizeof(encrypted_len));
+  memwipe(encoded_enc_cell, 0, sizeof(encoded_enc_cell));
+  tor_free(encrypted);
+}
+
+/* Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
+ * set it, encrypt it and encode it. */
+static void
+introduce1_set_encrypted(trn_cell_introduce1_t *cell,
+                         const hs_cell_introduce1_data_t *data)
+{
+  trn_cell_introduce_encrypted_t *enc_cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(cell);
+  tor_assert(data);
+
+  enc_cell = trn_cell_introduce_encrypted_new();
+  tor_assert(enc_cell);
+
+  /* Set extension data. None are used. */
+  ext = trn_cell_extension_new();
+  tor_assert(ext);
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_encrypted_set_extensions(enc_cell, ext);
+
+  /* Set the rendezvous cookie. */
+  memcpy(trn_cell_introduce_encrypted_getarray_rend_cookie(enc_cell),
+         data->rendezvous_cookie, REND_COOKIE_LEN);
+
+  /* Set the onion public key. */
+  introduce1_set_encrypted_onion_key(enc_cell, data->onion_pk->public_key);
+
+  /* Set the link specifiers. */
+  introduce1_set_encrypted_link_spec(enc_cell, data->link_specifiers);
+
+  /* Set padding. */
+  introduce1_set_encrypted_padding(cell, enc_cell);
+
+  /* Encrypt and encode it in the cell. */
+  introduce1_encrypt_and_encode(cell, enc_cell, data);
+
+  /* Cleanup. */
+  trn_cell_introduce_encrypted_free(enc_cell);
+}
+
+/* Set the authentication key in the INTRODUCE1 cell from the given data. */
+static void
+introduce1_set_auth_key(trn_cell_introduce1_t *cell,
+                        const hs_cell_introduce1_data_t *data)
+{
+  tor_assert(cell);
+  tor_assert(data);
+  /* There is only one possible type for a non legacy cell. */
+  trn_cell_introduce1_set_auth_key_type(cell, HS_INTRO_AUTH_KEY_TYPE_ED25519);
+  trn_cell_introduce1_set_auth_key_len(cell, ED25519_PUBKEY_LEN);
+  trn_cell_introduce1_setlen_auth_key(cell, ED25519_PUBKEY_LEN);
+  memcpy(trn_cell_introduce1_getarray_auth_key(cell),
+         data->auth_pk->pubkey, trn_cell_introduce1_getlen_auth_key(cell));
+}
+
+/* Set the legacy ID field in the INTRODUCE1 cell from the given data. */
+static void
+introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
+                         const hs_cell_introduce1_data_t *data)
+{
+  tor_assert(cell);
+  tor_assert(data);
+
+  if (data->is_legacy) {
+    uint8_t digest[DIGEST_LEN];
+    if (BUG(crypto_pk_get_digest(data->legacy_key, (char *) digest) < 0)) {
+      return;
+    }
+    memcpy(trn_cell_introduce1_getarray_legacy_key_id(cell),
+           digest, trn_cell_introduce1_getlen_legacy_key_id(cell));
+  } else {
+    /* We have to zeroed the LEGACY_KEY_ID field. */
+    memset(trn_cell_introduce1_getarray_legacy_key_id(cell), 0,
+           trn_cell_introduce1_getlen_legacy_key_id(cell));
+  }
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
+ * object. The encoded cell is put in cell_out that MUST at least be of the
+ * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
+ * a negative value and cell_out is untouched. This function also supports
+ * legacy cell creation. */
+ssize_t
+hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_intro_point_t *ip,
+                              uint8_t *cell_out)
+{
+  ssize_t cell_len = -1;
+  uint16_t sig_len = ED25519_SIG_LEN;
+  trn_cell_extension_t *ext;
+  trn_cell_establish_intro_t *cell = NULL;
+
+  tor_assert(circ_nonce);
+  tor_assert(ip);
+
+  /* Quickly handle the legacy IP. */
+  if (ip->base.is_only_legacy) {
+    tor_assert(ip->legacy_key);
+    cell_len = build_legacy_establish_intro(circ_nonce, ip->legacy_key,
+                                            cell_out);
+    tor_assert(cell_len <= RELAY_PAYLOAD_SIZE);
+    /* Success or not we are done here. */
+    goto done;
+  }
+
+  /* Set extension data. None used here. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  cell = trn_cell_establish_intro_new();
+  trn_cell_establish_intro_set_extensions(cell, ext);
+  /* Set signature size. Array is then allocated in the cell. We need to do
+   * this early so we can use trunnel API to get the signature length. */
+  trn_cell_establish_intro_set_sig_len(cell, sig_len);
+  trn_cell_establish_intro_setlen_sig(cell, sig_len);
+
+  /* Set AUTH_KEY_TYPE: 2 means ed25519 */
+  trn_cell_establish_intro_set_auth_key_type(cell,
+                                             HS_INTRO_AUTH_KEY_TYPE_ED25519);
+
+  /* Set AUTH_KEY and AUTH_KEY_LEN field. Must also set byte-length of
+   * AUTH_KEY to match */
+  {
+    uint16_t auth_key_len = ED25519_PUBKEY_LEN;
+    trn_cell_establish_intro_set_auth_key_len(cell, auth_key_len);
+    trn_cell_establish_intro_setlen_auth_key(cell, auth_key_len);
+    /* We do this call _after_ setting the length because it's reallocated at
+     * that point only. */
+    uint8_t *auth_key_ptr = trn_cell_establish_intro_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, ip->auth_key_kp.pubkey.pubkey, auth_key_len);
+  }
+
+  /* Calculate HANDSHAKE_AUTH field (MAC). */
+  {
+    ssize_t tmp_cell_enc_len = 0;
+    ssize_t tmp_cell_mac_offset =
+      sig_len + sizeof(cell->sig_len) +
+      trn_cell_establish_intro_getlen_handshake_mac(cell);
+    uint8_t tmp_cell_enc[RELAY_PAYLOAD_SIZE] = {0};
+    uint8_t mac[TRUNNEL_SHA3_256_LEN], *handshake_ptr;
+
+    /* We first encode the current fields we have in the cell so we can
+     * compute the MAC using the raw bytes. */
+    tmp_cell_enc_len = trn_cell_establish_intro_encode(tmp_cell_enc,
+                                                       sizeof(tmp_cell_enc),
+                                                       cell);
+    if (BUG(tmp_cell_enc_len < 0)) {
+      goto done;
+    }
+    /* Sanity check. */
+    tor_assert(tmp_cell_enc_len > tmp_cell_mac_offset);
+
+    /* Circuit nonce is always DIGEST_LEN according to tor-spec.txt. */
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        (uint8_t *) circ_nonce, DIGEST_LEN,
+                        tmp_cell_enc, tmp_cell_enc_len - tmp_cell_mac_offset);
+    handshake_ptr = trn_cell_establish_intro_getarray_handshake_mac(cell);
+    memcpy(handshake_ptr, mac, sizeof(mac));
+
+    memwipe(mac, 0, sizeof(mac));
+    memwipe(tmp_cell_enc, 0, sizeof(tmp_cell_enc));
+  }
+
+  /* Calculate the cell signature SIG. */
+  {
+    ssize_t tmp_cell_enc_len = 0;
+    ssize_t tmp_cell_sig_offset = (sig_len + sizeof(cell->sig_len));
+    uint8_t tmp_cell_enc[RELAY_PAYLOAD_SIZE] = {0}, *sig_ptr;
+    ed25519_signature_t sig;
+
+    /* We first encode the current fields we have in the cell so we can
+     * compute the signature from the raw bytes of the cell. */
+    tmp_cell_enc_len = trn_cell_establish_intro_encode(tmp_cell_enc,
+                                                       sizeof(tmp_cell_enc),
+                                                       cell);
+    if (BUG(tmp_cell_enc_len < 0)) {
+      goto done;
+    }
+
+    if (ed25519_sign_prefixed(&sig, tmp_cell_enc,
+                              tmp_cell_enc_len - tmp_cell_sig_offset,
+                              ESTABLISH_INTRO_SIG_PREFIX, &ip->auth_key_kp)) {
+      log_warn(LD_BUG, "Unable to make signature for ESTABLISH_INTRO cell.");
+      goto done;
+    }
+    /* Copy the signature into the cell. */
+    sig_ptr = trn_cell_establish_intro_getarray_sig(cell);
+    memcpy(sig_ptr, sig.sig, sig_len);
+
+    memwipe(tmp_cell_enc, 0, sizeof(tmp_cell_enc));
+  }
+
+  /* Encode the cell. Can't be bigger than a standard cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_out, RELAY_PAYLOAD_SIZE,
+                                             cell);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  return cell_len;
+}
+
+/* Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
+ * are successful at parsing it, return the length of the parsed cell else a
+ * negative value on error. */
+ssize_t
+hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len)
+{
+  ssize_t ret;
+  trn_cell_intro_established_t *cell = NULL;
+
+  tor_assert(payload);
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. */
+  ret = trn_cell_intro_established_parse(&cell, payload, payload_len);
+  if (ret >= 0) {
+    /* On success, we do not keep the cell, we just notify the caller that it
+     * was successfully parsed. */
+    trn_cell_intro_established_free(cell);
+  }
+  return ret;
+}
+
+/* Parsse the INTRODUCE2 cell using data which contains everything we need to
+ * do so and contains the destination buffers of information we extract and
+ * compute from the cell. Return 0 on success else a negative value. The
+ * service and circ are only used for logging purposes. */
+ssize_t
+hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
+                         const origin_circuit_t *circ,
+                         const hs_service_t *service)
+{
+  int ret = -1;
+  time_t elapsed;
+  uint8_t *decrypted = NULL;
+  size_t encrypted_section_len;
+  const uint8_t *encrypted_section;
+  trn_cell_introduce1_t *cell = NULL;
+  trn_cell_introduce_encrypted_t *enc_cell = NULL;
+  hs_ntor_intro_cell_keys_t *intro_keys = NULL;
+
+  tor_assert(data);
+  tor_assert(circ);
+  tor_assert(service);
+
+  /* Parse the cell into a decoded data structure pointed by cell_ptr. */
+  if (parse_introduce2_cell(service, circ, data->payload, data->payload_len,
+                            &cell) < 0) {
+    goto done;
+  }
+
+  log_info(LD_REND, "Received a decodable INTRODUCE2 cell on circuit %u "
+                    "for service %s. Decoding encrypted section...",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+
+  encrypted_section = trn_cell_introduce1_getconstarray_encrypted(cell);
+  encrypted_section_len = trn_cell_introduce1_getlen_encrypted(cell);
+
+  /* Encrypted section must at least contain the CLIENT_PK and MAC which is
+   * defined in section 3.3.2 of the specification. */
+  if (encrypted_section_len < (CURVE25519_PUBKEY_LEN + DIGEST256_LEN)) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted section length "
+                      "for service %s. Dropping cell.",
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Check our replay cache for this introduction point. */
+  if (replaycache_add_test_and_elapsed(data->replay_cache, encrypted_section,
+                                       encrypted_section_len, &elapsed)) {
+    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the"
+                      "same ENCRYPTED section was seen %ld seconds ago. "
+                      "Dropping cell.", (long int) elapsed);
+    goto done;
+  }
+
+  /* Build the key material out of the key material found in the cell. */
+  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
+                                           data->subcredential,
+                                           encrypted_section,
+                                           &data->client_pk);
+  if (intro_keys == NULL) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
+                      "compute key material on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Validate MAC from the cell and our computed key material. The MAC field
+   * in the cell is at the end of the encrypted section. */
+  {
+    uint8_t mac[DIGEST256_LEN];
+    /* The MAC field is at the very end of the ENCRYPTED section. */
+    size_t mac_offset = encrypted_section_len - sizeof(mac);
+    /* Compute the MAC. Use the entire encoded payload with a length up to the
+     * ENCRYPTED section. */
+    compute_introduce_mac(data->payload,
+                          data->payload_len - encrypted_section_len,
+                          encrypted_section, encrypted_section_len,
+                          intro_keys->mac_key, sizeof(intro_keys->mac_key),
+                          mac, sizeof(mac));
+    if (tor_memcmp(mac, encrypted_section + mac_offset, sizeof(mac))) {
+      log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell on "
+                        "circuit %u for service %s",
+               TO_CIRCUIT(circ)->n_circ_id,
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+  }
+
+  {
+    /* The ENCRYPTED_DATA section starts just after the CLIENT_PK. */
+    const uint8_t *encrypted_data =
+      encrypted_section + sizeof(data->client_pk);
+    /* It's symmetric encryption so it's correct to use the ENCRYPTED length
+     * for decryption. Computes the length of ENCRYPTED_DATA meaning removing
+     * the CLIENT_PK and MAC length. */
+    size_t encrypted_data_len =
+      encrypted_section_len - (sizeof(data->client_pk) + DIGEST256_LEN);
+
+    /* This decrypts the ENCRYPTED_DATA section of the cell. */
+    decrypted = decrypt_introduce2(intro_keys->enc_key,
+                                   encrypted_data, encrypted_data_len);
+    if (decrypted == NULL) {
+      log_info(LD_REND, "Unable to decrypt the ENCRYPTED section of an "
+                        "INTRODUCE2 cell on circuit %u for service %s",
+               TO_CIRCUIT(circ)->n_circ_id,
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+
+    /* Parse this blob into an encrypted cell structure so we can then extract
+     * the data we need out of it. */
+    enc_cell = parse_introduce2_encrypted(decrypted, encrypted_data_len,
+                                          circ, service);
+    memwipe(decrypted, 0, encrypted_data_len);
+    if (enc_cell == NULL) {
+      goto done;
+    }
+  }
+
+  /* XXX: Implement client authorization checks. */
+
+  /* Extract onion key and rendezvous cookie from the cell used for the
+   * rendezvous point circuit e2e encryption. */
+  memcpy(data->onion_pk.public_key,
+         trn_cell_introduce_encrypted_getconstarray_onion_key(enc_cell),
+         CURVE25519_PUBKEY_LEN);
+  memcpy(data->rendezvous_cookie,
+         trn_cell_introduce_encrypted_getconstarray_rend_cookie(enc_cell),
+         sizeof(data->rendezvous_cookie));
+
+  /* Extract rendezvous link specifiers. */
+  for (size_t idx = 0;
+       idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
+    link_specifier_t *lspec =
+      trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
+    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+  }
+
+  /* Success. */
+  ret = 0;
+  log_info(LD_REND, "Valid INTRODUCE2 cell. Launching rendezvous circuit.");
+
+ done:
+  if (intro_keys) {
+    memwipe(intro_keys, 0, sizeof(hs_ntor_intro_cell_keys_t));
+    tor_free(intro_keys);
+  }
+  tor_free(decrypted);
+  trn_cell_introduce_encrypted_free(enc_cell);
+  trn_cell_introduce1_free(cell);
+  return ret;
+}
+
+/* Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
+ * info. The encoded cell is put in cell_out and the length of the data is
+ * returned. This can't fail. */
+ssize_t
+hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
+                          size_t rendezvous_cookie_len,
+                          const uint8_t *rendezvous_handshake_info,
+                          size_t rendezvous_handshake_info_len,
+                          uint8_t *cell_out)
+{
+  ssize_t cell_len;
+  trn_cell_rendezvous1_t *cell;
+
+  tor_assert(rendezvous_cookie);
+  tor_assert(rendezvous_handshake_info);
+  tor_assert(cell_out);
+
+  cell = trn_cell_rendezvous1_new();
+  /* Set the RENDEZVOUS_COOKIE. */
+  memcpy(trn_cell_rendezvous1_getarray_rendezvous_cookie(cell),
+         rendezvous_cookie, rendezvous_cookie_len);
+  /* Set the HANDSHAKE_INFO. */
+  trn_cell_rendezvous1_setlen_handshake_info(cell,
+                                            rendezvous_handshake_info_len);
+  memcpy(trn_cell_rendezvous1_getarray_handshake_info(cell),
+         rendezvous_handshake_info, rendezvous_handshake_info_len);
+  /* Encoding. */
+  cell_len = trn_cell_rendezvous1_encode(cell_out, RELAY_PAYLOAD_SIZE, cell);
+  tor_assert(cell_len > 0);
+
+  trn_cell_rendezvous1_free(cell);
+  return cell_len;
+}
+
+/* Build an INTRODUCE1 cell from the given data. The encoded cell is put in
+ * cell_out which must be of at least size RELAY_PAYLOAD_SIZE. On success, the
+ * encoded length is returned else a negative value and the content of
+ * cell_out should be ignored. */
+ssize_t
+hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
+                         uint8_t *cell_out)
+{
+  ssize_t cell_len;
+  trn_cell_introduce1_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(data);
+  tor_assert(cell_out);
+
+  cell = trn_cell_introduce1_new();
+  tor_assert(cell);
+
+  /* Set extension data. None are used. */
+  ext = trn_cell_extension_new();
+  tor_assert(ext);
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce1_set_extensions(cell, ext);
+
+  /* Set the legacy ID field. */
+  introduce1_set_legacy_id(cell, data);
+
+  /* Set the authentication key. */
+  introduce1_set_auth_key(cell, data);
+
+  /* Set the encrypted section. This will set, encrypt and encode the
+   * ENCRYPTED section in the cell. After this, we'll be ready to encode. */
+  introduce1_set_encrypted(cell, data);
+
+  /* Final encoding. */
+  cell_len = trn_cell_introduce1_encode(cell_out, RELAY_PAYLOAD_SIZE, cell);
+
+  trn_cell_introduce1_free(cell);
+  return cell_len;
+}
+
+/* Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
+ * encoded cell is put in cell_out which must be of at least
+ * RELAY_PAYLOAD_SIZE. On success, the encoded length is returned and the
+ * caller should clear up the content of the cell.
+ *
+ * This function can't fail. */
+ssize_t
+hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
+                                   uint8_t *cell_out)
+{
+  tor_assert(rendezvous_cookie);
+  tor_assert(cell_out);
+
+  memcpy(cell_out, rendezvous_cookie, HS_REND_COOKIE_LEN);
+  return HS_REND_COOKIE_LEN;
+}
+
+/* Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
+ * Return the status code on success else a negative value if the cell as not
+ * decodable. */
+int
+hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+  trn_cell_introduce_ack_t *cell = NULL;
+
+  tor_assert(payload);
+
+  /* If it is a legacy IP, rend-spec.txt specifies that a ACK is 0 byte and a
+   * NACK is 1 byte. We can't use the legacy function for this so we have to
+   * do a special case. */
+  if (payload_len <= 1) {
+    if (payload_len == 0) {
+      ret = HS_CELL_INTRO_ACK_SUCCESS;
+    } else {
+      ret = HS_CELL_INTRO_ACK_FAILURE;
+    }
+    goto end;
+  }
+
+  if (trn_cell_introduce_ack_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_REND, "Invalid INTRODUCE_ACK cell. Unable to parse it.");
+    goto end;
+  }
+
+  ret = trn_cell_introduce_ack_get_status(cell);
+
+ end:
+  trn_cell_introduce_ack_free(cell);
+  return ret;
+}
+
+/* Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
+ * success, handshake_info contains the data in the HANDSHAKE_INFO field, and
+ * 0 is returned. On error, a negative value is returned. */
+int
+hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
+                          uint8_t *handshake_info, size_t handshake_info_len)
+{
+  int ret = -1;
+  trn_cell_rendezvous2_t *cell = NULL;
+
+  tor_assert(payload);
+  tor_assert(handshake_info);
+
+  if (trn_cell_rendezvous2_parse(&cell, payload, payload_len) < 0) {
+    log_info(LD_REND, "Invalid RENDEZVOUS2 cell. Unable to parse it.");
+    goto end;
+  }
+
+  /* Static size, we should never have an issue with this else we messed up
+   * our code flow. */
+  tor_assert(trn_cell_rendezvous2_getlen_handshake_info(cell) ==
+             handshake_info_len);
+  memcpy(handshake_info,
+         trn_cell_rendezvous2_getconstarray_handshake_info(cell),
+         handshake_info_len);
+  ret = 0;
+
+ end:
+  trn_cell_rendezvous2_free(cell);
+  return ret;
+}
+
+/* Clear the given INTRODUCE1 data structure data. */
+void
+hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
+{
+  if (data == NULL) {
+    return;
+  }
+  /* Object in this list have been moved to the cell object when building it
+   * so they've been freed earlier. We do that in order to avoid duplicating
+   * them leading to more memory and CPU time being used for nothing. */
+  smartlist_free(data->link_specifiers);
+  /* The data object has no ownership of any members. */
+  memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
+}
+
diff --git a/src/or/hs_cell.h b/src/or/hs_cell.h
new file mode 100644
index 0000000000..958dde4ffc
--- /dev/null
+++ b/src/or/hs_cell.h
@@ -0,0 +1,122 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cell.h
+ * \brief Header file containing cell data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_CELL_H
+#define TOR_HS_CELL_H
+
+#include "or.h"
+#include "hs_service.h"
+
+/* An INTRODUCE1 cell requires at least this amount of bytes (see section
+ * 3.2.2 of the specification). Below this value, the cell must be padded. */
+#define HS_CELL_INTRODUCE1_MIN_SIZE 246
+
+/* Status code of an INTRODUCE_ACK cell. */
+typedef enum {
+  HS_CELL_INTRO_ACK_SUCCESS = 0x0000, /* Cell relayed to service. */
+  HS_CELL_INTRO_ACK_FAILURE = 0x0001, /* Service ID not recognized */
+  HS_CELL_INTRO_ACK_BADFMT  = 0x0002, /* Bad message format */
+  HS_CELL_INTRO_ACK_NORELAY = 0x0003, /* Can't relay cell to service */
+} hs_cell_introd_ack_status_t;
+
+/* Onion key type found in the INTRODUCE1 cell. */
+typedef enum {
+  HS_CELL_ONION_KEY_TYPE_NTOR = 1,
+} hs_cell_onion_key_type_t;
+
+/* This data structure contains data that we need to build an INTRODUCE1 cell
+ * used by the INTRODUCE1 build function. */
+typedef struct hs_cell_introduce1_data_t {
+  /* Is this a legacy introduction point? */
+  unsigned int is_legacy : 1;
+  /* (Legacy only) The encryption key for a legacy intro point. Only set if
+   * is_legacy is true. */
+  const crypto_pk_t *legacy_key;
+  /* Introduction point authentication public key. */
+  const ed25519_public_key_t *auth_pk;
+  /* Introduction point encryption public key. */
+  const curve25519_public_key_t *enc_pk;
+  /* Subcredentials of the service. */
+  const uint8_t *subcredential;
+  /* Onion public key for the ntor handshake. */
+  const curve25519_public_key_t *onion_pk;
+  /* Rendezvous cookie. */
+  const uint8_t *rendezvous_cookie;
+  /* Public key put before the encrypted data (CLIENT_PK). */
+  const curve25519_keypair_t *client_kp;
+  /* Rendezvous point link specifiers. */
+  smartlist_t *link_specifiers;
+} hs_cell_introduce1_data_t;
+
+/* This data structure contains data that we need to parse an INTRODUCE2 cell
+ * which is used by the INTRODUCE2 cell parsing function. On a successful
+ * parsing, the onion_pk and rendezvous_cookie will be populated with the
+ * computed key material from the cell data. This structure is only used during
+ * INTRO2 parsing and discarded after that. */
+typedef struct hs_cell_introduce2_data_t {
+  /*** Immutable Section: Set on structure init. ***/
+
+  /* Introduction point authentication public key. Pointer owned by the
+     introduction point object through which we received the INTRO2 cell. */
+  const ed25519_public_key_t *auth_pk;
+  /* Introduction point encryption keypair for the ntor handshake. Pointer
+     owned by the introduction point object through which we received the
+     INTRO2 cell*/
+  const curve25519_keypair_t *enc_kp;
+  /* Subcredentials of the service. Pointer owned by the descriptor that owns
+     the introduction point through which we received the INTRO2 cell. */
+  const uint8_t *subcredential;
+  /* Payload of the received encoded cell. */
+  const uint8_t *payload;
+  /* Size of the payload of the received encoded cell. */
+  size_t payload_len;
+
+  /*** Mutable Section: Set upon parsing INTRODUCE2 cell. ***/
+
+  /* Onion public key computed using the INTRODUCE2 encrypted section. */
+  curve25519_public_key_t onion_pk;
+  /* Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
+  uint8_t rendezvous_cookie[REND_COOKIE_LEN];
+  /* Client public key from the INTRODUCE2 encrypted section. */
+  curve25519_public_key_t client_pk;
+  /* Link specifiers of the rendezvous point. Contains link_specifier_t. */
+  smartlist_t *link_specifiers;
+  /* Replay cache of the introduction point. */
+  replaycache_t *replay_cache;
+} hs_cell_introduce2_data_t;
+
+/* Build cell API. */
+ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_intro_point_t *ip,
+                                      uint8_t *cell_out);
+ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
+                                  size_t rendezvous_cookie_len,
+                                  const uint8_t *rendezvous_handshake_info,
+                                  size_t rendezvous_handshake_info_len,
+                                  uint8_t *cell_out);
+ssize_t hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
+                                 uint8_t *cell_out);
+ssize_t hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
+                                           uint8_t *cell_out);
+
+/* Parse cell API. */
+ssize_t hs_cell_parse_intro_established(const uint8_t *payload,
+                                        size_t payload_len);
+ssize_t hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
+                                 const origin_circuit_t *circ,
+                                 const hs_service_t *service);
+int hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len);
+int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
+                              uint8_t *handshake_info,
+                              size_t handshake_info_len);
+
+/* Util API. */
+void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
+
+#endif /* !defined(TOR_HS_CELL_H) */
+
diff --git a/src/or/hs_circuit.c b/src/or/hs_circuit.c
new file mode 100644
index 0000000000..d911f5fde4
--- /dev/null
+++ b/src/or/hs_circuit.c
@@ -0,0 +1,1250 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuit.c
+ **/
+
+#define HS_CIRCUIT_PRIVATE
+
+#include "or.h"
+#include "circpathbias.h"
+#include "circuitbuild.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "config.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "nodelist.h"
+#include "policies.h"
+#include "relay.h"
+#include "rendservice.h"
+#include "rephist.h"
+#include "router.h"
+
+#include "hs_cell.h"
+#include "hs_ident.h"
+#include "hs_ntor.h"
+#include "hs_service.h"
+#include "hs_circuit.h"
+
+/* Trunnel. */
+#include "ed25519_cert.h"
+#include "hs/cell_common.h"
+#include "hs/cell_establish_intro.h"
+
+/* A circuit is about to become an e2e rendezvous circuit. Check
+ * <b>circ_purpose</b> and ensure that it's properly set. Return true iff
+ * circuit purpose is properly set, otherwise return false. */
+static int
+circuit_purpose_is_correct_for_rend(unsigned int circ_purpose,
+                                    int is_service_side)
+{
+  if (is_service_side) {
+    if (circ_purpose != CIRCUIT_PURPOSE_S_CONNECT_REND) {
+      log_warn(LD_BUG,
+            "HS e2e circuit setup with wrong purpose (%d)", circ_purpose);
+      return 0;
+    }
+  }
+
+  if (!is_service_side) {
+    if (circ_purpose != CIRCUIT_PURPOSE_C_REND_READY &&
+        circ_purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) {
+      log_warn(LD_BUG,
+            "Client e2e circuit setup with wrong purpose (%d)", circ_purpose);
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
+/* Create and return a crypt path for the final hop of a v3 prop224 rendezvous
+ * circuit. Initialize the crypt path crypto using the output material from the
+ * ntor key exchange at <b>ntor_key_seed</b>.
+ *
+ * If <b>is_service_side</b> is set, we are the hidden service and the final
+ * hop of the rendezvous circuit is the client on the other side. */
+static crypt_path_t *
+create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
+                  int is_service_side)
+{
+  uint8_t keys[HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN];
+  crypt_path_t *cpath = NULL;
+
+  /* Do the key expansion */
+  if (hs_ntor_circuit_key_expansion(ntor_key_seed, seed_len,
+                                    keys, sizeof(keys)) < 0) {
+    goto err;
+  }
+
+  /* Setup the cpath */
+  cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath->magic = CRYPT_PATH_MAGIC;
+
+  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+                                is_service_side, 1) < 0) {
+    tor_free(cpath);
+    goto err;
+  }
+
+ err:
+  memwipe(keys, 0, sizeof(keys));
+  return cpath;
+}
+
+/* We are a v2 legacy HS client: Create and return a crypt path for the hidden
+ * service on the other side of the rendezvous circuit <b>circ</b>. Initialize
+ * the crypt path crypto using the body of the RENDEZVOUS1 cell at
+ * <b>rend_cell_body</b> (which must be at least DH_KEY_LEN+DIGEST_LEN bytes).
+ */
+static crypt_path_t *
+create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
+{
+  crypt_path_t *hop = NULL;
+  char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
+
+  /* first DH_KEY_LEN bytes are g^y from the service. Finish the dh
+   * handshake...*/
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->pending_final_cpath);
+  hop = circ->build_state->pending_final_cpath;
+
+  tor_assert(hop->rend_dh_handshake_state);
+  if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->rend_dh_handshake_state,
+                               (char*)rend_cell_body, DH_KEY_LEN,
+                               keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
+    log_warn(LD_GENERAL, "Couldn't complete DH handshake.");
+    goto err;
+  }
+  /* ... and set up cpath. */
+  if (circuit_init_cpath_crypto(hop,
+                                keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
+                                0, 0) < 0)
+    goto err;
+
+  /* Check whether the digest is right... */
+  if (tor_memneq(keys, rend_cell_body+DH_KEY_LEN, DIGEST_LEN)) {
+    log_warn(LD_PROTOCOL, "Incorrect digest of key material.");
+    goto err;
+  }
+
+  /* clean up the crypto stuff we just made */
+  crypto_dh_free(hop->rend_dh_handshake_state);
+  hop->rend_dh_handshake_state = NULL;
+
+  goto done;
+
+ err:
+  hop = NULL;
+
+ done:
+  memwipe(keys, 0, sizeof(keys));
+  return hop;
+}
+
+/* Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
+ * <b>circ</b> ready for use to transfer HS relay cells. */
+static void
+finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
+                      int is_service_side)
+{
+  tor_assert(circ);
+  tor_assert(hop);
+
+  /* Notify the circuit state machine that we are splicing this circuit */
+  int new_circ_purpose = is_service_side ?
+    CIRCUIT_PURPOSE_S_REND_JOINED : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circuit_change_purpose(TO_CIRCUIT(circ), new_circ_purpose);
+
+  /* All is well. Extend the circuit. */
+  hop->state = CPATH_STATE_OPEN;
+  /* Set the windows to default. */
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  /* Now that this circuit has finished connecting to its destination,
+   * make sure circuit_get_open_circ_or_launch is willing to return it
+   * so we can actually use it. */
+  circ->hs_circ_has_timed_out = 0;
+
+  /* Append the hop to the cpath of this circuit */
+  onion_append_to_cpath(&circ->cpath, hop);
+
+  /* In legacy code, 'pending_final_cpath' points to the final hop we just
+   * appended to the cpath. We set the original pointer to NULL so that we
+   * don't double free it. */
+  if (circ->build_state) {
+    circ->build_state->pending_final_cpath = NULL;
+  }
+
+  /* Finally, mark circuit as ready to be used for client streams */
+  if (!is_service_side) {
+    circuit_try_attaching_streams(circ);
+  }
+}
+
+/* For a given circuit and a service introduction point object, register the
+ * intro circuit to the circuitmap. This supports legacy intro point. */
+static void
+register_intro_circ(const hs_service_intro_point_t *ip,
+                    origin_circuit_t *circ)
+{
+  tor_assert(ip);
+  tor_assert(circ);
+
+  if (ip->base.is_only_legacy) {
+    hs_circuitmap_register_intro_circ_v2_service_side(circ,
+                                                      ip->legacy_key_digest);
+  } else {
+    hs_circuitmap_register_intro_circ_v3_service_side(circ,
+                                         &ip->auth_key_kp.pubkey);
+  }
+}
+
+/* Return the number of opened introduction circuit for the given circuit that
+ * is matching its identity key. */
+static unsigned int
+count_opened_desc_intro_point_circuits(const hs_service_t *service,
+                                       const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    const circuit_t *circ;
+    const origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc == NULL) {
+      continue;
+    }
+    circ = TO_CIRCUIT(ocirc);
+    tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+               circ->purpose == CIRCUIT_PURPOSE_S_INTRO);
+    /* Having a circuit not for the requested service is really bad. */
+    tor_assert(ed25519_pubkey_eq(&service->keys.identity_pk,
+                                 &ocirc->hs_ident->identity_pk));
+    /* Only count opened circuit and skip circuit that will be closed. */
+    if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN) {
+      count++;
+    }
+  } DIGEST256MAP_FOREACH_END;
+  return count;
+}
+
+/* From a given service, rendezvous cookie and handshake info, create a
+ * rendezvous point circuit identifier. This can't fail. */
+STATIC hs_ident_circuit_t *
+create_rp_circuit_identifier(const hs_service_t *service,
+                             const uint8_t *rendezvous_cookie,
+                             const curve25519_public_key_t *server_pk,
+                             const hs_ntor_rend_cell_keys_t *keys)
+{
+  hs_ident_circuit_t *ident;
+  uint8_t handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
+
+  tor_assert(service);
+  tor_assert(rendezvous_cookie);
+  tor_assert(server_pk);
+  tor_assert(keys);
+
+  ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
+  memcpy(ident->rendezvous_cookie, rendezvous_cookie,
+         sizeof(ident->rendezvous_cookie));
+  /* Build the HANDSHAKE_INFO which looks like this:
+   *    SERVER_PK        [32 bytes]
+   *    AUTH_INPUT_MAC   [32 bytes]
+   */
+  memcpy(handshake_info, server_pk->public_key, CURVE25519_PUBKEY_LEN);
+  memcpy(handshake_info + CURVE25519_PUBKEY_LEN, keys->rend_cell_auth_mac,
+         DIGEST256_LEN);
+  tor_assert(sizeof(ident->rendezvous_handshake_info) ==
+             sizeof(handshake_info));
+  memcpy(ident->rendezvous_handshake_info, handshake_info,
+         sizeof(ident->rendezvous_handshake_info));
+  /* Finally copy the NTOR_KEY_SEED for e2e encryption on the circuit. */
+  tor_assert(sizeof(ident->rendezvous_ntor_key_seed) ==
+             sizeof(keys->ntor_key_seed));
+  memcpy(ident->rendezvous_ntor_key_seed, keys->ntor_key_seed,
+         sizeof(ident->rendezvous_ntor_key_seed));
+  return ident;
+}
+
+/* From a given service and service intro point, create an introduction point
+ * circuit identifier. This can't fail. */
+static hs_ident_circuit_t *
+create_intro_circuit_identifier(const hs_service_t *service,
+                                const hs_service_intro_point_t *ip)
+{
+  hs_ident_circuit_t *ident;
+
+  tor_assert(service);
+  tor_assert(ip);
+
+  ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                               HS_IDENT_CIRCUIT_INTRO);
+  ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
+
+  return ident;
+}
+
+/* For a given introduction point and an introduction circuit, send the
+ * ESTABLISH_INTRO cell. The service object is used for logging. This can fail
+ * and if so, the circuit is closed and the intro point object is flagged
+ * that the circuit is not established anymore which is important for the
+ * retry mechanism. */
+static void
+send_establish_intro(const hs_service_t *service,
+                     hs_service_intro_point_t *ip, origin_circuit_t *circ)
+{
+  ssize_t cell_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(circ);
+
+  /* Encode establish intro cell. */
+  cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
+                                           ip, payload);
+  if (cell_len < 0) {
+    log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
+                      "on circuit %u. Closing circuit.",
+             safe_str_client(service->onion_address),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+
+  /* Send the cell on the circuit. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_INTRO,
+                                   (char *) payload, cell_len,
+                                   circ->cpath->prev) < 0) {
+    log_info(LD_REND, "Unable to send ESTABLISH_INTRO cell for service %s "
+                      "on circuit %u.",
+             safe_str_client(service->onion_address),
+             TO_CIRCUIT(circ)->n_circ_id);
+    /* On error, the circuit has been closed. */
+    goto done;
+  }
+
+  /* Record the attempt to use this circuit. */
+  pathbias_count_use_attempt(circ);
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+ done:
+  memwipe(payload, 0, sizeof(payload));
+}
+
+/* Return a string constant describing the anonymity of service. */
+static const char *
+get_service_anonymity_string(const hs_service_t *service)
+{
+  if (service->config.is_single_onion) {
+    return "single onion";
+  } else {
+    return "hidden";
+  }
+}
+
+/* For a given service, the ntor onion key and a rendezvous cookie, launch a
+ * circuit to the rendezvous point specified by the link specifiers. On
+ * success, a circuit identifier is attached to the circuit with the needed
+ * data. This function will try to open a circuit for a maximum value of
+ * MAX_REND_FAILURES then it will give up. */
+static void
+launch_rendezvous_point_circuit(const hs_service_t *service,
+                                const hs_service_intro_point_t *ip,
+                                const hs_cell_introduce2_data_t *data)
+{
+  int circ_needs_uptime;
+  time_t now = time(NULL);
+  extend_info_t *info = NULL;
+  origin_circuit_t *circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(data);
+
+  circ_needs_uptime = hs_service_requires_uptime_circ(service->config.ports);
+
+  /* Get the extend info data structure for the chosen rendezvous point
+   * specified by the given link specifiers. */
+  info = hs_get_extend_info_from_lspecs(data->link_specifiers,
+                                        &data->onion_pk,
+                                        service->config.is_single_onion);
+  if (info == NULL) {
+    /* We are done here, we can't extend to the rendezvous point.
+     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
+     * 0.3.2 clients, and somehow disable the option check, it will fail here.
+     */
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Not enough info to open a circuit to a rendezvous point for "
+           "%s service %s.",
+           get_service_anonymity_string(service),
+           safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  for (int i = 0; i < MAX_REND_FAILURES; i++) {
+    int circ_flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
+    if (circ_needs_uptime) {
+      circ_flags |= CIRCLAUNCH_NEED_UPTIME;
+    }
+    /* Firewall and policies are checked when getting the extend info. */
+    if (service->config.is_single_onion) {
+      circ_flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
+    }
+
+    circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND, info,
+                                         circ_flags);
+    if (circ != NULL) {
+      /* Stop retrying, we have a circuit! */
+      break;
+    }
+  }
+  if (circ == NULL) {
+    log_warn(LD_REND, "Giving up on launching a rendezvous circuit to %s "
+                      "for %s service %s",
+             safe_str_client(extend_info_describe(info)),
+             get_service_anonymity_string(service),
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+  log_info(LD_REND, "Rendezvous circuit launched to %s with cookie %s "
+                    "for %s service %s",
+           safe_str_client(extend_info_describe(info)),
+           safe_str_client(hex_str((const char *) data->rendezvous_cookie,
+                                   REND_COOKIE_LEN)),
+           get_service_anonymity_string(service),
+           safe_str_client(service->onion_address));
+  tor_assert(circ->build_state);
+  /* Rendezvous circuit have a specific timeout for the time spent on trying
+   * to connect to the rendezvous point. */
+  circ->build_state->expiry_time = now + MAX_REND_TIMEOUT;
+
+  /* Create circuit identifier and key material. */
+  {
+    hs_ntor_rend_cell_keys_t keys;
+    curve25519_keypair_t ephemeral_kp;
+    /* No need for extra strong, this is only for this circuit life time. This
+     * key will be used for the RENDEZVOUS1 cell that will be sent on the
+     * circuit once opened. */
+    curve25519_keypair_generate(&ephemeral_kp, 0);
+    if (hs_ntor_service_get_rendezvous1_keys(&ip->auth_key_kp.pubkey,
+                                             &ip->enc_key_kp,
+                                             &ephemeral_kp, &data->client_pk,
+                                             &keys) < 0) {
+      /* This should not really happened but just in case, don't make tor
+       * freak out, close the circuit and move on. */
+      log_info(LD_REND, "Unable to get RENDEZVOUS1 key material for "
+                        "service %s",
+               safe_str_client(service->onion_address));
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+      goto end;
+    }
+    circ->hs_ident = create_rp_circuit_identifier(service,
+                                                  data->rendezvous_cookie,
+                                                  &ephemeral_kp.pubkey, &keys);
+    memwipe(&ephemeral_kp, 0, sizeof(ephemeral_kp));
+    memwipe(&keys, 0, sizeof(keys));
+    tor_assert(circ->hs_ident);
+  }
+
+ end:
+  extend_info_free(info);
+}
+
+/* Return true iff the given service rendezvous circuit circ is allowed for a
+ * relaunch to the rendezvous point. */
+static int
+can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  /* This is initialized when allocating an origin circuit. */
+  tor_assert(circ->build_state);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* XXX: Retrying under certain condition. This is related to #22455. */
+
+  /* Avoid to relaunch twice a circuit to the same rendezvous point at the
+   * same time. */
+  if (circ->hs_service_side_rend_circ_has_been_relaunched) {
+    log_info(LD_REND, "Rendezvous circuit to %s has already been retried. "
+                      "Skipping retry.",
+             safe_str_client(
+                  extend_info_describe(circ->build_state->chosen_exit)));
+    goto disallow;
+  }
+
+  /* We check failure_count >= hs_get_service_max_rend_failures()-1 below, and
+   * the -1 is because we increment the failure count for our current failure
+   * *after* this clause. */
+  int max_rend_failures = hs_get_service_max_rend_failures() - 1;
+
+  /* A failure count that has reached maximum allowed or circuit that expired,
+   * we skip relaunching. */
+  if (circ->build_state->failure_count > max_rend_failures ||
+      circ->build_state->expiry_time <= time(NULL)) {
+    log_info(LD_REND, "Attempt to build a rendezvous circuit to %s has "
+                      "failed with %d attempts and expiry time %ld. "
+                      "Giving up building.",
+             safe_str_client(
+                  extend_info_describe(circ->build_state->chosen_exit)),
+             circ->build_state->failure_count,
+             (long int) circ->build_state->expiry_time);
+    goto disallow;
+  }
+
+  /* Allowed to relaunch. */
+  return 1;
+ disallow:
+  return 0;
+}
+
+/* Retry the rendezvous point of circ by launching a new circuit to it. */
+static void
+retry_service_rendezvous_point(const origin_circuit_t *circ)
+{
+  int flags = 0;
+  origin_circuit_t *new_circ;
+  cpath_build_state_t *bstate;
+
+  tor_assert(circ);
+  /* This is initialized when allocating an origin circuit. */
+  tor_assert(circ->build_state);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Ease our life. */
+  bstate = circ->build_state;
+
+  log_info(LD_REND, "Retrying rendezvous point circuit to %s",
+           safe_str_client(extend_info_describe(bstate->chosen_exit)));
+
+  /* Get the current build state flags for the next circuit. */
+  flags |= (bstate->need_uptime) ? CIRCLAUNCH_NEED_UPTIME : 0;
+  flags |= (bstate->need_capacity) ? CIRCLAUNCH_NEED_CAPACITY : 0;
+  flags |= (bstate->is_internal) ? CIRCLAUNCH_IS_INTERNAL : 0;
+
+  /* We do NOT add the onehop tunnel flag even though it might be a single
+   * onion service. The reason is that if we failed once to connect to the RP
+   * with a direct connection, we consider that chances are that we will fail
+   * again so try a 3-hop circuit and hope for the best. Because the service
+   * has no anonymity (single onion), this change of behavior won't affect
+   * security directly. */
+
+  new_circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
+                                           bstate->chosen_exit, flags);
+  if (new_circ == NULL) {
+    log_warn(LD_REND, "Failed to launch rendezvous circuit to %s",
+             safe_str_client(extend_info_describe(bstate->chosen_exit)));
+    goto done;
+  }
+
+  /* Transfer build state information to the new circuit state in part to
+   * catch any other failures. */
+  new_circ->build_state->failure_count = bstate->failure_count+1;
+  new_circ->build_state->expiry_time = bstate->expiry_time;
+  new_circ->hs_ident = hs_ident_circuit_dup(circ->hs_ident);
+
+ done:
+  return;
+}
+
+/* Add all possible link specifiers in node to lspecs:
+ *  - legacy ID is mandatory thus MUST be present in node;
+ *  - include ed25519 link specifier if present in the node, and the node
+ *    supports ed25519 link authentication, even if its link versions are not
+ *    compatible with us;
+ *  - include IPv4 link specifier, if the primary address is not IPv4, log a
+ *    BUG() warning, and return an empty smartlist;
+ *  - include IPv6 link specifier if present in the node. */
+static void
+get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+
+  tor_assert(node);
+  tor_assert(lspecs);
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it, and the node declares a
+     protocol version that supports ed25519 link authentication, even if that
+     link version is not compatible with us. (We are sending the ed25519 key
+     to another tor, which may support different link versions.) */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
+      node_supports_ed25519_link_authentication(node, 0)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+}
+
+/* Using the given descriptor intro point ip, the node of the
+ * rendezvous point rp_node and the service's subcredential, populate the
+ * already allocated intro1_data object with the needed key material and link
+ * specifiers.
+ *
+ * If rp_node has an invalid primary address, intro1_data->link_specifiers
+ * will be an empty list. Otherwise, this function can't fail. The ip
+ * MUST be a valid object containing the needed keys and authentication
+ * method. */
+static void
+setup_introduce1_data(const hs_desc_intro_point_t *ip,
+                      const node_t *rp_node,
+                      const uint8_t *subcredential,
+                      hs_cell_introduce1_data_t *intro1_data)
+{
+  smartlist_t *rp_lspecs;
+
+  tor_assert(ip);
+  tor_assert(rp_node);
+  tor_assert(subcredential);
+  tor_assert(intro1_data);
+
+  /* Build the link specifiers from the extend information of the rendezvous
+   * circuit that we've picked previously. */
+  rp_lspecs = smartlist_new();
+  get_lspecs_from_node(rp_node, rp_lspecs);
+
+  /* Populate the introduce1 data object. */
+  memset(intro1_data, 0, sizeof(hs_cell_introduce1_data_t));
+  if (ip->legacy.key != NULL) {
+    intro1_data->is_legacy = 1;
+    intro1_data->legacy_key = ip->legacy.key;
+  }
+  intro1_data->auth_pk = &ip->auth_key_cert->signed_key;
+  intro1_data->enc_pk = &ip->enc_key;
+  intro1_data->subcredential = subcredential;
+  intro1_data->onion_pk = node_get_curve25519_onion_key(rp_node);
+  intro1_data->link_specifiers = rp_lspecs;
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* Return an introduction point circuit matching the given intro point object.
+ * NULL is returned is no such circuit can be found. */
+origin_circuit_t *
+hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
+{
+  tor_assert(ip);
+
+  if (ip->base.is_only_legacy) {
+    return hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest);
+  } else {
+    return hs_circuitmap_get_intro_circ_v3_service_side(
+                                        &ip->auth_key_kp.pubkey);
+  }
+}
+
+/* Called when we fail building a rendezvous circuit at some point other than
+ * the last hop: launches a new circuit to the same rendezvous point. This
+ * supports legacy service.
+ *
+ * We currently relaunch connections to rendezvous points if:
+ * - A rendezvous circuit timed out before connecting to RP.
+ * - The rendezvous circuit failed to connect to the RP.
+ *
+ * We avoid relaunching a connection to this rendezvous point if:
+ * - We have already tried MAX_REND_FAILURES times to connect to this RP,
+ * - We've been trying to connect to this RP for more than MAX_REND_TIMEOUT
+ *   seconds, or
+ * - We've already retried this specific rendezvous circuit.
+ */
+void
+hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Check if we are allowed to relaunch to the rendezvous point of circ. */
+  if (!can_relaunch_service_rendezvous_point(circ)) {
+    goto done;
+  }
+
+  /* Flag the circuit that we are relaunching, to avoid to relaunch twice a
+   * circuit to the same rendezvous point at the same time. */
+  circ->hs_service_side_rend_circ_has_been_relaunched = 1;
+
+  /* Legacy services don't have a hidden service ident. */
+  if (circ->hs_ident) {
+    retry_service_rendezvous_point(circ);
+  } else {
+    rend_service_relaunch_rendezvous(circ);
+  }
+
+ done:
+  return;
+}
+
+/* For a given service and a service intro point, launch a circuit to the
+ * extend info ei. If the service is a single onion, a one-hop circuit will be
+ * requested. Return 0 if the circuit was successfully launched and tagged
+ * with the correct identifier. On error, a negative value is returned. */
+int
+hs_circ_launch_intro_point(hs_service_t *service,
+                           const hs_service_intro_point_t *ip,
+                           extend_info_t *ei)
+{
+  /* Standard flags for introduction circuit. */
+  int ret = -1, circ_flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  origin_circuit_t *circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(ei);
+
+  /* Update circuit flags in case of a single onion service that requires a
+   * direct connection. */
+  if (service->config.is_single_onion) {
+    circ_flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
+  }
+
+  log_info(LD_REND, "Launching a circuit to intro point %s for service %s.",
+           safe_str_client(extend_info_describe(ei)),
+           safe_str_client(service->onion_address));
+
+  /* Note down the launch for the retry period. Even if the circuit fails to
+   * be launched, we still want to respect the retry period to avoid stress on
+   * the circuit subsystem. */
+  service->state.num_intro_circ_launched++;
+  circ = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                       ei, circ_flags);
+  if (circ == NULL) {
+    goto end;
+  }
+
+  /* Setup the circuit identifier and attach it to it. */
+  circ->hs_ident = create_intro_circuit_identifier(service, ip);
+  tor_assert(circ->hs_ident);
+  /* Register circuit in the global circuitmap. */
+  register_intro_circ(ip, circ);
+
+  /* Success. */
+  ret = 0;
+ end:
+  return ret;
+}
+
+/* Called when a service introduction point circuit is done building. Given
+ * the service and intro point object, this function will send the
+ * ESTABLISH_INTRO cell on the circuit. Return 0 on success. Return 1 if the
+ * circuit has been repurposed to General because we already have too many
+ * opened. */
+int
+hs_circ_service_intro_has_opened(hs_service_t *service,
+                                 hs_service_intro_point_t *ip,
+                                 const hs_service_descriptor_t *desc,
+                                 origin_circuit_t *circ)
+{
+  int ret = 0;
+  unsigned int num_intro_circ, num_needed_circ;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(desc);
+  tor_assert(circ);
+
+  /* Cound opened circuits that have sent ESTABLISH_INTRO cells or are already
+   * established introduction circuits */
+  num_intro_circ = count_opened_desc_intro_point_circuits(service, desc);
+  num_needed_circ = service->config.num_intro_points;
+  if (num_intro_circ > num_needed_circ) {
+    /* There are too many opened valid intro circuit for what the service
+     * needs so repurpose this one. */
+
+    /* XXX: Legacy code checks options->ExcludeNodes and if not NULL it just
+     * closes the circuit. I have NO idea why it does that so it hasn't been
+     * added here. I can only assume in case our ExcludeNodes list changes but
+     * in that case, all circuit are flagged unusable (config.c). --dgoulet */
+
+    log_info(LD_CIRC | LD_REND, "Introduction circuit just opened but we "
+                                "have enough for service %s. Repurposing "
+                                "it to general and leaving internal.",
+             safe_str_client(service->onion_address));
+    tor_assert(circ->build_state->is_internal);
+    /* Remove it from the circuitmap. */
+    hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
+    /* Cleaning up the hidden service identifier and repurpose. */
+    hs_ident_circuit_free(circ->hs_ident);
+    circ->hs_ident = NULL;
+    if (circuit_should_use_vanguards(TO_CIRCUIT(circ)->purpose))
+      circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_HS_VANGUARDS);
+    else
+      circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_GENERAL);
+
+    /* Inform that this circuit just opened for this new purpose. */
+    circuit_has_opened(circ);
+    /* This return value indicate to the caller that the IP object should be
+     * removed from the service because it's corresponding circuit has just
+     * been repurposed. */
+    ret = 1;
+    goto done;
+  }
+
+  log_info(LD_REND, "Introduction circuit %u established for service %s.",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+  circuit_log_path(LOG_INFO, LD_REND, circ);
+
+  /* Time to send an ESTABLISH_INTRO cell on this circuit. On error, this call
+   * makes sure the circuit gets closed. */
+  send_establish_intro(service, ip, circ);
+
+ done:
+  return ret;
+}
+
+/* Called when a service rendezvous point circuit is done building. Given the
+ * service and the circuit, this function will send a RENDEZVOUS1 cell on the
+ * circuit using the information in the circuit identifier. If the cell can't
+ * be sent, the circuit is closed. */
+void
+hs_circ_service_rp_has_opened(const hs_service_t *service,
+                              origin_circuit_t *circ)
+{
+  size_t payload_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(circ->hs_ident);
+
+  /* Some useful logging. */
+  log_info(LD_REND, "Rendezvous circuit %u has opened with cookie %s "
+                    "for service %s",
+           TO_CIRCUIT(circ)->n_circ_id,
+           hex_str((const char *) circ->hs_ident->rendezvous_cookie,
+                   REND_COOKIE_LEN),
+           safe_str_client(service->onion_address));
+  circuit_log_path(LOG_INFO, LD_REND, circ);
+
+  /* This can't fail. */
+  payload_len = hs_cell_build_rendezvous1(
+                        circ->hs_ident->rendezvous_cookie,
+                        sizeof(circ->hs_ident->rendezvous_cookie),
+                        circ->hs_ident->rendezvous_handshake_info,
+                        sizeof(circ->hs_ident->rendezvous_handshake_info),
+                        payload);
+
+  /* Pad the payload with random bytes so it matches the size of a legacy cell
+   * which is normally always bigger. Also, the size of a legacy cell is
+   * always smaller than the RELAY_PAYLOAD_SIZE so this is safe. */
+  if (payload_len < HS_LEGACY_RENDEZVOUS_CELL_SIZE) {
+    crypto_rand((char *) payload + payload_len,
+                HS_LEGACY_RENDEZVOUS_CELL_SIZE - payload_len);
+    payload_len = HS_LEGACY_RENDEZVOUS_CELL_SIZE;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_RENDEZVOUS1,
+                                   (const char *) payload, payload_len,
+                                   circ->cpath->prev) < 0) {
+    /* On error, circuit is closed. */
+    log_warn(LD_REND, "Unable to send RENDEZVOUS1 cell on circuit %u "
+                      "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Setup end-to-end rendezvous circuit between the client and us. */
+  if (hs_circuit_setup_e2e_rend_circ(circ,
+                       circ->hs_ident->rendezvous_ntor_key_seed,
+                       sizeof(circ->hs_ident->rendezvous_ntor_key_seed),
+                       1) < 0) {
+    log_warn(LD_GENERAL, "Failed to setup circ");
+    goto done;
+  }
+
+ done:
+  memwipe(payload, 0, sizeof(payload));
+}
+
+/* Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
+ * the INTRO_ESTABLISHED cell payload of length payload_len arriving on the
+ * given introduction circuit circ. The service is only used for logging
+ * purposes. Return 0 on success else a negative value. */
+int
+hs_circ_handle_intro_established(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                 origin_circuit_t *circ,
+                                 const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(service);
+  tor_assert(ip);
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)) {
+    goto done;
+  }
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. For a legacy node, it's an empty payload so as long as we
+   * have the cell, we are good. */
+  if (!ip->base.is_only_legacy &&
+      hs_cell_parse_intro_established(payload, payload_len) < 0) {
+    log_warn(LD_REND, "Unable to parse the INTRO_ESTABLISHED cell on "
+                      "circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto done;
+  }
+
+  /* Switch the purpose to a fully working intro point. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_S_INTRO);
+  /* Getting a valid INTRODUCE_ESTABLISHED means we've successfully used the
+   * circuit so update our pathbias subsystem. */
+  pathbias_mark_use_success(circ);
+  /* Success. */
+  ret = 0;
+
+ done:
+  return ret;
+}
+
+/* We just received an INTRODUCE2 cell on the established introduction circuit
+ * circ.  Handle the INTRODUCE2 payload of size payload_len for the given
+ * circuit and service. This cell is associated with the intro point object ip
+ * and the subcredential. Return 0 on success else a negative value. */
+int
+hs_circ_handle_introduce2(const hs_service_t *service,
+                          const origin_circuit_t *circ,
+                          hs_service_intro_point_t *ip,
+                          const uint8_t *subcredential,
+                          const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+  time_t elapsed;
+  hs_cell_introduce2_data_t data;
+
+  tor_assert(service);
+  tor_assert(circ);
+  tor_assert(ip);
+  tor_assert(subcredential);
+  tor_assert(payload);
+
+  /* Populate the data structure with everything we need for the cell to be
+   * parsed, decrypted and key material computed correctly. */
+  data.auth_pk = &ip->auth_key_kp.pubkey;
+  data.enc_kp = &ip->enc_key_kp;
+  data.subcredential = subcredential;
+  data.payload = payload;
+  data.payload_len = payload_len;
+  data.link_specifiers = smartlist_new();
+  data.replay_cache = ip->replay_cache;
+
+  if (hs_cell_parse_introduce2(&data, circ, service) < 0) {
+    goto done;
+  }
+
+  /* Check whether we've seen this REND_COOKIE before to detect repeats. */
+  if (replaycache_add_test_and_elapsed(
+           service->state.replay_cache_rend_cookie,
+           data.rendezvous_cookie, sizeof(data.rendezvous_cookie),
+           &elapsed)) {
+    /* A Tor client will send a new INTRODUCE1 cell with the same REND_COOKIE
+     * as its previous one if its intro circ times out while in state
+     * CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT. If we received the first
+     * INTRODUCE1 cell (the intro-point relay converts it into an INTRODUCE2
+     * cell), we are already trying to connect to that rend point (and may
+     * have already succeeded); drop this cell. */
+    log_info(LD_REND, "We received an INTRODUCE2 cell with same REND_COOKIE "
+                      "field %ld seconds ago. Dropping cell.",
+             (long int) elapsed);
+    goto done;
+  }
+
+  /* At this point, we just confirmed that the full INTRODUCE2 cell is valid
+   * so increment our counter that we've seen one on this intro point. */
+  ip->introduce2_count++;
+
+  /* Launch rendezvous circuit with the onion key and rend cookie. */
+  launch_rendezvous_point_circuit(service, ip, &data);
+  /* Success. */
+  ret = 0;
+
+ done:
+  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(data.link_specifiers);
+  memwipe(&data, 0, sizeof(data));
+  return ret;
+}
+
+/* Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
+ * exchange output material at <b>ntor_key_seed</b> and setup <b>circ</b> to
+ * serve as a rendezvous end-to-end circuit between the client and the
+ * service. If <b>is_service_side</b> is set, then we are the hidden service
+ * and the other side is the client.
+ *
+ * Return 0 if the operation went well; in case of error return -1. */
+int
+hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
+                               const uint8_t *ntor_key_seed, size_t seed_len,
+                               int is_service_side)
+{
+  if (BUG(!circuit_purpose_is_correct_for_rend(TO_CIRCUIT(circ)->purpose,
+                                        is_service_side))) {
+    return -1;
+  }
+
+  crypt_path_t *hop = create_rend_cpath(ntor_key_seed, seed_len,
+                                        is_service_side);
+  if (!hop) {
+    log_warn(LD_REND, "Couldn't get v3 %s cpath!",
+             is_service_side ? "service-side" : "client-side");
+    return -1;
+  }
+
+  finalize_rend_circuit(circ, hop, is_service_side);
+
+  return 0;
+}
+
+/* We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
+ * <b>rend_cell_body</b> on <b>circ</b>. Finish up the DH key exchange and then
+ * extend the crypt path of <b>circ</b> so that the hidden service is on the
+ * other side. */
+int
+hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
+                                             const uint8_t *rend_cell_body)
+{
+
+  if (BUG(!circuit_purpose_is_correct_for_rend(
+                                      TO_CIRCUIT(circ)->purpose, 0))) {
+    return -1;
+  }
+
+  crypt_path_t *hop = create_rend_cpath_legacy(circ, rend_cell_body);
+  if (!hop) {
+    log_warn(LD_GENERAL, "Couldn't get v2 cpath.");
+    return -1;
+  }
+
+  finalize_rend_circuit(circ, hop, 0);
+
+  return 0;
+}
+
+/* Given the introduction circuit intro_circ, the rendezvous circuit
+ * rend_circ, a descriptor intro point object ip and the service's
+ * subcredential, send an INTRODUCE1 cell on intro_circ.
+ *
+ * This will also setup the circuit identifier on rend_circ containing the key
+ * material for the handshake and e2e encryption. Return 0 on success else
+ * negative value. Because relay_send_command_from_edge() closes the circuit
+ * on error, it is possible that intro_circ is closed on error. */
+int
+hs_circ_send_introduce1(origin_circuit_t *intro_circ,
+                        origin_circuit_t *rend_circ,
+                        const hs_desc_intro_point_t *ip,
+                        const uint8_t *subcredential)
+{
+  int ret = -1;
+  ssize_t payload_len;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  hs_cell_introduce1_data_t intro1_data;
+
+  tor_assert(intro_circ);
+  tor_assert(rend_circ);
+  tor_assert(ip);
+  tor_assert(subcredential);
+
+  /* It is undefined behavior in hs_cell_introduce1_data_clear() if intro1_data
+   * has been declared on the stack but not initialized. Here, we set it to 0.
+   */
+  memset(&intro1_data, 0, sizeof(hs_cell_introduce1_data_t));
+
+  /* This takes various objects in order to populate the introduce1 data
+   * object which is used to build the content of the cell. */
+  const node_t *exit_node = build_state_get_exit_node(rend_circ->build_state);
+  if (exit_node == NULL) {
+    log_info(LD_REND, "Unable to get rendezvous point for circuit %u. "
+             "Failing.", TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+  setup_introduce1_data(ip, exit_node, subcredential, &intro1_data);
+  /* If we didn't get any link specifiers, it's because our node was
+   * bad. */
+  if (BUG(!intro1_data.link_specifiers) ||
+      !smartlist_len(intro1_data.link_specifiers)) {
+    log_warn(LD_REND, "Unable to get link specifiers for INTRODUCE1 cell on "
+             "circuit %u.", TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+
+  /* Final step before we encode a cell, we setup the circuit identifier which
+   * will generate both the rendezvous cookie and client keypair for this
+   * connection. Those are put in the ident. */
+  intro1_data.rendezvous_cookie = rend_circ->hs_ident->rendezvous_cookie;
+  intro1_data.client_kp = &rend_circ->hs_ident->rendezvous_client_kp;
+
+  memcpy(intro_circ->hs_ident->rendezvous_cookie,
+         rend_circ->hs_ident->rendezvous_cookie,
+         sizeof(intro_circ->hs_ident->rendezvous_cookie));
+
+  /* From the introduce1 data object, this will encode the INTRODUCE1 cell
+   * into payload which is then ready to be sent as is. */
+  payload_len = hs_cell_build_introduce1(&intro1_data, payload);
+  if (BUG(payload_len < 0)) {
+    goto done;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(intro_circ),
+                                   RELAY_COMMAND_INTRODUCE1,
+                                   (const char *) payload, payload_len,
+                                   intro_circ->cpath->prev) < 0) {
+    /* On error, circuit is closed. */
+    log_warn(LD_REND, "Unable to send INTRODUCE1 cell on circuit %u.",
+             TO_CIRCUIT(intro_circ)->n_circ_id);
+    goto done;
+  }
+
+  /* Success. */
+  ret = 0;
+  goto done;
+
+ done:
+  hs_cell_introduce1_data_clear(&intro1_data);
+  memwipe(payload, 0, sizeof(payload));
+  return ret;
+}
+
+/* Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
+ * success, 0 is returned else -1 and the circuit is marked for close. */
+int
+hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
+{
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  log_info(LD_REND, "Send an ESTABLISH_RENDEZVOUS cell on circuit %u",
+           TO_CIRCUIT(circ)->n_circ_id);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it,
+   * and the rend cookie also means we've used the circ. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+
+  /* We've attempted to use this circuit. Probe it if we fail */
+  pathbias_count_use_attempt(circ);
+
+  /* Generate the RENDEZVOUS_COOKIE and place it in the identifier so we can
+   * complete the handshake when receiving the acknowledgement. */
+  crypto_rand((char *) circ->hs_ident->rendezvous_cookie, HS_REND_COOKIE_LEN);
+  /* Generate the client keypair. No need to be extra strong, not long term */
+  curve25519_keypair_generate(&circ->hs_ident->rendezvous_client_kp, 0);
+
+  cell_len =
+    hs_cell_build_establish_rendezvous(circ->hs_ident->rendezvous_cookie,
+                                       cell);
+  if (BUG(cell_len < 0)) {
+    goto err;
+  }
+
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                   RELAY_COMMAND_ESTABLISH_RENDEZVOUS,
+                                   (const char *) cell, cell_len,
+                                   circ->cpath->prev) < 0) {
+    /* Circuit has been marked for close */
+    log_warn(LD_REND, "Unable to send ESTABLISH_RENDEZVOUS cell on "
+                      "circuit %u", TO_CIRCUIT(circ)->n_circ_id);
+    memwipe(cell, 0, cell_len);
+    goto err;
+  }
+
+  memwipe(cell, 0, cell_len);
+  return 0;
+ err:
+  return -1;
+}
+
+/* We are about to close or free this <b>circ</b>. Clean it up from any
+ * related HS data structures. This function can be called multiple times
+ * safely for the same circuit. */
+void
+hs_circ_cleanup(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* If it's a service-side intro circ, notify the HS subsystem for the intro
+   * point circuit closing so it can be dealt with cleanly. */
+  if (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
+      circ->purpose == CIRCUIT_PURPOSE_S_INTRO) {
+    hs_service_intro_circ_has_closed(TO_ORIGIN_CIRCUIT(circ));
+  }
+
+  /* Clear HS circuitmap token for this circ (if any). Very important to be
+   * done after the HS subsystem has been notified of the close else the
+   * circuit will not be found.
+   *
+   * We do this at the close if possible because from that point on, the
+   * circuit is good as dead. We can't rely on removing it in the circuit
+   * free() function because we open a race window between the close and free
+   * where we can't register a new circuit for the same intro point. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
+
diff --git a/src/or/hs_circuit.h b/src/or/hs_circuit.h
new file mode 100644
index 0000000000..f69137e1d5
--- /dev/null
+++ b/src/or/hs_circuit.h
@@ -0,0 +1,75 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuit.h
+ * \brief Header file containing circuit data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_CIRCUIT_H
+#define TOR_HS_CIRCUIT_H
+
+#include "or.h"
+#include "crypto_ed25519.h"
+
+#include "hs_service.h"
+
+/* Cleanup function when the circuit is closed or/and freed. */
+void hs_circ_cleanup(circuit_t *circ);
+
+/* Circuit API. */
+int hs_circ_service_intro_has_opened(hs_service_t *service,
+                                     hs_service_intro_point_t *ip,
+                                     const hs_service_descriptor_t *desc,
+                                     origin_circuit_t *circ);
+void hs_circ_service_rp_has_opened(const hs_service_t *service,
+                                   origin_circuit_t *circ);
+int hs_circ_launch_intro_point(hs_service_t *service,
+                               const hs_service_intro_point_t *ip,
+                               extend_info_t *ei);
+int hs_circ_launch_rendezvous_point(const hs_service_t *service,
+                                    const curve25519_public_key_t *onion_key,
+                                    const uint8_t *rendezvous_cookie);
+void hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ);
+
+origin_circuit_t *hs_circ_service_get_intro_circ(
+                                      const hs_service_intro_point_t *ip);
+
+/* Cell API. */
+int hs_circ_handle_intro_established(const hs_service_t *service,
+                                     const hs_service_intro_point_t *ip,
+                                     origin_circuit_t *circ,
+                                     const uint8_t *payload,
+                                     size_t payload_len);
+int hs_circ_handle_introduce2(const hs_service_t *service,
+                              const origin_circuit_t *circ,
+                              hs_service_intro_point_t *ip,
+                              const uint8_t *subcredential,
+                              const uint8_t *payload, size_t payload_len);
+int hs_circ_send_introduce1(origin_circuit_t *intro_circ,
+                            origin_circuit_t *rend_circ,
+                            const hs_desc_intro_point_t *ip,
+                            const uint8_t *subcredential);
+int hs_circ_send_establish_rendezvous(origin_circuit_t *circ);
+
+/* e2e circuit API. */
+
+int hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
+                                   const uint8_t *ntor_key_seed,
+                                   size_t seed_len,
+                                   int is_service_side);
+int hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
+                                          const uint8_t *rend_cell_body);
+
+#ifdef HS_CIRCUIT_PRIVATE
+
+STATIC hs_ident_circuit_t *
+create_rp_circuit_identifier(const hs_service_t *service,
+                             const uint8_t *rendezvous_cookie,
+                             const curve25519_public_key_t *server_pk,
+                             const hs_ntor_rend_cell_keys_t *keys);
+
+#endif /* defined(HS_CIRCUIT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CIRCUIT_H) */
+
diff --git a/src/or/hs_circuitmap.c b/src/or/hs_circuitmap.c
new file mode 100644
index 0000000000..112c8bdced
--- /dev/null
+++ b/src/or/hs_circuitmap.c
@@ -0,0 +1,583 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.c
+ *
+ * \brief Hidden service circuitmap: A hash table that maps binary tokens to
+ *  introduction and rendezvous circuits; it's used:
+ *  (a) by relays acting as intro points and rendezvous points
+ *  (b) by hidden services to find intro and rend circuits and
+ *  (c) by HS clients to find rendezvous circuits.
+ **/
+
+#define HS_CIRCUITMAP_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "hs_circuitmap.h"
+
+/************************** HS circuitmap code *******************************/
+
+/* This is the hidden service circuitmap. It's a hash table that maps
+   introduction and rendezvous tokens to specific circuits such that given a
+   token it's easy to find the corresponding circuit. */
+static struct hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+/* This is a helper function used by the hash table code (HT_). It returns 1 if
+ * two circuits have the same HS token. */
+static int
+hs_circuits_have_same_token(const circuit_t *first_circuit,
+                            const circuit_t *second_circuit)
+{
+  const hs_token_t *first_token;
+  const hs_token_t *second_token;
+
+  tor_assert(first_circuit);
+  tor_assert(second_circuit);
+
+  first_token = first_circuit->hs_token;
+  second_token = second_circuit->hs_token;
+
+  /* Both circs must have a token */
+  if (BUG(!first_token) || BUG(!second_token)) {
+    return 0;
+  }
+
+  if (first_token->type != second_token->type) {
+    return 0;
+  }
+
+  if (first_token->token_len != second_token->token_len)
+    return 0;
+
+  return tor_memeq(first_token->token,
+                   second_token->token,
+                   first_token->token_len);
+}
+
+/* This is a helper function for the hash table code (HT_). It hashes a circuit
+ * HS token into an unsigned int for use as a key by the hash table routines.*/
+static inline unsigned int
+hs_circuit_hash_token(const circuit_t *circuit)
+{
+  tor_assert(circuit->hs_token);
+
+  return (unsigned) siphash24g(circuit->hs_token->token,
+                               circuit->hs_token->token_len);
+}
+
+/* Register the circuitmap hash table */
+HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct
+             circuit_t,    // The name of the element struct,
+             hs_circuitmap_node,        // The name of HT_ENTRY member
+             hs_circuit_hash_token, hs_circuits_have_same_token)
+
+HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node,
+             hs_circuit_hash_token, hs_circuits_have_same_token,
+             0.6, tor_reallocarray, tor_free_)
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global HS circuitmap. Used by unittests. */
+hs_circuitmap_ht *
+get_hs_circuitmap(void)
+{
+  return the_hs_circuitmap;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/****************** HS circuitmap utility functions **************************/
+
+/** Return a new HS token of type <b>type</b> containing <b>token</b>. */
+static hs_token_t *
+hs_token_new(hs_token_type_t type, size_t token_len,
+             const uint8_t *token)
+{
+  tor_assert(token);
+
+  hs_token_t *hs_token = tor_malloc_zero(sizeof(hs_token_t));
+  hs_token->type = type;
+  hs_token->token_len = token_len;
+  hs_token->token = tor_memdup(token, token_len);
+
+  return hs_token;
+}
+
+#define hs_token_free(val) \
+  FREE_AND_NULL(hs_token_t, hs_token_free_, (val))
+
+/** Free memory allocated by this <b>hs_token</b>. */
+static void
+hs_token_free_(hs_token_t *hs_token)
+{
+  if (!hs_token) {
+    return;
+  }
+
+  tor_free(hs_token->token);
+  tor_free(hs_token);
+}
+
+/** Return the circuit from the circuitmap with token <b>search_token</b>. */
+static circuit_t *
+get_circuit_with_token(hs_token_t *search_token)
+{
+  tor_assert(the_hs_circuitmap);
+
+  /* We use a dummy circuit object for the hash table search routine. */
+  circuit_t search_circ;
+  search_circ.hs_token = search_token;
+  return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ);
+}
+
+/* Helper function that registers <b>circ</b> with <b>token</b> on the HS
+   circuitmap. This function steals reference of <b>token</b>. */
+static void
+hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token)
+{
+  tor_assert(circ);
+  tor_assert(token);
+  tor_assert(the_hs_circuitmap);
+
+  /* If this circuit already has a token, clear it. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+
+  /* Kill old circuits with the same token. We want new intro/rend circuits to
+     take precedence over old ones, so that HSes and clients and reestablish
+     killed circuits without changing the HS token. */
+  {
+    circuit_t *found_circ;
+    found_circ = get_circuit_with_token(token);
+    if (found_circ) {
+      hs_circuitmap_remove_circuit(found_circ);
+      if (!found_circ->marked_for_close) {
+        circuit_mark_for_close(found_circ, END_CIRC_REASON_FINISHED);
+      }
+    }
+  }
+
+  /* Register circuit and token to circuitmap. */
+  circ->hs_token = token;
+  HT_INSERT(hs_circuitmap_ht, the_hs_circuitmap, circ);
+}
+
+/** Helper function: Register <b>circ</b> of <b>type</b> on the HS
+ *  circuitmap. Use the HS <b>token</b> as the key to the hash table.  If
+ *  <b>token</b> is not set, clear the circuit of any HS tokens. */
+static void
+hs_circuitmap_register_circuit(circuit_t *circ,
+                               hs_token_type_t type, size_t token_len,
+                               const uint8_t *token)
+{
+  hs_token_t *hs_token = NULL;
+
+  /* Create a new token and register it to the circuitmap */
+  tor_assert(token);
+  hs_token = hs_token_new(type, token_len, token);
+  tor_assert(hs_token);
+  hs_circuitmap_register_impl(circ, hs_token);
+}
+
+/* Helper function for hs_circuitmap_get_origin_circuit() and
+ * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the
+ * circuitmap, this function returns object type so the specialized functions
+ * using this helper can upcast it to the right type.
+ *
+ * Return NULL if not such circuit is found. */
+static circuit_t *
+hs_circuitmap_get_circuit_impl(hs_token_type_t type,
+                               size_t token_len,
+                               const uint8_t *token,
+                               uint8_t wanted_circ_purpose)
+{
+  circuit_t *found_circ = NULL;
+
+  tor_assert(the_hs_circuitmap);
+
+  /* Check the circuitmap if we have a circuit with this token */
+  {
+    hs_token_t *search_hs_token = hs_token_new(type, token_len, token);
+    tor_assert(search_hs_token);
+    found_circ = get_circuit_with_token(search_hs_token);
+    hs_token_free(search_hs_token);
+  }
+
+  /* Check that the circuit is useful to us */
+  if (!found_circ ||
+      found_circ->purpose != wanted_circ_purpose ||
+      found_circ->marked_for_close) {
+    return NULL;
+  }
+
+  return found_circ;
+}
+
+/* Helper function: Query circuitmap for origin circuit with <b>token</b> of
+ * size <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose
+ * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked
+ * for close. Return NULL if no such circuit is found. */
+static origin_circuit_t *
+hs_circuitmap_get_origin_circuit(hs_token_type_t type,
+                                 size_t token_len,
+                                 const uint8_t *token,
+                                 uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  return TO_ORIGIN_CIRCUIT(circ);
+}
+
+/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size
+ * <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose equal
+ * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for
+ * close. Return NULL if no such circuit is found. */
+static or_circuit_t *
+hs_circuitmap_get_or_circuit(hs_token_type_t type,
+                             size_t token_len,
+                             const uint8_t *token,
+                             uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(!CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORCIRC(circ));
+  return TO_OR_CIRCUIT(circ);
+}
+
+/************** Public circuitmap API ****************************************/
+
+/**** Public relay-side getters: */
+
+/* Public function: Return a v3 introduction circuit to this relay with
+ * <b>auth_key</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(
+                                          const ed25519_public_key_t *auth_key)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                      ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return v2 introduction circuit to this relay with
+ * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                      REND_TOKEN_LEN, digest,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return rendezvous circuit to this relay with rendezvous
+ * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_REND_RELAY_SIDE,
+                                      REND_TOKEN_LEN, cookie,
+                                      CIRCUIT_PURPOSE_REND_POINT_WAITING);
+}
+
+/** Public relay-side setters: */
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
+                                            const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_RELAY_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
+                                                const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/**** Public servide-side getters: */
+
+/* Public function: Return v3 introduction circuit with <b>auth_key</b>
+ * originating from this hidden service. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return v2 introduction circuit originating from this hidden
+ * service with <b>digest</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return rendezvous circuit originating from this hidden
+ * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Try to check if we have a connecting circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_CONNECT_REND);
+  if (circ) {
+    return circ;
+  }
+
+  /* Then try for connected circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_REND_JOINED);
+  return circ;
+}
+
+/* Public function: Return client-side rendezvous circuit with rendezvous
+ * <b>cookie</b>. It will look for circuits with the following purposes:
+
+ * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
+ *    RENDEZVOUS_ESTABLISHED). Waiting for RENDEZVOUS2 from service, and for
+ *    INTRODUCE_ACK from intro point.
+ *
+ * b) CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: Established rend circuit and
+ *    introduce circuit acked. Waiting for RENDEZVOUS2 from service.
+ *
+ * c) CIRCUIT_PURPOSE_C_REND_JOINED: Established rend circuit and received
+ *    RENDEZVOUS2 from service.
+ *
+ * d) CIRCUIT_PURPOSE_C_ESTABLISH_REND: Rend circuit open but not yet
+ *    established.
+ *
+ * Return NULL if no such circuit is found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = hs_circuitmap_get_established_rend_circ_client_side(cookie);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  return circ;
+}
+
+/*  Public function: Return client-side established rendezvous circuit with
+ *  rendezvous <b>cookie</b>. It will look for circuits with the following
+ *  purposes:
+ *
+ * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
+ *    RENDEZVOUS_ESTABLISHED). Waiting for RENDEZVOUS2 from service, and for
+ *    INTRODUCE_ACK from intro point.
+ *
+ * b) CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: Established rend circuit and
+ *    introduce circuit acked. Waiting for RENDEZVOUS2 from service.
+ *
+ * c) CIRCUIT_PURPOSE_C_REND_JOINED: Established rend circuit and received
+ *    RENDEZVOUS2 from service.
+ *
+ * Return NULL if no such circuit is found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_REND_READY);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                 CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+  if (circ) {
+    return circ;
+  }
+
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_CLIENT_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_C_REND_JOINED);
+  return circ;
+}
+
+/**** Public servide-side setters: */
+
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
+                                                  const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
+                                              const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * client-side circuitmap. */
+void
+hs_circuitmap_register_rend_circ_client_side(origin_circuit_t *or_circ,
+                                             const uint8_t *cookie)
+{
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+  { /* Basic circ purpose sanity checking */
+    tor_assert_nonfatal(circ->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+  }
+
+  hs_circuitmap_register_circuit(circ, HS_TOKEN_REND_CLIENT_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/**** Misc public functions: */
+
+/** Public function: Remove this circuit from the HS circuitmap. Clear its HS
+ *  token, and remove it from the hashtable. */
+void
+hs_circuitmap_remove_circuit(circuit_t *circ)
+{
+  tor_assert(the_hs_circuitmap);
+
+  if (!circ || !circ->hs_token) {
+    return;
+  }
+
+  /* Remove circ from circuitmap */
+  circuit_t *tmp;
+  tmp = HT_REMOVE(hs_circuitmap_ht, the_hs_circuitmap, circ);
+  /* ... and ensure the removal was successful. */
+  if (tmp) {
+    tor_assert(tmp == circ);
+  } else {
+    log_warn(LD_BUG, "Could not find circuit (%u) in circuitmap.",
+             circ->n_circ_id);
+  }
+
+  /* Clear token from circ */
+  hs_token_free(circ->hs_token);
+  circ->hs_token = NULL;
+}
+
+/* Public function: Initialize the global HS circuitmap. */
+void
+hs_circuitmap_init(void)
+{
+  tor_assert(!the_hs_circuitmap);
+
+  the_hs_circuitmap = tor_malloc_zero(sizeof(struct hs_circuitmap_ht));
+  HT_INIT(hs_circuitmap_ht, the_hs_circuitmap);
+}
+
+/* Public function: Free all memory allocated by the global HS circuitmap. */
+void
+hs_circuitmap_free_all(void)
+{
+  if (the_hs_circuitmap) {
+    HT_CLEAR(hs_circuitmap_ht, the_hs_circuitmap);
+    tor_free(the_hs_circuitmap);
+  }
+}
+
diff --git a/src/or/hs_circuitmap.h b/src/or/hs_circuitmap.h
new file mode 100644
index 0000000000..9e653480b5
--- /dev/null
+++ b/src/or/hs_circuitmap.h
@@ -0,0 +1,113 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.h
+ * \brief Header file for hs_circuitmap.c.
+ **/
+
+#ifndef TOR_HS_CIRCUITMAP_H
+#define TOR_HS_CIRCUITMAP_H
+
+typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht;
+
+typedef struct hs_token_s hs_token_t;
+struct or_circuit_t;
+struct origin_circuit_t;
+
+/** Public HS circuitmap API: */
+
+/** Public relay-side API: */
+
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(const
+                                           ed25519_public_key_t *auth_key);
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest);
+struct or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ,
+                                                 const uint8_t *cookie);
+void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
+                                                     const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
+                                         const ed25519_public_key_t *auth_key);
+
+/** Public service-side API: */
+
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key);
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie);
+struct origin_circuit_t *
+hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_intro_circ_v2_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const ed25519_public_key_t *auth_key);
+void hs_circuitmap_register_rend_circ_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *cookie);
+void hs_circuitmap_register_rend_circ_client_side(
+                                      struct origin_circuit_t *circ,
+                                      const uint8_t *cookie);
+
+void hs_circuitmap_remove_circuit(struct circuit_t *circ);
+
+void hs_circuitmap_init(void);
+void hs_circuitmap_free_all(void);
+
+#ifdef HS_CIRCUITMAP_PRIVATE
+
+/** Represents the type of HS token. */
+typedef enum {
+  /** A rendezvous cookie on a relay (128bit)*/
+  HS_TOKEN_REND_RELAY_SIDE,
+  /** A v2 introduction point pubkey on a relay (160bit) */
+  HS_TOKEN_INTRO_V2_RELAY_SIDE,
+  /** A v3 introduction point pubkey on a relay (256bit) */
+  HS_TOKEN_INTRO_V3_RELAY_SIDE,
+
+  /** A rendezvous cookie on a hidden service (128bit)*/
+  HS_TOKEN_REND_SERVICE_SIDE,
+  /** A v2 introduction point pubkey on a hidden service (160bit) */
+  HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+  /** A v3 introduction point pubkey on a hidden service (256bit) */
+  HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+
+  /** A rendezvous cookie on the client side (128bit) */
+  HS_TOKEN_REND_CLIENT_SIDE,
+} hs_token_type_t;
+
+/** Represents a token used in the HS protocol. Each such token maps to a
+ *  specific introduction or rendezvous circuit. */
+struct hs_token_s {
+  /* Type of HS token. */
+  hs_token_type_t type;
+
+  /* The size of the token (depends on the type). */
+  size_t token_len;
+
+  /* The token itself. Memory allocated at runtime. */
+  uint8_t *token;
+};
+
+#endif /* defined(HS_CIRCUITMAP_PRIVATE) */
+
+#ifdef TOR_UNIT_TESTS
+
+hs_circuitmap_ht *get_hs_circuitmap(void);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* !defined(TOR_HS_CIRCUITMAP_H) */
+
diff --git a/src/or/hs_client.c b/src/or/hs_client.c
new file mode 100644
index 0000000000..8ecefc1209
--- /dev/null
+++ b/src/or/hs_client.c
@@ -0,0 +1,1613 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_client.c
+ * \brief Implement next generation hidden service client functionality
+ **/
+
+#define HS_CLIENT_PRIVATE
+
+#include "or.h"
+#include "circpathbias.h"
+#include "circuitbuild.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "config.h"
+#include "connection.h"
+#include "connection_edge.h"
+#include "container.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "directory.h"
+#include "hs_cache.h"
+#include "hs_cell.h"
+#include "hs_circuit.h"
+#include "hs_client.h"
+#include "hs_control.h"
+#include "hs_descriptor.h"
+#include "hs_ident.h"
+#include "hs_ntor.h"
+#include "networkstatus.h"
+#include "nodelist.h"
+#include "reasons.h"
+#include "rendclient.h"
+#include "router.h"
+#include "routerset.h"
+
+/* Return a human-readable string for the client fetch status code. */
+static const char *
+fetch_status_to_string(hs_client_fetch_status_t status)
+{
+  switch (status) {
+  case HS_CLIENT_FETCH_ERROR:
+    return "Internal error";
+  case HS_CLIENT_FETCH_LAUNCHED:
+    return "Descriptor fetch launched";
+  case HS_CLIENT_FETCH_HAVE_DESC:
+    return "Already have descriptor";
+  case HS_CLIENT_FETCH_NO_HSDIRS:
+    return "No more HSDir available to query";
+  case HS_CLIENT_FETCH_NOT_ALLOWED:
+    return "Fetching descriptors is not allowed";
+  case HS_CLIENT_FETCH_MISSING_INFO:
+    return "Missing directory information";
+  case HS_CLIENT_FETCH_PENDING:
+    return "Pending descriptor fetch";
+  default:
+    return "(Unknown client fetch status code)";
+  }
+}
+
+/* Return true iff tor should close the SOCKS request(s) for the descriptor
+ * fetch that ended up with this given status code. */
+static int
+fetch_status_should_close_socks(hs_client_fetch_status_t status)
+{
+  switch (status) {
+  case HS_CLIENT_FETCH_NO_HSDIRS:
+    /* No more HSDir to query, we can't complete the SOCKS request(s). */
+  case HS_CLIENT_FETCH_ERROR:
+    /* The fetch triggered an internal error. */
+  case HS_CLIENT_FETCH_NOT_ALLOWED:
+    /* Client is not allowed to fetch (FetchHidServDescriptors 0). */
+    goto close;
+  case HS_CLIENT_FETCH_MISSING_INFO:
+  case HS_CLIENT_FETCH_HAVE_DESC:
+  case HS_CLIENT_FETCH_PENDING:
+  case HS_CLIENT_FETCH_LAUNCHED:
+    /* The rest doesn't require tor to close the SOCKS request(s). */
+    goto no_close;
+  }
+
+ no_close:
+  return 0;
+ close:
+  return 1;
+}
+
+/* Cancel all descriptor fetches currently in progress. */
+static void
+cancel_descriptor_fetches(void)
+{
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
+    if (BUG(ident == NULL)) {
+      /* A directory connection fetching a service descriptor can't have an
+       * empty hidden service identifier. */
+      continue;
+    }
+    log_debug(LD_REND, "Marking for close a directory connection fetching "
+                       "a hidden service descriptor for service %s.",
+              safe_str_client(ed25519_fmt(&ident->identity_pk)));
+    connection_mark_for_close(conn);
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* No ownership of the objects in this list. */
+  smartlist_free(conns);
+  log_info(LD_REND, "Hidden service client descriptor fetches cancelled.");
+}
+
+/* Get all connections that are waiting on a circuit and flag them back to
+ * waiting for a hidden service descriptor for the given service key
+ * service_identity_pk. */
+static void
+flag_all_conn_wait_desc(const ed25519_public_key_t *service_identity_pk)
+{
+  tor_assert(service_identity_pk);
+
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    edge_connection_t *edge_conn;
+    if (BUG(!CONN_IS_EDGE(conn))) {
+      continue;
+    }
+    edge_conn = TO_EDGE_CONN(conn);
+    if (edge_conn->hs_ident &&
+        ed25519_pubkey_eq(&edge_conn->hs_ident->identity_pk,
+                          service_identity_pk)) {
+      connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
+      conn->state = AP_CONN_STATE_RENDDESC_WAIT;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  smartlist_free(conns);
+}
+
+/* Remove tracked HSDir requests from our history for this hidden service
+ * identity public key. */
+static void
+purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+  ed25519_public_key_t blinded_pk;
+
+  tor_assert(identity_pk);
+
+  /* Get blinded pubkey of hidden service. It is possible that we just moved
+   * to a new time period meaning that we won't be able to purge the request
+   * from the previous time period. That is fine because they will expire at
+   * some point and we don't care about those anymore. */
+  hs_build_blinded_pubkey(identity_pk, NULL, 0,
+                          hs_get_time_period_num(0), &blinded_pk);
+  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
+    return;
+  }
+  /* Purge last hidden service request from cache for this blinded key. */
+  hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
+}
+
+/* Return true iff there is at least one pending directory descriptor request
+ * for the service identity_pk. */
+static int
+directory_request_is_pending(const ed25519_public_key_t *identity_pk)
+{
+  int ret = 0;
+  smartlist_t *conns =
+    connection_list_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+    const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
+    if (BUG(ident == NULL)) {
+      /* A directory connection fetching a service descriptor can't have an
+       * empty hidden service identifier. */
+      continue;
+    }
+    if (!ed25519_pubkey_eq(identity_pk, &ident->identity_pk)) {
+      continue;
+    }
+    ret = 1;
+    break;
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* No ownership of the objects in this list. */
+  smartlist_free(conns);
+  return ret;
+}
+
+/* We failed to fetch a descriptor for the service with <b>identity_pk</b>
+ * because of <b>status</b>. Find all pending SOCKS connections for this
+ * service that are waiting on the descriptor and close them with
+ * <b>reason</b>. */
+static void
+close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
+                                       hs_client_fetch_status_t status,
+                                       int reason)
+{
+  unsigned int count = 0;
+  time_t now = approx_time();
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we tried to get the descriptor */
+    if (!edge_conn->hs_ident ||
+        !ed25519_pubkey_eq(identity_pk,
+                           &edge_conn->hs_ident->identity_pk)) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+    /* Unattach the entry connection which will close for the reason. */
+    connection_mark_unattached_ap(entry_conn, reason);
+    count++;
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  if (count > 0) {
+    char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    hs_build_address(identity_pk, HS_VERSION_THREE, onion_address);
+    log_notice(LD_REND, "Closed %u streams for service %s.onion "
+                        "for reason %s. Fetch status: %s.",
+               count, safe_str_client(onion_address),
+               stream_end_reason_to_string(reason),
+               fetch_status_to_string(status));
+  }
+
+  /* No ownership of the object(s) in this list. */
+  smartlist_free(conns);
+}
+
+/* Find all pending SOCKS connection waiting for a descriptor and retry them
+ * all. This is called when the directory information changed. */
+static void
+retry_all_socks_conn_waiting_for_desc(void)
+{
+  smartlist_t *conns =
+    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    hs_client_fetch_status_t status;
+    const edge_connection_t *edge_conn =
+      ENTRY_TO_EDGE_CONN(TO_ENTRY_CONN(base_conn));
+
+    /* Ignore non HS or non v3 connection. */
+    if (edge_conn->hs_ident == NULL) {
+      continue;
+    }
+    /* In this loop, we will possibly try to fetch a descriptor for the
+     * pending connections because we just got more directory information.
+     * However, the refetch process can cleanup all SOCKS request so the same
+     * service if an internal error happens. Thus, we can end up with closed
+     * connections in our list. */
+    if (base_conn->marked_for_close) {
+      continue;
+    }
+
+    /* XXX: There is an optimization we could do which is that for a service
+     * key, we could check if we can fetch and remember that decision. */
+
+    /* Order a refetch in case it works this time. */
+    status = hs_client_refetch_hsdesc(&edge_conn->hs_ident->identity_pk);
+    if (BUG(status == HS_CLIENT_FETCH_HAVE_DESC)) {
+      /* This case is unique because it can NOT happen in theory. Once we get
+       * a new descriptor, the HS client subsystem is notified immediately and
+       * the connections waiting for it are handled which means the state will
+       * change from renddesc wait state. Log this and continue to next
+       * connection. */
+      continue;
+    }
+    /* In the case of an error, either all SOCKS connections have been
+     * closed or we are still missing directory information. Leave the
+     * connection in renddesc wait state so when we get more info, we'll be
+     * able to try it again. */
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  /* We don't have ownership of those objects. */
+  smartlist_free(conns);
+}
+
+/* A v3 HS circuit successfully connected to the hidden service. Update the
+ * stream state at <b>hs_conn_ident</b> appropriately. */
+static void
+note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
+{
+  tor_assert(hs_conn_ident);
+
+  /* Remove from the hid serv cache all requests for that service so we can
+   * query the HSDir again later on for various reasons. */
+  purge_hid_serv_request(&hs_conn_ident->identity_pk);
+
+  /* The v2 subsystem cleans up the intro point time out flag at this stage.
+   * We don't try to do it here because we still need to keep intact the intro
+   * point state for future connections. Even though we are able to connect to
+   * the service, doesn't mean we should reset the timed out intro points.
+   *
+   * It is not possible to have successfully connected to an intro point
+   * present in our cache that was on error or timed out. Every entry in that
+   * cache have a 2 minutes lifetime so ultimately the intro point(s) state
+   * will be reset and thus possible to be retried. */
+}
+
+/* Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
+ * descriptor by launching a dir connection to <b>hsdir</b>. Return a
+ * hs_client_fetch_status_t status code depending on how it went. */
+static hs_client_fetch_status_t
+directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const routerstatus_t *hsdir)
+{
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  ed25519_public_key_t blinded_pubkey;
+  char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
+  hs_ident_dir_conn_t hs_conn_dir_ident;
+  int retval;
+
+  tor_assert(hsdir);
+  tor_assert(onion_identity_pk);
+
+  /* Get blinded pubkey */
+  hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
+                          current_time_period, &blinded_pubkey);
+  /* ...and base64 it. */
+  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
+  if (BUG(retval < 0)) {
+    return HS_CLIENT_FETCH_ERROR;
+  }
+
+  /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
+  hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
+                         &hs_conn_dir_ident);
+
+  /* Setup directory request */
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_HSDESC);
+  directory_request_set_routerstatus(req, hsdir);
+  directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+  directory_request_set_resource(req, base64_blinded_pubkey);
+  directory_request_fetch_set_hs_ident(req, &hs_conn_dir_ident);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
+  log_info(LD_REND, "Descriptor fetch request for service %s with blinded "
+                    "key %s to directory %s",
+           safe_str_client(ed25519_fmt(onion_identity_pk)),
+           safe_str_client(base64_blinded_pubkey),
+           safe_str_client(routerstatus_describe(hsdir)));
+
+  /* Fire a REQUESTED event on the control port. */
+  hs_control_desc_event_requested(onion_identity_pk, base64_blinded_pubkey,
+                                  hsdir);
+
+  /* Cleanup memory. */
+  memwipe(&blinded_pubkey, 0, sizeof(blinded_pubkey));
+  memwipe(base64_blinded_pubkey, 0, sizeof(base64_blinded_pubkey));
+  memwipe(&hs_conn_dir_ident, 0, sizeof(hs_conn_dir_ident));
+
+  return HS_CLIENT_FETCH_LAUNCHED;
+}
+
+/** Return the HSDir we should use to fetch the descriptor of the hidden
+ *  service with identity key <b>onion_identity_pk</b>. */
+STATIC routerstatus_t *
+pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
+{
+  int retval;
+  char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  smartlist_t *responsible_hsdirs = NULL;
+  ed25519_public_key_t blinded_pubkey;
+  routerstatus_t *hsdir_rs = NULL;
+
+  tor_assert(onion_identity_pk);
+
+  /* Get blinded pubkey of hidden service */
+  hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
+                          current_time_period, &blinded_pubkey);
+  /* ...and base64 it. */
+  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
+  if (BUG(retval < 0)) {
+    return NULL;
+  }
+
+  /* Get responsible hsdirs of service for this time period */
+  responsible_hsdirs = smartlist_new();
+
+  hs_get_responsible_hsdirs(&blinded_pubkey, current_time_period,
+                            0, 1, responsible_hsdirs);
+
+  log_debug(LD_REND, "Found %d responsible HSDirs and about to pick one.",
+           smartlist_len(responsible_hsdirs));
+
+  /* Pick an HSDir from the responsible ones. The ownership of
+   * responsible_hsdirs is given to this function so no need to free it. */
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+
+  return hsdir_rs;
+}
+
+/** Fetch a v3 descriptor using the given <b>onion_identity_pk</b>.
+ *
+ * On success, HS_CLIENT_FETCH_LAUNCHED is returned. Otherwise, an error from
+ * hs_client_fetch_status_t is returned. */
+MOCK_IMPL(STATIC hs_client_fetch_status_t,
+fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
+{
+  routerstatus_t *hsdir_rs =NULL;
+
+  tor_assert(onion_identity_pk);
+
+  hsdir_rs = pick_hsdir_v3(onion_identity_pk);
+  if (!hsdir_rs) {
+    log_info(LD_REND, "Couldn't pick a v3 hsdir.");
+    return HS_CLIENT_FETCH_NO_HSDIRS;
+  }
+
+  return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
+}
+
+/* Make sure that the given v3 origin circuit circ is a valid correct
+ * introduction circuit. This will BUG() on any problems and hard assert if
+ * the anonymity of the circuit is not ok. Return 0 on success else -1 where
+ * the circuit should be mark for closed immediately. */
+static int
+intro_circ_is_ok(const origin_circuit_t *circ)
+{
+  int ret = 0;
+
+  tor_assert(circ);
+
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCING &&
+          TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT &&
+          TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) {
+    ret = -1;
+  }
+  if (BUG(circ->hs_ident == NULL)) {
+    ret = -1;
+  }
+  if (BUG(!hs_ident_intro_circ_is_valid(circ->hs_ident))) {
+    ret = -1;
+  }
+
+  /* This can stop the tor daemon but we want that since if we don't have
+   * anonymity on this circuit, something went really wrong. */
+  assert_circ_anonymity_ok(circ, get_options());
+  return ret;
+}
+
+/* Find a descriptor intro point object that matches the given ident in the
+ * given descriptor desc. Return NULL if not found. */
+static const hs_desc_intro_point_t *
+find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
+                               const hs_descriptor_t *desc)
+{
+  const hs_desc_intro_point_t *intro_point = NULL;
+
+  tor_assert(ident);
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    if (ed25519_pubkey_eq(&ident->intro_auth_pk,
+                          &ip->auth_key_cert->signed_key)) {
+      intro_point = ip;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  return intro_point;
+}
+
+/* Find a descriptor intro point object from the descriptor object desc that
+ * matches the given legacy identity digest in legacy_id. Return NULL if not
+ * found. */
+static hs_desc_intro_point_t *
+find_desc_intro_point_by_legacy_id(const char *legacy_id,
+                                   const hs_descriptor_t *desc)
+{
+  hs_desc_intro_point_t *ret_ip = NULL;
+
+  tor_assert(legacy_id);
+  tor_assert(desc);
+
+  /* We will go over every intro point and try to find which one is linked to
+   * that circuit. Those lists are small so it's not that expensive. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
+                            const hs_desc_link_specifier_t *, lspec) {
+      /* Not all tor node have an ed25519 identity key so we still rely on the
+       * legacy identity digest. */
+      if (lspec->type != LS_LEGACY_ID) {
+        continue;
+      }
+      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+        break;
+      }
+      /* Found it. */
+      ret_ip = ip;
+      goto end;
+    } SMARTLIST_FOREACH_END(lspec);
+  } SMARTLIST_FOREACH_END(ip);
+
+ end:
+  return ret_ip;
+}
+
+/* Send an INTRODUCE1 cell along the intro circuit and populate the rend
+ * circuit identifier with the needed key material for the e2e encryption.
+ * Return 0 on success, -1 if there is a transient error such that an action
+ * has been taken to recover and -2 if there is a permanent error indicating
+ * that both circuits were closed. */
+static int
+send_introduce1(origin_circuit_t *intro_circ,
+                origin_circuit_t *rend_circ)
+{
+  int status;
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  const ed25519_public_key_t *service_identity_pk = NULL;
+  const hs_desc_intro_point_t *ip;
+
+  tor_assert(rend_circ);
+  if (intro_circ_is_ok(intro_circ) < 0) {
+    goto perm_err;
+  }
+
+  service_identity_pk = &intro_circ->hs_ident->identity_pk;
+  /* For logging purposes. There will be a time where the hs_ident will have a
+   * version number but for now there is none because it's all v3. */
+  hs_build_address(service_identity_pk, HS_VERSION_THREE, onion_address);
+
+  log_info(LD_REND, "Sending INTRODUCE1 cell to service %s on circuit %u",
+           safe_str_client(onion_address), TO_CIRCUIT(intro_circ)->n_circ_id);
+
+  /* 1) Get descriptor from our cache. */
+  const hs_descriptor_t *desc =
+    hs_cache_lookup_as_client(service_identity_pk);
+  if (desc == NULL || !hs_client_any_intro_points_usable(service_identity_pk,
+                                                         desc)) {
+    log_info(LD_REND, "Request to %s %s. Trying to fetch a new descriptor.",
+             safe_str_client(onion_address),
+             (desc) ? "didn't have usable intro points" :
+             "didn't have a descriptor");
+    hs_client_refetch_hsdesc(service_identity_pk);
+    /* We just triggered a refetch, make sure every connections are back
+     * waiting for that descriptor. */
+    flag_all_conn_wait_desc(service_identity_pk);
+    /* We just asked for a refetch so this is a transient error. */
+    goto tran_err;
+  }
+
+  /* We need to find which intro point in the descriptor we are connected to
+   * on intro_circ. */
+  ip = find_desc_intro_point_by_ident(intro_circ->hs_ident, desc);
+  if (BUG(ip == NULL)) {
+    /* If we can find a descriptor from this introduction circuit ident, we
+     * must have a valid intro point object. Permanent error. */
+    goto perm_err;
+  }
+
+  /* Send the INTRODUCE1 cell. */
+  if (hs_circ_send_introduce1(intro_circ, rend_circ, ip,
+                              desc->subcredential) < 0) {
+    /* Unable to send the cell, the intro circuit has been marked for close so
+     * this is a permanent error. */
+    tor_assert_nonfatal(TO_CIRCUIT(intro_circ)->marked_for_close);
+    goto perm_err;
+  }
+
+  /* Cell has been sent successfully. Copy the introduction point
+   * authentication and encryption key in the rendezvous circuit identifier so
+   * we can compute the ntor keys when we receive the RENDEZVOUS2 cell. */
+  memcpy(&rend_circ->hs_ident->intro_enc_pk, &ip->enc_key,
+         sizeof(rend_circ->hs_ident->intro_enc_pk));
+  ed25519_pubkey_copy(&rend_circ->hs_ident->intro_auth_pk,
+                      &intro_circ->hs_ident->intro_auth_pk);
+
+  /* Now, we wait for an ACK or NAK on this circuit. */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT);
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the _C_INTRODUCE_ACK_WAIT state. */
+  TO_CIRCUIT(intro_circ)->timestamp_dirty = time(NULL);
+  pathbias_count_use_attempt(intro_circ);
+
+  /* Success. */
+  status = 0;
+  goto end;
+
+ perm_err:
+  /* Permanent error: it is possible that the intro circuit was closed prior
+   * because we weren't able to send the cell. Make sure we don't double close
+   * it which would result in a warning. */
+  if (!TO_CIRCUIT(intro_circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_INTERNAL);
+  }
+  circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_INTERNAL);
+  status = -2;
+  goto end;
+
+ tran_err:
+  status = -1;
+
+ end:
+  memwipe(onion_address, 0, sizeof(onion_address));
+  return status;
+}
+
+/* Using the introduction circuit circ, setup the authentication key of the
+ * intro point this circuit has extended to. */
+static void
+setup_intro_circ_auth_key(origin_circuit_t *circ)
+{
+  const hs_descriptor_t *desc;
+  const hs_desc_intro_point_t *ip;
+
+  tor_assert(circ);
+
+  desc = hs_cache_lookup_as_client(&circ->hs_ident->identity_pk);
+  if (BUG(desc == NULL)) {
+    /* Opening intro circuit without the descriptor is no good... */
+    goto end;
+  }
+
+  /* We will go over every intro point and try to find which one is linked to
+   * that circuit. Those lists are small so it's not that expensive. */
+  ip = find_desc_intro_point_by_legacy_id(
+                       circ->build_state->chosen_exit->identity_digest, desc);
+  if (ip) {
+    /* We got it, copy its authentication key to the identifier. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+    goto end;
+  }
+
+  /* Reaching this point means we didn't find any intro point for this circuit
+   * which is not suppose to happen. */
+  tor_assert_nonfatal_unreached();
+
+ end:
+  return;
+}
+
+/* Called when an introduction circuit has opened. */
+static void
+client_intro_circ_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
+  log_info(LD_REND, "Introduction circuit %u has opened. Attaching streams.",
+           (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+
+  /* This is an introduction circuit so we'll attach the correct
+   * authentication key to the circuit identifier so it can be identified
+   * properly later on. */
+  setup_intro_circ_auth_key(circ);
+
+  connection_ap_attach_pending(1);
+}
+
+/* Called when a rendezvous circuit has opened. */
+static void
+client_rendezvous_circ_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND);
+
+  const extend_info_t *rp_ei = circ->build_state->chosen_exit;
+
+  /* Check that we didn't accidentally choose a node that does not understand
+   * the v3 rendezvous protocol */
+  if (rp_ei) {
+    const node_t *rp_node = node_get_by_id(rp_ei->identity_digest);
+    if (rp_node) {
+      if (BUG(!node_supports_v3_rendezvous_point(rp_node))) {
+        return;
+      }
+    }
+  }
+
+  log_info(LD_REND, "Rendezvous circuit has opened to %s.",
+           safe_str_client(extend_info_describe(rp_ei)));
+
+  /* Ignore returned value, nothing we can really do. On failure, the circuit
+   * will be marked for close. */
+  hs_circ_send_establish_rendezvous(circ);
+
+  /* Register rend circuit in circuitmap if it's still alive. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    hs_circuitmap_register_rend_circ_client_side(circ,
+                                     circ->hs_ident->rendezvous_cookie);
+  }
+}
+
+/* This is an helper function that convert a descriptor intro point object ip
+ * to a newly allocated extend_info_t object fully initialized. Return NULL if
+ * we can't convert it for which chances are that we are missing or malformed
+ * link specifiers. */
+STATIC extend_info_t *
+desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
+{
+  extend_info_t *ei;
+  smartlist_t *lspecs = smartlist_new();
+
+  tor_assert(ip);
+
+  /* We first encode the descriptor link specifiers into the binary
+   * representation which is a trunnel object. */
+  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
+                          const hs_desc_link_specifier_t *, desc_lspec) {
+    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
+    smartlist_add(lspecs, lspec);
+  } SMARTLIST_FOREACH_END(desc_lspec);
+
+  /* Explicitly put the direct connection option to 0 because this is client
+   * side and there is no such thing as a non anonymous client. */
+  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
+  smartlist_free(lspecs);
+  return ei;
+}
+
+/* Return true iff the intro point ip for the service service_pk is usable.
+ * This function checks if the intro point is in the client intro state cache
+ * and checks at the failures. It is considered usable if:
+ *   - No error happened (INTRO_POINT_FAILURE_GENERIC)
+ *   - It is not flagged as timed out (INTRO_POINT_FAILURE_TIMEOUT)
+ *   - The unreachable count is lower than
+ *     MAX_INTRO_POINT_REACHABILITY_FAILURES (INTRO_POINT_FAILURE_UNREACHABLE)
+ */
+static int
+intro_point_is_usable(const ed25519_public_key_t *service_pk,
+                      const hs_desc_intro_point_t *ip)
+{
+  const hs_cache_intro_state_t *state;
+
+  tor_assert(service_pk);
+  tor_assert(ip);
+
+  state = hs_cache_client_intro_state_find(service_pk,
+                                           &ip->auth_key_cert->signed_key);
+  if (state == NULL) {
+    /* This means we've never encountered any problem thus usable. */
+    goto usable;
+  }
+  if (state->error) {
+    log_info(LD_REND, "Intro point with auth key %s had an error. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+  if (state->timed_out) {
+    log_info(LD_REND, "Intro point with auth key %s timed out. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+  if (state->unreachable_count >= MAX_INTRO_POINT_REACHABILITY_FAILURES) {
+    log_info(LD_REND, "Intro point with auth key %s unreachable. Not usable",
+             safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)));
+    goto not_usable;
+  }
+
+ usable:
+  return 1;
+ not_usable:
+  return 0;
+}
+
+/* Using a descriptor desc, return a newly allocated extend_info_t object of a
+ * randomly picked introduction point from its list. Return NULL if none are
+ * usable. */
+STATIC extend_info_t *
+client_get_random_intro(const ed25519_public_key_t *service_pk)
+{
+  extend_info_t *ei = NULL, *ei_excluded = NULL;
+  smartlist_t *usable_ips = NULL;
+  const hs_descriptor_t *desc;
+  const hs_desc_encrypted_data_t *enc_data;
+  const or_options_t *options = get_options();
+  /* Calculate the onion address for logging purposes */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(service_pk);
+
+  desc = hs_cache_lookup_as_client(service_pk);
+  /* Assume the service is v3 if the descriptor is missing. This is ok,
+   * because we only use the address in log messages */
+  hs_build_address(service_pk,
+                   desc ? desc->plaintext_data.version : HS_VERSION_THREE,
+                   onion_address);
+  if (desc == NULL || !hs_client_any_intro_points_usable(service_pk,
+                                                         desc)) {
+    log_info(LD_REND, "Unable to randomly select an introduction point "
+             "for service %s because descriptor %s. We can't connect.",
+             safe_str_client(onion_address),
+             (desc) ? "doesn't have any usable intro points"
+                    : "is missing (assuming v3 onion address)");
+    goto end;
+  }
+
+  enc_data = &desc->encrypted_data;
+  usable_ips = smartlist_new();
+  smartlist_add_all(usable_ips, enc_data->intro_points);
+  while (smartlist_len(usable_ips) != 0) {
+    int idx;
+    const hs_desc_intro_point_t *ip;
+
+    /* Pick a random intro point and immediately remove it from the usable
+     * list so we don't pick it again if we have to iterate more. */
+    idx = crypto_rand_int(smartlist_len(usable_ips));
+    ip = smartlist_get(usable_ips, idx);
+    smartlist_del(usable_ips, idx);
+
+    /* We need to make sure we have a usable intro points which is in a good
+     * state in our cache. */
+    if (!intro_point_is_usable(service_pk, ip)) {
+      continue;
+    }
+
+    /* Generate an extend info object from the intro point object. */
+    ei = desc_intro_point_to_extend_info(ip);
+    if (ei == NULL) {
+      /* We can get here for instance if the intro point is a private address
+       * and we aren't allowed to extend to those. */
+      log_info(LD_REND, "Unable to select introduction point with auth key %s "
+               "for service %s, because we could not extend to it.",
+               safe_str_client(ed25519_fmt(&ip->auth_key_cert->signed_key)),
+               safe_str_client(onion_address));
+      continue;
+    }
+
+    /* Test the pick against ExcludeNodes. */
+    if (routerset_contains_extendinfo(options->ExcludeNodes, ei)) {
+      /* If this pick is in the ExcludeNodes list, we keep its reference so if
+       * we ever end up not being able to pick anything else and StrictNodes is
+       * unset, we'll use it. */
+      if (ei_excluded) {
+        /* If something was already here free it. After the loop is gone we
+         * will examine the last excluded intro point, and that's fine since
+         * that's random anyway */
+        extend_info_free(ei_excluded);
+      }
+      ei_excluded = ei;
+      continue;
+    }
+
+    /* Good pick! Let's go with this. */
+    goto end;
+  }
+
+  /* Reaching this point means a couple of things. Either we can't use any of
+   * the intro point listed because the IP address can't be extended to or it
+   * is listed in the ExcludeNodes list. In the later case, if StrictNodes is
+   * set, we are forced to not use anything. */
+  ei = ei_excluded;
+  if (options->StrictNodes) {
+    log_warn(LD_REND, "Every introduction point for service %s is in the "
+             "ExcludeNodes set and StrictNodes is set. We can't connect.",
+             safe_str_client(onion_address));
+    extend_info_free(ei);
+    ei = NULL;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Every introduction point for service "
+           "%s is unusable or we can't extend to it. We can't connect.",
+           safe_str_client(onion_address));
+  }
+
+ end:
+  smartlist_free(usable_ips);
+  memwipe(onion_address, 0, sizeof(onion_address));
+  return ei;
+}
+
+/* For this introduction circuit, we'll look at if we have any usable
+ * introduction point left for this service. If so, we'll use the circuit to
+ * re-extend to a new intro point. Else, we'll close the circuit and its
+ * corresponding rendezvous circuit. Return 0 if we are re-extending else -1
+ * if we are closing the circuits.
+ *
+ * This is called when getting an INTRODUCE_ACK cell with a NACK. */
+static int
+close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
+{
+  int ret = -1;
+  const hs_descriptor_t *desc;
+  origin_circuit_t *rend_circ;
+
+  tor_assert(intro_circ);
+
+  desc = hs_cache_lookup_as_client(&intro_circ->hs_ident->identity_pk);
+  if (BUG(desc == NULL)) {
+    /* We can't continue without a descriptor. */
+    goto close;
+  }
+  /* We still have the descriptor, great! Let's try to see if we can
+   * re-extend by looking up if there are any usable intro points. */
+  if (!hs_client_any_intro_points_usable(&intro_circ->hs_ident->identity_pk,
+                                         desc)) {
+    goto close;
+  }
+  /* Try to re-extend now. */
+  if (hs_client_reextend_intro_circuit(intro_circ) < 0) {
+    goto close;
+  }
+  /* Success on re-extending. Don't return an error. */
+  ret = 0;
+  goto end;
+
+ close:
+  /* Change the intro circuit purpose before so we don't report an intro point
+   * failure again triggering an extra descriptor fetch. The circuit can
+   * already be closed on failure to re-extend. */
+  if (!TO_CIRCUIT(intro_circ)->marked_for_close) {
+    circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                           CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+    circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_FINISHED);
+  }
+  /* Close the related rendezvous circuit. */
+  rend_circ = hs_circuitmap_get_rend_circ_client_side(
+                                     intro_circ->hs_ident->rendezvous_cookie);
+  /* The rendezvous circuit might have collapsed while the INTRODUCE_ACK was
+   * inflight so we can't expect one every time. */
+  if (rend_circ) {
+    circuit_mark_for_close(TO_CIRCUIT(rend_circ), END_CIRC_REASON_FINISHED);
+  }
+
+ end:
+  return ret;
+}
+
+/* Called when we get an INTRODUCE_ACK success status code. Do the appropriate
+ * actions for the rendezvous point and finally close intro_circ. */
+static void
+handle_introduce_ack_success(origin_circuit_t *intro_circ)
+{
+  origin_circuit_t *rend_circ = NULL;
+
+  tor_assert(intro_circ);
+
+  log_info(LD_REND, "Received INTRODUCE_ACK ack! Informing rendezvous");
+
+  /* Get the rendezvous circuit for this rendezvous cookie. */
+  uint8_t *rendezvous_cookie = intro_circ->hs_ident->rendezvous_cookie;
+  rend_circ =
+  hs_circuitmap_get_established_rend_circ_client_side(rendezvous_cookie);
+  if (rend_circ == NULL) {
+    log_warn(LD_REND, "Can't find any rendezvous circuit. Stopping");
+    goto end;
+  }
+
+  assert_circ_anonymity_ok(rend_circ, get_options());
+
+  /* It is possible to get a RENDEZVOUS2 cell before the INTRODUCE_ACK which
+   * means that the circuit will be joined and already transmitting data. In
+   * that case, simply skip the purpose change and close the intro circuit
+   * like it should be. */
+  if (TO_CIRCUIT(rend_circ)->purpose == CIRCUIT_PURPOSE_C_REND_JOINED) {
+    goto end;
+  }
+  circuit_change_purpose(TO_CIRCUIT(rend_circ),
+                         CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the
+   * CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED state. */
+  TO_CIRCUIT(rend_circ)->timestamp_dirty = time(NULL);
+
+ end:
+  /* We don't need the intro circuit anymore. It did what it had to do! */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ),
+                         CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
+  circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_FINISHED);
+
+  /* XXX: Close pending intro circuits we might have in parallel. */
+  return;
+}
+
+/* Called when we get an INTRODUCE_ACK failure status code. Depending on our
+ * failure cache status, either close the circuit or re-extend to a new
+ * introduction point. */
+static void
+handle_introduce_ack_bad(origin_circuit_t *circ, int status)
+{
+  tor_assert(circ);
+
+  log_info(LD_REND, "Received INTRODUCE_ACK nack by %s. Reason: %u",
+      safe_str_client(extend_info_describe(circ->build_state->chosen_exit)),
+      status);
+
+  /* It's a NAK. The introduction point didn't relay our request. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
+
+  /* Note down this failure in the intro point failure cache. Depending on how
+   * many times we've tried this intro point, close it or reextend. */
+  hs_cache_client_intro_state_note(&circ->hs_ident->identity_pk,
+                                   &circ->hs_ident->intro_auth_pk,
+                                   INTRO_POINT_FAILURE_GENERIC);
+}
+
+/* Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
+ * cell is in payload of length payload_len. Return 0 on success else a
+ * negative value. The circuit is either close or reuse to re-extend to a new
+ * introduction point. */
+static int
+handle_introduce_ack(origin_circuit_t *circ, const uint8_t *payload,
+                     size_t payload_len)
+{
+  int status, ret = -1;
+
+  tor_assert(circ);
+  tor_assert(circ->build_state);
+  tor_assert(circ->build_state->chosen_exit);
+  assert_circ_anonymity_ok(circ, get_options());
+  tor_assert(payload);
+
+  status = hs_cell_parse_introduce_ack(payload, payload_len);
+  switch (status) {
+  case HS_CELL_INTRO_ACK_SUCCESS:
+    ret = 0;
+    handle_introduce_ack_success(circ);
+    goto end;
+  case HS_CELL_INTRO_ACK_FAILURE:
+  case HS_CELL_INTRO_ACK_BADFMT:
+  case HS_CELL_INTRO_ACK_NORELAY:
+    handle_introduce_ack_bad(circ, status);
+    /* We are going to see if we have to close the circuits (IP and RP) or we
+     * can re-extend to a new intro point. */
+    ret = close_or_reextend_intro_circ(circ);
+    break;
+  default:
+    log_info(LD_PROTOCOL, "Unknown INTRODUCE_ACK status code %u from %s",
+        status,
+        safe_str_client(extend_info_describe(circ->build_state->chosen_exit)));
+    break;
+  }
+
+ end:
+  return ret;
+}
+
+/* Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
+ * encoded cell is in payload of length payload_len. Return 0 on success or a
+ * negative value on error. On error, the circuit is marked for close. */
+STATIC int
+handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
+                   size_t payload_len)
+{
+  int ret = -1;
+  curve25519_public_key_t server_pk;
+  uint8_t auth_mac[DIGEST256_LEN] = {0};
+  uint8_t handshake_info[CURVE25519_PUBKEY_LEN + sizeof(auth_mac)] = {0};
+  hs_ntor_rend_cell_keys_t keys;
+  const hs_ident_circuit_t *ident;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Make things easier. */
+  ident = circ->hs_ident;
+  tor_assert(ident);
+
+  if (hs_cell_parse_rendezvous2(payload, payload_len, handshake_info,
+                                sizeof(handshake_info)) < 0) {
+    goto err;
+  }
+  /* Get from the handshake info the SERVER_PK and AUTH_MAC. */
+  memcpy(&server_pk, handshake_info, CURVE25519_PUBKEY_LEN);
+  memcpy(auth_mac, handshake_info + CURVE25519_PUBKEY_LEN, sizeof(auth_mac));
+
+  /* Generate the handshake info. */
+  if (hs_ntor_client_get_rendezvous1_keys(&ident->intro_auth_pk,
+                                          &ident->rendezvous_client_kp,
+                                          &ident->intro_enc_pk, &server_pk,
+                                          &keys) < 0) {
+    log_info(LD_REND, "Unable to compute the rendezvous keys.");
+    goto err;
+  }
+
+  /* Critical check, make sure that the MAC matches what we got with what we
+   * computed just above. */
+  if (!hs_ntor_client_rendezvous2_mac_is_good(&keys, auth_mac)) {
+    log_info(LD_REND, "Invalid MAC in RENDEZVOUS2. Rejecting cell.");
+    goto err;
+  }
+
+  /* Setup the e2e encryption on the circuit and finalize its state. */
+  if (hs_circuit_setup_e2e_rend_circ(circ, keys.ntor_key_seed,
+                                     sizeof(keys.ntor_key_seed), 0) < 0) {
+    log_info(LD_REND, "Unable to setup the e2e encryption.");
+    goto err;
+  }
+  /* Success. Hidden service connection finalized! */
+  ret = 0;
+  goto end;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+ end:
+  memwipe(&keys, 0, sizeof(keys));
+  return ret;
+}
+
+/* Return true iff the client can fetch a descriptor for this service public
+ * identity key and status_out if not NULL is untouched. If the client can
+ * _not_ fetch the descriptor and if status_out is not NULL, it is set with
+ * the fetch status code. */
+static unsigned int
+can_client_refetch_desc(const ed25519_public_key_t *identity_pk,
+                        hs_client_fetch_status_t *status_out)
+{
+  hs_client_fetch_status_t status;
+
+  tor_assert(identity_pk);
+
+  /* Are we configured to fetch descriptors? */
+  if (!get_options()->FetchHidServDescriptors) {
+    log_warn(LD_REND, "We received an onion address for a hidden service "
+                      "descriptor but we are configured to not fetch.");
+    status = HS_CLIENT_FETCH_NOT_ALLOWED;
+    goto cannot;
+  }
+
+  /* Without a live consensus we can't do any client actions. It is needed to
+   * compute the hashring for a service. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    log_info(LD_REND, "Can't fetch descriptor for service %s because we "
+                      "are missing a live consensus. Stalling connection.",
+             safe_str_client(ed25519_fmt(identity_pk)));
+    status = HS_CLIENT_FETCH_MISSING_INFO;
+    goto cannot;
+  }
+
+  if (!router_have_minimum_dir_info()) {
+    log_info(LD_REND, "Can't fetch descriptor for service %s because we "
+                      "dont have enough descriptors. Stalling connection.",
+             safe_str_client(ed25519_fmt(identity_pk)));
+    status = HS_CLIENT_FETCH_MISSING_INFO;
+    goto cannot;
+  }
+
+  /* Check if fetching a desc for this HS is useful to us right now */
+  {
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(identity_pk);
+    if (cached_desc && hs_client_any_intro_points_usable(identity_pk,
+                                                         cached_desc)) {
+      log_info(LD_GENERAL, "We would fetch a v3 hidden service descriptor "
+                           "but we already have a usable descriptor.");
+      status = HS_CLIENT_FETCH_HAVE_DESC;
+      goto cannot;
+    }
+  }
+
+  /* Don't try to refetch while we have a pending request for it. */
+  if (directory_request_is_pending(identity_pk)) {
+    log_info(LD_REND, "Already a pending directory request. Waiting on it.");
+    status = HS_CLIENT_FETCH_PENDING;
+    goto cannot;
+  }
+
+  /* Yes, client can fetch! */
+  return 1;
+ cannot:
+  if (status_out) {
+    *status_out = status;
+  }
+  return 0;
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/** A circuit just finished connecting to a hidden service that the stream
+ *  <b>conn</b> has been waiting for. Let the HS subsystem know about this. */
+void
+hs_client_note_connection_attempt_succeeded(const edge_connection_t *conn)
+{
+  tor_assert(connection_edge_is_rendezvous_stream(conn));
+
+  if (BUG(conn->rend_data && conn->hs_ident)) {
+    log_warn(LD_BUG, "Stream had both rend_data and hs_ident..."
+             "Prioritizing hs_ident");
+  }
+
+  if (conn->hs_ident) { /* It's v3: pass it to the prop224 handler */
+    note_connection_attempt_succeeded(conn->hs_ident);
+    return;
+  } else if (conn->rend_data) { /* It's v2: pass it to the legacy handler */
+    rend_client_note_connection_attempt_ended(conn->rend_data);
+    return;
+  }
+}
+
+/* With the given encoded descriptor in desc_str and the service key in
+ * service_identity_pk, decode the descriptor and set the desc pointer with a
+ * newly allocated descriptor object.
+ *
+ * Return 0 on success else a negative value and desc is set to NULL. */
+int
+hs_client_decode_descriptor(const char *desc_str,
+                            const ed25519_public_key_t *service_identity_pk,
+                            hs_descriptor_t **desc)
+{
+  int ret;
+  uint8_t subcredential[DIGEST256_LEN];
+  ed25519_public_key_t blinded_pubkey;
+
+  tor_assert(desc_str);
+  tor_assert(service_identity_pk);
+  tor_assert(desc);
+
+  /* Create subcredential for this HS so that we can decrypt */
+  {
+    uint64_t current_time_period = hs_get_time_period_num(0);
+    hs_build_blinded_pubkey(service_identity_pk, NULL, 0, current_time_period,
+                            &blinded_pubkey);
+    hs_get_subcredential(service_identity_pk, &blinded_pubkey, subcredential);
+  }
+
+  /* Parse descriptor */
+  ret = hs_desc_decode_descriptor(desc_str, subcredential, desc);
+  memwipe(subcredential, 0, sizeof(subcredential));
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* Make sure the descriptor signing key cross certifies with the computed
+   * blinded key. Without this validation, anyone knowing the subcredential
+   * and onion address can forge a descriptor. */
+  tor_cert_t *cert = (*desc)->plaintext_data.signing_key_cert;
+  if (tor_cert_checksig(cert,
+                        &blinded_pubkey, approx_time()) < 0) {
+    log_warn(LD_GENERAL, "Descriptor signing key certificate signature "
+             "doesn't validate with computed blinded key: %s",
+             tor_cert_describe_signature_status(cert));
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
+/* Return true iff there are at least one usable intro point in the service
+ * descriptor desc. */
+int
+hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
+                                  const hs_descriptor_t *desc)
+{
+  tor_assert(service_pk);
+  tor_assert(desc);
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    if (intro_point_is_usable(service_pk, ip)) {
+      goto usable;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  return 0;
+ usable:
+  return 1;
+}
+
+/** Launch a connection to a hidden service directory to fetch a hidden
+ * service descriptor using <b>identity_pk</b> to get the necessary keys.
+ *
+ * A hs_client_fetch_status_t code is returned. */
+int
+hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk)
+{
+  hs_client_fetch_status_t status;
+
+  tor_assert(identity_pk);
+
+  if (!can_client_refetch_desc(identity_pk, &status)) {
+    return status;
+  }
+
+  /* Try to fetch the desc and if we encounter an unrecoverable error, mark
+   * the desc as unavailable for now. */
+  status = fetch_v3_desc(identity_pk);
+  if (fetch_status_should_close_socks(status)) {
+    close_all_socks_conns_waiting_for_desc(identity_pk, status,
+                                           END_STREAM_REASON_RESOLVEFAILED);
+    /* Remove HSDir fetch attempts so that we can retry later if the user
+     * wants us to regardless of if we closed any connections. */
+    purge_hid_serv_request(identity_pk);
+  }
+  return status;
+}
+
+/* This is called when we are trying to attach an AP connection to these
+ * hidden service circuits from connection_ap_handshake_attach_circuit().
+ * Return 0 on success, -1 for a transient error that is actions were
+ * triggered to recover or -2 for a permenent error where both circuits will
+ * marked for close.
+ *
+ * The following supports every hidden service version. */
+int
+hs_client_send_introduce1(origin_circuit_t *intro_circ,
+                          origin_circuit_t *rend_circ)
+{
+  return (intro_circ->hs_ident) ? send_introduce1(intro_circ, rend_circ) :
+                                  rend_client_send_introduction(intro_circ,
+                                                                rend_circ);
+}
+
+/* Called when the client circuit circ has been established. It can be either
+ * an introduction or rendezvous circuit. This function handles all hidden
+ * service versions. */
+void
+hs_client_circuit_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  switch (TO_CIRCUIT(circ)->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    if (circ->hs_ident) {
+      client_intro_circ_has_opened(circ);
+    } else {
+      rend_client_introcirc_has_opened(circ);
+    }
+    break;
+  case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
+    if (circ->hs_ident) {
+      client_rendezvous_circ_has_opened(circ);
+    } else {
+      rend_client_rendcirc_has_opened(circ);
+    }
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+}
+
+/* Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
+ * the circuit to CIRCUIT_PURPOSE_C_REND_READY. Return 0 on success else a
+ * negative value and the circuit marked for close. */
+int
+hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
+                                   const uint8_t *payload, size_t payload_len)
+{
+  tor_assert(circ);
+  tor_assert(payload);
+
+  (void) payload_len;
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_ESTABLISH_REND) {
+    log_warn(LD_PROTOCOL, "Got a RENDEZVOUS_ESTABLISHED but we were not "
+                          "expecting one. Closing circuit.");
+    goto err;
+  }
+
+  log_info(LD_REND, "Received an RENDEZVOUS_ESTABLISHED. This circuit is "
+                    "now ready for rendezvous.");
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_READY);
+
+  /* Set timestamp_dirty, because circuit_expire_building expects it to
+   * specify when a circuit entered the _C_REND_READY state. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+
+  /* From a path bias point of view, this circuit is now successfully used.
+   * Waiting any longer opens us up to attacks from malicious hidden services.
+   * They could induce the client to attempt to connect to their hidden
+   * service and never reply to the client's rend requests */
+  pathbias_mark_use_success(circ);
+
+  /* If we already have the introduction circuit built, make sure we send
+   * the INTRODUCE cell _now_ */
+  connection_ap_attach_pending(1);
+
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* This is called when a descriptor has arrived following a fetch request and
+ * has been stored in the client cache. Every entry connection that matches
+ * the service identity key in the ident will get attached to the hidden
+ * service circuit. */
+void
+hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident)
+{
+  time_t now = time(NULL);
+  smartlist_t *conns = NULL;
+
+  tor_assert(ident);
+
+  conns = connection_list_by_type_state(CONN_TYPE_AP,
+                                        AP_CONN_STATE_RENDDESC_WAIT);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    const hs_descriptor_t *desc;
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we just fetched its descriptor. */
+    if (!edge_conn->hs_ident ||
+        !ed25519_pubkey_eq(&ident->identity_pk,
+                           &edge_conn->hs_ident->identity_pk)) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+
+    /* We were just called because we stored the descriptor for this service
+     * so not finding a descriptor means we have a bigger problem. */
+    desc = hs_cache_lookup_as_client(&ident->identity_pk);
+    if (BUG(desc == NULL)) {
+      goto end;
+    }
+
+    if (!hs_client_any_intro_points_usable(&ident->identity_pk, desc)) {
+      log_info(LD_REND, "Hidden service descriptor is unusable. "
+                        "Closing streams.");
+      connection_mark_unattached_ap(entry_conn,
+                                    END_STREAM_REASON_RESOLVEFAILED);
+      /* We are unable to use the descriptor so remove the directory request
+       * from the cache so the next connection can try again. */
+      note_connection_attempt_succeeded(edge_conn->hs_ident);
+      continue;
+    }
+
+    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
+
+    /* Because the connection can now proceed to opening circuit and
+     * ultimately connect to the service, reset those timestamp so the
+     * connection is considered "fresh" and can continue without being closed
+     * too early. */
+    base_conn->timestamp_created = now;
+    base_conn->timestamp_last_read_allowed = now;
+    base_conn->timestamp_last_write_allowed = now;
+    /* Change connection's state into waiting for a circuit. */
+    base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
+
+    connection_ap_mark_as_pending_circuit(entry_conn);
+  } SMARTLIST_FOREACH_END(base_conn);
+
+ end:
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(conns);
+}
+
+/* Return a newly allocated extend_info_t for a randomly chosen introduction
+ * point for the given edge connection identifier ident. Return NULL if we
+ * can't pick any usable introduction points. */
+extend_info_t *
+hs_client_get_random_intro_from_edge(const edge_connection_t *edge_conn)
+{
+  tor_assert(edge_conn);
+
+  return (edge_conn->hs_ident) ?
+    client_get_random_intro(&edge_conn->hs_ident->identity_pk) :
+    rend_client_get_random_intro(edge_conn->rend_data);
+}
+/* Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
+ * Return 0 on success else a negative value is returned. The circuit will be
+ * closed or reuse to extend again to another intro point. */
+int
+hs_client_receive_introduce_ack(origin_circuit_t *circ,
+                                const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
+    log_warn(LD_PROTOCOL, "Unexpected INTRODUCE_ACK on circuit %u.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+    goto end;
+  }
+
+  ret = (circ->hs_ident) ? handle_introduce_ack(circ, payload, payload_len) :
+                           rend_client_introduction_acked(circ, payload,
+                                                          payload_len);
+  /* For path bias: This circuit was used successfully. NACK or ACK counts. */
+  pathbias_mark_use_success(circ);
+
+ end:
+  return ret;
+}
+
+/* Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
+ * 0 on success else a negative value is returned. The circuit will be closed
+ * on error. */
+int
+hs_client_receive_rendezvous2(origin_circuit_t *circ,
+                              const uint8_t *payload, size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Circuit can possibly be in both state because we could receive a
+   * RENDEZVOUS2 cell before the INTRODUCE_ACK has been received. */
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_REND_READY &&
+      TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) {
+    log_warn(LD_PROTOCOL, "Unexpected RENDEZVOUS2 cell on circuit %u. "
+                          "Closing circuit.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+    goto end;
+  }
+
+  log_info(LD_REND, "Got RENDEZVOUS2 cell from hidden service on circuit %u.",
+           TO_CIRCUIT(circ)->n_circ_id);
+
+  ret = (circ->hs_ident) ? handle_rendezvous2(circ, payload, payload_len) :
+                           rend_client_receive_rendezvous(circ, payload,
+                                                          payload_len);
+ end:
+  return ret;
+}
+
+/* Extend the introduction circuit circ to another valid introduction point
+ * for the hidden service it is trying to connect to, or mark it and launch a
+ * new circuit if we can't extend it.  Return 0 on success or possible
+ * success. Return -1 and mark the introduction circuit for close on permanent
+ * failure.
+ *
+ * On failure, the caller is responsible for marking the associated rendezvous
+ * circuit for close. */
+int
+hs_client_reextend_intro_circuit(origin_circuit_t *circ)
+{
+  int ret = -1;
+  extend_info_t *ei;
+
+  tor_assert(circ);
+
+  ei = (circ->hs_ident) ?
+    client_get_random_intro(&circ->hs_ident->identity_pk) :
+    rend_client_get_random_intro(circ->rend_data);
+  if (ei == NULL) {
+    log_warn(LD_REND, "No usable introduction points left. Closing.");
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    goto end;
+  }
+
+  if (circ->remaining_relay_early_cells) {
+    log_info(LD_REND, "Re-extending circ %u, this time to %s.",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(extend_info_describe(ei)));
+    ret = circuit_extend_to_new_exit(circ, ei);
+    if (ret == 0) {
+      /* We were able to extend so update the timestamp so we avoid expiring
+       * this circuit too early. The intro circuit is short live so the
+       * linkability issue is minimized, we just need the circuit to hold a
+       * bit longer so we can introduce. */
+      TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+    }
+  } else {
+    log_info(LD_REND, "Closing intro circ %u (out of RELAY_EARLY cells).",
+             (unsigned int) TO_CIRCUIT(circ)->n_circ_id);
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
+    /* connection_ap_handshake_attach_circuit will launch a new intro circ. */
+    ret = 0;
+  }
+
+ end:
+  extend_info_free(ei);
+  return ret;
+}
+
+/* Release all the storage held by the client subsystem. */
+void
+hs_client_free_all(void)
+{
+  /* Purge the hidden service request cache. */
+  hs_purge_last_hid_serv_requests();
+}
+
+/* Purge all potentially remotely-detectable state held in the hidden
+ * service client code. Called on SIGNAL NEWNYM. */
+void
+hs_client_purge_state(void)
+{
+  /* v2 subsystem. */
+  rend_client_purge_state();
+
+  /* Cancel all descriptor fetches. Do this first so once done we are sure
+   * that our descriptor cache won't modified. */
+  cancel_descriptor_fetches();
+  /* Purge the introduction point state cache. */
+  hs_cache_client_intro_state_purge();
+  /* Purge the descriptor cache. */
+  hs_cache_purge_as_client();
+  /* Purge the last hidden service request cache. */
+  hs_purge_last_hid_serv_requests();
+
+  log_info(LD_REND, "Hidden service client state has been purged.");
+}
+
+/* Called when our directory information has changed. */
+void
+hs_client_dir_info_changed(void)
+{
+  /* We have possibly reached the minimum directory information or new
+   * consensus so retry all pending SOCKS connection in
+   * AP_CONN_STATE_RENDDESC_WAIT state in order to fetch the descriptor. */
+  retry_all_socks_conn_waiting_for_desc();
+}
+
diff --git a/src/or/hs_client.h b/src/or/hs_client.h
new file mode 100644
index 0000000000..2523568ad1
--- /dev/null
+++ b/src/or/hs_client.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_client.h
+ * \brief Header file containing client data for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_CLIENT_H
+#define TOR_HS_CLIENT_H
+
+#include "crypto_ed25519.h"
+#include "hs_descriptor.h"
+#include "hs_ident.h"
+
+/* Status code of a descriptor fetch request. */
+typedef enum {
+  /* Something internally went wrong. */
+  HS_CLIENT_FETCH_ERROR        = -1,
+  /* The fetch request has been launched successfully. */
+  HS_CLIENT_FETCH_LAUNCHED     = 0,
+  /* We already have a usable descriptor. No fetch. */
+  HS_CLIENT_FETCH_HAVE_DESC    = 1,
+  /* No more HSDir available to query. */
+  HS_CLIENT_FETCH_NO_HSDIRS    = 2,
+  /* The fetch request is not allowed. */
+  HS_CLIENT_FETCH_NOT_ALLOWED  = 3,
+  /* We are missing information to be able to launch a request. */
+  HS_CLIENT_FETCH_MISSING_INFO = 4,
+  /* There is a pending fetch for the requested service. */
+  HS_CLIENT_FETCH_PENDING      = 5,
+} hs_client_fetch_status_t;
+
+void hs_client_note_connection_attempt_succeeded(
+                                       const edge_connection_t *conn);
+
+int hs_client_decode_descriptor(
+                     const char *desc_str,
+                     const ed25519_public_key_t *service_identity_pk,
+                     hs_descriptor_t **desc);
+int hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
+                                      const hs_descriptor_t *desc);
+int hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk);
+void hs_client_dir_info_changed(void);
+
+int hs_client_send_introduce1(origin_circuit_t *intro_circ,
+                              origin_circuit_t *rend_circ);
+
+void hs_client_circuit_has_opened(origin_circuit_t *circ);
+
+int hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
+                                       const uint8_t *payload,
+                                       size_t payload_len);
+int hs_client_receive_introduce_ack(origin_circuit_t *circ,
+                                    const uint8_t *payload,
+                                    size_t payload_len);
+int hs_client_receive_rendezvous2(origin_circuit_t *circ,
+                                  const uint8_t *payload,
+                                  size_t payload_len);
+
+void hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident);
+
+extend_info_t *hs_client_get_random_intro_from_edge(
+                                          const edge_connection_t *edge_conn);
+
+int hs_client_reextend_intro_circuit(origin_circuit_t *circ);
+
+void hs_client_purge_state(void);
+
+void hs_client_free_all(void);
+
+#ifdef HS_CLIENT_PRIVATE
+
+STATIC routerstatus_t *
+pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk);
+
+STATIC extend_info_t *
+client_get_random_intro(const ed25519_public_key_t *service_pk);
+
+STATIC extend_info_t *
+desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip);
+
+STATIC int handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
+                              size_t payload_len);
+
+MOCK_DECL(STATIC hs_client_fetch_status_t,
+          fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk));
+
+#endif /* defined(HS_CLIENT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_CLIENT_H) */
+
diff --git a/src/or/hs_common.c b/src/or/hs_common.c
new file mode 100644
index 0000000000..c42ef0c97b
--- /dev/null
+++ b/src/or/hs_common.c
@@ -0,0 +1,1819 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.c
+ * \brief Contains code shared between different HS protocol version as well
+ *        as useful data structures and accessors used by other subsystems.
+ *        The rendcommon.c should only contains code relating to the v2
+ *        protocol.
+ **/
+
+#define HS_COMMON_PRIVATE
+
+#include "or.h"
+
+#include "config.h"
+#include "circuitbuild.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "networkstatus.h"
+#include "nodelist.h"
+#include "hs_cache.h"
+#include "hs_common.h"
+#include "hs_client.h"
+#include "hs_ident.h"
+#include "hs_service.h"
+#include "hs_circuitmap.h"
+#include "policies.h"
+#include "rendcommon.h"
+#include "rendservice.h"
+#include "routerset.h"
+#include "router.h"
+#include "shared_random_client.h"
+#include "dirauth/shared_random_state.h"
+
+/* Trunnel */
+#include "ed25519_cert.h"
+
+/* Ed25519 Basepoint value. Taken from section 5 of
+ * https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-03 */
+static const char *str_ed25519_basepoint =
+  "(15112221349535400772501151409588531511"
+  "454012693041857206046113283949847762202, "
+  "463168356949264781694283940034751631413"
+  "07993866256225615783033603165251855960)";
+
+#ifdef HAVE_SYS_UN_H
+
+/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t,
+ * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success
+ * else return -ENOSYS if AF_UNIX is not supported (see function in the
+ * #else statement below). */
+static int
+add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
+{
+  tor_assert(ports);
+  tor_assert(p);
+  tor_assert(p->is_unix_addr);
+
+  smartlist_add(ports, p);
+  return 0;
+}
+
+/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0
+ * on success else return -ENOSYS if AF_UNIX is not supported (see function
+ * in the #else statement below). */
+static int
+set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
+{
+  tor_assert(conn);
+  tor_assert(p);
+  tor_assert(p->is_unix_addr);
+
+  conn->base_.socket_family = AF_UNIX;
+  tor_addr_make_unspec(&conn->base_.addr);
+  conn->base_.port = 1;
+  conn->base_.address = tor_strdup(p->unix_addr);
+  return 0;
+}
+
+#else /* !(defined(HAVE_SYS_UN_H)) */
+
+static int
+set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
+{
+  (void) conn;
+  (void) p;
+  return -ENOSYS;
+}
+
+static int
+add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
+{
+  (void) ports;
+  (void) p;
+  return -ENOSYS;
+}
+
+#endif /* defined(HAVE_SYS_UN_H) */
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * current hsdir_index. */
+static int
+compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN);
+}
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * next hsdir_index. */
+static int
+compare_digest_to_store_first_hsdir_index(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN);
+}
+
+/* Helper function: The key is a digest that we compare to a node_t object
+ * next hsdir_index. */
+static int
+compare_digest_to_store_second_hsdir_index(const void *_key,
+                                          const void **_member)
+{
+  const char *key = _key;
+  const node_t *node = *_member;
+  return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects current hsdir_index. */
+static int
+compare_node_fetch_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.fetch,
+                    node2->hsdir_index.fetch,
+                    DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects next hsdir_index. */
+static int
+compare_node_store_first_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.store_first,
+                    node2->hsdir_index.store_first,
+                    DIGEST256_LEN);
+}
+
+/* Helper function: Compare two node_t objects next hsdir_index. */
+static int
+compare_node_store_second_hsdir_index(const void **a, const void **b)
+{
+  const node_t *node1= *a;
+  const node_t *node2 = *b;
+  return tor_memcmp(node1->hsdir_index.store_second,
+                    node2->hsdir_index.store_second,
+                    DIGEST256_LEN);
+}
+
+/* Allocate and return a string containing the path to filename in directory.
+ * This function will never return NULL. The caller must free this path. */
+char *
+hs_path_from_filename(const char *directory, const char *filename)
+{
+  char *file_path = NULL;
+
+  tor_assert(directory);
+  tor_assert(filename);
+
+  tor_asprintf(&file_path, "%s%s%s", directory, PATH_SEPARATOR, filename);
+  return file_path;
+}
+
+/* Make sure that the directory for <b>service</b> is private, using the config
+ * <b>username</b>.
+ * If <b>create</b> is true:
+ *  - if the directory exists, change permissions if needed,
+ *  - if the directory does not exist, create it with the correct permissions.
+ * If <b>create</b> is false:
+ *  - if the directory exists, check permissions,
+ *  - if the directory does not exist, check if we think we can create it.
+ * Return 0 on success, -1 on failure. */
+int
+hs_check_service_private_dir(const char *username, const char *path,
+                             unsigned int dir_group_readable,
+                             unsigned int create)
+{
+  cpd_check_t check_opts = CPD_NONE;
+
+  tor_assert(path);
+
+  if (create) {
+    check_opts |= CPD_CREATE;
+  } else {
+    check_opts |= CPD_CHECK_MODE_ONLY;
+    check_opts |= CPD_CHECK;
+  }
+  if (dir_group_readable) {
+    check_opts |= CPD_GROUP_READ;
+  }
+  /* Check/create directory */
+  if (check_private_dir(path, check_opts, username) < 0) {
+    return -1;
+  }
+  return 0;
+}
+
+/* Default, minimum, and maximum values for the maximum rendezvous failures
+ * consensus parameter. */
+#define MAX_REND_FAILURES_DEFAULT 2
+#define MAX_REND_FAILURES_MIN 1
+#define MAX_REND_FAILURES_MAX 10
+
+/** How many times will a hidden service operator attempt to connect to
+ * a requested rendezvous point before giving up? */
+int
+hs_get_service_max_rend_failures(void)
+{
+  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
+                                 MAX_REND_FAILURES_DEFAULT,
+                                 MAX_REND_FAILURES_MIN,
+                                 MAX_REND_FAILURES_MAX);
+}
+
+/** Get the default HS time period length in minutes from the consensus. */
+STATIC uint64_t
+get_time_period_length(void)
+{
+  /* If we are on a test network, make the time period smaller than normal so
+     that we actually see it rotate. Specifically, make it the same length as
+     an SRV protocol run. */
+  if (get_options()->TestingTorNetwork) {
+    unsigned run_duration = sr_state_get_protocol_run_duration();
+    /* An SRV run should take more than a minute (it's 24 rounds) */
+    tor_assert_nonfatal(run_duration > 60);
+    /* Turn it from seconds to minutes before returning: */
+    return sr_state_get_protocol_run_duration() / 60;
+  }
+
+  int32_t time_period_length = networkstatus_get_param(NULL, "hsdir_interval",
+                                             HS_TIME_PERIOD_LENGTH_DEFAULT,
+                                             HS_TIME_PERIOD_LENGTH_MIN,
+                                             HS_TIME_PERIOD_LENGTH_MAX);
+  /* Make sure it's a positive value. */
+  tor_assert(time_period_length >= 0);
+  /* uint64_t will always be able to contain a int32_t */
+  return (uint64_t) time_period_length;
+}
+
+/** Get the HS time period number at time <b>now</b>. If <b>now</b> is not set,
+ *  we try to get the time ourselves from a live consensus. */
+uint64_t
+hs_get_time_period_num(time_t now)
+{
+  uint64_t time_period_num;
+  time_t current_time;
+
+  /* If no time is specified, set current time based on consensus time, and
+   * only fall back to system time if that fails. */
+  if (now != 0) {
+    current_time = now;
+  } else {
+    networkstatus_t *ns = networkstatus_get_live_consensus(approx_time());
+    current_time = ns ? ns->valid_after : approx_time();
+  }
+
+  /* Start by calculating minutes since the epoch */
+  uint64_t time_period_length = get_time_period_length();
+  uint64_t minutes_since_epoch = current_time / 60;
+
+  /* Apply the rotation offset as specified by prop224 (section
+   * [TIME-PERIODS]), so that new time periods synchronize nicely with SRV
+   * publication */
+  unsigned int time_period_rotation_offset = sr_state_get_phase_duration();
+  time_period_rotation_offset /= 60; /* go from seconds to minutes */
+  tor_assert(minutes_since_epoch > time_period_rotation_offset);
+  minutes_since_epoch -= time_period_rotation_offset;
+
+  /* Calculate the time period */
+  time_period_num = minutes_since_epoch / time_period_length;
+  return time_period_num;
+}
+
+/** Get the number of the _upcoming_ HS time period, given that the current
+ *  time is <b>now</b>. If <b>now</b> is not set, we try to get the time from a
+ *  live consensus. */
+uint64_t
+hs_get_next_time_period_num(time_t now)
+{
+  return hs_get_time_period_num(now) + 1;
+}
+
+/* Get the number of the _previous_ HS time period, given that the current time
+ * is <b>now</b>. If <b>now</b> is not set, we try to get the time from a live
+ * consensus. */
+uint64_t
+hs_get_previous_time_period_num(time_t now)
+{
+  return hs_get_time_period_num(now) - 1;
+}
+
+/* Return the start time of the upcoming time period based on <b>now</b>. If
+   <b>now</b> is not set, we try to get the time ourselves from a live
+   consensus. */
+time_t
+hs_get_start_time_of_next_time_period(time_t now)
+{
+  uint64_t time_period_length = get_time_period_length();
+
+  /* Get start time of next time period */
+  uint64_t next_time_period_num = hs_get_next_time_period_num(now);
+  uint64_t start_of_next_tp_in_mins = next_time_period_num *time_period_length;
+
+  /* Apply rotation offset as specified by prop224 section [TIME-PERIODS] */
+  unsigned int time_period_rotation_offset = sr_state_get_phase_duration();
+  return (time_t)(start_of_next_tp_in_mins * 60 + time_period_rotation_offset);
+}
+
+/* Create a new rend_data_t for a specific given <b>version</b>.
+ * Return a pointer to the newly allocated data structure. */
+static rend_data_t *
+rend_data_alloc(uint32_t version)
+{
+  rend_data_t *rend_data = NULL;
+
+  switch (version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2 = tor_malloc_zero(sizeof(*v2));
+    v2->base_.version = HS_VERSION_TWO;
+    v2->base_.hsdirs_fp = smartlist_new();
+    rend_data = &v2->base_;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return rend_data;
+}
+
+/** Free all storage associated with <b>data</b> */
+void
+rend_data_free_(rend_data_t *data)
+{
+  if (!data) {
+    return;
+  }
+  /* By using our allocation function, this should always be set. */
+  tor_assert(data->hsdirs_fp);
+  /* Cleanup the HSDir identity digest. */
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(data->hsdirs_fp);
+  /* Depending on the version, cleanup. */
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(data);
+    tor_free(v2_data);
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Allocate and return a deep copy of <b>data</b>. */
+rend_data_t *
+rend_data_dup(const rend_data_t *data)
+{
+  rend_data_t *data_dup = NULL;
+  smartlist_t *hsdirs_fp = smartlist_new();
+
+  tor_assert(data);
+  tor_assert(data->hsdirs_fp);
+
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
+                    smartlist_add(hsdirs_fp, tor_memdup(fp, DIGEST_LEN)));
+
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = tor_memdup(TO_REND_DATA_V2(data),
+                                         sizeof(*v2_data));
+    data_dup = &v2_data->base_;
+    data_dup->hsdirs_fp = hsdirs_fp;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return data_dup;
+}
+
+/* Compute the descriptor ID for each HS descriptor replica and save them. A
+ * valid onion address must be present in the <b>rend_data</b>.
+ *
+ * Return 0 on success else -1. */
+static int
+compute_desc_id(rend_data_t *rend_data)
+{
+  int ret = 0;
+  unsigned replica;
+  time_t now = time(NULL);
+
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    /* Compute descriptor ID for each replicas. */
+    for (replica = 0; replica < ARRAY_LENGTH(v2_data->descriptor_id);
+         replica++) {
+      ret = rend_compute_v2_desc_id(v2_data->descriptor_id[replica],
+                                    v2_data->onion_address,
+                                    v2_data->descriptor_cookie,
+                                    now, replica);
+      if (ret < 0) {
+        goto end;
+      }
+    }
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+
+ end:
+  return ret;
+}
+
+/* Allocate and initialize a rend_data_t object for a service using the
+ * provided arguments. All arguments are optional (can be NULL), except from
+ * <b>onion_address</b> which MUST be set. The <b>pk_digest</b> is the hash of
+ * the service private key. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation this service is configured with.
+ *
+ * Return a valid rend_data_t pointer. This only returns a version 2 object of
+ * rend_data_t. */
+rend_data_t *
+rend_data_service_create(const char *onion_address, const char *pk_digest,
+                         const uint8_t *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL);
+
+  if (pk_digest) {
+    memcpy(v2->rend_pk_digest, pk_digest, sizeof(v2->rend_pk_digest));
+  }
+  if (cookie) {
+    memcpy(rend_data->rend_cookie, cookie, sizeof(rend_data->rend_cookie));
+  }
+
+  strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+  v2->auth_type = auth_type;
+
+  return rend_data;
+}
+
+/* Allocate and initialize a rend_data_t object for a client request using the
+ * given arguments. Either an onion address or a descriptor ID is needed. Both
+ * can be given but in this case only the onion address will be used to make
+ * the descriptor fetch. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation the service is configured with.
+ *
+ * Return a valid rend_data_t pointer or NULL on error meaning the
+ * descriptor IDs couldn't be computed from the given data. */
+rend_data_t *
+rend_data_client_create(const char *onion_address, const char *desc_id,
+                        const char *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL || desc_id != NULL);
+
+  if (cookie) {
+    memcpy(v2->descriptor_cookie, cookie, sizeof(v2->descriptor_cookie));
+  }
+  if (desc_id) {
+    memcpy(v2->desc_id_fetch, desc_id, sizeof(v2->desc_id_fetch));
+  }
+  if (onion_address) {
+    strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+    if (compute_desc_id(rend_data) < 0) {
+      goto error;
+    }
+  }
+
+  v2->auth_type = auth_type;
+
+  return rend_data;
+
+ error:
+  rend_data_free(rend_data);
+  return NULL;
+}
+
+/* Return the onion address from the rend data. Depending on the version,
+ * the size of the address can vary but it's always NUL terminated. */
+const char *
+rend_data_get_address(const rend_data_t *rend_data)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    return TO_REND_DATA_V2(rend_data)->onion_address;
+  default:
+    /* We should always have a supported version. */
+    tor_assert_unreached();
+  }
+}
+
+/* Return the descriptor ID for a specific replica number from the rend
+ * data. The returned data is a binary digest and depending on the version its
+ * size can vary. The size of the descriptor ID is put in <b>len_out</b> if
+ * non NULL. */
+const char *
+rend_data_get_desc_id(const rend_data_t *rend_data, uint8_t replica,
+                      size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    tor_assert(replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS);
+    if (len_out) {
+      *len_out = DIGEST_LEN;
+    }
+    return TO_REND_DATA_V2(rend_data)->descriptor_id[replica];
+  default:
+    /* We should always have a supported version. */
+    tor_assert_unreached();
+  }
+}
+
+/* Return the public key digest using the given <b>rend_data</b>. The size of
+ * the digest is put in <b>len_out</b> (if set) which can differ depending on
+ * the version. */
+const uint8_t *
+rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    const rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    if (len_out) {
+      *len_out = sizeof(v2_data->rend_pk_digest);
+    }
+    return (const uint8_t *) v2_data->rend_pk_digest;
+  }
+  default:
+    /* We should always have a supported version. */
+    tor_assert_unreached();
+  }
+}
+
+/* Using the given time period number, compute the disaster shared random
+ * value and put it in srv_out. It MUST be at least DIGEST256_LEN bytes. */
+static void
+compute_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(srv_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+
+  /* Start setting up payload:
+   *  H("shared-random-disaster" | INT_8(period_length) | INT_8(period_num)) */
+  crypto_digest_add_bytes(digest, HS_SRV_DISASTER_PREFIX,
+                          HS_SRV_DISASTER_PREFIX_LEN);
+
+  /* Setup INT_8(period_length) | INT_8(period_num) */
+  {
+    uint64_t time_period_length = get_time_period_length();
+    char period_stuff[sizeof(uint64_t)*2];
+    size_t offset = 0;
+    set_uint64(period_stuff, tor_htonll(time_period_length));
+    offset += sizeof(uint64_t);
+    set_uint64(period_stuff+offset, tor_htonll(time_period_num));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(period_stuff));
+
+    crypto_digest_add_bytes(digest, period_stuff,  sizeof(period_stuff));
+  }
+
+  crypto_digest_get_digest(digest, (char *) srv_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/** Due to the high cost of computing the disaster SRV and that potentially we
+ *  would have to do it thousands of times in a row, we always cache the
+ *  computer disaster SRV (and its corresponding time period num) in case we
+ *  want to reuse it soon after. We need to cache two SRVs, one for each active
+ *  time period.
+ */
+static uint8_t cached_disaster_srv[2][DIGEST256_LEN];
+static uint64_t cached_time_period_nums[2] = {0};
+
+/** Compute the disaster SRV value for this <b>time_period_num</b> and put it
+ *  in <b>srv_out</b> (of size at least DIGEST256_LEN). First check our caches
+ *  to see if we have already computed it. */
+STATIC void
+get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
+{
+  if (time_period_num == cached_time_period_nums[0]) {
+    memcpy(srv_out, cached_disaster_srv[0], DIGEST256_LEN);
+    return;
+  } else if (time_period_num == cached_time_period_nums[1]) {
+    memcpy(srv_out, cached_disaster_srv[1], DIGEST256_LEN);
+    return;
+  } else {
+    int replace_idx;
+    // Replace the lower period number.
+    if (cached_time_period_nums[0] <= cached_time_period_nums[1]) {
+      replace_idx = 0;
+    } else {
+      replace_idx = 1;
+    }
+    cached_time_period_nums[replace_idx] = time_period_num;
+    compute_disaster_srv(time_period_num, cached_disaster_srv[replace_idx]);
+    memcpy(srv_out, cached_disaster_srv[replace_idx], DIGEST256_LEN);
+    return;
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Get the first cached disaster SRV. Only used by unittests. */
+STATIC uint8_t *
+get_first_cached_disaster_srv(void)
+{
+  return cached_disaster_srv[0];
+}
+
+/** Get the second cached disaster SRV. Only used by unittests. */
+STATIC uint8_t *
+get_second_cached_disaster_srv(void)
+{
+  return cached_disaster_srv[1];
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/* When creating a blinded key, we need a parameter which construction is as
+ * follow: H(pubkey | [secret] | ed25519-basepoint | nonce).
+ *
+ * The nonce has a pre-defined format which uses the time period number
+ * period_num and the start of the period in second start_time_period.
+ *
+ * The secret of size secret_len is optional meaning that it can be NULL and
+ * thus will be ignored for the param construction.
+ *
+ * The result is put in param_out. */
+static void
+build_blinded_key_param(const ed25519_public_key_t *pubkey,
+                        const uint8_t *secret, size_t secret_len,
+                        uint64_t period_num, uint64_t period_length,
+                        uint8_t *param_out)
+{
+  size_t offset = 0;
+  const char blind_str[] = "Derive temporary signing key";
+  uint8_t nonce[HS_KEYBLIND_NONCE_LEN];
+  crypto_digest_t *digest;
+
+  tor_assert(pubkey);
+  tor_assert(param_out);
+
+  /* Create the nonce N. The construction is as follow:
+   *    N = "key-blind" || INT_8(period_num) || INT_8(period_length) */
+  memcpy(nonce, HS_KEYBLIND_NONCE_PREFIX, HS_KEYBLIND_NONCE_PREFIX_LEN);
+  offset += HS_KEYBLIND_NONCE_PREFIX_LEN;
+  set_uint64(nonce + offset, tor_htonll(period_num));
+  offset += sizeof(uint64_t);
+  set_uint64(nonce + offset, tor_htonll(period_length));
+  offset += sizeof(uint64_t);
+  tor_assert(offset == HS_KEYBLIND_NONCE_LEN);
+
+  /* Generate the parameter h and the construction is as follow:
+   *    h = H(BLIND_STRING | pubkey | [secret] | ed25519-basepoint | N) */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, blind_str, sizeof(blind_str));
+  crypto_digest_add_bytes(digest, (char *) pubkey, ED25519_PUBKEY_LEN);
+  /* Optional secret. */
+  if (secret) {
+    crypto_digest_add_bytes(digest, (char *) secret, secret_len);
+  }
+  crypto_digest_add_bytes(digest, str_ed25519_basepoint,
+                          strlen(str_ed25519_basepoint));
+  crypto_digest_add_bytes(digest, (char *) nonce, sizeof(nonce));
+
+  /* Extract digest and put it in the param. */
+  crypto_digest_get_digest(digest, (char *) param_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  memwipe(nonce, 0, sizeof(nonce));
+}
+
+/* Using an ed25519 public key and version to build the checksum of an
+ * address. Put in checksum_out. Format is:
+ *    SHA3-256(".onion checksum" || PUBKEY || VERSION)
+ *
+ * checksum_out must be large enough to receive 32 bytes (DIGEST256_LEN). */
+static void
+build_hs_checksum(const ed25519_public_key_t *key, uint8_t version,
+                  uint8_t *checksum_out)
+{
+  size_t offset = 0;
+  char data[HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN];
+
+  /* Build checksum data. */
+  memcpy(data, HS_SERVICE_ADDR_CHECKSUM_PREFIX,
+         HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN);
+  offset += HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN;
+  memcpy(data + offset, key->pubkey, ED25519_PUBKEY_LEN);
+  offset += ED25519_PUBKEY_LEN;
+  set_uint8(data + offset, version);
+  offset += sizeof(version);
+  tor_assert(offset == HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN);
+
+  /* Hash the data payload to create the checksum. */
+  crypto_digest256((char *) checksum_out, data, sizeof(data),
+                   DIGEST_SHA3_256);
+}
+
+/* Using an ed25519 public key, checksum and version to build the binary
+ * representation of a service address. Put in addr_out. Format is:
+ *    addr_out = PUBKEY || CHECKSUM || VERSION
+ *
+ * addr_out must be large enough to receive HS_SERVICE_ADDR_LEN bytes. */
+static void
+build_hs_address(const ed25519_public_key_t *key, const uint8_t *checksum,
+                 uint8_t version, char *addr_out)
+{
+  size_t offset = 0;
+
+  tor_assert(key);
+  tor_assert(checksum);
+
+  memcpy(addr_out, key->pubkey, ED25519_PUBKEY_LEN);
+  offset += ED25519_PUBKEY_LEN;
+  memcpy(addr_out + offset, checksum, HS_SERVICE_ADDR_CHECKSUM_LEN_USED);
+  offset += HS_SERVICE_ADDR_CHECKSUM_LEN_USED;
+  set_uint8(addr_out + offset, version);
+  offset += sizeof(uint8_t);
+  tor_assert(offset == HS_SERVICE_ADDR_LEN);
+}
+
+/* Helper for hs_parse_address(): Using a binary representation of a service
+ * address, parse its content into the key_out, checksum_out and version_out.
+ * Any out variable can be NULL in case the caller would want only one field.
+ * checksum_out MUST at least be 2 bytes long. address must be at least
+ * HS_SERVICE_ADDR_LEN bytes but doesn't need to be NUL terminated. */
+static void
+hs_parse_address_impl(const char *address, ed25519_public_key_t *key_out,
+                      uint8_t *checksum_out, uint8_t *version_out)
+{
+  size_t offset = 0;
+
+  tor_assert(address);
+
+  if (key_out) {
+    /* First is the key. */
+    memcpy(key_out->pubkey, address, ED25519_PUBKEY_LEN);
+  }
+  offset += ED25519_PUBKEY_LEN;
+  if (checksum_out) {
+    /* Followed by a 2 bytes checksum. */
+    memcpy(checksum_out, address + offset, HS_SERVICE_ADDR_CHECKSUM_LEN_USED);
+  }
+  offset += HS_SERVICE_ADDR_CHECKSUM_LEN_USED;
+  if (version_out) {
+    /* Finally, version value is 1 byte. */
+    *version_out = get_uint8(address + offset);
+  }
+  offset += sizeof(uint8_t);
+  /* Extra safety. */
+  tor_assert(offset == HS_SERVICE_ADDR_LEN);
+}
+
+/* Using the given identity public key and a blinded public key, compute the
+ * subcredential and put it in subcred_out (must be of size DIGEST256_LEN).
+ * This can't fail. */
+void
+hs_get_subcredential(const ed25519_public_key_t *identity_pk,
+                     const ed25519_public_key_t *blinded_pk,
+                     uint8_t *subcred_out)
+{
+  uint8_t credential[DIGEST256_LEN];
+  crypto_digest_t *digest;
+
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(subcred_out);
+
+  /* First, build the credential. Construction is as follow:
+   *  credential = H("credential" | public-identity-key) */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_CREDENTIAL_PREFIX,
+                          HS_CREDENTIAL_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) identity_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_get_digest(digest, (char *) credential, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  /* Now, compute the subcredential. Construction is as follow:
+   *  subcredential = H("subcredential" | credential | blinded-public-key). */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_SUBCREDENTIAL_PREFIX,
+                          HS_SUBCREDENTIAL_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) credential,
+                          sizeof(credential));
+  crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_get_digest(digest, (char *) subcred_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+
+  memwipe(credential, 0, sizeof(credential));
+}
+
+/* From the given list of hidden service ports, find the ones that much the
+ * given edge connection conn, pick one at random and use it to set the
+ * connection address. Return 0 on success or -1 if none. */
+int
+hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn)
+{
+  rend_service_port_config_t *chosen_port;
+  unsigned int warn_once = 0;
+  smartlist_t *matching_ports;
+
+  tor_assert(ports);
+  tor_assert(conn);
+
+  matching_ports = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(ports, rend_service_port_config_t *, p) {
+    if (TO_CONN(conn)->port != p->virtual_port) {
+      continue;
+    }
+    if (!(p->is_unix_addr)) {
+      smartlist_add(matching_ports, p);
+    } else {
+      if (add_unix_port(matching_ports, p)) {
+        if (!warn_once) {
+          /* Unix port not supported so warn only once. */
+          log_warn(LD_REND, "Saw AF_UNIX virtual port mapping for port %d "
+                            "which is unsupported on this platform. "
+                            "Ignoring it.",
+                   TO_CONN(conn)->port);
+        }
+        warn_once++;
+      }
+    }
+  } SMARTLIST_FOREACH_END(p);
+
+  chosen_port = smartlist_choose(matching_ports);
+  smartlist_free(matching_ports);
+  if (chosen_port) {
+    if (!(chosen_port->is_unix_addr)) {
+      /* Get a non-AF_UNIX connection ready for connection_exit_connect() */
+      tor_addr_copy(&TO_CONN(conn)->addr, &chosen_port->real_addr);
+      TO_CONN(conn)->port = chosen_port->real_port;
+    } else {
+      if (set_unix_port(conn, chosen_port)) {
+        /* Simply impossible to end up here else we were able to add a Unix
+         * port without AF_UNIX support... ? */
+        tor_assert(0);
+      }
+    }
+  }
+  return (chosen_port) ? 0 : -1;
+}
+
+/* Using a base32 representation of a service address, parse its content into
+ * the key_out, checksum_out and version_out. Any out variable can be NULL in
+ * case the caller would want only one field. checksum_out MUST at least be 2
+ * bytes long.
+ *
+ * Return 0 if parsing went well; return -1 in case of error. */
+int
+hs_parse_address(const char *address, ed25519_public_key_t *key_out,
+                 uint8_t *checksum_out, uint8_t *version_out)
+{
+  char decoded[HS_SERVICE_ADDR_LEN];
+
+  tor_assert(address);
+
+  /* Obvious length check. */
+  if (strlen(address) != HS_SERVICE_ADDR_LEN_BASE32) {
+    log_warn(LD_REND, "Service address %s has an invalid length. "
+                      "Expected %lu but got %lu.",
+             escaped_safe_str(address),
+             (unsigned long) HS_SERVICE_ADDR_LEN_BASE32,
+             (unsigned long) strlen(address));
+    goto invalid;
+  }
+
+  /* Decode address so we can extract needed fields. */
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
+    log_warn(LD_REND, "Service address %s can't be decoded.",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Parse the decoded address into the fields we need. */
+  hs_parse_address_impl(decoded, key_out, checksum_out, version_out);
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Validate a given onion address. The length, the base32 decoding and
+ * checksum are validated. Return 1 if valid else 0. */
+int
+hs_address_is_valid(const char *address)
+{
+  uint8_t version;
+  uint8_t checksum[HS_SERVICE_ADDR_CHECKSUM_LEN_USED];
+  uint8_t target_checksum[DIGEST256_LEN];
+  ed25519_public_key_t service_pubkey;
+
+  /* Parse the decoded address into the fields we need. */
+  if (hs_parse_address(address, &service_pubkey, checksum, &version) < 0) {
+    goto invalid;
+  }
+
+  /* Get the checksum it's suppose to be and compare it with what we have
+   * encoded in the address. */
+  build_hs_checksum(&service_pubkey, version, target_checksum);
+  if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
+    log_warn(LD_REND, "Service address %s invalid checksum.",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Validate that this pubkey does not have a torsion component. We need to do
+   * this on the prop224 client-side so that attackers can't give equivalent
+   * forms of an onion address to users. */
+  if (ed25519_validate_pubkey(&service_pubkey) < 0) {
+    log_warn(LD_REND, "Service address %s has bad pubkey .",
+             escaped_safe_str(address));
+    goto invalid;
+  }
+
+  /* Valid address. */
+  return 1;
+ invalid:
+  return 0;
+}
+
+/* Build a service address using an ed25519 public key and a given version.
+ * The returned address is base32 encoded and put in addr_out. The caller MUST
+ * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
+ *
+ * Format is as follow:
+ *     base32(PUBKEY || CHECKSUM || VERSION)
+ *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
+ * */
+void
+hs_build_address(const ed25519_public_key_t *key, uint8_t version,
+                 char *addr_out)
+{
+  uint8_t checksum[DIGEST256_LEN];
+  char address[HS_SERVICE_ADDR_LEN];
+
+  tor_assert(key);
+  tor_assert(addr_out);
+
+  /* Get the checksum of the address. */
+  build_hs_checksum(key, version, checksum);
+  /* Get the binary address representation. */
+  build_hs_address(key, checksum, version, address);
+
+  /* Encode the address. addr_out will be NUL terminated after this. */
+  base32_encode(addr_out, HS_SERVICE_ADDR_LEN_BASE32 + 1, address,
+                sizeof(address));
+  /* Validate what we just built. */
+  tor_assert(hs_address_is_valid(addr_out));
+}
+
+/* Return a newly allocated copy of lspec. */
+link_specifier_t *
+hs_link_specifier_dup(const link_specifier_t *lspec)
+{
+  link_specifier_t *result = link_specifier_new();
+  memcpy(result, lspec, sizeof(*result));
+  /* The unrecognized field is a dynamic array so make sure to copy its
+   * content and not the pointer. */
+  link_specifier_setlen_un_unrecognized(
+                  result, link_specifier_getlen_un_unrecognized(lspec));
+  if (link_specifier_getlen_un_unrecognized(result)) {
+    memcpy(link_specifier_getarray_un_unrecognized(result),
+           link_specifier_getconstarray_un_unrecognized(lspec),
+           link_specifier_getlen_un_unrecognized(result));
+  }
+  return result;
+}
+
+/* From a given ed25519 public key pk and an optional secret, compute a
+ * blinded public key and put it in blinded_pk_out. This is only useful to
+ * the client side because the client only has access to the identity public
+ * key of the service. */
+void
+hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
+                        const uint8_t *secret, size_t secret_len,
+                        uint64_t time_period_num,
+                        ed25519_public_key_t *blinded_pk_out)
+{
+  /* Our blinding key API requires a 32 bytes parameter. */
+  uint8_t param[DIGEST256_LEN];
+
+  tor_assert(pk);
+  tor_assert(blinded_pk_out);
+  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+
+  build_blinded_key_param(pk, secret, secret_len,
+                          time_period_num, get_time_period_length(), param);
+  ed25519_public_blind(blinded_pk_out, pk, param);
+
+  memwipe(param, 0, sizeof(param));
+}
+
+/* From a given ed25519 keypair kp and an optional secret, compute a blinded
+ * keypair for the current time period and put it in blinded_kp_out. This is
+ * only useful by the service side because the client doesn't have access to
+ * the identity secret key. */
+void
+hs_build_blinded_keypair(const ed25519_keypair_t *kp,
+                         const uint8_t *secret, size_t secret_len,
+                         uint64_t time_period_num,
+                         ed25519_keypair_t *blinded_kp_out)
+{
+  /* Our blinding key API requires a 32 bytes parameter. */
+  uint8_t param[DIGEST256_LEN];
+
+  tor_assert(kp);
+  tor_assert(blinded_kp_out);
+  /* Extra safety. A zeroed key is bad. */
+  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+
+  build_blinded_key_param(&kp->pubkey, secret, secret_len,
+                          time_period_num, get_time_period_length(), param);
+  ed25519_keypair_blind(blinded_kp_out, kp, param);
+
+  memwipe(param, 0, sizeof(param));
+}
+
+/* Return true if we are currently in the time segment between a new time
+ * period and a new SRV (in the real network that happens between 12:00 and
+ * 00:00 UTC). Here is a diagram showing exactly when this returns true:
+ *
+ *    +------------------------------------------------------------------+
+ *    |                                                                  |
+ *    | 00:00      12:00       00:00       12:00       00:00       12:00 |
+ *    | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+ *    |                                                                  |
+ *    |  $==========|-----------$===========|-----------$===========|    |
+ *    |             ^^^^^^^^^^^^            ^^^^^^^^^^^^                 |
+ *    |                                                                  |
+ *    +------------------------------------------------------------------+
+ */
+MOCK_IMPL(int,
+hs_in_period_between_tp_and_srv,(const networkstatus_t *consensus, time_t now))
+{
+  time_t valid_after;
+  time_t srv_start_time, tp_start_time;
+
+  if (!consensus) {
+    consensus = networkstatus_get_live_consensus(now);
+    if (!consensus) {
+      return 0;
+    }
+  }
+
+  /* Get start time of next TP and of current SRV protocol run, and check if we
+   * are between them. */
+  valid_after = consensus->valid_after;
+  srv_start_time =
+    sr_state_get_start_time_of_current_protocol_run(valid_after);
+  tp_start_time = hs_get_start_time_of_next_time_period(srv_start_time);
+
+  if (valid_after >= srv_start_time && valid_after < tp_start_time) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* Return 1 if any virtual port in ports needs a circuit with good uptime.
+ * Else return 0. */
+int
+hs_service_requires_uptime_circ(const smartlist_t *ports)
+{
+  tor_assert(ports);
+
+  SMARTLIST_FOREACH_BEGIN(ports, rend_service_port_config_t *, p) {
+    if (smartlist_contains_int_as_string(get_options()->LongLivedPorts,
+                                         p->virtual_port)) {
+      return 1;
+    }
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
+
+/* Build hs_index which is used to find the responsible hsdirs. This index
+ * value is used to select the responsible HSDir where their hsdir_index is
+ * closest to this value.
+ *    SHA3-256("store-at-idx" | blinded_public_key |
+ *             INT_8(replicanum) | INT_8(period_length) | INT_8(period_num) )
+ *
+ * hs_index_out must be large enough to receive DIGEST256_LEN bytes. */
+void
+hs_build_hs_index(uint64_t replica, const ed25519_public_key_t *blinded_pk,
+                  uint64_t period_num, uint8_t *hs_index_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(blinded_pk);
+  tor_assert(hs_index_out);
+
+  /* Build hs_index. See construction at top of function comment. */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HS_INDEX_PREFIX, HS_INDEX_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+
+  /* Now setup INT_8(replicanum) | INT_8(period_length) | INT_8(period_num) */
+  {
+    uint64_t period_length = get_time_period_length();
+    char buf[sizeof(uint64_t)*3];
+    size_t offset = 0;
+    set_uint64(buf, tor_htonll(replica));
+    offset += sizeof(uint64_t);
+    set_uint64(buf+offset, tor_htonll(period_length));
+    offset += sizeof(uint64_t);
+    set_uint64(buf+offset, tor_htonll(period_num));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(buf));
+
+    crypto_digest_add_bytes(digest, buf, sizeof(buf));
+  }
+
+  crypto_digest_get_digest(digest, (char *) hs_index_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/* Build hsdir_index which is used to find the responsible hsdirs. This is the
+ * index value that is compare to the hs_index when selecting an HSDir.
+ *    SHA3-256("node-idx" | node_identity |
+ *             shared_random_value | INT_8(period_length) | INT_8(period_num) )
+ *
+ * hsdir_index_out must be large enough to receive DIGEST256_LEN bytes. */
+void
+hs_build_hsdir_index(const ed25519_public_key_t *identity_pk,
+                     const uint8_t *srv_value, uint64_t period_num,
+                     uint8_t *hsdir_index_out)
+{
+  crypto_digest_t *digest;
+
+  tor_assert(identity_pk);
+  tor_assert(srv_value);
+  tor_assert(hsdir_index_out);
+
+  /* Build hsdir_index. See construction at top of function comment. */
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  crypto_digest_add_bytes(digest, HSDIR_INDEX_PREFIX, HSDIR_INDEX_PREFIX_LEN);
+  crypto_digest_add_bytes(digest, (const char *) identity_pk->pubkey,
+                          ED25519_PUBKEY_LEN);
+  crypto_digest_add_bytes(digest, (const char *) srv_value, DIGEST256_LEN);
+
+  {
+    uint64_t time_period_length = get_time_period_length();
+    char period_stuff[sizeof(uint64_t)*2];
+    size_t offset = 0;
+    set_uint64(period_stuff, tor_htonll(period_num));
+    offset += sizeof(uint64_t);
+    set_uint64(period_stuff+offset, tor_htonll(time_period_length));
+    offset += sizeof(uint64_t);
+    tor_assert(offset == sizeof(period_stuff));
+
+    crypto_digest_add_bytes(digest, period_stuff,  sizeof(period_stuff));
+  }
+
+  crypto_digest_get_digest(digest, (char *) hsdir_index_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
+}
+
+/* Return a newly allocated buffer containing the current shared random value
+ * or if not present, a disaster value is computed using the given time period
+ * number. If a consensus is provided in <b>ns</b>, use it to get the SRV
+ * value. This function can't fail. */
+uint8_t *
+hs_get_current_srv(uint64_t time_period_num, const networkstatus_t *ns)
+{
+  uint8_t *sr_value = tor_malloc_zero(DIGEST256_LEN);
+  const sr_srv_t *current_srv = sr_get_current(ns);
+
+  if (current_srv) {
+    memcpy(sr_value, current_srv->value, sizeof(current_srv->value));
+  } else {
+    /* Disaster mode. */
+    get_disaster_srv(time_period_num, sr_value);
+  }
+  return sr_value;
+}
+
+/* Return a newly allocated buffer containing the previous shared random
+ * value or if not present, a disaster value is computed using the given time
+ * period number. This function can't fail. */
+uint8_t *
+hs_get_previous_srv(uint64_t time_period_num, const networkstatus_t *ns)
+{
+  uint8_t *sr_value = tor_malloc_zero(DIGEST256_LEN);
+  const sr_srv_t *previous_srv = sr_get_previous(ns);
+
+  if (previous_srv) {
+    memcpy(sr_value, previous_srv->value, sizeof(previous_srv->value));
+  } else {
+    /* Disaster mode. */
+    get_disaster_srv(time_period_num, sr_value);
+  }
+  return sr_value;
+}
+
+/* Return the number of replicas defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_n_replicas(void)
+{
+  /* The [1,16] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_n_replicas",
+                                 HS_DEFAULT_HSDIR_N_REPLICAS, 1, 16);
+}
+
+/* Return the spread fetch value defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_spread_fetch(void)
+{
+  /* The [1,128] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_spread_fetch",
+                                 HS_DEFAULT_HSDIR_SPREAD_FETCH, 1, 128);
+}
+
+/* Return the spread store value defined by a consensus parameter or the
+ * default value. */
+int32_t
+hs_get_hsdir_spread_store(void)
+{
+  /* The [1,128] range is a specification requirement. */
+  return networkstatus_get_param(NULL, "hsdir_spread_store",
+                                 HS_DEFAULT_HSDIR_SPREAD_STORE, 1, 128);
+}
+
+/** <b>node</b> is an HSDir so make sure that we have assigned an hsdir index.
+ *  Return 0 if everything is as expected, else return -1. */
+static int
+node_has_hsdir_index(const node_t *node)
+{
+  tor_assert(node_supports_v3_hsdir(node));
+
+  /* A node can't have an HSDir index without a descriptor since we need desc
+   * to get its ed25519 key.  for_direct_connect should be zero, since we
+   * always use the consensus-indexed node's keys to build the hash ring, even
+   * if some of the consensus-indexed nodes are also bridges. */
+  if (!node_has_preferred_descriptor(node, 0)) {
+    return 0;
+  }
+
+  /* At this point, since the node has a desc, this node must also have an
+   * hsdir index. If not, something went wrong, so BUG out. */
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+                          DIGEST256_LEN))) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/* For a given blinded key and time period number, get the responsible HSDir
+ * and put their routerstatus_t object in the responsible_dirs list. If
+ * 'use_second_hsdir_index' is true, use the second hsdir_index of the node_t
+ * is used. If 'for_fetching' is true, the spread fetch consensus parameter is
+ * used else the spread store is used which is only for upload. This function
+ * can't fail but it is possible that the responsible_dirs list contains fewer
+ * nodes than expected.
+ *
+ * This function goes over the latest consensus routerstatus list and sorts it
+ * by their node_t hsdir_index then does a binary search to find the closest
+ * node. All of this makes it a bit CPU intensive so use it wisely. */
+void
+hs_get_responsible_hsdirs(const ed25519_public_key_t *blinded_pk,
+                          uint64_t time_period_num, int use_second_hsdir_index,
+                          int for_fetching, smartlist_t *responsible_dirs)
+{
+  smartlist_t *sorted_nodes;
+  /* The compare function used for the smartlist bsearch. We have two
+   * different depending on is_next_period. */
+  int (*cmp_fct)(const void *, const void **);
+
+  tor_assert(blinded_pk);
+  tor_assert(responsible_dirs);
+
+  sorted_nodes = smartlist_new();
+
+  /* Make sure we actually have a live consensus */
+  networkstatus_t *c = networkstatus_get_live_consensus(approx_time());
+  if (!c || smartlist_len(c->routerstatus_list) == 0) {
+      log_warn(LD_REND, "No live consensus so we can't get the responsible "
+               "hidden service directories.");
+      goto done;
+  }
+
+  /* Ensure the nodelist is fresh, since it contains the HSDir indices. */
+  nodelist_ensure_freshness(c);
+
+  /* Add every node_t that support HSDir v3 for which we do have a valid
+   * hsdir_index already computed for them for this consensus. */
+  {
+    SMARTLIST_FOREACH_BEGIN(c->routerstatus_list, const routerstatus_t *, rs) {
+      /* Even though this node_t object won't be modified and should be const,
+       * we can't add const object in a smartlist_t. */
+      node_t *n = node_get_mutable_by_id(rs->identity_digest);
+      tor_assert(n);
+      if (node_supports_v3_hsdir(n) && rs->is_hs_dir) {
+        if (!node_has_hsdir_index(n)) {
+          log_info(LD_GENERAL, "Node %s was found without hsdir index.",
+                   node_describe(n));
+          continue;
+        }
+        smartlist_add(sorted_nodes, n);
+      }
+    } SMARTLIST_FOREACH_END(rs);
+  }
+  if (smartlist_len(sorted_nodes) == 0) {
+    log_warn(LD_REND, "No nodes found to be HSDir or supporting v3.");
+    goto done;
+  }
+
+  /* First thing we have to do is sort all node_t by hsdir_index. The
+   * is_next_period tells us if we want the current or the next one. Set the
+   * bsearch compare function also while we are at it. */
+  if (for_fetching) {
+    smartlist_sort(sorted_nodes, compare_node_fetch_hsdir_index);
+    cmp_fct = compare_digest_to_fetch_hsdir_index;
+  } else if (use_second_hsdir_index) {
+    smartlist_sort(sorted_nodes, compare_node_store_second_hsdir_index);
+    cmp_fct = compare_digest_to_store_second_hsdir_index;
+  } else {
+    smartlist_sort(sorted_nodes, compare_node_store_first_hsdir_index);
+    cmp_fct = compare_digest_to_store_first_hsdir_index;
+  }
+
+  /* For all replicas, we'll select a set of HSDirs using the consensus
+   * parameters and the sorted list. The replica starting at value 1 is
+   * defined by the specification. */
+  for (int replica = 1; replica <= hs_get_hsdir_n_replicas(); replica++) {
+    int idx, start, found, n_added = 0;
+    uint8_t hs_index[DIGEST256_LEN] = {0};
+    /* Number of node to add to the responsible dirs list depends on if we are
+     * trying to fetch or store. A client always fetches. */
+    int n_to_add = (for_fetching) ? hs_get_hsdir_spread_fetch() :
+                                    hs_get_hsdir_spread_store();
+
+    /* Get the index that we should use to select the node. */
+    hs_build_hs_index(replica, blinded_pk, time_period_num, hs_index);
+    /* The compare function pointer has been set correctly earlier. */
+    start = idx = smartlist_bsearch_idx(sorted_nodes, hs_index, cmp_fct,
+                                        &found);
+    /* Getting the length of the list if no member is greater than the key we
+     * are looking for so start at the first element. */
+    if (idx == smartlist_len(sorted_nodes)) {
+      start = idx = 0;
+    }
+    while (n_added < n_to_add) {
+      const node_t *node = smartlist_get(sorted_nodes, idx);
+      /* If the node has already been selected which is possible between
+       * replicas, the specification says to skip over. */
+      if (!smartlist_contains(responsible_dirs, node->rs)) {
+        smartlist_add(responsible_dirs, node->rs);
+        ++n_added;
+      }
+      if (++idx == smartlist_len(sorted_nodes)) {
+        /* Wrap if we've reached the end of the list. */
+        idx = 0;
+      }
+      if (idx == start) {
+        /* We've gone over the whole list, stop and avoid infinite loop. */
+        break;
+      }
+    }
+  }
+
+ done:
+  smartlist_free(sorted_nodes);
+}
+
+/*********************** HSDir request tracking ***************************/
+
+/** Return the period for which a hidden service directory cannot be queried
+ * for the same descriptor ID again, taking TestingTorNetwork into account. */
+time_t
+hs_hsdir_requery_period(const or_options_t *options)
+{
+  tor_assert(options);
+
+  if (options->TestingTorNetwork) {
+    return REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING;
+  } else {
+    return REND_HID_SERV_DIR_REQUERY_PERIOD;
+  }
+}
+
+/** Tracks requests for fetching hidden service descriptors. It's used by
+ *  hidden service clients, to avoid querying HSDirs that have already failed
+ *  giving back a descriptor. The same data structure is used to track both v2
+ *  and v3 HS descriptor requests.
+ *
+ * The string map is a key/value store that contains the last request times to
+ * hidden service directories for certain queries. Specifically:
+ *
+ *   key = base32(hsdir_identity) + base32(hs_identity)
+ *   value = time_t of last request for that hs_identity to that HSDir
+ *
+ * where 'hsdir_identity' is the identity digest of the HSDir node, and
+ * 'hs_identity' is the descriptor ID of the HS in the v2 case, or the ed25519
+ * blinded public key of the HS in the v3 case. */
+static strmap_t *last_hid_serv_requests_ = NULL;
+
+/** Returns last_hid_serv_requests_, initializing it to a new strmap if
+ * necessary. */
+STATIC strmap_t *
+get_last_hid_serv_requests(void)
+{
+  if (!last_hid_serv_requests_)
+    last_hid_serv_requests_ = strmap_new();
+  return last_hid_serv_requests_;
+}
+
+/** Look up the last request time to hidden service directory <b>hs_dir</b>
+ * for descriptor request key <b>req_key_str</b> which is the descriptor ID
+ * for a v2 service or the blinded key for v3. If <b>set</b> is non-zero,
+ * assign the current time <b>now</b> and return that.  Otherwise, return the
+ * most recent request time, or 0 if no such request has been sent before. */
+time_t
+hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
+                                const char *req_key_str,
+                                time_t now, int set)
+{
+  char hsdir_id_base32[BASE32_DIGEST_LEN + 1];
+  char *hsdir_desc_comb_id = NULL;
+  time_t *last_request_ptr;
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+
+  /* Create the key */
+  base32_encode(hsdir_id_base32, sizeof(hsdir_id_base32),
+                hs_dir->identity_digest, DIGEST_LEN);
+  tor_asprintf(&hsdir_desc_comb_id, "%s%s", hsdir_id_base32, req_key_str);
+
+  if (set) {
+    time_t *oldptr;
+    last_request_ptr = tor_malloc_zero(sizeof(time_t));
+    *last_request_ptr = now;
+    oldptr = strmap_set(last_hid_serv_requests, hsdir_desc_comb_id,
+                        last_request_ptr);
+    tor_free(oldptr);
+  } else {
+    last_request_ptr = strmap_get(last_hid_serv_requests,
+                                  hsdir_desc_comb_id);
+  }
+
+  tor_free(hsdir_desc_comb_id);
+  return (last_request_ptr) ? *last_request_ptr : 0;
+}
+
+/** Clean the history of request times to hidden service directories, so that
+ * it does not contain requests older than REND_HID_SERV_DIR_REQUERY_PERIOD
+ * seconds any more. */
+void
+hs_clean_last_hid_serv_requests(time_t now)
+{
+  strmap_iter_t *iter;
+  time_t cutoff = now - hs_hsdir_requery_period(get_options());
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+  for (iter = strmap_iter_init(last_hid_serv_requests);
+       !strmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    time_t *ent;
+    strmap_iter_get(iter, &key, &val);
+    ent = (time_t *) val;
+    if (*ent < cutoff) {
+      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
+      tor_free(ent);
+    } else {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+    }
+  }
+}
+
+/** Remove all requests related to the descriptor request key string
+ * <b>req_key_str</b> from the history of times of requests to hidden service
+ * directories.
+ *
+ * This is called from rend_client_note_connection_attempt_ended(), which
+ * must be idempotent, so any future changes to this function must leave it
+ * idempotent too. */
+void
+hs_purge_hid_serv_from_last_hid_serv_requests(const char *req_key_str)
+{
+  strmap_iter_t *iter;
+  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
+
+  for (iter = strmap_iter_init(last_hid_serv_requests);
+       !strmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    strmap_iter_get(iter, &key, &val);
+
+    /* XXX: The use of REND_DESC_ID_V2_LEN_BASE32 is very wrong in terms of
+     * semantic, see #23305. */
+
+    /* This strmap contains variable-sized elements so this is a basic length
+     * check on the strings we are about to compare. The key is variable sized
+     * since it's composed as follows:
+     *   key = base32(hsdir_identity) + base32(req_key_str)
+     * where 'req_key_str' is the descriptor ID of the HS in the v2 case, or
+     * the ed25519 blinded public key of the HS in the v3 case. */
+    if (strlen(key) < REND_DESC_ID_V2_LEN_BASE32 + strlen(req_key_str)) {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+      continue;
+    }
+
+    /* Check if the tracked request matches our request key */
+    if (tor_memeq(key + REND_DESC_ID_V2_LEN_BASE32, req_key_str,
+                  strlen(req_key_str))) {
+      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
+      tor_free(val);
+    } else {
+      iter = strmap_iter_next(last_hid_serv_requests, iter);
+    }
+  }
+}
+
+/** Purge the history of request times to hidden service directories,
+ * so that future lookups of an HS descriptor will not fail because we
+ * accessed all of the HSDir relays responsible for the descriptor
+ * recently. */
+void
+hs_purge_last_hid_serv_requests(void)
+{
+ /* Don't create the table if it doesn't exist yet (and it may very
+   * well not exist if the user hasn't accessed any HSes)... */
+  strmap_t *old_last_hid_serv_requests = last_hid_serv_requests_;
+  /* ... and let get_last_hid_serv_requests re-create it for us if
+   * necessary. */
+  last_hid_serv_requests_ = NULL;
+
+  if (old_last_hid_serv_requests != NULL) {
+    log_info(LD_REND, "Purging client last-HS-desc-request-time table");
+    strmap_free(old_last_hid_serv_requests, tor_free_);
+  }
+}
+
+/***********************************************************************/
+
+/** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
+ *  one that we should use to fetch a descriptor right now. Take into account
+ *  previous failed attempts at fetching this descriptor from HSDirs using the
+ *  string identifier <b>req_key_str</b>.
+ *
+ *  Steals ownership of <b>responsible_dirs</b>.
+ *
+ *  Return the routerstatus of the chosen HSDir if successful, otherwise return
+ *  NULL if no HSDirs are worth trying right now. */
+routerstatus_t *
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+{
+  smartlist_t *usable_responsible_dirs = smartlist_new();
+  const or_options_t *options = get_options();
+  routerstatus_t *hs_dir;
+  time_t now = time(NULL);
+  int excluded_some;
+
+  tor_assert(req_key_str);
+
+  /* Clean outdated request history first. */
+  hs_clean_last_hid_serv_requests(now);
+
+  /* Only select those hidden service directories to which we did not send a
+   * request recently and for which we have a router descriptor here.
+   *
+   * Use for_direct_connect==0 even if we will be connecting to the node
+   * directly, since we always use the key information in the
+   * consensus-indexed node descriptors for building the index.
+   **/
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, routerstatus_t *, dir) {
+    time_t last = hs_lookup_last_hid_serv_request(dir, req_key_str, 0, 0);
+    const node_t *node = node_get_by_id(dir->identity_digest);
+    if (last + hs_hsdir_requery_period(options) >= now ||
+        !node || !node_has_preferred_descriptor(node, 0)) {
+      SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      continue;
+    }
+    if (!routerset_contains_node(options->ExcludeNodes, node)) {
+      smartlist_add(usable_responsible_dirs, dir);
+    }
+  } SMARTLIST_FOREACH_END(dir);
+
+  excluded_some =
+    smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
+
+  hs_dir = smartlist_choose(usable_responsible_dirs);
+  if (!hs_dir && !options->StrictNodes) {
+    hs_dir = smartlist_choose(responsible_dirs);
+  }
+
+  smartlist_free(responsible_dirs);
+  smartlist_free(usable_responsible_dirs);
+  if (!hs_dir) {
+    log_info(LD_REND, "Could not pick one of the responsible hidden "
+                      "service directories, because we requested them all "
+                      "recently without success.");
+    if (options->StrictNodes && excluded_some) {
+      log_warn(LD_REND, "Could not pick a hidden service directory for the "
+               "requested hidden service: they are all either down or "
+               "excluded, and StrictNodes is set.");
+    }
+  } else {
+    /* Remember that we are requesting a descriptor from this hidden service
+     * directory now. */
+    hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
+  }
+
+  return hs_dir;
+}
+
+/* From a list of link specifier, an onion key and if we are requesting a
+ * direct connection (ex: single onion service), return a newly allocated
+ * extend_info_t object. This function always returns an extend info with
+ * an IPv4 address, or NULL.
+ *
+ * It performs the following checks:
+ *  if either IPv4 or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ */
+extend_info_t *
+hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
+                               const curve25519_public_key_t *onion_key,
+                               int direct_conn)
+{
+  int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
+  char legacy_id[DIGEST_LEN] = {0};
+  uint16_t port_v4 = 0;
+  tor_addr_t addr_v4;
+  ed25519_public_key_t ed25519_pk;
+  extend_info_t *info = NULL;
+
+  tor_assert(lspecs);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_LEGACY_ID:
+      /* Make sure we do have enough bytes for the legacy ID. */
+      if (link_specifier_getlen_un_legacy_id(ls) < sizeof(legacy_id)) {
+        break;
+      }
+      memcpy(legacy_id, link_specifier_getconstarray_un_legacy_id(ls),
+             sizeof(legacy_id));
+      have_legacy_id = 1;
+      break;
+    case LS_ED25519_ID:
+      memcpy(ed25519_pk.pubkey,
+             link_specifier_getconstarray_un_ed25519_id(ls),
+             ED25519_PUBKEY_LEN);
+      have_ed25519_id = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* Legacy ID is mandatory, and we require IPv4. */
+  if (!have_v4 || !have_legacy_id) {
+    goto done;
+  }
+
+  /* We know we have IPv4, because we just checked. */
+  if (!direct_conn) {
+    /* All clients can extend to any IPv4 via a 3-hop path. */
+    goto validate;
+  } else if (direct_conn &&
+             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
+                                                  FIREWALL_OR_CONNECTION,
+                                                  0, 0)) {
+    /* Direct connection and we can reach it in IPv4 so go for it. */
+    goto validate;
+
+    /* We will add support for falling back to a 3-hop path in a later
+     * release. */
+  } else {
+    /* If we can't reach IPv4, return NULL. */
+    goto done;
+  }
+
+  /* We will add support for IPv6 in a later release. */
+
+ validate:
+  /* We'll validate now that the address we've picked isn't a private one. If
+   * it is, are we allowing to extend to private address? */
+  if (!extend_info_addr_is_allowed(&addr_v4)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Requested address is private and we are not allowed to extend to "
+           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+    goto done;
+  }
+
+  /* We do have everything for which we think we can connect successfully. */
+  info = extend_info_new(NULL, legacy_id,
+                         (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
+                         onion_key, &addr_v4, port_v4);
+ done:
+  return info;
+}
+
+/***********************************************************************/
+
+/* Initialize the entire HS subsytem. This is called in tor_init() before any
+ * torrc options are loaded. Only for >= v3. */
+void
+hs_init(void)
+{
+  hs_circuitmap_init();
+  hs_service_init();
+  hs_cache_init();
+}
+
+/* Release and cleanup all memory of the HS subsystem (all version). This is
+ * called by tor_free_all(). */
+void
+hs_free_all(void)
+{
+  hs_circuitmap_free_all();
+  hs_service_free_all();
+  hs_cache_free_all();
+  hs_client_free_all();
+}
+
+/* For the given origin circuit circ, decrement the number of rendezvous
+ * stream counter. This handles every hidden service version. */
+void
+hs_dec_rdv_stream_counter(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circ->rend_data) {
+    circ->rend_data->nr_streams--;
+  } else if (circ->hs_ident) {
+    circ->hs_ident->num_rdv_streams--;
+  } else {
+    /* Should not be called if this circuit is not for hidden service. */
+    tor_assert_nonfatal_unreached();
+  }
+}
+
+/* For the given origin circuit circ, increment the number of rendezvous
+ * stream counter. This handles every hidden service version. */
+void
+hs_inc_rdv_stream_counter(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circ->rend_data) {
+    circ->rend_data->nr_streams++;
+  } else if (circ->hs_ident) {
+    circ->hs_ident->num_rdv_streams++;
+  } else {
+    /* Should not be called if this circuit is not for hidden service. */
+    tor_assert_nonfatal_unreached();
+  }
+}
diff --git a/src/or/hs_common.h b/src/or/hs_common.h
new file mode 100644
index 0000000000..ef7d5dca2b
--- /dev/null
+++ b/src/or/hs_common.h
@@ -0,0 +1,284 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.h
+ * \brief Header file containing common data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_COMMON_H
+#define TOR_HS_COMMON_H
+
+#include "or.h"
+
+/* Trunnel */
+#include "ed25519_cert.h"
+
+/* Protocol version 2. Use this instead of hardcoding "2" in the code base,
+ * this adds a clearer semantic to the value when used. */
+#define HS_VERSION_TWO 2
+/* Version 3 of the protocol (prop224). */
+#define HS_VERSION_THREE 3
+/* Earliest and latest version we support. */
+#define HS_VERSION_MIN HS_VERSION_TWO
+#define HS_VERSION_MAX HS_VERSION_THREE
+
+/** Try to maintain this many intro points per service by default. */
+#define NUM_INTRO_POINTS_DEFAULT 3
+/** Maximum number of intro points per generic and version 2 service. */
+#define NUM_INTRO_POINTS_MAX 10
+/** Number of extra intro points we launch if our set of intro nodes is empty.
+ * See proposal 155, section 4. */
+#define NUM_INTRO_POINTS_EXTRA 2
+
+/** If we can't build our intro circuits, don't retry for this long. */
+#define INTRO_CIRC_RETRY_PERIOD (60*5)
+/** Don't try to build more than this many circuits before giving up for a
+ * while.*/
+#define MAX_INTRO_CIRCS_PER_PERIOD 10
+/** How many times will a hidden service operator attempt to connect to a
+ * requested rendezvous point before giving up? */
+#define MAX_REND_FAILURES 1
+/** How many seconds should we spend trying to connect to a requested
+ * rendezvous point before giving up? */
+#define MAX_REND_TIMEOUT 30
+
+/* String prefix for the signature of ESTABLISH_INTRO */
+#define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1"
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+
+/* Prefix of the onion address checksum. */
+#define HS_SERVICE_ADDR_CHECKSUM_PREFIX ".onion checksum"
+/* Length of the checksum prefix minus the NUL terminated byte. */
+#define HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN \
+  (sizeof(HS_SERVICE_ADDR_CHECKSUM_PREFIX) - 1)
+/* Length of the resulting checksum of the address. The construction of this
+ * checksum looks like:
+ *   CHECKSUM = ".onion checksum" || PUBKEY || VERSION
+ * where VERSION is 1 byte. This is pre-hashing. */
+#define HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN \
+  (HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN + ED25519_PUBKEY_LEN + sizeof(uint8_t))
+/* The amount of bytes we use from the address checksum. */
+#define HS_SERVICE_ADDR_CHECKSUM_LEN_USED 2
+/* Length of the binary encoded service address which is of course before the
+ * base32 encoding. Construction is:
+ *    PUBKEY || CHECKSUM || VERSION
+ * with 1 byte VERSION and 2 bytes CHECKSUM. The following is 35 bytes. */
+#define HS_SERVICE_ADDR_LEN \
+  (ED25519_PUBKEY_LEN + HS_SERVICE_ADDR_CHECKSUM_LEN_USED + sizeof(uint8_t))
+/* Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
+ * length ends up to 56 bytes (not counting the terminated NUL byte.) */
+#define HS_SERVICE_ADDR_LEN_BASE32 \
+  (CEIL_DIV(HS_SERVICE_ADDR_LEN * 8, 5))
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */
+
+/* Keyblinding parameter construction is as follow:
+ *    "key-blind" || INT_8(period_num) || INT_8(start_period_sec) */
+#define HS_KEYBLIND_NONCE_PREFIX "key-blind"
+#define HS_KEYBLIND_NONCE_PREFIX_LEN (sizeof(HS_KEYBLIND_NONCE_PREFIX) - 1)
+#define HS_KEYBLIND_NONCE_LEN \
+  (HS_KEYBLIND_NONCE_PREFIX_LEN + sizeof(uint64_t) + sizeof(uint64_t))
+
+/* Credential and subcredential prefix value. */
+#define HS_CREDENTIAL_PREFIX "credential"
+#define HS_CREDENTIAL_PREFIX_LEN (sizeof(HS_CREDENTIAL_PREFIX) - 1)
+#define HS_SUBCREDENTIAL_PREFIX "subcredential"
+#define HS_SUBCREDENTIAL_PREFIX_LEN (sizeof(HS_SUBCREDENTIAL_PREFIX) - 1)
+
+/* Node hidden service stored at index prefix value. */
+#define HS_INDEX_PREFIX "store-at-idx"
+#define HS_INDEX_PREFIX_LEN (sizeof(HS_INDEX_PREFIX) - 1)
+
+/* Node hidden service directory index prefix value. */
+#define HSDIR_INDEX_PREFIX "node-idx"
+#define HSDIR_INDEX_PREFIX_LEN (sizeof(HSDIR_INDEX_PREFIX) - 1)
+
+/* Prefix of the shared random value disaster mode. */
+#define HS_SRV_DISASTER_PREFIX "shared-random-disaster"
+#define HS_SRV_DISASTER_PREFIX_LEN (sizeof(HS_SRV_DISASTER_PREFIX) - 1)
+
+/* Default value of number of hsdir replicas (hsdir_n_replicas). */
+#define HS_DEFAULT_HSDIR_N_REPLICAS 2
+/* Default value of hsdir spread store (hsdir_spread_store). */
+#define HS_DEFAULT_HSDIR_SPREAD_STORE 4
+/* Default value of hsdir spread fetch (hsdir_spread_fetch). */
+#define HS_DEFAULT_HSDIR_SPREAD_FETCH 3
+
+/* The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
+ * bigger than the 84 bytes needed for version 3 so we need to pad up to that
+ * length so it is indistinguishable between versions. */
+#define HS_LEGACY_RENDEZVOUS_CELL_SIZE \
+  (REND_COOKIE_LEN + DH_KEY_LEN + DIGEST_LEN)
+
+/* Type of authentication key used by an introduction point. */
+typedef enum {
+  HS_AUTH_KEY_TYPE_LEGACY  = 1,
+  HS_AUTH_KEY_TYPE_ED25519 = 2,
+} hs_auth_key_type_t;
+
+/* Return value when adding an ephemeral service through the ADD_ONION
+ * control port command. Both v2 and v3 share these. */
+typedef enum {
+  RSAE_BADAUTH     = -5, /**< Invalid auth_type/auth_clients */
+  RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */
+  RSAE_ADDREXISTS  = -3, /**< Onion address collision */
+  RSAE_BADPRIVKEY  = -2, /**< Invalid public key */
+  RSAE_INTERNAL    = -1, /**< Internal error */
+  RSAE_OKAY        = 0   /**< Service added as expected */
+} hs_service_add_ephemeral_status_t;
+
+/* Represents the mapping from a virtual port of a rendezvous service to a
+ * real port on some IP. */
+typedef struct rend_service_port_config_t {
+  /* The incoming HS virtual port we're mapping */
+  uint16_t virtual_port;
+  /* Is this an AF_UNIX port? */
+  unsigned int is_unix_addr:1;
+  /* The outgoing TCP port to use, if !is_unix_addr */
+  uint16_t real_port;
+  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
+  tor_addr_t real_addr;
+  /* The socket path to connect to, if is_unix_addr */
+  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
+} rend_service_port_config_t;
+
+void hs_init(void);
+void hs_free_all(void);
+
+void hs_cleanup_circ(circuit_t *circ);
+
+int hs_check_service_private_dir(const char *username, const char *path,
+                                 unsigned int dir_group_readable,
+                                 unsigned int create);
+int hs_get_service_max_rend_failures(void);
+
+char *hs_path_from_filename(const char *directory, const char *filename);
+void hs_build_address(const ed25519_public_key_t *key, uint8_t version,
+                      char *addr_out);
+int hs_address_is_valid(const char *address);
+int hs_parse_address(const char *address, ed25519_public_key_t *key_out,
+                     uint8_t *checksum_out, uint8_t *version_out);
+
+void hs_build_blinded_pubkey(const ed25519_public_key_t *pubkey,
+                             const uint8_t *secret, size_t secret_len,
+                             uint64_t time_period_num,
+                             ed25519_public_key_t *pubkey_out);
+void hs_build_blinded_keypair(const ed25519_keypair_t *kp,
+                              const uint8_t *secret, size_t secret_len,
+                              uint64_t time_period_num,
+                              ed25519_keypair_t *kp_out);
+int hs_service_requires_uptime_circ(const smartlist_t *ports);
+
+void rend_data_free_(rend_data_t *data);
+#define rend_data_free(data) \
+  FREE_AND_NULL(rend_data_t, rend_data_free_, (data))
+rend_data_t *rend_data_dup(const rend_data_t *data);
+rend_data_t *rend_data_client_create(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *cookie,
+                                     rend_auth_type_t auth_type);
+rend_data_t *rend_data_service_create(const char *onion_address,
+                                      const char *pk_digest,
+                                      const uint8_t *cookie,
+                                      rend_auth_type_t auth_type);
+const char *rend_data_get_address(const rend_data_t *rend_data);
+const char *rend_data_get_desc_id(const rend_data_t *rend_data,
+                                  uint8_t replica, size_t *len_out);
+const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data,
+                                       size_t *len_out);
+
+routerstatus_t *pick_hsdir(const char *desc_id, const char *desc_id_base32);
+
+void hs_get_subcredential(const ed25519_public_key_t *identity_pk,
+                          const ed25519_public_key_t *blinded_pk,
+                          uint8_t *subcred_out);
+
+uint64_t hs_get_previous_time_period_num(time_t now);
+uint64_t hs_get_time_period_num(time_t now);
+uint64_t hs_get_next_time_period_num(time_t now);
+time_t hs_get_start_time_of_next_time_period(time_t now);
+
+link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
+
+MOCK_DECL(int, hs_in_period_between_tp_and_srv,
+          (const networkstatus_t *consensus, time_t now));
+
+uint8_t *hs_get_current_srv(uint64_t time_period_num,
+                            const networkstatus_t *ns);
+uint8_t *hs_get_previous_srv(uint64_t time_period_num,
+                             const networkstatus_t *ns);
+
+void hs_build_hsdir_index(const ed25519_public_key_t *identity_pk,
+                          const uint8_t *srv, uint64_t period_num,
+                          uint8_t *hsdir_index_out);
+void hs_build_hs_index(uint64_t replica,
+                       const ed25519_public_key_t *blinded_pk,
+                       uint64_t period_num, uint8_t *hs_index_out);
+
+int32_t hs_get_hsdir_n_replicas(void);
+int32_t hs_get_hsdir_spread_fetch(void);
+int32_t hs_get_hsdir_spread_store(void);
+
+void hs_get_responsible_hsdirs(const ed25519_public_key_t *blinded_pk,
+                              uint64_t time_period_num,
+                              int use_second_hsdir_index,
+                              int for_fetching, smartlist_t *responsible_dirs);
+routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
+                              const char *req_key_str);
+
+time_t hs_hsdir_requery_period(const or_options_t *options);
+time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
+                                       const char *desc_id_base32,
+                                       time_t now, int set);
+void hs_clean_last_hid_serv_requests(time_t now);
+void hs_purge_hid_serv_from_last_hid_serv_requests(const char *desc_id);
+void hs_purge_last_hid_serv_requests(void);
+
+int hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn);
+
+void hs_inc_rdv_stream_counter(origin_circuit_t *circ);
+void hs_dec_rdv_stream_counter(origin_circuit_t *circ);
+
+extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
+                                  const curve25519_public_key_t *onion_key,
+                                  int direct_conn);
+
+#ifdef HS_COMMON_PRIVATE
+
+STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
+
+/** The period for which a hidden service directory cannot be queried for
+ * the same descriptor ID again. */
+#define REND_HID_SERV_DIR_REQUERY_PERIOD (15 * 60)
+/** Test networks generate a new consensus every 5 or 10 seconds.
+ * So allow them to requery HSDirs much faster. */
+#define REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING (5)
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC strmap_t *get_last_hid_serv_requests(void);
+STATIC uint64_t get_time_period_length(void);
+
+STATIC uint8_t *get_first_cached_disaster_srv(void);
+STATIC uint8_t *get_second_cached_disaster_srv(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_COMMON_PRIVATE) */
+
+#endif /* !defined(TOR_HS_COMMON_H) */
+
diff --git a/src/or/hs_config.c b/src/or/hs_config.c
new file mode 100644
index 0000000000..be223503a0
--- /dev/null
+++ b/src/or/hs_config.c
@@ -0,0 +1,590 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_config.c
+ * \brief Implement hidden service configuration subsystem.
+ *
+ * \details
+ *
+ * This file has basically one main entry point: hs_config_service_all(). It
+ * takes the torrc options and configure hidden service from it. In validate
+ * mode, nothing is added to the global service list or keys are not generated
+ * nor loaded.
+ *
+ * A service is configured in two steps. It is first created using the tor
+ * options and then put in a staging list. It will stay there until
+ * hs_service_load_all_keys() is called. That function is responsible to
+ * load/generate the keys for the service in the staging list and if
+ * successful, transfert the service to the main global service list where
+ * at that point it is ready to be used.
+ *
+ * Configuration functions are per-version and there is a main generic one for
+ * every option that is common to all version (config_generic_service).
+ **/
+
+#define HS_CONFIG_PRIVATE
+
+#include "hs_common.h"
+#include "hs_config.h"
+#include "hs_service.h"
+#include "rendservice.h"
+
+/* Using the given list of services, stage them into our global state. Every
+ * service version are handled. This function can remove entries in the given
+ * service_list.
+ *
+ * Staging a service means that we take all services in service_list and we
+ * put them in the staging list (global) which acts as a temporary list that
+ * is used by the service loading key process. In other words, staging a
+ * service puts it in a list to be considered when loading the keys and then
+ * moved to the main global list. */
+static void
+stage_services(smartlist_t *service_list)
+{
+  tor_assert(service_list);
+
+  /* This is v2 specific. Trigger service pruning which will make sure the
+   * just configured services end up in the main global list. It should only
+   * be done in non validation mode because v2 subsystem handles service
+   * object differently. */
+  rend_service_prune_list();
+
+  /* Cleanup v2 service from the list, we don't need those object anymore
+   * because we validated them all against the others and we want to stage
+   * only >= v3 service. And remember, v2 has a different object type which is
+   * shadow copied from an hs_service_t type. */
+  SMARTLIST_FOREACH_BEGIN(service_list, hs_service_t *, s) {
+    if (s->config.version == HS_VERSION_TWO) {
+      SMARTLIST_DEL_CURRENT(service_list, s);
+      hs_service_free(s);
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  /* This is >= v3 specific. Using the newly configured service list, stage
+   * them into our global state. Every object ownership is lost after. */
+  hs_service_stage_services(service_list);
+}
+
+/* Validate the given service against all service in the given list. If the
+ * service is ephemeral, this function ignores it. Services with the same
+ * directory path aren't allowed and will return an error. If a duplicate is
+ * found, 1 is returned else 0 if none found. */
+static int
+service_is_duplicate_in_list(const smartlist_t *service_list,
+                             const hs_service_t *service)
+{
+  int ret = 0;
+
+  tor_assert(service_list);
+  tor_assert(service);
+
+  /* Ephemeral service don't have a directory configured so no need to check
+   * for a service in the list having the same path. */
+  if (service->config.is_ephemeral) {
+    goto end;
+  }
+
+  /* XXX: Validate if we have any service that has the given service dir path.
+   * This has two problems:
+   *
+   * a) It's O(n^2), but the same comment from the bottom of
+   *    rend_config_services() should apply.
+   *
+   * b) We only compare directory paths as strings, so we can't
+   *    detect two distinct paths that specify the same directory
+   *    (which can arise from symlinks, case-insensitivity, bind
+   *    mounts, etc.).
+   *
+   * It also can't detect that two separate Tor instances are trying
+   * to use the same HiddenServiceDir; for that, we would need a
+   * lock file.  But this is enough to detect a simple mistake that
+   * at least one person has actually made. */
+  SMARTLIST_FOREACH_BEGIN(service_list, const hs_service_t *, s) {
+    if (!strcmp(s->config.directory_path, service->config.directory_path)) {
+      log_warn(LD_REND, "Another hidden service is already configured "
+                        "for directory %s",
+               escaped(service->config.directory_path));
+      ret = 1;
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+ end:
+  return ret;
+}
+
+/* Helper function: Given an configuration option name, its value, a minimum
+ * min and a maxium max, parse the value as a uint64_t. On success, ok is set
+ * to 1 and ret is the parsed value. On error, ok is set to 0 and ret must be
+ * ignored. This function logs both on error and success. */
+static uint64_t
+helper_parse_uint64(const char *opt, const char *value, uint64_t min,
+                    uint64_t max, int *ok)
+{
+  uint64_t ret = 0;
+
+  tor_assert(opt);
+  tor_assert(value);
+  tor_assert(ok);
+
+  *ok = 0;
+  ret = tor_parse_uint64(value, 10, min, max, ok, NULL);
+  if (!*ok) {
+    log_warn(LD_CONFIG, "%s must be between %" PRIu64 " and %"PRIu64
+                        ", not %s.",
+             opt, min, max, value);
+    goto err;
+  }
+  log_info(LD_CONFIG, "%s was parsed to %" PRIu64, opt, ret);
+ err:
+  return ret;
+}
+
+/* Return true iff the given options starting at line_ for a hidden service
+ * contains at least one invalid option. Each hidden service option don't
+ * apply to all versions so this function can find out. The line_ MUST start
+ * right after the HiddenServiceDir line of this service.
+ *
+ * This is mainly for usability so we can inform the user of any invalid
+ * option for the hidden service version instead of silently ignoring. */
+static int
+config_has_invalid_options(const config_line_t *line_,
+                           const hs_service_t *service)
+{
+  int ret = 0;
+  const char **optlist;
+  const config_line_t *line;
+
+  tor_assert(service);
+  tor_assert(service->config.version <= HS_VERSION_MAX);
+
+  /* List of options that a v3 service doesn't support thus must exclude from
+   * its configuration. */
+  const char *opts_exclude_v3[] = {
+    "HiddenServiceAuthorizeClient",
+    NULL /* End marker. */
+  };
+
+  /* Defining the size explicitly allows us to take advantage of the compiler
+   * which warns us if we ever bump the max version but forget to grow this
+   * array. The plus one is because we have a version 0 :). */
+  struct {
+    const char **list;
+  } exclude_lists[HS_VERSION_MAX + 1] = {
+    { NULL }, /* v0. */
+    { NULL }, /* v1. */
+    { NULL }, /* v2 */
+    { opts_exclude_v3 }, /* v3. */
+  };
+
+  optlist = exclude_lists[service->config.version].list;
+  if (optlist == NULL) {
+    /* No exclude options to look at for this version. */
+    goto end;
+  }
+  for (int i = 0; optlist[i]; i++) {
+    const char *opt = optlist[i];
+    for (line = line_; line; line = line->next) {
+      if (!strcasecmp(line->key, "HiddenServiceDir")) {
+        /* We just hit the next hidden service, stop right now. */
+        goto end;
+      }
+      if (!strcasecmp(line->key, opt)) {
+        log_warn(LD_CONFIG, "Hidden service option %s is incompatible with "
+                            "version %" PRIu32 " of service in %s",
+                 opt, service->config.version,
+                 service->config.directory_path);
+        ret = 1;
+        /* Continue the loop so we can find all possible options. */
+        continue;
+      }
+    }
+  }
+ end:
+  return ret;
+}
+
+/* Validate service configuration. This is used when loading the configuration
+ * and once we've setup a service object, it's config object is passed to this
+ * function for further validation. This does not validate service key
+ * material. Return 0 if valid else -1 if invalid. */
+static int
+config_validate_service(const hs_service_config_t *config)
+{
+  tor_assert(config);
+
+  /* Amount of ports validation. */
+  if (!config->ports || smartlist_len(config->ports) == 0) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
+             escaped(config->directory_path));
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* Configuration funcion for a version 3 service. The line_ must be pointing
+ * to the directive directly after a HiddenServiceDir. That way, when hitting
+ * the next HiddenServiceDir line or reaching the end of the list of lines, we
+ * know that we have to stop looking for more options. The given service
+ * object must be already allocated and passed through
+ * config_generic_service() prior to calling this function.
+ *
+ * Return 0 on success else a negative value. */
+static int
+config_service_v3(const config_line_t *line_,
+                  hs_service_config_t *config)
+{
+  int have_num_ip = 0;
+  const char *dup_opt_seen = NULL;
+  const config_line_t *line;
+
+  tor_assert(config);
+
+  for (line = line_; line; line = line->next) {
+    int ok = 0;
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* We just hit the next hidden service, stop right now. */
+      break;
+    }
+    /* Number of introduction points. */
+    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+      config->num_intro_points =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                                           NUM_INTRO_POINTS_DEFAULT,
+                                           HS_CONFIG_V3_MAX_INTRO_POINTS,
+                                           &ok);
+      if (!ok || have_num_ip) {
+        if (have_num_ip)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_num_ip = 1;
+      continue;
+    }
+  }
+
+  /* We do not load the key material for the service at this stage. This is
+   * done later once tor can confirm that it is in a running state. */
+
+  /* We are about to return a fully configured service so do one last pass of
+   * validation at it. */
+  if (config_validate_service(config) < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  if (dup_opt_seen) {
+    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
+  }
+  return -1;
+}
+
+/* Configure a service using the given options in line_ and options. This is
+ * called for any service regardless of its version which means that all
+ * directives in this function are generic to any service version. This
+ * function will also check the validity of the service directory path.
+ *
+ * The line_ must be pointing to the directive directly after a
+ * HiddenServiceDir. That way, when hitting the next HiddenServiceDir line or
+ * reaching the end of the list of lines, we know that we have to stop looking
+ * for more options.
+ *
+ * Return 0 on success else -1. */
+static int
+config_generic_service(const config_line_t *line_,
+                       const or_options_t *options,
+                       hs_service_t *service)
+{
+  int dir_seen = 0;
+  const config_line_t *line;
+  hs_service_config_t *config;
+  /* If this is set, we've seen a duplicate of this option. Keep the string
+   * so we can log the directive. */
+  const char *dup_opt_seen = NULL;
+  /* These variables will tell us if we ever have duplicate. */
+  int have_version = 0, have_allow_unknown_ports = 0;
+  int have_dir_group_read = 0, have_max_streams = 0;
+  int have_max_streams_close = 0;
+
+  tor_assert(line_);
+  tor_assert(options);
+  tor_assert(service);
+
+  /* Makes thing easier. */
+  config = &service->config;
+
+  /* The first line starts with HiddenServiceDir so we consider what's next is
+   * the configuration of the service. */
+  for (line = line_; line ; line = line->next) {
+    int ok = 0;
+
+    /* This indicate that we have a new service to configure. */
+    if (!strcasecmp(line->key, "HiddenServiceDir")) {
+      /* This function only configures one service at a time so if we've
+       * already seen one, stop right now. */
+      if (dir_seen) {
+        break;
+      }
+      /* Ok, we've seen one and we are about to configure it. */
+      dir_seen = 1;
+      config->directory_path = tor_strdup(line->value);
+      log_info(LD_CONFIG, "HiddenServiceDir=%s. Configuring...",
+               escaped(config->directory_path));
+      continue;
+    }
+    if (BUG(!dir_seen)) {
+      goto err;
+    }
+    /* Version of the service. */
+    if (!strcasecmp(line->key, "HiddenServiceVersion")) {
+      service->config.version =
+        (uint32_t) helper_parse_uint64(line->key, line->value, HS_VERSION_MIN,
+                                       HS_VERSION_MAX, &ok);
+      if (!ok || have_version) {
+        if (have_version)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_version = 1;
+      continue;
+    }
+    /* Virtual port. */
+    if (!strcasecmp(line->key, "HiddenServicePort")) {
+      char *err_msg = NULL;
+      /* XXX: Can we rename this? */
+      rend_service_port_config_t *portcfg =
+        rend_service_parse_port_config(line->value, " ", &err_msg);
+      if (!portcfg) {
+        if (err_msg) {
+          log_warn(LD_CONFIG, "%s", err_msg);
+        }
+        tor_free(err_msg);
+        goto err;
+      }
+      tor_assert(!err_msg);
+      smartlist_add(config->ports, portcfg);
+      log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
+               line->value, escaped(config->directory_path));
+      continue;
+    }
+    /* Do we allow unknown ports. */
+    if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
+      config->allow_unknown_ports =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_allow_unknown_ports) {
+        if (have_allow_unknown_ports)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_allow_unknown_ports = 1;
+      continue;
+    }
+    /* Directory group readable. */
+    if (!strcasecmp(line->key, "HiddenServiceDirGroupReadable")) {
+      config->dir_group_readable =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_dir_group_read) {
+        if (have_dir_group_read)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_dir_group_read = 1;
+      continue;
+    }
+    /* Maximum streams per circuit. */
+    if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
+      config->max_streams_per_rdv_circuit =
+        helper_parse_uint64(line->key, line->value, 0,
+                            HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT, &ok);
+      if (!ok || have_max_streams) {
+        if (have_max_streams)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_max_streams = 1;
+      continue;
+    }
+    /* Maximum amount of streams before we close the circuit. */
+    if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
+      config->max_streams_close_circuit =
+        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
+      if (!ok || have_max_streams_close) {
+        if (have_max_streams_close)
+          dup_opt_seen = line->key;
+        goto err;
+      }
+      have_max_streams_close = 1;
+      continue;
+    }
+  }
+
+  /* Check if we are configured in non anonymous mode meaning every service
+   * becomes a single onion service. */
+  if (rend_service_non_anonymous_mode_enabled(options)) {
+    config->is_single_onion = 1;
+    /* We will add support for IPv6-only v3 single onion services in a future
+     * Tor version. This won't catch "ReachableAddresses reject *4", but that
+     * option doesn't work anyway. */
+    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
+      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
+               "supported. Set HiddenServiceSingleHopMode 0 and "
+               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
+      goto err;
+    }
+  }
+
+  /* Success */
+  return 0;
+ err:
+  if (dup_opt_seen) {
+    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
+  }
+  return -1;
+}
+
+/* Configure a service using the given line and options. This function will
+ * call the corresponding configuration function for a specific service
+ * version and validate the service against the other ones. On success, add
+ * the service to the given list and return 0. On error, nothing is added to
+ * the list and a negative value is returned. */
+static int
+config_service(const config_line_t *line, const or_options_t *options,
+               smartlist_t *service_list)
+{
+  int ret;
+  hs_service_t *service = NULL;
+
+  tor_assert(line);
+  tor_assert(options);
+  tor_assert(service_list);
+
+  /* We have a new hidden service. */
+  service = hs_service_new(options);
+  /* We'll configure that service as a generic one and then pass it to a
+   * specific function according to the configured version number. */
+  if (config_generic_service(line, options, service) < 0) {
+    goto err;
+  }
+  tor_assert(service->config.version <= HS_VERSION_MAX);
+  /* Before we configure the service on a per-version basis, we'll make
+   * sure that this set of options for a service are valid that is for
+   * instance an option only for v2 is not used for v3. */
+  if (config_has_invalid_options(line->next, service)) {
+    goto err;
+  }
+  /* Check permission on service directory that was just parsed. And this must
+   * be done regardless of the service version. Do not ask for the directory
+   * to be created, this is done when the keys are loaded because we could be
+   * in validation mode right now. */
+  if (hs_check_service_private_dir(options->User,
+                                   service->config.directory_path,
+                                   service->config.dir_group_readable,
+                                   0) < 0) {
+    goto err;
+  }
+  /* Different functions are in charge of specific options for a version. We
+   * start just after the service directory line so once we hit another
+   * directory line, the function knows that it has to stop parsing. */
+  switch (service->config.version) {
+  case HS_VERSION_TWO:
+    ret = rend_config_service(line->next, options, &service->config);
+    break;
+  case HS_VERSION_THREE:
+    ret = config_service_v3(line->next, &service->config);
+    break;
+  default:
+    /* We do validate before if we support the parsed version. */
+    tor_assert_nonfatal_unreached();
+    goto err;
+  }
+  if (ret < 0) {
+    goto err;
+  }
+  /* We'll check if this service can be kept depending on the others
+   * configured previously. */
+  if (service_is_duplicate_in_list(service_list, service)) {
+    goto err;
+  }
+  /* Passes, add it to the given list. */
+  smartlist_add(service_list, service);
+  return 0;
+
+ err:
+  hs_service_free(service);
+  return -1;
+}
+
+/* From a set of <b>options</b>, setup every hidden service found. Return 0 on
+ * success or -1 on failure. If <b>validate_only</b> is set, parse, warn and
+ * return as normal, but don't actually change the configured services. */
+int
+hs_config_service_all(const or_options_t *options, int validate_only)
+{
+  int dir_option_seen = 0, ret = -1;
+  const config_line_t *line;
+  smartlist_t *new_service_list = NULL;
+
+  tor_assert(options);
+
+  /* Newly configured service are put in that list which is then used for
+   * validation and staging for >= v3. */
+  new_service_list = smartlist_new();
+
+  for (line = options->RendConfigLines; line; line = line->next) {
+    /* Ignore all directives that aren't the start of a service. */
+    if (strcasecmp(line->key, "HiddenServiceDir")) {
+      if (!dir_option_seen) {
+        log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
+                 line->key);
+        goto err;
+      }
+      continue;
+    }
+    /* Flag that we've seen a directory directive and we'll use it to make
+     * sure that the torrc options ordering is actually valid. */
+    dir_option_seen = 1;
+
+    /* Try to configure this service now. On success, it will be added to the
+     * list and validated against the service in that same list. */
+    if (config_service(line, options, new_service_list) < 0) {
+      goto err;
+    }
+  }
+
+  /* In non validation mode, we'll stage those services we just successfully
+   * configured. Service ownership is transferred from the list to the global
+   * state. If any service is invalid, it will be removed from the list and
+   * freed. All versions are handled in that function. */
+  if (!validate_only) {
+    stage_services(new_service_list);
+  } else {
+    /* We've just validated that we were able to build a clean working list of
+     * services. We don't need those objects anymore. */
+    SMARTLIST_FOREACH(new_service_list, hs_service_t *, s,
+                      hs_service_free(s));
+    /* For the v2 subsystem, the configuration function adds the service
+     * object to the staging list and it is transferred in the main list
+     * through the prunning process. In validation mode, we thus have to purge
+     * the staging list so it's not kept in memory as valid service. */
+    rend_service_free_staging_list();
+  }
+
+  /* Success. Note that the service list has no ownership of its content. */
+  ret = 0;
+  goto end;
+
+ err:
+  SMARTLIST_FOREACH(new_service_list, hs_service_t *, s, hs_service_free(s));
+
+ end:
+  smartlist_free(new_service_list);
+  /* Tor main should call the free all function on error. */
+  return ret;
+}
+
diff --git a/src/or/hs_config.h b/src/or/hs_config.h
new file mode 100644
index 0000000000..6cd7aed460
--- /dev/null
+++ b/src/or/hs_config.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_config.h
+ * \brief Header file containing configuration ABI/API for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_CONFIG_H
+#define TOR_HS_CONFIG_H
+
+#include "or.h"
+
+/* Max value for HiddenServiceMaxStreams */
+#define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
+/* Maximum number of intro points per version 3 services. */
+#define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+
+/* API */
+
+int hs_config_service_all(const or_options_t *options, int validate_only);
+
+#endif /* !defined(TOR_HS_CONFIG_H) */
+
diff --git a/src/or/hs_control.c b/src/or/hs_control.c
new file mode 100644
index 0000000000..6b9b95c6d8
--- /dev/null
+++ b/src/or/hs_control.c
@@ -0,0 +1,258 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_control.c
+ * \brief Contains control port event related code.
+ **/
+
+#include "or.h"
+#include "control.h"
+#include "crypto_util.h"
+#include "hs_common.h"
+#include "hs_control.h"
+#include "hs_descriptor.h"
+#include "hs_service.h"
+#include "nodelist.h"
+
+/* Send on the control port the "HS_DESC REQUESTED [...]" event.
+ *
+ * The onion_pk is the onion service public key, base64_blinded_pk is the
+ * base64 encoded blinded key for the service and hsdir_rs is the routerstatus
+ * object of the HSDir that this request is for. */
+void
+hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
+                                const char *base64_blinded_pk,
+                                const routerstatus_t *hsdir_rs)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  const uint8_t *hsdir_index;
+  const node_t *hsdir_node;
+
+  tor_assert(onion_pk);
+  tor_assert(base64_blinded_pk);
+  tor_assert(hsdir_rs);
+
+  hs_build_address(onion_pk, HS_VERSION_THREE, onion_address);
+
+  /* Get the node from the routerstatus object to get the HSDir index used for
+   * this request. We can't have a routerstatus entry without a node and we
+   * can't pick a node without an hsdir_index. */
+  hsdir_node = node_get_by_id(hsdir_rs->identity_digest);
+  tor_assert(hsdir_node);
+  /* This is a fetch event. */
+  hsdir_index = hsdir_node->hsdir_index.fetch;
+
+  /* Trigger the event. */
+  control_event_hs_descriptor_requested(onion_address, REND_NO_AUTH,
+                                        hsdir_rs->identity_digest,
+                                        base64_blinded_pk,
+                                        hex_str((const char *) hsdir_index,
+                                                DIGEST256_LEN));
+  memwipe(onion_address, 0, sizeof(onion_address));
+}
+
+/* Send on the control port the "HS_DESC FAILED [...]" event.
+ *
+ * Using a directory connection identifier, the HSDir identity digest and a
+ * reason for the failure. None can be NULL. */
+void
+hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
+                             const char *hsdir_id_digest,
+                             const char *reason)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+  tor_assert(reason);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
+                                       hsdir_id_digest, reason);
+}
+
+/* Send on the control port the "HS_DESC RECEIVED [...]" event.
+ *
+ * Using a directory connection identifier and the HSDir identity digest.
+ * None can be NULL. */
+void
+hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
+                               const char *hsdir_id_digest)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
+                                         hsdir_id_digest);
+}
+
+/* Send on the control port the "HS_DESC CREATED [...]" event.
+ *
+ * Using the onion address of the descriptor's service and the blinded public
+ * key of the descriptor as a descriptor ID. None can be NULL. */
+void
+hs_control_desc_event_created(const char *onion_address,
+                              const ed25519_public_key_t *blinded_pk)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(onion_address);
+  tor_assert(blinded_pk);
+
+  /* Build base64 encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
+    return;
+  }
+
+  /* Version 3 doesn't use the replica number in its descriptor ID computation
+   * so we pass negative value so the control port subsystem can ignore it. */
+  control_event_hs_descriptor_created(onion_address, base64_blinded_pk, -1);
+}
+
+/* Send on the control port the "HS_DESC UPLOAD [...]" event.
+ *
+ * Using the onion address of the descriptor's service, the HSDir identity
+ * digest, the blinded public key of the descriptor as a descriptor ID and the
+ * HSDir index for this particular request. None can be NULL. */
+void
+hs_control_desc_event_upload(const char *onion_address,
+                             const char *hsdir_id_digest,
+                             const ed25519_public_key_t *blinded_pk,
+                             const uint8_t *hsdir_index)
+{
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(onion_address);
+  tor_assert(hsdir_id_digest);
+  tor_assert(blinded_pk);
+  tor_assert(hsdir_index);
+
+  /* Build base64 encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
+                                     base64_blinded_pk,
+                                     hex_str((const char *) hsdir_index,
+                                             DIGEST256_LEN));
+}
+
+/* Send on the control port the "HS_DESC UPLOADED [...]" event.
+ *
+ * Using the directory connection identifier and the HSDir identity digest.
+ * None can be NULL. */
+void
+hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
+                               const char *hsdir_id_digest)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hs_descriptor_uploaded(hsdir_id_digest, onion_address);
+}
+
+/* Send on the control port the "HS_DESC_CONTENT [...]" event.
+ *
+ * Using the directory connection identifier, the HSDir identity digest and
+ * the body of the descriptor (as it was received from the directory). None
+ * can be NULL. */
+void
+hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
+                              const char *hsdir_id_digest,
+                              const char *body)
+{
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+
+  tor_assert(ident);
+  tor_assert(hsdir_id_digest);
+
+  /* Build onion address and encoded blinded key. */
+  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
+                                       &ident->blinded_pk) < 0) {
+    return;
+  }
+  hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
+
+  control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
+                                      hsdir_id_digest, body);
+}
+
+/* Handle the "HSPOST [...]" command. The body is an encoded descriptor for
+ * the given onion_address. The descriptor will be uploaded to each directory
+ * in hsdirs_rs. If NULL, the responsible directories for the current time
+ * period will be selected.
+ *
+ * Return -1 on if the descriptor plaintext section is not decodable. Else, 0
+ * on success. */
+int
+hs_control_hspost_command(const char *body, const char *onion_address,
+                          const smartlist_t *hsdirs_rs)
+{
+  int ret = -1;
+  ed25519_public_key_t identity_pk;
+  hs_desc_plaintext_data_t plaintext;
+  smartlist_t *hsdirs = NULL;
+
+  tor_assert(body);
+  tor_assert(onion_address);
+
+  /* This can't fail because we require the caller to pass us a valid onion
+   * address that has passed hs_address_is_valid(). */
+  if (BUG(hs_parse_address(onion_address, &identity_pk, NULL, NULL) < 0)) {
+    goto done; // LCOV_EXCL_LINE
+  }
+
+  /* Only decode the plaintext part which is what the directory will do to
+   * validate before caching. */
+  if (hs_desc_decode_plaintext(body, &plaintext) < 0) {
+    goto done;
+  }
+
+  /* No HSDir(s) given, we'll compute what the current ones should be. */
+  if (hsdirs_rs == NULL) {
+    hsdirs = smartlist_new();
+    hs_get_responsible_hsdirs(&plaintext.blinded_pubkey,
+                              hs_get_time_period_num(0),
+                              0, /* Always the current descriptor which uses
+                                  * the first hsdir index. */
+                              0, /* It is for storing on a directory. */
+                              hsdirs);
+    hsdirs_rs = hsdirs;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(hsdirs_rs, const routerstatus_t *, rs) {
+    hs_service_upload_desc_to_dir(body, plaintext.version, &identity_pk,
+                                  &plaintext.blinded_pubkey, rs);
+  } SMARTLIST_FOREACH_END(rs);
+  ret = 0;
+
+ done:
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(hsdirs);
+  return ret;
+}
+
diff --git a/src/or/hs_control.h b/src/or/hs_control.h
new file mode 100644
index 0000000000..95c46e655e
--- /dev/null
+++ b/src/or/hs_control.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_control.h
+ * \brief Header file containing control port event related code.
+ **/
+
+#ifndef TOR_HS_CONTROL_H
+#define TOR_HS_CONTROL_H
+
+#include "hs_ident.h"
+
+/* Event "HS_DESC REQUESTED [...]" */
+void hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
+                                     const char *base64_blinded_pk,
+                                     const routerstatus_t *hsdir_rs);
+
+/* Event "HS_DESC FAILED [...]" */
+void hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
+                                  const char *hsdir_id_digest,
+                                  const char *reason);
+
+/* Event "HS_DESC RECEIVED [...]" */
+void hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
+                                    const char *hsdir_id_digest);
+
+/* Event "HS_DESC CREATED [...]" */
+void hs_control_desc_event_created(const char *onion_address,
+                                   const ed25519_public_key_t *blinded_pk);
+
+/* Event "HS_DESC UPLOAD [...]" */
+void hs_control_desc_event_upload(const char *onion_address,
+                                  const char *hsdir_id_digest,
+                                  const ed25519_public_key_t *blinded_pk,
+                                  const uint8_t *hsdir_index);
+
+/* Event "HS_DESC UPLOADED [...]" */
+void hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
+                                    const char *hsdir_id_digest);
+
+/* Event "HS_DESC_CONTENT [...]" */
+void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
+                                   const char *hsdir_id_digest,
+                                   const char *body);
+
+/* Command "HSPOST [...]" */
+int hs_control_hspost_command(const char *body, const char *onion_address,
+                              const smartlist_t *hsdirs_rs);
+
+#endif /* !defined(TOR_HS_CONTROL_H) */
+
diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c
new file mode 100644
index 0000000000..e85c29a24e
--- /dev/null
+++ b/src/or/hs_descriptor.c
@@ -0,0 +1,2611 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.c
+ * \brief Handle hidden service descriptor encoding/decoding.
+ *
+ * \details
+ * Here is a graphical depiction of an HS descriptor and its layers:
+ *
+ *      +------------------------------------------------------+
+ *      |DESCRIPTOR HEADER:                                    |
+ *      |  hs-descriptor 3                                     |
+ *      |  descriptor-lifetime 180                             |
+ *      |  ...                                                 |
+ *      |  superencrypted                                      |
+ *      |+---------------------------------------------------+ |
+ *      ||SUPERENCRYPTED LAYER (aka OUTER ENCRYPTED LAYER):  | |
+ *      ||  desc-auth-type x25519                            | |
+ *      ||  desc-auth-ephemeral-key                          | |
+ *      ||  auth-client                                      | |
+ *      ||  auth-client                                      | |
+ *      ||  ...                                              | |
+ *      ||  encrypted                                        | |
+ *      ||+-------------------------------------------------+| |
+ *      |||ENCRYPTED LAYER (aka INNER ENCRYPTED LAYER):     || |
+ *      |||  create2-formats                                || |
+ *      |||  intro-auth-required                            || |
+ *      |||  introduction-point                             || |
+ *      |||  introduction-point                             || |
+ *      |||  ...                                            || |
+ *      ||+-------------------------------------------------+| |
+ *      |+---------------------------------------------------+ |
+ *      +------------------------------------------------------+
+ *
+ * The DESCRIPTOR HEADER section is completely unencrypted and contains generic
+ * descriptor metadata.
+ *
+ * The SUPERENCRYPTED LAYER section is the first layer of encryption, and it's
+ * encrypted using the blinded public key of the hidden service to protect
+ * against entities who don't know its onion address. The clients of the hidden
+ * service know its onion address and blinded public key, whereas third-parties
+ * (like HSDirs) don't know it (except if it's a public hidden service).
+ *
+ * The ENCRYPTED LAYER section is the second layer of encryption, and it's
+ * encrypted using the client authorization key material (if those exist). When
+ * client authorization is enabled, this second layer of encryption protects
+ * the descriptor content from unauthorized entities. If client authorization
+ * is disabled, this second layer of encryption does not provide any extra
+ * security but is still present. The plaintext of this layer contains all the
+ * information required to connect to the hidden service like its list of
+ * introduction points.
+ **/
+
+/* For unit tests.*/
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "or.h"
+#include "ed25519_cert.h" /* Trunnel interface. */
+#include "hs_descriptor.h"
+#include "circuitbuild.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "parsecommon.h"
+#include "rendcache.h"
+#include "hs_cache.h"
+#include "hs_config.h"
+#include "torcert.h" /* tor_cert_encode_ed22519() */
+
+/* Constant string value used for the descriptor format. */
+#define str_hs_desc "hs-descriptor"
+#define str_desc_cert "descriptor-signing-key-cert"
+#define str_rev_counter "revision-counter"
+#define str_superencrypted "superencrypted"
+#define str_encrypted "encrypted"
+#define str_signature "signature"
+#define str_lifetime "descriptor-lifetime"
+/* Constant string value for the encrypted part of the descriptor. */
+#define str_create2_formats "create2-formats"
+#define str_intro_auth_required "intro-auth-required"
+#define str_single_onion "single-onion-service"
+#define str_intro_point "introduction-point"
+#define str_ip_onion_key "onion-key"
+#define str_ip_auth_key "auth-key"
+#define str_ip_enc_key "enc-key"
+#define str_ip_enc_key_cert "enc-key-cert"
+#define str_ip_legacy_key "legacy-key"
+#define str_ip_legacy_key_cert "legacy-key-cert"
+#define str_intro_point_start "\n" str_intro_point " "
+/* Constant string value for the construction to encrypt the encrypted data
+ * section. */
+#define str_enc_const_superencryption "hsdir-superencrypted-data"
+#define str_enc_const_encryption "hsdir-encrypted-data"
+/* Prefix required to compute/verify HS desc signatures */
+#define str_desc_sig_prefix "Tor onion service descriptor sig v3"
+#define str_desc_auth_type "desc-auth-type"
+#define str_desc_auth_key "desc-auth-ephemeral-key"
+#define str_desc_auth_client "auth-client"
+#define str_encrypted "encrypted"
+
+/* Authentication supported types. */
+static const struct {
+  hs_desc_auth_type_t type;
+  const char *identifier;
+} intro_auth_types[] = {
+  { HS_DESC_AUTH_ED25519, "ed25519" },
+  /* Indicate end of array. */
+  { 0, NULL }
+};
+
+/* Descriptor ruleset. */
+static token_rule_t hs_desc_v3_token_table[] = {
+  T1_START(str_hs_desc, R_HS_DESCRIPTOR, EQ(1), NO_OBJ),
+  T1(str_lifetime, R3_DESC_LIFETIME, EQ(1), NO_OBJ),
+  T1(str_desc_cert, R3_DESC_SIGNING_CERT, NO_ARGS, NEED_OBJ),
+  T1(str_rev_counter, R3_REVISION_COUNTER, EQ(1), NO_OBJ),
+  T1(str_superencrypted, R3_SUPERENCRYPTED, NO_ARGS, NEED_OBJ),
+  T1_END(str_signature, R3_SIGNATURE, EQ(1), NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the superencrypted section. */
+static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
+  T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ),
+  T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ),
+  T1N(str_desc_auth_client, R3_DESC_AUTH_CLIENT, GE(3), NO_OBJ),
+  T1(str_encrypted, R3_ENCRYPTED, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the encrypted section. */
+static token_rule_t hs_desc_encrypted_v3_token_table[] = {
+  T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ),
+  T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, ARGS, NO_OBJ),
+  T01(str_single_onion, R3_SINGLE_ONION_SERVICE, ARGS, NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the introduction points section. */
+static token_rule_t hs_desc_intro_point_v3_token_table[] = {
+  T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ),
+  T1N(str_ip_onion_key, R3_INTRO_ONION_KEY, GE(2), OBJ_OK),
+  T1(str_ip_auth_key, R3_INTRO_AUTH_KEY, NO_ARGS, NEED_OBJ),
+  T1(str_ip_enc_key, R3_INTRO_ENC_KEY, GE(2), OBJ_OK),
+  T1(str_ip_enc_key_cert, R3_INTRO_ENC_KEY_CERT, ARGS, OBJ_OK),
+  T01(str_ip_legacy_key, R3_INTRO_LEGACY_KEY, ARGS, NEED_KEY_1024),
+  T01(str_ip_legacy_key_cert, R3_INTRO_LEGACY_KEY_CERT, ARGS, OBJ_OK),
+  END_OF_TABLE
+};
+
+/* Free the content of the plaintext section of a descriptor. */
+STATIC void
+desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->superencrypted_blob) {
+    tor_free(desc->superencrypted_blob);
+  }
+  tor_cert_free(desc->signing_key_cert);
+
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Free the content of the encrypted section of a descriptor. */
+static void
+desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->intro_auth_types) {
+    SMARTLIST_FOREACH(desc->intro_auth_types, char *, a, tor_free(a));
+    smartlist_free(desc->intro_auth_types);
+  }
+  if (desc->intro_points) {
+    SMARTLIST_FOREACH(desc->intro_points, hs_desc_intro_point_t *, ip,
+                      hs_desc_intro_point_free(ip));
+    smartlist_free(desc->intro_points);
+  }
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
+ * We use SHA3-256 for the MAC computation.
+ * This function can't fail. */
+static void
+build_mac(const uint8_t *mac_key, size_t mac_key_len,
+          const uint8_t *salt, size_t salt_len,
+          const uint8_t *encrypted, size_t encrypted_len,
+          uint8_t *mac_out, size_t mac_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t mac_len_netorder = tor_htonll(mac_key_len);
+  const uint64_t salt_len_netorder = tor_htonll(salt_len);
+
+  tor_assert(mac_key);
+  tor_assert(salt);
+  tor_assert(encrypted);
+  tor_assert(mac_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  /* As specified in section 2.5 of proposal 224, first add the mac key
+   * then add the salt first and then the encrypted section. */
+
+  crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len);
+  crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) salt, salt_len);
+  crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, mac_len);
+  crypto_digest_free(digest);
+}
+
+/* Using a given decriptor object, build the secret input needed for the
+ * KDF and put it in the dst pointer which is an already allocated buffer
+ * of size dstlen. */
+static void
+build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen)
+{
+  size_t offset = 0;
+
+  tor_assert(desc);
+  tor_assert(dst);
+  tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen);
+
+  /* XXX use the destination length as the memcpy length */
+  /* Copy blinded public key. */
+  memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey,
+         sizeof(desc->plaintext_data.blinded_pubkey.pubkey));
+  offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey);
+  /* Copy subcredential. */
+  memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential));
+  offset += sizeof(desc->subcredential);
+  /* Copy revision counter value. */
+  set_uint64(dst + offset, tor_htonll(desc->plaintext_data.revision_counter));
+  offset += sizeof(uint64_t);
+  tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset);
+}
+
+/* Do the KDF construction and put the resulting data in key_out which is of
+ * key_out_len length. It uses SHAKE-256 as specified in the spec. */
+static void
+build_kdf_key(const hs_descriptor_t *desc,
+              const uint8_t *salt, size_t salt_len,
+              uint8_t *key_out, size_t key_out_len,
+              int is_superencrypted_layer)
+{
+  uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN];
+  crypto_xof_t *xof;
+
+  tor_assert(desc);
+  tor_assert(salt);
+  tor_assert(key_out);
+
+  /* Build the secret input for the KDF computation. */
+  build_secret_input(desc, secret_input, sizeof(secret_input));
+
+  xof = crypto_xof_new();
+  /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */
+  crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input));
+  crypto_xof_add_bytes(xof, salt, salt_len);
+
+  /* Feed in the right string constant based on the desc layer */
+  if (is_superencrypted_layer) {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_superencryption,
+                         strlen(str_enc_const_superencryption));
+  } else {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_encryption,
+                         strlen(str_enc_const_encryption));
+  }
+
+  /* Eat from our KDF. */
+  crypto_xof_squeeze_bytes(xof, key_out, key_out_len);
+  crypto_xof_free(xof);
+  memwipe(secret_input,  0, sizeof(secret_input));
+}
+
+/* Using the given descriptor and salt, run it through our KDF function and
+ * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out.
+ * This function can't fail. */
+static void
+build_secret_key_iv_mac(const hs_descriptor_t *desc,
+                        const uint8_t *salt, size_t salt_len,
+                        uint8_t *key_out, size_t key_len,
+                        uint8_t *iv_out, size_t iv_len,
+                        uint8_t *mac_out, size_t mac_len,
+                        int is_superencrypted_layer)
+{
+  size_t offset = 0;
+  uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN];
+
+  tor_assert(desc);
+  tor_assert(salt);
+  tor_assert(key_out);
+  tor_assert(iv_out);
+  tor_assert(mac_out);
+
+  build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key),
+                is_superencrypted_layer);
+  /* Copy the bytes we need for both the secret key and IV. */
+  memcpy(key_out, kdf_key, key_len);
+  offset += key_len;
+  memcpy(iv_out, kdf_key + offset, iv_len);
+  offset += iv_len;
+  memcpy(mac_out, kdf_key + offset, mac_len);
+  /* Extra precaution to make sure we are not out of bound. */
+  tor_assert((offset + mac_len) == sizeof(kdf_key));
+  memwipe(kdf_key, 0, sizeof(kdf_key));
+}
+
+/* === ENCODING === */
+
+/* Encode the given link specifier objects into a newly allocated string.
+ * This can't fail so caller can always assume a valid string being
+ * returned. */
+STATIC char *
+encode_link_specifiers(const smartlist_t *specs)
+{
+  char *encoded_b64 = NULL;
+  link_specifier_list_t *lslist = link_specifier_list_new();
+
+  tor_assert(specs);
+  /* No link specifiers is a code flow error, can't happen. */
+  tor_assert(smartlist_len(specs) > 0);
+  tor_assert(smartlist_len(specs) <= UINT8_MAX);
+
+  link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
+
+  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+                          spec) {
+    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
+    if (ls) {
+      link_specifier_list_add_spec(lslist, ls);
+    }
+  } SMARTLIST_FOREACH_END(spec);
+
+  {
+    uint8_t *encoded;
+    ssize_t encoded_len, encoded_b64_len, ret;
+
+    encoded_len = link_specifier_list_encoded_len(lslist);
+    tor_assert(encoded_len > 0);
+    encoded = tor_malloc_zero(encoded_len);
+    ret = link_specifier_list_encode(encoded, encoded_len, lslist);
+    tor_assert(ret == encoded_len);
+
+    /* Base64 encode our binary format. Add extra NUL byte for the base64
+     * encoded value. */
+    encoded_b64_len = base64_encode_size(encoded_len, 0) + 1;
+    encoded_b64 = tor_malloc_zero(encoded_b64_len);
+    ret = base64_encode(encoded_b64, encoded_b64_len, (const char *) encoded,
+                        encoded_len, 0);
+    tor_assert(ret == (encoded_b64_len - 1));
+    tor_free(encoded);
+  }
+
+  link_specifier_list_free(lslist);
+  return encoded_b64;
+}
+
+/* Encode an introduction point legacy key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_legacy_key(const hs_desc_intro_point_t *ip)
+{
+  char *key_str, b64_cert[256], *encoded = NULL;
+  size_t key_str_len;
+
+  tor_assert(ip);
+
+  /* Encode cross cert. */
+  if (base64_encode(b64_cert, sizeof(b64_cert),
+                    (const char *) ip->legacy.cert.encoded,
+                    ip->legacy.cert.len, BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy crosscert.");
+    goto done;
+  }
+  /* Convert the encryption key to PEM format NUL terminated. */
+  if (crypto_pk_write_public_key_to_string(ip->legacy.key, &key_str,
+                                           &key_str_len) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy encryption key.");
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s \n%s"  /* Newline is added by the call above. */
+               "%s\n"
+               "-----BEGIN CROSSCERT-----\n"
+               "%s"
+               "-----END CROSSCERT-----",
+               str_ip_legacy_key, key_str,
+               str_ip_legacy_key_cert, b64_cert);
+  tor_free(key_str);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point encryption key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_enc_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL, *encoded_cert;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "enc-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
+    goto done;
+  }
+  if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s ntor %s\n"
+               "%s\n%s",
+               str_ip_enc_key, key_b64,
+               str_ip_enc_key_cert, encoded_cert);
+  tor_free(encoded_cert);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point onion key. Return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_onion_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "onion-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point object and return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_intro_point(const ed25519_public_key_t *sig_key,
+                   const hs_desc_intro_point_t *ip)
+{
+  char *encoded_ip = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(ip);
+  tor_assert(sig_key);
+
+  /* Encode link specifier. */
+  {
+    char *ls_str = encode_link_specifiers(ip->link_specifiers);
+    smartlist_add_asprintf(lines, "%s %s", str_intro_point, ls_str);
+    tor_free(ls_str);
+  }
+
+  /* Onion key encoding. */
+  {
+    char *encoded_onion_key = encode_onion_key(ip);
+    if (encoded_onion_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_onion_key);
+    tor_free(encoded_onion_key);
+  }
+
+  /* Authentication key encoding. */
+  {
+    char *encoded_cert;
+    if (tor_cert_encode_ed22519(ip->auth_key_cert, &encoded_cert) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s\n%s", str_ip_auth_key, encoded_cert);
+    tor_free(encoded_cert);
+  }
+
+  /* Encryption key encoding. */
+  {
+    char *encoded_enc_key = encode_enc_key(ip);
+    if (encoded_enc_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_enc_key);
+    tor_free(encoded_enc_key);
+  }
+
+  /* Legacy key if any. */
+  if (ip->legacy.key != NULL) {
+    /* Strong requirement else the IP creation was badly done. */
+    tor_assert(ip->legacy.cert.encoded);
+    char *encoded_legacy_key = encode_legacy_key(ip);
+    if (encoded_legacy_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_legacy_key);
+    tor_free(encoded_legacy_key);
+  }
+
+  /* Join them all in one blob of text. */
+  encoded_ip = smartlist_join_strings(lines, "\n", 1, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return encoded_ip;
+}
+
+/* Given a source length, return the new size including padding for the
+ * plaintext encryption. */
+static size_t
+compute_padded_plaintext_length(size_t plaintext_len)
+{
+  size_t plaintext_padded_len;
+  const int padding_block_length = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+
+  /* Make sure we won't overflow. */
+  tor_assert(plaintext_len <= (SIZE_T_CEILING - padding_block_length));
+
+  /* Get the extra length we need to add. For example, if srclen is 10200
+   * bytes, this will expand to (2 * 10k) == 20k thus an extra 9800 bytes. */
+  plaintext_padded_len = CEIL_DIV(plaintext_len, padding_block_length) *
+                         padding_block_length;
+  /* Can never be extra careful. Make sure we are _really_ padded. */
+  tor_assert(!(plaintext_padded_len % padding_block_length));
+  return plaintext_padded_len;
+}
+
+/* Given a buffer, pad it up to the encrypted section padding requirement. Set
+ * the newly allocated string in padded_out and return the length of the
+ * padded buffer. */
+STATIC size_t
+build_plaintext_padding(const char *plaintext, size_t plaintext_len,
+                        uint8_t **padded_out)
+{
+  size_t padded_len;
+  uint8_t *padded;
+
+  tor_assert(plaintext);
+  tor_assert(padded_out);
+
+  /* Allocate the final length including padding. */
+  padded_len = compute_padded_plaintext_length(plaintext_len);
+  tor_assert(padded_len >= plaintext_len);
+  padded = tor_malloc_zero(padded_len);
+
+  memcpy(padded, plaintext, plaintext_len);
+  *padded_out = padded;
+  return padded_len;
+}
+
+/* Using a key, IV and plaintext data of length plaintext_len, create the
+ * encrypted section by encrypting it and setting encrypted_out with the
+ * data. Return size of the encrypted data buffer. */
+static size_t
+build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext,
+                size_t plaintext_len, uint8_t **encrypted_out,
+                int is_superencrypted_layer)
+{
+  size_t encrypted_len;
+  uint8_t *padded_plaintext, *encrypted;
+  crypto_cipher_t *cipher;
+
+  tor_assert(key);
+  tor_assert(iv);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* If we are encrypting the middle layer of the descriptor, we need to first
+     pad the plaintext */
+  if (is_superencrypted_layer) {
+    encrypted_len = build_plaintext_padding(plaintext, plaintext_len,
+                                            &padded_plaintext);
+    /* Extra precautions that we have a valid padding length. */
+    tor_assert(!(encrypted_len % HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE));
+  } else { /* No padding required for inner layers */
+    padded_plaintext = tor_memdup(plaintext, plaintext_len);
+    encrypted_len = plaintext_len;
+  }
+
+  /* This creates a cipher for AES. It can't fail. */
+  cipher = crypto_cipher_new_with_iv_and_bits(key, iv,
+                                              HS_DESC_ENCRYPTED_BIT_SIZE);
+  /* We use a stream cipher so the encrypted length will be the same as the
+   * plaintext padded length. */
+  encrypted = tor_malloc_zero(encrypted_len);
+  /* This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted,
+                        (const char *) padded_plaintext, encrypted_len);
+  *encrypted_out = encrypted;
+  /* Cleanup. */
+  crypto_cipher_free(cipher);
+  tor_free(padded_plaintext);
+  return encrypted_len;
+}
+
+/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> to get the
+ * keys. Set encrypted_out with the encrypted data and return the length of
+ * it. <b>is_superencrypted_layer</b> is set if this is the outer encrypted
+ * layer of the descriptor. */
+static size_t
+encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext,
+                        char **encrypted_out, int is_superencrypted_layer)
+{
+  char *final_blob;
+  size_t encrypted_len, final_blob_len, offset = 0;
+  uint8_t *encrypted;
+  uint8_t salt[HS_DESC_ENCRYPTED_SALT_LEN];
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], mac[DIGEST256_LEN];
+
+  tor_assert(desc);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* Get our salt. The returned bytes are already hashed. */
+  crypto_strongest_rand(salt, sizeof(salt));
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the encryption. */
+  build_secret_key_iv_mac(desc, salt, sizeof(salt),
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build the encrypted part that is do the actual encryption. */
+  encrypted_len = build_encrypted(secret_key, secret_iv, plaintext,
+                                  strlen(plaintext), &encrypted,
+                                  is_superencrypted_layer);
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  /* This construction is specified in section 2.5 of proposal 224. */
+  final_blob_len = sizeof(salt) + encrypted_len + DIGEST256_LEN;
+  final_blob = tor_malloc_zero(final_blob_len);
+
+  /* Build the MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, sizeof(salt),
+            encrypted, encrypted_len, mac, sizeof(mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+
+  /* The salt is the first value. */
+  memcpy(final_blob, salt, sizeof(salt));
+  offset = sizeof(salt);
+  /* Second value is the encrypted data. */
+  memcpy(final_blob + offset, encrypted, encrypted_len);
+  offset += encrypted_len;
+  /* Third value is the MAC. */
+  memcpy(final_blob + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  /* Cleanup the buffers. */
+  memwipe(salt, 0, sizeof(salt));
+  memwipe(encrypted, 0, encrypted_len);
+  tor_free(encrypted);
+  /* Extra precaution. */
+  tor_assert(offset == final_blob_len);
+
+  *encrypted_out = final_blob;
+  return final_blob_len;
+}
+
+/* Create and return a string containing a fake client-auth entry. It's the
+ * responsibility of the caller to free the returned string. This function will
+ * never fail. */
+static char *
+get_fake_auth_client_str(void)
+{
+  char *auth_client_str = NULL;
+  /* We are gonna fill these arrays with fake base64 data. They are all double
+   * the size of their binary representation to fit the base64 overhead. */
+  char client_id_b64[8*2];
+  char iv_b64[16*2];
+  char encrypted_cookie_b64[16*2];
+  int retval;
+
+  /* This is a macro to fill a field with random data and then base64 it. */
+#define FILL_WITH_FAKE_DATA_AND_BASE64(field) STMT_BEGIN         \
+  crypto_rand((char *)field, sizeof(field));                     \
+  retval = base64_encode_nopad(field##_b64, sizeof(field##_b64), \
+                               field, sizeof(field));            \
+  tor_assert(retval > 0);                                        \
+  STMT_END
+
+  { /* Get those fakes! */
+    uint8_t client_id[8]; /* fake client-id */
+    uint8_t iv[16]; /* fake IV (initialization vector) */
+    uint8_t encrypted_cookie[16]; /* fake encrypted cookie */
+
+    FILL_WITH_FAKE_DATA_AND_BASE64(client_id);
+    FILL_WITH_FAKE_DATA_AND_BASE64(iv);
+    FILL_WITH_FAKE_DATA_AND_BASE64(encrypted_cookie);
+  }
+
+  /* Build the final string */
+  tor_asprintf(&auth_client_str, "%s %s %s %s", str_desc_auth_client,
+               client_id_b64, iv_b64, encrypted_cookie_b64);
+
+#undef FILL_WITH_FAKE_DATA_AND_BASE64
+
+  return auth_client_str;
+}
+
+/** How many lines of "client-auth" we want in our descriptors; fake or not. */
+#define CLIENT_AUTH_ENTRIES_BLOCK_SIZE 16
+
+/** Create the "client-auth" part of the descriptor and return a
+ *  newly-allocated string with it. It's the responsibility of the caller to
+ *  free the returned string. */
+static char *
+get_fake_auth_client_lines(void)
+{
+  /* XXX: Client authorization is still not implemented, so all this function
+     does is make fake clients */
+  int i = 0;
+  smartlist_t *auth_client_lines = smartlist_new();
+  char *auth_client_lines_str = NULL;
+
+  /* Make a line for each fake client */
+  const int num_fake_clients = CLIENT_AUTH_ENTRIES_BLOCK_SIZE;
+  for (i = 0; i < num_fake_clients; i++) {
+    char *auth_client_str = get_fake_auth_client_str();
+    tor_assert(auth_client_str);
+    smartlist_add(auth_client_lines, auth_client_str);
+  }
+
+  /* Join all lines together to form final string */
+  auth_client_lines_str = smartlist_join_strings(auth_client_lines,
+                                                 "\n", 1, NULL);
+  /* Cleanup the mess */
+  SMARTLIST_FOREACH(auth_client_lines, char *, a, tor_free(a));
+  smartlist_free(auth_client_lines);
+
+  return auth_client_lines_str;
+}
+
+/* Create the inner layer of the descriptor (which includes the intro points,
+ * etc.). Return a newly-allocated string with the layer plaintext, or NULL if
+ * an error occurred. It's the responsibility of the caller to free the
+ * returned string. */
+static char *
+get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
+{
+  char *encoded_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* Build the start of the section prior to the introduction points. */
+  {
+    if (!desc->encrypted_data.create2_ntor) {
+      log_err(LD_BUG, "HS desc doesn't have recognized handshake type.");
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s %d\n", str_create2_formats,
+                           ONION_HANDSHAKE_TYPE_NTOR);
+
+    if (desc->encrypted_data.intro_auth_types &&
+        smartlist_len(desc->encrypted_data.intro_auth_types)) {
+      /* Put the authentication-required line. */
+      char *buf = smartlist_join_strings(desc->encrypted_data.intro_auth_types,
+                                         " ", 0, NULL);
+      smartlist_add_asprintf(lines, "%s %s\n", str_intro_auth_required, buf);
+      tor_free(buf);
+    }
+
+    if (desc->encrypted_data.single_onion_service) {
+      smartlist_add_asprintf(lines, "%s\n", str_single_onion);
+    }
+  }
+
+  /* Build the introduction point(s) section. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    char *encoded_ip = encode_intro_point(&desc->plaintext_data.signing_pubkey,
+                                          ip);
+    if (encoded_ip == NULL) {
+      log_err(LD_BUG, "HS desc intro point is malformed.");
+      goto err;
+    }
+    smartlist_add(lines, encoded_ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Build the entire encrypted data section into one encoded plaintext and
+   * then encrypt it. */
+  encoded_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+
+  return encoded_str;
+}
+
+/* Create the middle layer of the descriptor, which includes the client auth
+ * data and the encrypted inner layer (provided as a base64 string at
+ * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
+ * layer plaintext, or NULL if an error occurred. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
+                                    const char *layer2_b64_ciphertext)
+{
+  char *layer1_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* XXX: Disclaimer: This function generates only _fake_ client auth
+   * data. Real client auth is not yet implemented, but client auth data MUST
+   * always be present in descriptors. In the future this function will be
+   * refactored to use real client auth data if they exist (#20700). */
+  (void) *desc;
+
+  /* Specify auth type */
+  smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_type, "x25519");
+
+  {  /* Create fake ephemeral x25519 key */
+    char fake_key_base64[CURVE25519_BASE64_PADDED_LEN + 1];
+    curve25519_keypair_t fake_x25519_keypair;
+    if (curve25519_keypair_generate(&fake_x25519_keypair, 0) < 0) {
+      goto done;
+    }
+    if (curve25519_public_to_base64(fake_key_base64,
+                                    &fake_x25519_keypair.pubkey) < 0) {
+      goto done;
+    }
+    smartlist_add_asprintf(lines, "%s %s\n",
+                           str_desc_auth_key, fake_key_base64);
+    /* No need to memwipe any of these fake keys. They will go unused. */
+  }
+
+  {  /* Create fake auth-client lines. */
+    char *auth_client_lines = get_fake_auth_client_lines();
+    tor_assert(auth_client_lines);
+    smartlist_add(lines, auth_client_lines);
+  }
+
+  /* create encrypted section */
+  {
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_encrypted, layer2_b64_ciphertext);
+  }
+
+  layer1_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ done:
+  SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
+  smartlist_free(lines);
+
+  return layer1_str;
+}
+
+/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
+ * returning it. <b>desc</b> is provided to derive the encryption
+ * keys. <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the
+ * middle (superencrypted) layer of the descriptor. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+encrypt_desc_data_and_base64(const hs_descriptor_t *desc,
+                             const char *encoded_str,
+                             int is_superencrypted_layer)
+{
+  char *enc_b64;
+  ssize_t enc_b64_len, ret_len, enc_len;
+  char *encrypted_blob = NULL;
+
+  enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob,
+                                    is_superencrypted_layer);
+  /* Get the encoded size plus a NUL terminating byte. */
+  enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1;
+  enc_b64 = tor_malloc_zero(enc_b64_len);
+  /* Base64 the encrypted blob before returning it. */
+  ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len,
+                          BASE64_ENCODE_MULTILINE);
+  /* Return length doesn't count the NUL byte. */
+  tor_assert(ret_len == (enc_b64_len - 1));
+  tor_free(encrypted_blob);
+
+  return enc_b64;
+}
+
+/* Generate and encode the superencrypted portion of <b>desc</b>. This also
+ * involves generating the encrypted portion of the descriptor, and performing
+ * the superencryption. A newly allocated NUL-terminated string pointer
+ * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0
+ * on success else a negative value. */
+static int
+encode_superencrypted_data(const hs_descriptor_t *desc,
+                           char **encrypted_blob_out)
+{
+  int ret = -1;
+  char *layer2_str = NULL;
+  char *layer2_b64_ciphertext = NULL;
+  char *layer1_str = NULL;
+  char *layer1_b64_ciphertext = NULL;
+
+  tor_assert(desc);
+  tor_assert(encrypted_blob_out);
+
+  /* Func logic: We first create the inner layer of the descriptor (layer2).
+   * We then encrypt it and use it to create the middle layer of the descriptor
+   * (layer1).  Finally we superencrypt the middle layer and return it to our
+   * caller. */
+
+  /* Create inner descriptor layer */
+  layer2_str = get_inner_encrypted_layer_plaintext(desc);
+  if (!layer2_str) {
+    goto err;
+  }
+
+  /* Encrypt and b64 the inner layer */
+  layer2_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer2_str, 0);
+  if (!layer2_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Now create middle descriptor layer given the inner layer */
+  layer1_str = get_outer_encrypted_layer_plaintext(desc,layer2_b64_ciphertext);
+  if (!layer1_str) {
+    goto err;
+  }
+
+  /* Encrypt and base64 the middle layer */
+  layer1_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer1_str, 1);
+  if (!layer1_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(layer1_str);
+  tor_free(layer2_str);
+  tor_free(layer2_b64_ciphertext);
+
+  *encrypted_blob_out = layer1_b64_ciphertext;
+  return ret;
+}
+
+/* Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
+ * newly allocated string of the encoded descriptor. On error, -1 is returned
+ * and encoded_out is untouched. */
+static int
+desc_encode_v3(const hs_descriptor_t *desc,
+               const ed25519_keypair_t *signing_kp, char **encoded_out)
+{
+  int ret = -1;
+  char *encoded_str = NULL;
+  size_t encoded_len;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(signing_kp);
+  tor_assert(encoded_out);
+  tor_assert(desc->plaintext_data.version == 3);
+
+  if (BUG(desc->subcredential == NULL)) {
+    goto err;
+  }
+
+  /* Build the non-encrypted values. */
+  {
+    char *encoded_cert;
+    /* Encode certificate then create the first line of the descriptor. */
+    if (desc->plaintext_data.signing_key_cert->cert_type
+        != CERT_TYPE_SIGNING_HS_DESC) {
+      log_err(LD_BUG, "HS descriptor signing key has an unexpected cert type "
+              "(%d)", (int) desc->plaintext_data.signing_key_cert->cert_type);
+      goto err;
+    }
+    if (tor_cert_encode_ed22519(desc->plaintext_data.signing_key_cert,
+                                &encoded_cert) < 0) {
+      /* The function will print error logs. */
+      goto err;
+    }
+    /* Create the hs descriptor line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_hs_desc,
+                           desc->plaintext_data.version);
+    /* Add the descriptor lifetime line (in minutes). */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_lifetime,
+                           desc->plaintext_data.lifetime_sec / 60);
+    /* Create the descriptor certificate line. */
+    smartlist_add_asprintf(lines, "%s\n%s", str_desc_cert, encoded_cert);
+    tor_free(encoded_cert);
+    /* Create the revision counter line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu64, str_rev_counter,
+                           desc->plaintext_data.revision_counter);
+  }
+
+  /* Build the superencrypted data section. */
+  {
+    char *enc_b64_blob=NULL;
+    if (encode_superencrypted_data(desc, &enc_b64_blob) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_superencrypted, enc_b64_blob);
+    tor_free(enc_b64_blob);
+  }
+
+  /* Join all lines in one string so we can generate a signature and append
+   * it to the descriptor. */
+  encoded_str = smartlist_join_strings(lines, "\n", 1, &encoded_len);
+
+  /* Sign all fields of the descriptor with our short term signing key. */
+  {
+    ed25519_signature_t sig;
+    char ed_sig_b64[ED25519_SIG_BASE64_LEN + 1];
+    if (ed25519_sign_prefixed(&sig,
+                              (const uint8_t *) encoded_str, encoded_len,
+                              str_desc_sig_prefix, signing_kp) < 0) {
+      log_warn(LD_BUG, "Can't sign encoded HS descriptor!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
+      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    /* Create the signature line. */
+    smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
+  }
+  /* Free previous string that we used so compute the signature. */
+  tor_free(encoded_str);
+  encoded_str = smartlist_join_strings(lines, "\n", 1, NULL);
+  *encoded_out = encoded_str;
+
+  if (strlen(encoded_str) >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_GENERAL, "We just made an HS descriptor that's too big (%d)."
+             "Failing.", (int)strlen(encoded_str));
+    tor_free(encoded_str);
+    goto err;
+  }
+
+  /* XXX: Trigger a control port event. */
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return ret;
+}
+
+/* === DECODING === */
+
+/* Given an encoded string of the link specifiers, return a newly allocated
+ * list of decoded link specifiers. Return NULL on error. */
+STATIC smartlist_t *
+decode_link_specifiers(const char *encoded)
+{
+  int decoded_len;
+  size_t encoded_len, i;
+  uint8_t *decoded;
+  smartlist_t *results = NULL;
+  link_specifier_list_t *specs = NULL;
+
+  tor_assert(encoded);
+
+  encoded_len = strlen(encoded);
+  decoded = tor_malloc(encoded_len);
+  decoded_len = base64_decode((char *) decoded, encoded_len, encoded,
+                              encoded_len);
+  if (decoded_len < 0) {
+    goto err;
+  }
+
+  if (link_specifier_list_parse(&specs, decoded,
+                                (size_t) decoded_len) < decoded_len) {
+    goto err;
+  }
+  tor_assert(specs);
+  results = smartlist_new();
+
+  for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
+    hs_desc_link_specifier_t *hs_spec;
+    link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
+    tor_assert(ls);
+
+    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
+    hs_spec->type = link_specifier_get_ls_type(ls);
+    switch (hs_spec->type) {
+    case LS_IPV4:
+      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
+                          link_specifier_get_un_ipv4_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
+      break;
+    case LS_IPV6:
+      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
+                               link_specifier_getarray_un_ipv6_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
+      break;
+    case LS_LEGACY_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
+                 sizeof(hs_spec->u.legacy_id));
+      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
+             sizeof(hs_spec->u.legacy_id));
+      break;
+    case LS_ED25519_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
+                 sizeof(hs_spec->u.ed25519_id));
+      memcpy(hs_spec->u.ed25519_id,
+             link_specifier_getconstarray_un_ed25519_id(ls),
+             sizeof(hs_spec->u.ed25519_id));
+      break;
+    default:
+      tor_free(hs_spec);
+      goto err;
+    }
+
+    smartlist_add(results, hs_spec);
+  }
+
+  goto done;
+ err:
+  if (results) {
+    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    smartlist_free(results);
+    results = NULL;
+  }
+ done:
+  link_specifier_list_free(specs);
+  tor_free(decoded);
+  return results;
+}
+
+/* Given a list of authentication types, decode it and put it in the encrypted
+ * data section. Return 1 if we at least know one of the type or 0 if we know
+ * none of them. */
+static int
+decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  int match = 0;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  desc->intro_auth_types = smartlist_new();
+  smartlist_split_string(desc->intro_auth_types, list, " ", 0, 0);
+
+  /* Validate the types that we at least know about one. */
+  SMARTLIST_FOREACH_BEGIN(desc->intro_auth_types, const char *, auth) {
+    for (int idx = 0; intro_auth_types[idx].identifier; idx++) {
+      if (!strncmp(auth, intro_auth_types[idx].identifier,
+                   strlen(intro_auth_types[idx].identifier))) {
+        match = 1;
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(auth);
+
+  return match;
+}
+
+/* Parse a space-delimited list of integers representing CREATE2 formats into
+ * the bitfield in hs_desc_encrypted_data_t. Ignore unrecognized values. */
+static void
+decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  smartlist_t *tokens;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  tokens = smartlist_new();
+  smartlist_split_string(tokens, list, " ", 0, 0);
+
+  SMARTLIST_FOREACH_BEGIN(tokens, char *, s) {
+    int ok;
+    unsigned long type = tor_parse_ulong(s, 10, 1, UINT16_MAX, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_REND, "Unparseable value %s in create2 list", escaped(s));
+      continue;
+    }
+    switch (type) {
+    case ONION_HANDSHAKE_TYPE_NTOR:
+      desc->create2_ntor = 1;
+      break;
+    default:
+      /* We deliberately ignore unsupported handshake types */
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  SMARTLIST_FOREACH(tokens, char *, s, tor_free(s));
+  smartlist_free(tokens);
+}
+
+/* Given a certificate, validate the certificate for certain conditions which
+ * are if the given type matches the cert's one, if the signing key is
+ * included and if the that key was actually used to sign the certificate.
+ *
+ * Return 1 iff if all conditions pass or 0 if one of them fails. */
+STATIC int
+cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
+{
+  tor_assert(log_obj_type);
+
+  if (cert == NULL) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", log_obj_type);
+    goto err;
+  }
+  if (cert->cert_type != type) {
+    log_warn(LD_REND, "Invalid cert type %02x for %s.", cert->cert_type,
+             log_obj_type);
+    goto err;
+  }
+  /* All certificate must have its signing key included. */
+  if (!cert->signing_key_included) {
+    log_warn(LD_REND, "Signing key is NOT included for %s.", log_obj_type);
+    goto err;
+  }
+  /* The following will not only check if the signature matches but also the
+   * expiration date and overall validity. */
+  if (tor_cert_checksig(cert, &cert->signing_key, approx_time()) < 0) {
+    log_warn(LD_REND, "Invalid signature for %s: %s", log_obj_type,
+             tor_cert_describe_signature_status(cert));
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/* Given some binary data, try to parse it to get a certificate object. If we
+ * have a valid cert, validate it using the given wanted type. On error, print
+ * a log using the err_msg has the certificate identifier adding semantic to
+ * the log and cert_out is set to NULL. On success, 0 is returned and cert_out
+ * points to a newly allocated certificate object. */
+static int
+cert_parse_and_validate(tor_cert_t **cert_out, const char *data,
+                        size_t data_len, unsigned int cert_type_wanted,
+                        const char *err_msg)
+{
+  tor_cert_t *cert;
+
+  tor_assert(cert_out);
+  tor_assert(data);
+  tor_assert(err_msg);
+
+  /* Parse certificate. */
+  cert = tor_cert_parse((const uint8_t *) data, data_len);
+  if (!cert) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", err_msg);
+    goto err;
+  }
+
+  /* Validate certificate. */
+  if (!cert_is_valid(cert, cert_type_wanted, err_msg)) {
+    goto err;
+  }
+
+  *cert_out = cert;
+  return 0;
+
+ err:
+  tor_cert_free(cert);
+  *cert_out = NULL;
+  return -1;
+}
+
+/* Return true iff the given length of the encrypted data of a descriptor
+ * passes validation. */
+STATIC int
+encrypted_data_length_is_valid(size_t len)
+{
+  /* Make sure there is enough data for the salt and the mac. The equality is
+     there to ensure that there is at least one byte of encrypted data. */
+  if (len <= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN) {
+    log_warn(LD_REND, "Length of descriptor's encrypted data is too small. "
+                      "Got %lu but minimum value is %d",
+             (unsigned long)len, HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/** Decrypt an encrypted descriptor layer at <b>encrypted_blob</b> of size
+ *  <b>encrypted_blob_size</b>. Use the descriptor object <b>desc</b> to
+ *  generate the right decryption keys; set <b>decrypted_out</b> to the
+ *  plaintext. If <b>is_superencrypted_layer</b> is set, this is the outter
+ *  encrypted layer of the descriptor.
+ *
+ * On any error case, including an empty output, return 0 and set
+ * *<b>decrypted_out</b> to NULL.
+ */
+MOCK_IMPL(STATIC size_t,
+decrypt_desc_layer,(const hs_descriptor_t *desc,
+                    const uint8_t *encrypted_blob,
+                    size_t encrypted_blob_size,
+                    int is_superencrypted_layer,
+                    char **decrypted_out))
+{
+  uint8_t *decrypted = NULL;
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
+  const uint8_t *salt, *encrypted, *desc_mac;
+  size_t encrypted_len, result_len = 0;
+
+  tor_assert(decrypted_out);
+  tor_assert(desc);
+  tor_assert(encrypted_blob);
+
+  /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC .
+   * Make sure we have enough space for all these things. */
+  if (!encrypted_data_length_is_valid(encrypted_blob_size)) {
+    goto err;
+  }
+
+  /* Start of the blob thus the salt. */
+  salt = encrypted_blob;
+
+  /* Next is the encrypted data. */
+  encrypted = encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN;
+  encrypted_len = encrypted_blob_size -
+    (HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+  tor_assert(encrypted_len > 0); /* guaranteed by the check above */
+
+  /* And last comes the MAC. */
+  desc_mac = encrypted_blob + encrypted_blob_size - DIGEST256_LEN;
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the decryption. */
+  build_secret_key_iv_mac(desc, salt, HS_DESC_ENCRYPTED_SALT_LEN,
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, HS_DESC_ENCRYPTED_SALT_LEN,
+            encrypted, encrypted_len, our_mac, sizeof(our_mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+  /* Verify MAC; MAC is H(mac_key || salt || encrypted)
+   *
+   * This is a critical check that is making sure the computed MAC matches the
+   * one in the descriptor. */
+  if (!tor_memeq(our_mac, desc_mac, sizeof(our_mac))) {
+    log_info(LD_REND, "Encrypted service descriptor MAC check failed");
+    goto err;
+  }
+
+  {
+    /* Decrypt. Here we are assured that the encrypted length is valid for
+     * decryption. */
+    crypto_cipher_t *cipher;
+
+    cipher = crypto_cipher_new_with_iv_and_bits(secret_key, secret_iv,
+                                                HS_DESC_ENCRYPTED_BIT_SIZE);
+    /* Extra byte for the NUL terminated byte. */
+    decrypted = tor_malloc_zero(encrypted_len + 1);
+    crypto_cipher_decrypt(cipher, (char *) decrypted,
+                          (const char *) encrypted, encrypted_len);
+    crypto_cipher_free(cipher);
+  }
+
+  {
+    /* Adjust length to remove NUL padding bytes */
+    uint8_t *end = memchr(decrypted, 0, encrypted_len);
+    result_len = encrypted_len;
+    if (end) {
+      result_len = end - decrypted;
+    }
+  }
+
+  if (result_len == 0) {
+    /* Treat this as an error, so that somebody will free the output. */
+    goto err;
+  }
+
+  /* Make sure to NUL terminate the string. */
+  decrypted[encrypted_len] = '\0';
+  *decrypted_out = (char *) decrypted;
+  goto done;
+
+ err:
+  if (decrypted) {
+    tor_free(decrypted);
+  }
+  *decrypted_out = NULL;
+  result_len = 0;
+
+ done:
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  return result_len;
+}
+
+/* Basic validation that the superencrypted client auth portion of the
+ * descriptor is well-formed and recognized. Return True if so, otherwise
+ * return False. */
+static int
+superencrypted_auth_data_is_valid(smartlist_t *tokens)
+{
+  /* XXX: This is just basic validation for now. When we implement client auth,
+     we can refactor this function so that it actually parses and saves the
+     data. */
+
+  { /* verify desc auth type */
+    const directory_token_t *tok;
+    tok = find_by_keyword(tokens, R3_DESC_AUTH_TYPE);
+    tor_assert(tok->n_args >= 1);
+    if (strcmp(tok->args[0], "x25519")) {
+      log_warn(LD_DIR, "Unrecognized desc auth type");
+      return 0;
+    }
+  }
+
+  { /* verify desc auth key */
+    const directory_token_t *tok;
+    curve25519_public_key_t k;
+    tok = find_by_keyword(tokens, R3_DESC_AUTH_KEY);
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus desc auth key in HS desc");
+      return 0;
+    }
+  }
+
+  /* verify desc auth client items */
+  SMARTLIST_FOREACH_BEGIN(tokens, const directory_token_t *, tok) {
+    if (tok->tp == R3_DESC_AUTH_CLIENT) {
+      tor_assert(tok->n_args >= 3);
+    }
+  } SMARTLIST_FOREACH_END(tok);
+
+  return 1;
+}
+
+/* Parse <b>message</b>, the plaintext of the superencrypted portion of an HS
+ * descriptor. Set <b>encrypted_out</b> to the encrypted blob, and return its
+ * size */
+STATIC size_t
+decode_superencrypted(const char *message, size_t message_len,
+                     uint8_t **encrypted_out)
+{
+  int retval = 0;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len, tokens,
+                      hs_desc_superencrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Superencrypted portion is not parseable");
+    goto err;
+  }
+
+  /* Do some rudimentary validation of the authentication data */
+  if (!superencrypted_auth_data_is_valid(tokens)) {
+    log_warn(LD_REND, "Invalid auth data");
+    goto err;
+  }
+
+  /* Extract the encrypted data section. */
+  {
+    const directory_token_t *tok;
+    tok = find_by_keyword(tokens, R3_ENCRYPTED);
+    tor_assert(tok->object_body);
+    if (strcmp(tok->object_type, "MESSAGE") != 0) {
+      log_warn(LD_REND, "Desc superencrypted data section is invalid");
+      goto err;
+    }
+    /* Make sure the length of the encrypted blob is valid. */
+    if (!encrypted_data_length_is_valid(tok->object_size)) {
+      goto err;
+    }
+
+    /* Copy the encrypted blob to the descriptor object so we can handle it
+     * latter if needed. */
+    tor_assert(tok->object_size <= INT_MAX);
+    *encrypted_out = tor_memdup(tok->object_body, tok->object_size);
+    retval = (int) tok->object_size;
+  }
+
+ err:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return retval;
+}
+
+/* Decrypt both the superencrypted and the encrypted section of the descriptor
+ * using the given descriptor object <b>desc</b>. A newly allocated NUL
+ * terminated string is put in decrypted_out which contains the inner encrypted
+ * layer of the descriptor. Return the length of decrypted_out on success else
+ * 0 is returned and decrypted_out is set to NULL. */
+static size_t
+desc_decrypt_all(const hs_descriptor_t *desc, char **decrypted_out)
+{
+  size_t  decrypted_len = 0;
+  size_t encrypted_len = 0;
+  size_t superencrypted_len = 0;
+  char *superencrypted_plaintext = NULL;
+  uint8_t *encrypted_blob = NULL;
+
+  /** Function logic: This function takes us from the descriptor header to the
+   *  inner encrypted layer, by decrypting and decoding the middle descriptor
+   *  layer. In the end we return the contents of the inner encrypted layer to
+   *  our caller. */
+
+  /* 1. Decrypt middle layer of descriptor */
+  superencrypted_len = decrypt_desc_layer(desc,
+                                 desc->plaintext_data.superencrypted_blob,
+                                 desc->plaintext_data.superencrypted_blob_size,
+                                 1,
+                                 &superencrypted_plaintext);
+  if (!superencrypted_len) {
+    log_warn(LD_REND, "Decrypting superencrypted desc failed.");
+    goto err;
+  }
+  tor_assert(superencrypted_plaintext);
+
+  /* 2. Parse "superencrypted" */
+  encrypted_len = decode_superencrypted(superencrypted_plaintext,
+                                        superencrypted_len,
+                                        &encrypted_blob);
+  if (!encrypted_len) {
+    goto err;
+  }
+  tor_assert(encrypted_blob);
+
+  /* 3. Decrypt "encrypted" and set decrypted_out */
+  char *decrypted_desc;
+  decrypted_len = decrypt_desc_layer(desc,
+                                     encrypted_blob, encrypted_len,
+                                     0, &decrypted_desc);
+  if (!decrypted_len) {
+    log_warn(LD_REND, "Decrypting encrypted desc failed.");
+    goto err;
+  }
+  tor_assert(decrypted_desc);
+
+  *decrypted_out = decrypted_desc;
+
+ err:
+  tor_free(superencrypted_plaintext);
+  tor_free(encrypted_blob);
+
+  return decrypted_len;
+}
+
+/* Given the token tok for an intro point legacy key, the list of tokens, the
+ * introduction point ip being decoded and the descriptor desc from which it
+ * comes from, decode the legacy key and set the intro point object. Return 0
+ * on success else -1 on failure. */
+static int
+decode_intro_legacy_key(const directory_token_t *tok,
+                        smartlist_t *tokens,
+                        hs_desc_intro_point_t *ip,
+                        const hs_descriptor_t *desc)
+{
+  tor_assert(tok);
+  tor_assert(tokens);
+  tor_assert(ip);
+  tor_assert(desc);
+
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_REND, "Introduction point legacy key is invalid");
+    goto err;
+  }
+  ip->legacy.key = crypto_pk_dup_key(tok->key);
+  /* Extract the legacy cross certification cert which MUST be present if we
+   * have a legacy key. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY_CERT);
+  if (!tok) {
+    log_warn(LD_REND, "Introduction point legacy key cert is missing");
+    goto err;
+  }
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "CROSSCERT")) {
+    /* Info level because this might be an unknown field that we should
+     * ignore. */
+    log_info(LD_REND, "Introduction point legacy encryption key "
+                      "cross-certification has an unknown format.");
+    goto err;
+  }
+  /* Keep a copy of the certificate. */
+  ip->legacy.cert.encoded = tor_memdup(tok->object_body, tok->object_size);
+  ip->legacy.cert.len = tok->object_size;
+  /* The check on the expiration date is for the entire lifetime of a
+   * certificate which is 24 hours. However, a descriptor has a maximum
+   * lifetime of 12 hours meaning we have a 12h difference between the two
+   * which ultimately accommodate the clock skewed client. */
+  if (rsa_ed25519_crosscert_check(ip->legacy.cert.encoded,
+                                  ip->legacy.cert.len, ip->legacy.key,
+                                  &desc->plaintext_data.signing_pubkey,
+                                  approx_time() - HS_DESC_CERT_LIFETIME)) {
+    log_warn(LD_REND, "Unable to check cross-certification on the "
+                      "introduction point legacy encryption key.");
+    ip->cross_certified = 0;
+    goto err;
+  }
+
+  /* Success. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Dig into the descriptor <b>tokens</b> to find the onion key we should use
+ * for this intro point, and set it into <b>onion_key_out</b>. Return 0 if it
+ * was found and well-formed, otherwise return -1 in case of errors. */
+static int
+set_intro_point_onion_key(curve25519_public_key_t *onion_key_out,
+                          const smartlist_t *tokens)
+{
+  int retval = -1;
+  smartlist_t *onion_keys = NULL;
+
+  tor_assert(onion_key_out);
+
+  onion_keys = find_all_by_keyword(tokens, R3_INTRO_ONION_KEY);
+  if (!onion_keys) {
+    log_warn(LD_REND, "Descriptor did not contain intro onion keys");
+    goto err;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(onion_keys, directory_token_t *, tok) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    /* Try to find an ntor key, it's the only recognized type right now */
+    if (!strcmp(tok->args[0], "ntor")) {
+      if (curve25519_public_from_base64(onion_key_out, tok->args[1]) < 0) {
+        log_warn(LD_REND, "Introduction point ntor onion-key is invalid");
+        goto err;
+      }
+      /* Got the onion key! Set the appropriate retval */
+      retval = 0;
+    }
+  } SMARTLIST_FOREACH_END(tok);
+
+  /* Log an error if we didn't find it :( */
+  if (retval < 0) {
+    log_warn(LD_REND, "Descriptor did not contain ntor onion keys");
+  }
+
+ err:
+  smartlist_free(onion_keys);
+  return retval;
+}
+
+/* Given the start of a section and the end of it, decode a single
+ * introduction point from that section. Return a newly allocated introduction
+ * point object containing the decoded data. Return NULL if the section can't
+ * be decoded. */
+STATIC hs_desc_intro_point_t *
+decode_introduction_point(const hs_descriptor_t *desc, const char *start)
+{
+  hs_desc_intro_point_t *ip = NULL;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  const directory_token_t *tok;
+
+  tor_assert(desc);
+  tor_assert(start);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, start, start + strlen(start),
+                      tokens, hs_desc_intro_point_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Introduction point is not parseable");
+    goto err;
+  }
+
+  /* Ok we seem to have a well formed section containing enough tokens to
+   * parse. Allocate our IP object and try to populate it. */
+  ip = hs_desc_intro_point_new();
+
+  /* "introduction-point" SP link-specifiers NL */
+  tok = find_by_keyword(tokens, R3_INTRODUCTION_POINT);
+  tor_assert(tok->n_args == 1);
+  /* Our constructor creates this list by default so free it. */
+  smartlist_free(ip->link_specifiers);
+  ip->link_specifiers = decode_link_specifiers(tok->args[0]);
+  if (!ip->link_specifiers) {
+    log_warn(LD_REND, "Introduction point has invalid link specifiers");
+    goto err;
+  }
+
+  /* "onion-key" SP ntor SP key NL */
+  if (set_intro_point_onion_key(&ip->onion_key, tokens) < 0) {
+    goto err;
+  }
+
+  /* "auth-key" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_AUTH_KEY);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+    log_warn(LD_REND, "Unexpected object type for introduction auth key");
+    goto err;
+  }
+  /* Parse cert and do some validation. */
+  if (cert_parse_and_validate(&ip->auth_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_AUTH_HS_IP_KEY,
+                              "introduction point auth-key") < 0) {
+    goto err;
+  }
+  /* Validate authentication certificate with descriptor signing key. */
+  if (tor_cert_checksig(ip->auth_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid authentication key signature: %s",
+             tor_cert_describe_signature_status(ip->auth_key_cert));
+    goto err;
+  }
+
+  /* Exactly one "enc-key" SP "ntor" SP key NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY);
+  if (!strcmp(tok->args[0], "ntor")) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    if (curve25519_public_from_base64(&ip->enc_key, tok->args[1]) < 0) {
+      log_warn(LD_REND, "Introduction point ntor enc-key is invalid");
+      goto err;
+    }
+  } else {
+    /* Unknown key type so we can't use that introduction point. */
+    log_warn(LD_REND, "Introduction point encryption key is unrecognized.");
+    goto err;
+  }
+
+  /* Exactly once "enc-key-cert" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY_CERT);
+  tor_assert(tok->object_body);
+  /* Do the cross certification. */
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+      log_warn(LD_REND, "Introduction point ntor encryption key "
+                        "cross-certification has an unknown format.");
+      goto err;
+  }
+  if (cert_parse_and_validate(&ip->enc_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_CROSS_HS_IP_KEYS,
+                              "introduction point enc-key-cert") < 0) {
+    goto err;
+  }
+  if (tor_cert_checksig(ip->enc_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid encryption key signature: %s",
+             tor_cert_describe_signature_status(ip->enc_key_cert));
+    goto err;
+  }
+  /* It is successfully cross certified. Flag the object. */
+  ip->cross_certified = 1;
+
+  /* Do we have a "legacy-key" SP key NL ?*/
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY);
+  if (tok) {
+    if (decode_intro_legacy_key(tok, tokens, ip, desc) < 0) {
+      goto err;
+    }
+  }
+
+  /* Introduction point has been parsed successfully. */
+  goto done;
+
+ err:
+  hs_desc_intro_point_free(ip);
+  ip = NULL;
+
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return ip;
+}
+
+/* Given a descriptor string at <b>data</b>, decode all possible introduction
+ * points that we can find. Add the introduction point object to desc_enc as we
+ * find them. This function can't fail and it is possible that zero
+ * introduction points can be decoded. */
+static void
+decode_intro_points(const hs_descriptor_t *desc,
+                    hs_desc_encrypted_data_t *desc_enc,
+                    const char *data)
+{
+  smartlist_t *chunked_desc = smartlist_new();
+  smartlist_t *intro_points = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(desc_enc);
+  tor_assert(data);
+  tor_assert(desc_enc->intro_points);
+
+  /* Take the desc string, and extract the intro point substrings out of it */
+  {
+    /* Split the descriptor string using the intro point header as delimiter */
+    smartlist_split_string(chunked_desc, data, str_intro_point_start, 0, 0);
+
+    /* Check if there are actually any intro points included. The first chunk
+     * should be other descriptor fields (e.g. create2-formats), so it's not an
+     * intro point. */
+    if (smartlist_len(chunked_desc) < 2) {
+      goto done;
+    }
+  }
+
+  /* Take the intro point substrings, and prepare them for parsing */
+  {
+    int i = 0;
+    /* Prepend the introduction-point header to all the chunks, since
+       smartlist_split_string() devoured it. */
+    SMARTLIST_FOREACH_BEGIN(chunked_desc, char *, chunk) {
+      /* Ignore first chunk. It's other descriptor fields. */
+      if (i++ == 0) {
+        continue;
+      }
+
+      smartlist_add_asprintf(intro_points, "%s %s", str_intro_point, chunk);
+    } SMARTLIST_FOREACH_END(chunk);
+  }
+
+  /* Parse the intro points! */
+  SMARTLIST_FOREACH_BEGIN(intro_points, const char *, intro_point) {
+    hs_desc_intro_point_t *ip = decode_introduction_point(desc, intro_point);
+    if (!ip) {
+      /* Malformed introduction point section. We'll ignore this introduction
+       * point and continue parsing. New or unknown fields are possible for
+       * forward compatibility. */
+      continue;
+    }
+    smartlist_add(desc_enc->intro_points, ip);
+  } SMARTLIST_FOREACH_END(intro_point);
+
+ done:
+  SMARTLIST_FOREACH(chunked_desc, char *, a, tor_free(a));
+  smartlist_free(chunked_desc);
+  SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
+  smartlist_free(intro_points);
+}
+/* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
+ * descriptor in encoded_desc validates the descriptor content. */
+STATIC int
+desc_sig_is_valid(const char *b64_sig,
+                  const ed25519_public_key_t *signing_pubkey,
+                  const char *encoded_desc, size_t encoded_len)
+{
+  int ret = 0;
+  ed25519_signature_t sig;
+  const char *sig_start;
+
+  tor_assert(b64_sig);
+  tor_assert(signing_pubkey);
+  tor_assert(encoded_desc);
+  /* Verifying nothing won't end well :). */
+  tor_assert(encoded_len > 0);
+
+  /* Signature length check. */
+  if (strlen(b64_sig) != ED25519_SIG_BASE64_LEN) {
+    log_warn(LD_REND, "Service descriptor has an invalid signature length."
+                      "Exptected %d but got %lu",
+             ED25519_SIG_BASE64_LEN, (unsigned long) strlen(b64_sig));
+    goto err;
+  }
+
+  /* First, convert base64 blob to an ed25519 signature. */
+  if (ed25519_signature_from_base64(&sig, b64_sig) != 0) {
+    log_warn(LD_REND, "Service descriptor does not contain a valid "
+                      "signature");
+    goto err;
+  }
+
+  /* Find the start of signature. */
+  sig_start = tor_memstr(encoded_desc, encoded_len, "\n" str_signature " ");
+  /* Getting here means the token parsing worked for the signature so if we
+   * can't find the start of the signature, we have a code flow issue. */
+  if (!sig_start) {
+    log_warn(LD_GENERAL, "Malformed signature line. Rejecting.");
+    goto err;
+  }
+  /* Skip newline, it has to go in the signature check. */
+  sig_start++;
+
+  /* Validate signature with the full body of the descriptor. */
+  if (ed25519_checksig_prefixed(&sig,
+                                (const uint8_t *) encoded_desc,
+                                sig_start - encoded_desc,
+                                str_desc_sig_prefix,
+                                signing_pubkey) != 0) {
+    log_warn(LD_REND, "Invalid signature on service descriptor");
+    goto err;
+  }
+  /* Valid signature! All is good. */
+  ret = 1;
+
+ err:
+  return ret;
+}
+
+/* Decode descriptor plaintext data for version 3. Given a list of tokens, an
+ * allocated plaintext object that will be populated and the encoded
+ * descriptor with its length. The last one is needed for signature
+ * verification. Unknown tokens are simply ignored so this won't error on
+ * unknowns but requires that all v3 token be present and valid.
+ *
+ * Return 0 on success else a negative value. */
+static int
+desc_decode_plaintext_v3(smartlist_t *tokens,
+                         hs_desc_plaintext_data_t *desc,
+                         const char *encoded_desc, size_t encoded_len)
+{
+  int ok;
+  directory_token_t *tok;
+
+  tor_assert(tokens);
+  tor_assert(desc);
+  /* Version higher could still use this function to decode most of the
+   * descriptor and then they decode the extra part. */
+  tor_assert(desc->version >= 3);
+
+  /* Descriptor lifetime parsing. */
+  tok = find_by_keyword(tokens, R3_DESC_LIFETIME);
+  tor_assert(tok->n_args == 1);
+  desc->lifetime_sec = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor lifetime value is invalid");
+    goto err;
+  }
+  /* Put it from minute to second. */
+  desc->lifetime_sec *= 60;
+  if (desc->lifetime_sec > HS_DESC_MAX_LIFETIME) {
+    log_warn(LD_REND, "Service descriptor lifetime is too big. "
+                      "Got %" PRIu32 " but max is %d",
+             desc->lifetime_sec, HS_DESC_MAX_LIFETIME);
+    goto err;
+  }
+
+  /* Descriptor signing certificate. */
+  tok = find_by_keyword(tokens, R3_DESC_SIGNING_CERT);
+  tor_assert(tok->object_body);
+  /* Expecting a prop220 cert with the signing key extension, which contains
+   * the blinded public key. */
+  if (strcmp(tok->object_type, "ED25519 CERT") != 0) {
+    log_warn(LD_REND, "Service descriptor signing cert wrong type (%s)",
+             escaped(tok->object_type));
+    goto err;
+  }
+  if (cert_parse_and_validate(&desc->signing_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_SIGNING_HS_DESC,
+                              "service descriptor signing key") < 0) {
+    goto err;
+  }
+
+  /* Copy the public keys into signing_pubkey and blinded_pubkey */
+  memcpy(&desc->signing_pubkey, &desc->signing_key_cert->signed_key,
+         sizeof(ed25519_public_key_t));
+  memcpy(&desc->blinded_pubkey, &desc->signing_key_cert->signing_key,
+         sizeof(ed25519_public_key_t));
+
+  /* Extract revision counter value. */
+  tok = find_by_keyword(tokens, R3_REVISION_COUNTER);
+  tor_assert(tok->n_args == 1);
+  desc->revision_counter = tor_parse_uint64(tok->args[0], 10, 0,
+                                            UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor revision-counter is invalid");
+    goto err;
+  }
+
+  /* Extract the encrypted data section. */
+  tok = find_by_keyword(tokens, R3_SUPERENCRYPTED);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "MESSAGE") != 0) {
+    log_warn(LD_REND, "Service descriptor encrypted data section is invalid");
+    goto err;
+  }
+  /* Make sure the length of the encrypted blob is valid. */
+  if (!encrypted_data_length_is_valid(tok->object_size)) {
+    goto err;
+  }
+
+  /* Copy the encrypted blob to the descriptor object so we can handle it
+   * latter if needed. */
+  desc->superencrypted_blob = tor_memdup(tok->object_body, tok->object_size);
+  desc->superencrypted_blob_size = tok->object_size;
+
+  /* Extract signature and verify it. */
+  tok = find_by_keyword(tokens, R3_SIGNATURE);
+  tor_assert(tok->n_args == 1);
+  /* First arg here is the actual encoded signature. */
+  if (!desc_sig_is_valid(tok->args[0], &desc->signing_pubkey,
+                         encoded_desc, encoded_len)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Decode the version 3 encrypted section of the given descriptor desc. The
+ * desc_encrypted_out will be populated with the decoded data. Return 0 on
+ * success else -1. */
+static int
+desc_decode_encrypted_v3(const hs_descriptor_t *desc,
+                         hs_desc_encrypted_data_t *desc_encrypted_out)
+{
+  int result = -1;
+  char *message = NULL;
+  size_t message_len;
+  memarea_t *area = NULL;
+  directory_token_t *tok;
+  smartlist_t *tokens = NULL;
+
+  tor_assert(desc);
+  tor_assert(desc_encrypted_out);
+
+  /* Decrypt the superencrypted data that is located in the plaintext section
+   * in the descriptor as a blob of bytes. */
+  message_len = desc_decrypt_all(desc, &message);
+  if (!message_len) {
+    /* Inform at notice level that the onion address requested can't be
+     * reached without client authorization most likely. */
+    log_notice(LD_REND, "Fail to decrypt descriptor for requested onion "
+                        "address. It is likely requiring client "
+                        "authorization.");
+    goto err;
+  }
+  tor_assert(message);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len,
+                      tokens, hs_desc_encrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Encrypted service descriptor is not parseable.");
+    goto err;
+  }
+
+  /* CREATE2 supported cell format. It's mandatory. */
+  tok = find_by_keyword(tokens, R3_CREATE2_FORMATS);
+  tor_assert(tok);
+  decode_create2_list(desc_encrypted_out, tok->args[0]);
+  /* Must support ntor according to the specification */
+  if (!desc_encrypted_out->create2_ntor) {
+    log_warn(LD_REND, "Service create2-formats does not include ntor.");
+    goto err;
+  }
+
+  /* Authentication type. It's optional but only once. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_AUTH_REQUIRED);
+  if (tok) {
+    if (!decode_auth_type(desc_encrypted_out, tok->args[0])) {
+      log_warn(LD_REND, "Service descriptor authentication type has "
+                        "invalid entry(ies).");
+      goto err;
+    }
+  }
+
+  /* Is this service a single onion service? */
+  tok = find_opt_by_keyword(tokens, R3_SINGLE_ONION_SERVICE);
+  if (tok) {
+    desc_encrypted_out->single_onion_service = 1;
+  }
+
+  /* Initialize the descriptor's introduction point list before we start
+   * decoding. Having 0 intro point is valid. Then decode them all. */
+  desc_encrypted_out->intro_points = smartlist_new();
+  decode_intro_points(desc, desc_encrypted_out, message);
+
+  /* Validation of maximum introduction points allowed. */
+  if (smartlist_len(desc_encrypted_out->intro_points) >
+      HS_CONFIG_V3_MAX_INTRO_POINTS) {
+    log_warn(LD_REND, "Service descriptor contains too many introduction "
+                      "points. Maximum allowed is %d but we have %d",
+             HS_CONFIG_V3_MAX_INTRO_POINTS,
+             smartlist_len(desc_encrypted_out->intro_points));
+    goto err;
+  }
+
+  /* NOTE: Unknown fields are allowed because this function could be used to
+   * decode other descriptor version. */
+
+  result = 0;
+  goto done;
+
+ err:
+  tor_assert(result < 0);
+  desc_encrypted_data_free_contents(desc_encrypted_out);
+
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  if (message) {
+    tor_free(message);
+  }
+  return result;
+}
+
+/* Table of encrypted decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_encrypted_handlers[])(
+      const hs_descriptor_t *desc,
+      hs_desc_encrypted_data_t *desc_encrypted) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_encrypted_v3,
+};
+
+/* Decode the encrypted data section of the given descriptor and store the
+ * data in the given encrypted data object. Return 0 on success else a
+ * negative value on error. */
+int
+hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                         hs_desc_encrypted_data_t *desc_encrypted)
+{
+  int ret;
+  uint32_t version;
+
+  tor_assert(desc);
+  /* Ease our life a bit. */
+  version = desc->plaintext_data.version;
+  tor_assert(desc_encrypted);
+  /* Calling this function without an encrypted blob to parse is a code flow
+   * error. The plaintext parsing should never succeed in the first place
+   * without an encrypted section. */
+  tor_assert(desc->plaintext_data.superencrypted_blob);
+  /* Let's make sure we have a supported version as well. By correctly parsing
+   * the plaintext, this should not fail. */
+  if (BUG(!hs_desc_is_supported_version(version))) {
+    ret = -1;
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_encrypted_handlers) >= version);
+  tor_assert(decode_encrypted_handlers[version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_encrypted_handlers[version](desc, desc_encrypted);
+  if (ret < 0) {
+    goto err;
+  }
+
+ err:
+  return ret;
+}
+
+/* Table of plaintext decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_plaintext_handlers[])(
+      smartlist_t *tokens,
+      hs_desc_plaintext_data_t *desc,
+      const char *encoded_desc,
+      size_t encoded_len) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_plaintext_v3,
+};
+
+/* Fully decode the given descriptor plaintext and store the data in the
+ * plaintext data object. Returns 0 on success else a negative value. */
+int
+hs_desc_decode_plaintext(const char *encoded,
+                         hs_desc_plaintext_data_t *plaintext)
+{
+  int ok = 0, ret = -1;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  size_t encoded_len;
+  directory_token_t *tok;
+
+  tor_assert(encoded);
+  tor_assert(plaintext);
+
+  /* Check that descriptor is within size limits. */
+  encoded_len = strlen(encoded);
+  if (encoded_len >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_REND, "Service descriptor is too big (%lu bytes)",
+             (unsigned long) encoded_len);
+    goto err;
+  }
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  /* Tokenize the descriptor so we can start to parse it. */
+  if (tokenize_string(area, encoded, encoded + encoded_len, tokens,
+                      hs_desc_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Service descriptor is not parseable");
+    goto err;
+  }
+
+  /* Get the version of the descriptor which is the first mandatory field of
+   * the descriptor. From there, we'll decode the right descriptor version. */
+  tok = find_by_keyword(tokens, R_HS_DESCRIPTOR);
+  tor_assert(tok->n_args == 1);
+  plaintext->version = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor has unparseable version %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (!hs_desc_is_supported_version(plaintext->version)) {
+    log_warn(LD_REND, "Service descriptor has unsupported version %" PRIu32,
+             plaintext->version);
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_plaintext_handlers) >= plaintext->version);
+  tor_assert(decode_plaintext_handlers[plaintext->version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_plaintext_handlers[plaintext->version](tokens, plaintext,
+                                                      encoded, encoded_len);
+  if (ret < 0) {
+    goto err;
+  }
+  /* Success. Descriptor has been populated with the data. */
+  ret = 0;
+
+ err:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  return ret;
+}
+
+/* Fully decode an encoded descriptor and set a newly allocated descriptor
+ * object in desc_out. Subcredentials are used if not NULL else it's ignored.
+ *
+ * Return 0 on success. A negative value is returned on error and desc_out is
+ * set to NULL. */
+int
+hs_desc_decode_descriptor(const char *encoded,
+                          const uint8_t *subcredential,
+                          hs_descriptor_t **desc_out)
+{
+  int ret = -1;
+  hs_descriptor_t *desc;
+
+  tor_assert(encoded);
+
+  desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+
+  /* Subcredentials are optional. */
+  if (BUG(!subcredential)) {
+    log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
+    goto err;
+  }
+
+  memcpy(desc->subcredential, subcredential, sizeof(desc->subcredential));
+
+  ret = hs_desc_decode_plaintext(encoded, &desc->plaintext_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  ret = hs_desc_decode_encrypted(desc, &desc->encrypted_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  if (desc_out) {
+    *desc_out = desc;
+  } else {
+    hs_descriptor_free(desc);
+  }
+  return ret;
+
+ err:
+  hs_descriptor_free(desc);
+  if (desc_out) {
+    *desc_out = NULL;
+  }
+
+  tor_assert(ret < 0);
+  return ret;
+}
+
+/* Table of encode function version specific. The functions are indexed by the
+ * version number so v3 callback is at index 3 in the array. */
+static int
+  (*encode_handlers[])(
+      const hs_descriptor_t *desc,
+      const ed25519_keypair_t *signing_kp,
+      char **encoded_out) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_encode_v3,
+};
+
+/* Encode the given descriptor desc including signing with the given key pair
+ * signing_kp. On success, encoded_out points to a newly allocated NUL
+ * terminated string that contains the encoded descriptor as a string.
+ *
+ * Return 0 on success and encoded_out is a valid pointer. On error, -1 is
+ * returned and encoded_out is set to NULL. */
+MOCK_IMPL(int,
+hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
+                           const ed25519_keypair_t *signing_kp,
+                           char **encoded_out))
+{
+  int ret = -1;
+  uint32_t version;
+
+  tor_assert(desc);
+  tor_assert(encoded_out);
+
+  /* Make sure we support the version of the descriptor format. */
+  version = desc->plaintext_data.version;
+  if (!hs_desc_is_supported_version(version)) {
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(encode_handlers) >= version);
+  tor_assert(encode_handlers[version]);
+
+  ret = encode_handlers[version](desc, signing_kp, encoded_out);
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* Try to decode what we just encoded. Symmetry is nice! */
+  ret = hs_desc_decode_descriptor(*encoded_out, desc->subcredential, NULL);
+  if (BUG(ret < 0)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  *encoded_out = NULL;
+  return ret;
+}
+
+/* Free the descriptor plaintext data object. */
+void
+hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
+{
+  desc_plaintext_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the descriptor encrypted data object. */
+void
+hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
+{
+  desc_encrypted_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the given descriptor object. */
+void
+hs_descriptor_free_(hs_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  desc_plaintext_data_free_contents(&desc->plaintext_data);
+  desc_encrypted_data_free_contents(&desc->encrypted_data);
+  tor_free(desc);
+}
+
+/* Return the size in bytes of the given plaintext data object. A sizeof() is
+ * not enough because the object contains pointers and the encrypted blob.
+ * This is particularly useful for our OOM subsystem that tracks the HSDir
+ * cache size for instance. */
+size_t
+hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data)
+{
+  tor_assert(data);
+  return (sizeof(*data) + sizeof(*data->signing_key_cert) +
+          data->superencrypted_blob_size);
+}
+
+/* Return the size in bytes of the given encrypted data object. Used by OOM
+ * subsystem. */
+static size_t
+hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
+{
+  tor_assert(data);
+  size_t intro_size = 0;
+  if (data->intro_auth_types) {
+    intro_size +=
+      smartlist_len(data->intro_auth_types) * sizeof(intro_auth_types);
+  }
+  if (data->intro_points) {
+    /* XXX could follow pointers here and get more accurate size */
+    intro_size +=
+      smartlist_len(data->intro_points) * sizeof(hs_desc_intro_point_t);
+  }
+
+  return sizeof(*data) + intro_size;
+}
+
+/* Return the size in bytes of the given descriptor object. Used by OOM
+ * subsystem. */
+  size_t
+hs_desc_obj_size(const hs_descriptor_t *data)
+{
+  tor_assert(data);
+  return (hs_desc_plaintext_obj_size(&data->plaintext_data) +
+          hs_desc_encrypted_obj_size(&data->encrypted_data) +
+          sizeof(data->subcredential));
+}
+
+/* Return a newly allocated descriptor intro point. */
+hs_desc_intro_point_t *
+hs_desc_intro_point_new(void)
+{
+  hs_desc_intro_point_t *ip = tor_malloc_zero(sizeof(*ip));
+  ip->link_specifiers = smartlist_new();
+  return ip;
+}
+
+/* Free a descriptor intro point object. */
+void
+hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
+{
+  if (ip == NULL) {
+    return;
+  }
+  if (ip->link_specifiers) {
+    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
+                      ls, hs_desc_link_specifier_free(ls));
+    smartlist_free(ip->link_specifiers);
+  }
+  tor_cert_free(ip->auth_key_cert);
+  tor_cert_free(ip->enc_key_cert);
+  crypto_pk_free(ip->legacy.key);
+  tor_free(ip->legacy.cert.encoded);
+  tor_free(ip);
+}
+
+/* Free the given descriptor link specifier. */
+void
+hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
+{
+  if (ls == NULL) {
+    return;
+  }
+  tor_free(ls);
+}
+
+/* Return a newly allocated descriptor link specifier using the given extend
+ * info and requested type. Return NULL on error. */
+hs_desc_link_specifier_t *
+hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
+{
+  hs_desc_link_specifier_t *ls = NULL;
+
+  tor_assert(info);
+
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = type;
+  switch (ls->type) {
+  case LS_IPV4:
+    if (info->addr.family != AF_INET) {
+      goto err;
+    }
+    tor_addr_copy(&ls->u.ap.addr, &info->addr);
+    ls->u.ap.port = info->port;
+    break;
+  case LS_IPV6:
+    if (info->addr.family != AF_INET6) {
+      goto err;
+    }
+    tor_addr_copy(&ls->u.ap.addr, &info->addr);
+    ls->u.ap.port = info->port;
+    break;
+  case LS_LEGACY_ID:
+    /* Bug out if the identity digest is not set */
+    if (BUG(tor_mem_is_zero(info->identity_digest,
+                            sizeof(info->identity_digest)))) {
+      goto err;
+    }
+    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
+    break;
+  case LS_ED25519_ID:
+    /* ed25519 keys are optional for intro points */
+    if (ed25519_public_key_is_zero(&info->ed_identity)) {
+      goto err;
+    }
+    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
+           sizeof(ls->u.ed25519_id));
+    break;
+  default:
+    /* Unknown type is code flow error. */
+    tor_assert(0);
+  }
+
+  return ls;
+ err:
+  tor_free(ls);
+  return NULL;
+}
+
+/* From the given descriptor, remove and free every introduction point. */
+void
+hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
+{
+  smartlist_t *ips;
+
+  tor_assert(desc);
+
+  ips = desc->encrypted_data.intro_points;
+  if (ips) {
+    SMARTLIST_FOREACH(ips, hs_desc_intro_point_t *,
+                      ip, hs_desc_intro_point_free(ip));
+    smartlist_clear(ips);
+  }
+}
+
+/* From a descriptor link specifier object spec, returned a newly allocated
+ * link specifier object that is the encoded representation of spec. Return
+ * NULL on error. */
+link_specifier_t *
+hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
+{
+  tor_assert(spec);
+
+  link_specifier_t *ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, spec->type);
+
+  switch (spec->type) {
+  case LS_IPV4:
+    link_specifier_set_un_ipv4_addr(ls,
+                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
+    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
+    /* Four bytes IPv4 and two bytes port. */
+    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
+                                  sizeof(spec->u.ap.port));
+    break;
+  case LS_IPV6:
+  {
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
+    break;
+  }
+  case LS_LEGACY_ID:
+  {
+    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
+    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
+    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
+    link_specifier_set_ls_len(ls, legacy_id_len);
+    break;
+  }
+  case LS_ED25519_ID:
+  {
+    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
+    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
+    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
+    link_specifier_set_ls_len(ls, ed25519_id_len);
+    break;
+  }
+  default:
+    tor_assert_nonfatal_unreached();
+    link_specifier_free(ls);
+    ls = NULL;
+  }
+
+  return ls;
+}
+
diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h
new file mode 100644
index 0000000000..8195c6efbc
--- /dev/null
+++ b/src/or/hs_descriptor.h
@@ -0,0 +1,285 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.h
+ * \brief Header file for hs_descriptor.c
+ **/
+
+#ifndef TOR_HS_DESCRIPTOR_H
+#define TOR_HS_DESCRIPTOR_H
+
+#include <stdint.h>
+
+#include "or.h"
+#include "address.h"
+#include "container.h"
+#include "crypto.h"
+#include "crypto_ed25519.h"
+#include "ed25519_cert.h" /* needed for trunnel */
+#include "torcert.h"
+
+/* Trunnel */
+struct link_specifier_t;
+
+/* The earliest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MIN 3
+/* The latest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MAX 3
+
+/* Default lifetime of a descriptor in seconds. The valus is set at 3 hours
+ * which is 180 minutes or 10800 seconds. */
+#define HS_DESC_DEFAULT_LIFETIME (3 * 60 * 60)
+/* Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
+ * which is 720 minutes or 43200 seconds. */
+#define HS_DESC_MAX_LIFETIME (12 * 60 * 60)
+/* Lifetime of certificate in the descriptor. This defines the lifetime of the
+ * descriptor signing key and the cross certification cert of that key. It is
+ * set to 54 hours because a descriptor can be around for 48 hours and because
+ * consensuses are used after the hour, add an extra 6 hours to give some time
+ * for the service to stop using it. */
+#define HS_DESC_CERT_LIFETIME (54 * 60 * 60)
+/* Length of the salt needed for the encrypted section of a descriptor. */
+#define HS_DESC_ENCRYPTED_SALT_LEN 16
+/* Length of the secret input needed for the KDF construction which derives
+ * the encryption key for the encrypted data section of the descriptor. This
+ * adds up to 68 bytes being the blinded key, hashed subcredential and
+ * revision counter. */
+#define HS_DESC_ENCRYPTED_SECRET_INPUT_LEN \
+  ED25519_PUBKEY_LEN + DIGEST256_LEN + sizeof(uint64_t)
+/* Length of the KDF output value which is the length of the secret key,
+ * the secret IV and MAC key length which is the length of H() output. */
+#define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \
+  CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN
+/* Pad plaintext of superencrypted data section before encryption so that its
+ * length is a multiple of this value. */
+#define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000
+/* Maximum length in bytes of a full hidden service descriptor. */
+#define HS_DESC_MAX_LEN 50000 /* 50kb max size */
+
+/* Key length for the descriptor symmetric encryption. As specified in the
+ * protocol, we use AES-256 for the encrypted section of the descriptor. The
+ * following is the length in bytes and the bit size. */
+#define HS_DESC_ENCRYPTED_KEY_LEN CIPHER256_KEY_LEN
+#define HS_DESC_ENCRYPTED_BIT_SIZE (HS_DESC_ENCRYPTED_KEY_LEN * 8)
+
+/* Type of authentication in the descriptor. */
+typedef enum {
+  HS_DESC_AUTH_ED25519 = 1
+} hs_desc_auth_type_t;
+
+/* Link specifier object that contains information on how to extend to the
+ * relay that is the address, port and handshake type. */
+typedef struct hs_desc_link_specifier_t {
+  /* Indicate the type of link specifier. See trunnel ed25519_cert
+   * specification. */
+  uint8_t type;
+
+  /* It must be one of these types, can't be more than one. */
+  union {
+    /* IP address and port of the relay use to extend. */
+    tor_addr_port_t ap;
+    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
+    uint8_t legacy_id[DIGEST_LEN];
+    /* ed25519 identity. A 32-byte key. */
+    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
+  } u;
+} hs_desc_link_specifier_t;
+
+/* Introduction point information located in a descriptor. */
+typedef struct hs_desc_intro_point_t {
+  /* Link specifier(s) which details how to extend to the relay. This list
+   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+  smartlist_t *link_specifiers;
+
+  /* Onion key of the introduction point used to extend to it for the ntor
+   * handshake. */
+  curve25519_public_key_t onion_key;
+
+  /* Authentication key used to establish the introduction point circuit and
+   * cross-certifies the blinded public key for the replica thus signed by
+   * the blinded key and in turn signs it. */
+  tor_cert_t *auth_key_cert;
+
+  /* Encryption key for the "ntor" type. */
+  curve25519_public_key_t enc_key;
+
+  /* Certificate cross certifying the descriptor signing key by the encryption
+   * curve25519 key. This certificate contains the signing key and is of type
+   * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */
+  tor_cert_t *enc_key_cert;
+
+  /* (Optional): If this introduction point is a legacy one that is version <=
+   * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able
+   * to relay the cells to the service correctly. */
+  struct {
+    /* RSA public key. */
+    crypto_pk_t *key;
+
+    /* Cross certified cert with the descriptor signing key (RSA->Ed). Because
+     * of the cross certification API, we need to keep the certificate binary
+     * blob and its length in order to properly encode it after. */
+    struct {
+      uint8_t *encoded;
+      size_t len;
+    } cert;
+  } legacy;
+
+  /* True iff the introduction point has passed the cross certification. Upon
+   * decoding an intro point, this must be true. */
+  unsigned int cross_certified : 1;
+} hs_desc_intro_point_t;
+
+/* The encrypted data section of a descriptor. Obviously the data in this is
+ * in plaintext but encrypted once encoded. */
+typedef struct hs_desc_encrypted_data_t {
+  /* Bitfield of CREATE2 cell supported formats. The only currently supported
+   * format is ntor. */
+  unsigned int create2_ntor : 1;
+
+  /* A list of authentication types that a client must at least support one
+   * in order to contact the service. Contains NULL terminated strings. */
+  smartlist_t *intro_auth_types;
+
+  /* Is this descriptor a single onion service? */
+  unsigned int single_onion_service : 1;
+
+  /* A list of intro points. Contains hs_desc_intro_point_t objects. */
+  smartlist_t *intro_points;
+} hs_desc_encrypted_data_t;
+
+/* Plaintext data that is unencrypted information of the descriptor. */
+typedef struct hs_desc_plaintext_data_t {
+  /* Version of the descriptor format. Spec specifies this field as a
+   * positive integer. */
+  uint32_t version;
+
+  /* The lifetime of the descriptor in seconds. */
+  uint32_t lifetime_sec;
+
+  /* Certificate with the short-term ed22519 descriptor signing key for the
+   * replica which is signed by the blinded public key for that replica. */
+  tor_cert_t *signing_key_cert;
+
+  /* Signing public key which is used to sign the descriptor. Same public key
+   * as in the signing key certificate. */
+  ed25519_public_key_t signing_pubkey;
+
+  /* Blinded public key used for this descriptor derived from the master
+   * identity key and generated for a specific replica number. */
+  ed25519_public_key_t blinded_pubkey;
+
+  /* Revision counter is incremented at each upload, regardless of whether
+   * the descriptor has changed. This avoids leaking whether the descriptor
+   * has changed. Spec specifies this as a 8 bytes positive integer. */
+  uint64_t revision_counter;
+
+  /* Decoding only: The b64-decoded superencrypted blob from the descriptor */
+  uint8_t *superencrypted_blob;
+
+  /* Decoding only: Size of the superencrypted_blob */
+  size_t superencrypted_blob_size;
+} hs_desc_plaintext_data_t;
+
+/* Service descriptor in its decoded form. */
+typedef struct hs_descriptor_t {
+  /* Contains the plaintext part of the descriptor. */
+  hs_desc_plaintext_data_t plaintext_data;
+
+  /* The following contains what's in the encrypted part of the descriptor.
+   * It's only encrypted in the encoded version of the descriptor thus the
+   * data contained in that object is in plaintext. */
+  hs_desc_encrypted_data_t encrypted_data;
+
+  /* Subcredentials of a service, used by the client and service to decrypt
+   * the encrypted data. */
+  uint8_t subcredential[DIGEST256_LEN];
+} hs_descriptor_t;
+
+/* Return true iff the given descriptor format version is supported. */
+static inline int
+hs_desc_is_supported_version(uint32_t version)
+{
+  if (version < HS_DESC_SUPPORTED_FORMAT_VERSION_MIN ||
+      version > HS_DESC_SUPPORTED_FORMAT_VERSION_MAX) {
+    return 0;
+  }
+  return 1;
+}
+
+/* Public API. */
+
+void hs_descriptor_free_(hs_descriptor_t *desc);
+#define hs_descriptor_free(desc) \
+  FREE_AND_NULL(hs_descriptor_t, hs_descriptor_free_, (desc))
+void hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc);
+#define hs_desc_plaintext_data_free(desc) \
+  FREE_AND_NULL(hs_desc_plaintext_data_t, hs_desc_plaintext_data_free_, (desc))
+void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
+#define hs_desc_encrypted_data_free(desc) \
+  FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
+
+void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
+#define hs_desc_link_specifier_free(ls) \
+  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
+
+hs_desc_link_specifier_t *hs_desc_link_specifier_new(
+                                  const extend_info_t *info, uint8_t type);
+void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
+
+MOCK_DECL(int,
+          hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
+                                     const ed25519_keypair_t *signing_kp,
+                                     char **encoded_out));
+
+int hs_desc_decode_descriptor(const char *encoded,
+                              const uint8_t *subcredential,
+                              hs_descriptor_t **desc_out);
+int hs_desc_decode_plaintext(const char *encoded,
+                             hs_desc_plaintext_data_t *plaintext);
+int hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                             hs_desc_encrypted_data_t *desc_out);
+
+size_t hs_desc_obj_size(const hs_descriptor_t *data);
+size_t hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data);
+
+hs_desc_intro_point_t *hs_desc_intro_point_new(void);
+void hs_desc_intro_point_free_(hs_desc_intro_point_t *ip);
+#define hs_desc_intro_point_free(ip) \
+  FREE_AND_NULL(hs_desc_intro_point_t, hs_desc_intro_point_free_, (ip))
+
+link_specifier_t *hs_desc_lspec_to_trunnel(
+                                   const hs_desc_link_specifier_t *spec);
+
+#ifdef HS_DESCRIPTOR_PRIVATE
+
+/* Encoding. */
+STATIC char *encode_link_specifiers(const smartlist_t *specs);
+STATIC size_t build_plaintext_padding(const char *plaintext,
+                                      size_t plaintext_len,
+                                      uint8_t **padded_out);
+/* Decoding. */
+STATIC smartlist_t *decode_link_specifiers(const char *encoded);
+STATIC hs_desc_intro_point_t *decode_introduction_point(
+                                const hs_descriptor_t *desc,
+                                const char *text);
+STATIC int encrypted_data_length_is_valid(size_t len);
+STATIC int cert_is_valid(tor_cert_t *cert, uint8_t type,
+                         const char *log_obj_type);
+STATIC int desc_sig_is_valid(const char *b64_sig,
+                             const ed25519_public_key_t *signing_pubkey,
+                             const char *encoded_desc, size_t encoded_len);
+STATIC size_t decode_superencrypted(const char *message, size_t message_len,
+                                   uint8_t **encrypted_out);
+STATIC void desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc);
+
+MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
+                                             const uint8_t *encrypted_blob,
+                                             size_t encrypted_blob_size,
+                                             int is_superencrypted_layer,
+                                             char **decrypted_out));
+
+#endif /* defined(HS_DESCRIPTOR_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DESCRIPTOR_H) */
+
diff --git a/src/or/hs_ident.c b/src/or/hs_ident.c
new file mode 100644
index 0000000000..3603e329d4
--- /dev/null
+++ b/src/or/hs_ident.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ident.c
+ * \brief Contains circuit and connection identifier code for the whole HS
+ *        subsytem.
+ **/
+
+#include "crypto_util.h"
+#include "hs_ident.h"
+
+/* Return a newly allocated circuit identifier. The given public key is copied
+ * identity_pk into the identifier. */
+hs_ident_circuit_t *
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
+                     hs_ident_circuit_type_t circuit_type)
+{
+  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
+             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
+  hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  ident->circuit_type = circuit_type;
+  return ident;
+}
+
+/* Free the given circuit identifier. */
+void
+hs_ident_circuit_free_(hs_ident_circuit_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_circuit_t));
+  tor_free(ident);
+}
+
+/* For a given circuit identifier src, return a newly allocated copy of it.
+ * This can't fail. */
+hs_ident_circuit_t *
+hs_ident_circuit_dup(const hs_ident_circuit_t *src)
+{
+  hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
+  memcpy(ident, src, sizeof(*ident));
+  return ident;
+}
+
+/* For a given directory connection identifier src, return a newly allocated
+ * copy of it. This can't fail. */
+hs_ident_dir_conn_t *
+hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
+{
+  hs_ident_dir_conn_t *ident = tor_malloc_zero(sizeof(*ident));
+  memcpy(ident, src, sizeof(*ident));
+  return ident;
+}
+
+/* Free the given directory connection identifier. */
+void
+hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_dir_conn_t));
+  tor_free(ident);
+}
+
+/* Initialized the allocated ident object with identity_pk and blinded_pk.
+ * None of them can be NULL since a valid directory connection identifier must
+ * have all fields set. */
+void
+hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
+                       const ed25519_public_key_t *blinded_pk,
+                       hs_ident_dir_conn_t *ident)
+{
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(ident);
+
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  ed25519_pubkey_copy(&ident->blinded_pk, blinded_pk);
+}
+
+/* Return a newly allocated edge connection identifier. The given public key
+ * identity_pk is copied into the identifier. */
+hs_ident_edge_conn_t *
+hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
+{
+  hs_ident_edge_conn_t *ident = tor_malloc_zero(sizeof(*ident));
+  ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
+  return ident;
+}
+
+/* Free the given edge connection identifier. */
+void
+hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
+{
+  if (ident == NULL) {
+    return;
+  }
+  memwipe(ident, 0, sizeof(hs_ident_edge_conn_t));
+  tor_free(ident);
+}
+
+/* Return true if the given ident is valid for an introduction circuit. */
+int
+hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
+{
+  if (ident == NULL) {
+    goto invalid;
+  }
+
+  if (ed25519_public_key_is_zero(&ident->identity_pk)) {
+    goto invalid;
+  }
+
+  if (ed25519_public_key_is_zero(&ident->intro_auth_pk)) {
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 1;
+ invalid:
+  return 0;
+}
+
diff --git a/src/or/hs_ident.h b/src/or/hs_ident.h
new file mode 100644
index 0000000000..8f9da30c35
--- /dev/null
+++ b/src/or/hs_ident.h
@@ -0,0 +1,146 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ident.h
+ * \brief Header file containing circuit and connection identifier data for
+ *        the whole HS subsytem.
+ *
+ * \details
+ * This interface is used to uniquely identify a hidden service on a circuit
+ * or connection using the service identity public key. Once the circuit or
+ * connection subsystem calls in the hidden service one, we use those
+ * identifiers to lookup the corresponding objects like service, intro point
+ * and descriptor.
+ *
+ * Furthermore, the circuit identifier holds cryptographic material needed for
+ * the e2e encryption on the rendezvous circuit which is set once the
+ * rendezvous circuit has opened and ready to be used.
+ **/
+
+#ifndef TOR_HS_IDENT_H
+#define TOR_HS_IDENT_H
+
+#include "crypto_ed25519.h"
+
+#include "hs_common.h"
+
+/* Length of the rendezvous cookie that is used to connect circuits at the
+ * rendezvous point. */
+#define HS_REND_COOKIE_LEN DIGEST_LEN
+
+/* Type of circuit an hs_ident_t object is associated with. */
+typedef enum {
+  HS_IDENT_CIRCUIT_INTRO      = 1,
+  HS_IDENT_CIRCUIT_RENDEZVOUS = 2,
+} hs_ident_circuit_type_t;
+
+/* Client and service side circuit identifier that is used for hidden service
+ * circuit establishment. Not all fields contain data, it depends on the
+ * circuit purpose. This is attached to an origin_circuit_t. All fields are
+ * used by both client and service. */
+typedef struct hs_ident_circuit_t {
+  /* (All circuit) The public key used to uniquely identify the service. It is
+   * the one found in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* (All circuit) The type of circuit this identifier is attached to.
+   * Accessors of the fields in this object assert non fatal on this circuit
+   * type. In other words, if a rendezvous field is being accessed, the
+   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
+   * set when an object is initialized in its constructor. */
+  hs_ident_circuit_type_t circuit_type;
+
+  /* (All circuit) Introduction point authentication key. It's also needed on
+   * the rendezvous circuit for the ntor handshake. It's used as the unique key
+   * of the introduction point so it should not be shared between multiple
+   * intro points. */
+  ed25519_public_key_t intro_auth_pk;
+
+  /* (Only client rendezvous circuit) Introduction point encryption public
+   * key. We keep it in the rendezvous identifier for the ntor handshake. */
+  curve25519_public_key_t intro_enc_pk;
+
+  /* (Only rendezvous circuit) Rendezvous cookie sent from the client to the
+   * service with an INTRODUCE1 cell and used by the service in an
+   * RENDEZVOUS1 cell. */
+  uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
+
+  /* (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
+   * RENDEZVOUS1 cell of the service. The construction is as follows:
+   *    SERVER_PK   [32 bytes]
+   *    AUTH_MAC    [32 bytes]
+   */
+  uint8_t rendezvous_handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
+
+  /* (Only client rendezvous circuit) Client ephemeral keypair needed for the
+   * e2e encryption with the service. */
+  curve25519_keypair_t rendezvous_client_kp;
+
+  /* (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
+   * the e2e encryption with the client on the circuit. */
+  uint8_t rendezvous_ntor_key_seed[DIGEST256_LEN];
+
+  /* (Only rendezvous circuit) Number of streams associated with this
+   * rendezvous circuit. We track this because there is a check on a maximum
+   * value. */
+  uint64_t num_rdv_streams;
+} hs_ident_circuit_t;
+
+/* Client and service side directory connection identifier used for a
+ * directory connection to identify which service is being queried. This is
+ * attached to a dir_connection_t. */
+typedef struct hs_ident_dir_conn_t {
+  /* The public key used to uniquely identify the service. It is the one found
+   * in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* The blinded public key used to uniquely identify the descriptor that this
+   * directory connection identifier is for. Only used by the service-side code
+   * to fine control descriptor uploads. */
+  ed25519_public_key_t blinded_pk;
+
+  /* XXX: Client authorization. */
+} hs_ident_dir_conn_t;
+
+/* Client and service side edge connection identifier used for an edge
+ * connection to identify which service is being queried. This is attached to
+ * a edge_connection_t. */
+typedef struct hs_ident_edge_conn_t {
+  /* The public key used to uniquely identify the service. It is the one found
+   * in the onion address. */
+  ed25519_public_key_t identity_pk;
+
+  /* XXX: Client authorization. */
+} hs_ident_edge_conn_t;
+
+/* Circuit identifier API. */
+hs_ident_circuit_t *hs_ident_circuit_new(
+                             const ed25519_public_key_t *identity_pk,
+                             hs_ident_circuit_type_t circuit_type);
+void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
+#define hs_ident_circuit_free(id) \
+  FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
+hs_ident_circuit_t *hs_ident_circuit_dup(const hs_ident_circuit_t *src);
+
+/* Directory connection identifier API. */
+hs_ident_dir_conn_t *hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src);
+void hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident);
+#define hs_ident_dir_conn_free(id) \
+  FREE_AND_NULL(hs_ident_dir_conn_t, hs_ident_dir_conn_free_, (id))
+void hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
+                            const ed25519_public_key_t *blinded_pk,
+                            hs_ident_dir_conn_t *ident);
+
+/* Edge connection identifier API. */
+hs_ident_edge_conn_t *hs_ident_edge_conn_new(
+                                    const ed25519_public_key_t *identity_pk);
+void hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident);
+#define hs_ident_edge_conn_free(id) \
+  FREE_AND_NULL(hs_ident_edge_conn_t, hs_ident_edge_conn_free_, (id))
+
+/* Validators */
+int hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident);
+
+#endif /* !defined(TOR_HS_IDENT_H) */
+
diff --git a/src/or/hs_intropoint.c b/src/or/hs_intropoint.c
new file mode 100644
index 0000000000..ee79109ca9
--- /dev/null
+++ b/src/or/hs_intropoint.c
@@ -0,0 +1,611 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.c
+ * \brief Implement next generation introductions point functionality
+ **/
+
+#define HS_INTROPOINT_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "relay.h"
+#include "rendmid.h"
+#include "rephist.h"
+
+/* Trunnel */
+#include "ed25519_cert.h"
+#include "hs/cell_common.h"
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+
+#include "hs_circuitmap.h"
+#include "hs_descriptor.h"
+#include "hs_intropoint.h"
+#include "hs_common.h"
+
+/** Extract the authentication key from an ESTABLISH_INTRO or INTRODUCE1 using
+ * the given <b>cell_type</b> from <b>cell</b> and place it in
+ * <b>auth_key_out</b>. */
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell)
+{
+  size_t auth_key_len;
+  const uint8_t *key_array;
+
+  tor_assert(auth_key_out);
+  tor_assert(cell);
+
+  switch (cell_type) {
+  case RELAY_COMMAND_ESTABLISH_INTRO:
+  {
+    const trn_cell_establish_intro_t *c_cell = cell;
+    key_array = trn_cell_establish_intro_getconstarray_auth_key(c_cell);
+    auth_key_len = trn_cell_establish_intro_getlen_auth_key(c_cell);
+    break;
+  }
+  case RELAY_COMMAND_INTRODUCE1:
+  {
+    const trn_cell_introduce1_t *c_cell = cell;
+    key_array = trn_cell_introduce1_getconstarray_auth_key(cell);
+    auth_key_len = trn_cell_introduce1_getlen_auth_key(c_cell);
+    break;
+  }
+  default:
+    /* Getting here is really bad as it means we got a unknown cell type from
+     * this file where every call has an hardcoded value. */
+    tor_assert_unreached(); /* LCOV_EXCL_LINE */
+  }
+  tor_assert(key_array);
+  tor_assert(auth_key_len == sizeof(auth_key_out->pubkey));
+  memcpy(auth_key_out->pubkey, key_array, auth_key_len);
+}
+
+/** We received an ESTABLISH_INTRO <b>cell</b>. Verify its signature and MAC,
+ *  given <b>circuit_key_material</b>. Return 0 on success else -1 on error. */
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *cell,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len)
+{
+  /* We only reach this function if the first byte of the cell is 0x02 which
+   * means that auth_key_type is of ed25519 type, hence this check should
+   * always pass. See hs_intro_received_establish_intro().  */
+  if (BUG(cell->auth_key_type != HS_INTRO_AUTH_KEY_TYPE_ED25519)) {
+    return -1;
+  }
+
+  /* Make sure the auth key length is of the right size for this type. For
+   * EXTRA safety, we check both the size of the array and the length which
+   * must be the same. Safety first!*/
+  if (trn_cell_establish_intro_getlen_auth_key(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_establish_intro_get_auth_key_len(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "ESTABLISH_INTRO auth key length is invalid");
+    return -1;
+  }
+
+  const uint8_t *msg = cell->start_cell;
+
+  /* Verify the sig */
+  {
+    ed25519_signature_t sig_struct;
+    const uint8_t *sig_array =
+      trn_cell_establish_intro_getconstarray_sig(cell);
+
+    /* Make sure the signature length is of the right size. For EXTRA safety,
+     * we check both the size of the array and the length which must be the
+     * same. Safety first!*/
+    if (trn_cell_establish_intro_getlen_sig(cell) != sizeof(sig_struct.sig) ||
+        trn_cell_establish_intro_get_sig_len(cell) != sizeof(sig_struct.sig)) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO sig len is invalid");
+      return -1;
+    }
+    /* We are now sure that sig_len is of the right size. */
+    memcpy(sig_struct.sig, sig_array, cell->sig_len);
+
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO, cell);
+
+    const size_t sig_msg_len = cell->end_sig_fields - msg;
+    int sig_mismatch = ed25519_checksig_prefixed(&sig_struct,
+                                                 msg, sig_msg_len,
+                                                 ESTABLISH_INTRO_SIG_PREFIX,
+                                                 &auth_key);
+    if (sig_mismatch) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO signature not as expected");
+      return -1;
+    }
+  }
+
+  /* Verify the MAC */
+  {
+    const size_t auth_msg_len = cell->end_mac_fields - msg;
+    uint8_t mac[DIGEST256_LEN];
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        circuit_key_material, circuit_key_material_len,
+                        msg, auth_msg_len);
+    if (tor_memneq(mac, cell->handshake_mac, sizeof(mac))) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO handshake_auth not as expected");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/* Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
+MOCK_IMPL(int,
+hs_intro_send_intro_established_cell,(or_circuit_t *circ))
+{
+  int ret;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_intro_established_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Build the cell payload. */
+  cell = trn_cell_intro_established_new();
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_intro_established_set_extensions(cell, ext);
+  /* Encode the cell to binary format. */
+  encoded_len = trn_cell_intro_established_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_intro_established_encode(encoded_cell, encoded_len,
+                                                cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRO_ESTABLISHED,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_intro_established_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
+ *  well-formed and passed our verifications. Perform appropriate actions to
+ *  establish an intro point. */
+static int
+handle_verified_establish_intro_cell(or_circuit_t *circ,
+                               const trn_cell_establish_intro_t *parsed_cell)
+{
+  /* Get the auth key of this intro point */
+  ed25519_public_key_t auth_key;
+  get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                         parsed_cell);
+
+  /* Then notify the hidden service that the intro point is established by
+     sending an INTRO_ESTABLISHED cell */
+  if (hs_intro_send_intro_established_cell(circ)) {
+    log_warn(LD_PROTOCOL, "Couldn't send INTRO_ESTABLISHED cell.");
+    return -1;
+  }
+
+  /* Associate intro point auth key with this circuit. */
+  hs_circuitmap_register_intro_circ_v3_relay_side(circ, &auth_key);
+  /* Repurpose this circuit into an intro circuit. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  return 0;
+}
+
+/** We just received an ESTABLISH_INTRO cell in <b>circ</b> with payload in
+ *  <b>request</b>. Handle it by making <b>circ</b> an intro circuit. Return 0
+ *  if everything went well, or -1 if there were errors. */
+static int
+handle_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                       size_t request_len)
+{
+  int cell_ok, retval = -1;
+  trn_cell_establish_intro_t *parsed_cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  log_info(LD_REND, "Received an ESTABLISH_INTRO request on circuit %" PRIu32,
+           circ->p_circ_id);
+
+  /* Check that the circuit is in shape to become an intro point */
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
+    goto err;
+  }
+
+  /* Parse the cell */
+  ssize_t parsing_result = trn_cell_establish_intro_parse(&parsed_cell,
+                                                         request, request_len);
+  if (parsing_result < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s ESTABLISH_INTRO cell.",
+           parsing_result == -1 ? "invalid" : "truncated");
+    goto err;
+  }
+
+  cell_ok = verify_establish_intro_cell(parsed_cell,
+                                        (uint8_t *) circ->rend_circ_nonce,
+                                        sizeof(circ->rend_circ_nonce));
+  if (cell_ok < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Failed to verify ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* This cell is legit. Take the appropriate actions. */
+  cell_ok = handle_verified_establish_intro_cell(circ, parsed_cell);
+  if (cell_ok < 0) {
+    goto err;
+  }
+
+  /* We are done! */
+  retval = 0;
+  goto done;
+
+ err:
+  /* When sending the intro establish ack, on error the circuit can be marked
+   * as closed so avoid a double close. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  }
+
+ done:
+  trn_cell_establish_intro_free(parsed_cell);
+  return retval;
+}
+
+/* Return True if circuit is suitable for being an intro circuit. */
+static int
+circuit_is_suitable_intro_point(const or_circuit_t *circ,
+                                const char *log_cell_type_str)
+{
+  tor_assert(circ);
+  tor_assert(log_cell_type_str);
+
+  /* Basic circuit state sanity checks. */
+  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-OR circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  if (circ->base_.n_chan) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-edge circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  /* Suitable. */
+  return 1;
+}
+
+/* Return True if circuit is suitable for being service-side intro circuit. */
+int
+hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ)
+{
+  return circuit_is_suitable_intro_point(circ, "ESTABLISH_INTRO");
+}
+
+/* We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
+ * a legacy or a next gen cell, and pass it to the appropriate handler. */
+int
+hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                            size_t request_len)
+{
+  tor_assert(circ);
+  tor_assert(request);
+
+  if (request_len == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Empty ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* Using the first byte of the cell, figure out the version of
+   * ESTABLISH_INTRO and pass it to the appropriate cell handler */
+  const uint8_t first_byte = request[0];
+  switch (first_byte) {
+    case HS_INTRO_AUTH_KEY_TYPE_LEGACY0:
+    case HS_INTRO_AUTH_KEY_TYPE_LEGACY1:
+      return rend_mid_establish_intro_legacy(circ, request, request_len);
+    case HS_INTRO_AUTH_KEY_TYPE_ED25519:
+      return handle_establish_intro(circ, request, request_len);
+    default:
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unrecognized AUTH_KEY_TYPE %u.", first_byte);
+      goto err;
+  }
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
+ * value in <b>status</b>. Depending on the status, it can be ACK or a NACK.
+ * Return 0 on success else a negative value on error which will close the
+ * circuit. */
+static int
+send_introduce_ack_cell(or_circuit_t *circ, hs_intro_ack_status_t status)
+{
+  int ret = -1;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_introduce_ack_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Setup the INTRODUCE_ACK cell. We have no extensions so the N_EXTENSIONS
+   * field is set to 0 by default with a new object. */
+  cell = trn_cell_introduce_ack_new();
+  ret = trn_cell_introduce_ack_set_status(cell, status);
+  /* We have no cell extensions in an INTRODUCE_ACK cell. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_ack_set_extensions(cell, ext);
+  /* A wrong status is a very bad code flow error as this value is controlled
+   * by the code in this file and not an external input. This means we use a
+   * code that is not known by the trunnel ABI. */
+  tor_assert(ret == 0);
+  /* Encode the payload. We should never fail to get the encoded length. */
+  encoded_len = trn_cell_introduce_ack_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_introduce_ack_encode(encoded_cell, encoded_len, cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRODUCE_ACK,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_introduce_ack_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
+ * negative value for an invalid cell that should be NACKed. */
+STATIC int
+validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
+{
+  size_t legacy_key_id_len;
+  const uint8_t *legacy_key_id;
+
+  tor_assert(cell);
+
+  /* This code path SHOULD NEVER be reached if the cell is a legacy type so
+   * safety net here. The legacy ID must be zeroes in this case. */
+  legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
+  legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
+  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+    goto invalid;
+  }
+
+  /* The auth key of an INTRODUCE1 should be of type ed25519 thus leading to a
+   * known fixed length as well. */
+  if (trn_cell_introduce1_get_auth_key_type(cell) !=
+      HS_INTRO_AUTH_KEY_TYPE_ED25519) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key type. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_get_auth_key_len(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_introduce1_getlen_auth_key(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_getlen_encrypted(cell) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell encrypted length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
+ * the payload in <b>request</b> of size <b>request_len</b>. Return 0 if
+ * everything went well, or -1 if an error occurred. This function is in charge
+ * of sending back an INTRODUCE_ACK cell and will close client_circ on error.
+ */
+STATIC int
+handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
+                  size_t request_len)
+{
+  int ret = -1;
+  or_circuit_t *service_circ;
+  trn_cell_introduce1_t *parsed_cell;
+  hs_intro_ack_status_t status = HS_INTRO_ACK_STATUS_SUCCESS;
+
+  tor_assert(client_circ);
+  tor_assert(request);
+
+  /* Parse cell. Note that we can only parse the non encrypted section for
+   * which we'll use the authentication key to find the service introduction
+   * circuit and relay the cell on it. */
+  ssize_t cell_size = trn_cell_introduce1_parse(&parsed_cell, request,
+                                               request_len);
+  if (cell_size < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s INTRODUCE1 cell. Responding with NACK.",
+           cell_size == -1 ? "invalid" : "truncated");
+    /* Inform client that the INTRODUCE1 has a bad format. */
+    status = HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Once parsed validate the cell format. */
+  if (validate_introduce1_parsed_cell(parsed_cell) < 0) {
+    /* Inform client that the INTRODUCE1 has bad format. */
+    status = HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Find introduction circuit through our circuit map. */
+  {
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_INTRODUCE1, parsed_cell);
+    service_circ = hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    if (service_circ == NULL) {
+      char b64_key[ED25519_BASE64_LEN + 1];
+      ed25519_public_to_base64(b64_key, &auth_key);
+      log_info(LD_REND, "No intro circuit found for INTRODUCE1 cell "
+                        "with auth key %s from circuit %" PRIu32 ". "
+                        "Responding with NACK.",
+               safe_str(b64_key), client_circ->p_circ_id);
+      /* Inform the client that we don't know the requested service ID. */
+      status = HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+      goto send_ack;
+    }
+  }
+
+  /* Relay the cell to the service on its intro circuit with an INTRODUCE2
+   * cell which is the same exact payload. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
+                                   RELAY_COMMAND_INTRODUCE2,
+                                   (char *) request, request_len, NULL)) {
+    log_warn(LD_PROTOCOL, "Unable to send INTRODUCE2 cell to the service.");
+    /* Inform the client that we can't relay the cell. */
+    status = HS_INTRO_ACK_STATUS_CANT_RELAY;
+    goto send_ack;
+  }
+
+  /* Success! Send an INTRODUCE_ACK success status onto the client circuit. */
+  status = HS_INTRO_ACK_STATUS_SUCCESS;
+  ret = 0;
+
+ send_ack:
+  /* Send INTRODUCE_ACK or INTRODUCE_NACK to client */
+  if (send_introduce_ack_cell(client_circ, status) < 0) {
+    log_warn(LD_PROTOCOL, "Unable to send an INTRODUCE ACK status %d "
+                          "to client.", status);
+    /* Circuit has been closed on failure of transmission. */
+    goto done;
+  }
+  if (status != HS_INTRO_ACK_STATUS_SUCCESS) {
+    /* We just sent a NACK that is a non success status code so close the
+     * circuit because it's not useful to keep it open. Remember, a client can
+     * only send one INTRODUCE1 cell on a circuit. */
+    circuit_mark_for_close(TO_CIRCUIT(client_circ), END_CIRC_REASON_INTERNAL);
+  }
+ done:
+  trn_cell_introduce1_free(parsed_cell);
+  return ret;
+}
+
+/* Identify if the encoded cell we just received is a legacy one or not. The
+ * <b>request</b> should be at least DIGEST_LEN bytes long. */
+STATIC int
+introduce1_cell_is_legacy(const uint8_t *request)
+{
+  tor_assert(request);
+
+  /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
+   * indicates a legacy cell (v2). */
+  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+    /* Legacy cell. */
+    return 1;
+  }
+  /* Not a legacy cell. */
+  return 0;
+}
+
+/* Return true iff the circuit <b>circ</b> is suitable for receiving an
+ * INTRODUCE1 cell. */
+STATIC int
+circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Is this circuit an intro point circuit? */
+  if (!circuit_is_suitable_intro_point(circ, "INTRODUCE1")) {
+    return 0;
+  }
+
+  if (circ->already_received_introduce1) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Blocking multiple introductions on the same circuit. "
+           "Someone might be trying to attack a hidden service through "
+           "this relay.");
+    return 0;
+  }
+
+  return 1;
+}
+
+/* We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
+ * it is and pass it to the appropriate handler. Return 0 on success else a
+ * negative value and the circuit is closed. */
+int
+hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                             size_t request_len)
+{
+  int ret;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  /* A cell that can't hold a DIGEST_LEN is invalid as we need to check if
+   * it's a legacy cell or not using the first DIGEST_LEN bytes. */
+  if (request_len < DIGEST_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Invalid INTRODUCE1 cell length.");
+    goto err;
+  }
+
+  /* Make sure we have a circuit that can have an INTRODUCE1 cell on it. */
+  if (!circuit_is_suitable_for_introduce1(circ)) {
+    /* We do not send a NACK because the circuit is not suitable for any kind
+     * of response or transmission as it's a violation of the protocol. */
+    goto err;
+  }
+  /* Mark the circuit that we got this cell. None are allowed after this as a
+   * DoS mitigation since one circuit with one client can hammer a service. */
+  circ->already_received_introduce1 = 1;
+
+  /* We are sure here to have at least DIGEST_LEN bytes. */
+  if (introduce1_cell_is_legacy(request)) {
+    /* Handle a legacy cell. */
+    ret = rend_mid_introduce_legacy(circ, request, request_len);
+  } else {
+    /* Handle a non legacy cell. */
+    ret = handle_introduce1(circ, request, request_len);
+  }
+  return ret;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Clear memory allocated by the given intropoint object ip (but don't free the
+ * object itself). */
+void
+hs_intropoint_clear(hs_intropoint_t *ip)
+{
+  if (ip == NULL) {
+    return;
+  }
+  tor_cert_free(ip->auth_key_cert);
+  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
+                    hs_desc_link_specifier_free(ls));
+  smartlist_free(ip->link_specifiers);
+  memset(ip, 0, sizeof(hs_intropoint_t));
+}
diff --git a/src/or/hs_intropoint.h b/src/or/hs_intropoint.h
new file mode 100644
index 0000000000..749d1530e1
--- /dev/null
+++ b/src/or/hs_intropoint.h
@@ -0,0 +1,79 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.h
+ * \brief Header file for hs_intropoint.c.
+ **/
+
+#ifndef TOR_HS_INTRO_H
+#define TOR_HS_INTRO_H
+
+#include "crypto_curve25519.h"
+#include "torcert.h"
+
+/* Authentication key type in an ESTABLISH_INTRO cell. */
+typedef enum {
+  HS_INTRO_AUTH_KEY_TYPE_LEGACY0 = 0x00,
+  HS_INTRO_AUTH_KEY_TYPE_LEGACY1 = 0x01,
+  HS_INTRO_AUTH_KEY_TYPE_ED25519 = 0x02,
+} hs_intro_auth_key_type_t;
+
+/* INTRODUCE_ACK status code. */
+typedef enum {
+  HS_INTRO_ACK_STATUS_SUCCESS    = 0x0000,
+  HS_INTRO_ACK_STATUS_UNKNOWN_ID = 0x0001,
+  HS_INTRO_ACK_STATUS_BAD_FORMAT = 0x0002,
+  HS_INTRO_ACK_STATUS_CANT_RELAY = 0x0003,
+} hs_intro_ack_status_t;
+
+/* Object containing introduction point common data between the service and
+ * the client side. */
+typedef struct hs_intropoint_t {
+  /* Does this intro point only supports legacy ID ?. */
+  unsigned int is_only_legacy : 1;
+
+  /* Authentication key certificate from the descriptor. */
+  tor_cert_t *auth_key_cert;
+  /* A list of link specifier. */
+  smartlist_t *link_specifiers;
+} hs_intropoint_t;
+
+int hs_intro_received_establish_intro(or_circuit_t *circ,
+                                      const uint8_t *request,
+                                      size_t request_len);
+int hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                                 size_t request_len);
+
+MOCK_DECL(int, hs_intro_send_intro_established_cell,(or_circuit_t *circ));
+
+/* also used by rendservice.c */
+int hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ);
+
+hs_intropoint_t *hs_intro_new(void);
+void hs_intropoint_clear(hs_intropoint_t *ip);
+
+#ifdef HS_INTROPOINT_PRIVATE
+
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *out,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len);
+
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell);
+
+STATIC int introduce1_cell_is_legacy(const uint8_t *request);
+STATIC int handle_introduce1(or_circuit_t *client_circ,
+                             const uint8_t *request, size_t request_len);
+STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
+STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+
+#endif /* defined(HS_INTROPOINT_PRIVATE) */
+
+#endif /* !defined(TOR_HS_INTRO_H) */
+
diff --git a/src/or/hs_ntor.c b/src/or/hs_ntor.c
new file mode 100644
index 0000000000..809fa83bb8
--- /dev/null
+++ b/src/or/hs_ntor.c
@@ -0,0 +1,619 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** \file hs_ntor.c
+ *  \brief Implements the ntor variant used in Tor hidden services.
+ *
+ *  \details
+ *  This module handles the variant of the ntor handshake that is documented in
+ *  section [NTOR-WITH-EXTRA-DATA] of rend-spec-ng.txt .
+ *
+ *  The functions in this file provide an API that should be used when sending
+ *  or receiving INTRODUCE1/RENDEZVOUS1 cells to generate the various key
+ *  material required to create and handle those cells.
+ *
+ *  In the case of INTRODUCE1 it provides encryption and MAC keys to
+ *  encode/decode the encrypted blob (see hs_ntor_intro_cell_keys_t). The
+ *  relevant pub functions are hs_ntor_{client,service}_get_introduce1_keys().
+ *
+ *  In the case of RENDEZVOUS1 it calculates the MAC required to authenticate
+ *  the cell, and also provides the key seed that is used to derive the crypto
+ *  material for rendezvous encryption (see hs_ntor_rend_cell_keys_t). The
+ *  relevant pub functions are hs_ntor_{client,service}_get_rendezvous1_keys().
+ *  It also provides a function (hs_ntor_circuit_key_expansion()) that does the
+ *  rendezvous key expansion to setup end-to-end rend circuit keys.
+ */
+
+#include "or.h"
+#include "crypto_util.h"
+#include "hs_ntor.h"
+
+/* String constants used by the ntor HS protocol */
+#define PROTOID "tor-hs-ntor-curve25519-sha3-256-1"
+#define PROTOID_LEN (sizeof(PROTOID) - 1)
+#define SERVER_STR "Server"
+#define SERVER_STR_LEN (sizeof(SERVER_STR) - 1)
+
+/* Protocol-specific tweaks to our crypto inputs */
+#define T_HSENC PROTOID ":hs_key_extract"
+#define T_HSENC_LEN (sizeof(T_HSENC) - 1)
+#define T_HSVERIFY PROTOID ":hs_verify"
+#define T_HSMAC PROTOID ":hs_mac"
+#define M_HSEXPAND PROTOID ":hs_key_expand"
+#define M_HSEXPAND_LEN (sizeof(M_HSEXPAND) - 1)
+
+/************************* Helper functions: *******************************/
+
+/** Helper macro: copy <b>len</b> bytes from <b>inp</b> to <b>ptr</b> and
+ *advance <b>ptr</b> by the number of bytes copied. Stolen from onion_ntor.c */
+#define APPEND(ptr, inp, len)                   \
+  STMT_BEGIN {                                  \
+    memcpy(ptr, (inp), (len));                  \
+    ptr += len;                                 \
+  } STMT_END
+
+/* Length of EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID */
+#define REND_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN * 2 + \
+  ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN)
+/* Length of auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" */
+#define REND_AUTH_INPUT_LEN (DIGEST256_LEN + ED25519_PUBKEY_LEN + \
+  CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN + SERVER_STR_LEN)
+
+/** Helper function: Compute the last part of the HS ntor handshake which
+ *  derives key material necessary to create and handle RENDEZVOUS1
+ *  cells. Function used by both client and service. The actual calculations is
+ *  as follows:
+ *
+ *    NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *    verify = MAC(rend_secret_hs_input, t_hsverify)
+ *    auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *    auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ *  where in the above, AUTH_KEY is <b>intro_auth_pubkey</b>, B is
+ *  <b>intro_enc_pubkey</b>, Y is <b>service_ephemeral_rend_pubkey</b>, and X
+ *  is <b>client_ephemeral_enc_pubkey</b>. The provided
+ *  <b>rend_secret_hs_input</b> is of size REND_SECRET_HS_INPUT_LEN.
+ *
+ *  The final results of NTOR_KEY_SEED and auth_input_mac are placed in
+ *  <b>hs_ntor_rend_cell_keys_out</b>. Return 0 if everything went fine. */
+static int
+get_rendezvous1_key_material(const uint8_t *rend_secret_hs_input,
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t ntor_key_seed[DIGEST256_LEN];
+  uint8_t ntor_verify[DIGEST256_LEN];
+  uint8_t rend_auth_input[REND_AUTH_INPUT_LEN];
+  uint8_t rend_cell_auth[DIGEST256_LEN];
+  uint8_t *ptr;
+
+  /* Let's build NTOR_KEY_SEED */
+  crypto_mac_sha3_256(ntor_key_seed, sizeof(ntor_key_seed),
+                      rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN,
+                      (const uint8_t *)T_HSENC, strlen(T_HSENC));
+  bad |= safe_mem_is_zero(ntor_key_seed, DIGEST256_LEN);
+
+  /* Let's build ntor_verify */
+  crypto_mac_sha3_256(ntor_verify, sizeof(ntor_verify),
+                      rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN,
+                      (const uint8_t *)T_HSVERIFY, strlen(T_HSVERIFY));
+  bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN);
+
+  /* Let's build auth_input: */
+  ptr = rend_auth_input;
+  /* Append ntor_verify */
+  APPEND(ptr, ntor_verify, sizeof(ntor_verify));
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append Y */
+  APPEND(ptr,
+         service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr,
+         client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  /* Append "Server" */
+  APPEND(ptr, SERVER_STR, strlen(SERVER_STR));
+  tor_assert(ptr == rend_auth_input + sizeof(rend_auth_input));
+
+  /* Let's build auth_input_mac that goes in RENDEZVOUS1 cell */
+  crypto_mac_sha3_256(rend_cell_auth, sizeof(rend_cell_auth),
+                      rend_auth_input, sizeof(rend_auth_input),
+                      (const uint8_t *)T_HSMAC, strlen(T_HSMAC));
+  bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN);
+
+  { /* Get the computed RENDEZVOUS1 material! */
+    memcpy(&hs_ntor_rend_cell_keys_out->rend_cell_auth_mac,
+           rend_cell_auth, DIGEST256_LEN);
+    memcpy(&hs_ntor_rend_cell_keys_out->ntor_key_seed,
+           ntor_key_seed, DIGEST256_LEN);
+  }
+
+  memwipe(rend_cell_auth, 0, sizeof(rend_cell_auth));
+  memwipe(rend_auth_input, 0, sizeof(rend_auth_input));
+  memwipe(ntor_key_seed, 0, sizeof(ntor_key_seed));
+
+  return bad;
+}
+
+/** Length of secret_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID */
+#define INTRO_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN +ED25519_PUBKEY_LEN +\
+  CURVE25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN + PROTOID_LEN)
+/* Length of info = m_hsexpand | subcredential */
+#define INFO_BLOB_LEN (M_HSEXPAND_LEN + DIGEST256_LEN)
+/* Length of KDF input = intro_secret_hs_input | t_hsenc | info */
+#define KDF_INPUT_LEN (INTRO_SECRET_HS_INPUT_LEN + T_HSENC_LEN + INFO_BLOB_LEN)
+
+/** Helper function: Compute the part of the HS ntor handshake that generates
+ *  key material for creating and handling INTRODUCE1 cells. Function used
+ *  by both client and service. Specifically, calculate the following:
+ *
+ *     info = m_hsexpand | subcredential
+ *     hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *     ENC_KEY = hs_keys[0:S_KEY_LEN]
+ *     MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ *  where intro_secret_hs_input is <b>secret_input</b> (of size
+ *  INTRO_SECRET_HS_INPUT_LEN), and <b>subcredential</b> is of size
+ *  DIGEST256_LEN.
+ *
+ * If everything went well, fill <b>hs_ntor_intro_cell_keys_out</b> with the
+ * necessary key material, and return 0. */
+static void
+get_introduce1_key_material(const uint8_t *secret_input,
+                        const uint8_t *subcredential,
+                        hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
+  uint8_t info_blob[INFO_BLOB_LEN];
+  uint8_t kdf_input[KDF_INPUT_LEN];
+  crypto_xof_t *xof;
+  uint8_t *ptr;
+
+  /* Let's build info */
+  ptr = info_blob;
+  APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
+  APPEND(ptr, subcredential, DIGEST256_LEN);
+  tor_assert(ptr == info_blob + sizeof(info_blob));
+
+  /* Let's build the input to the KDF */
+  ptr = kdf_input;
+  APPEND(ptr, secret_input, INTRO_SECRET_HS_INPUT_LEN);
+  APPEND(ptr, T_HSENC, strlen(T_HSENC));
+  APPEND(ptr, info_blob, sizeof(info_blob));
+  tor_assert(ptr == kdf_input + sizeof(kdf_input));
+
+  /* Now we need to run kdf_input over SHAKE-256 */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
+  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
+  crypto_xof_free(xof);
+
+  { /* Get the keys */
+    memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
+    memcpy(&hs_ntor_intro_cell_keys_out->mac_key,
+           keystream+CIPHER256_KEY_LEN, DIGEST256_LEN);
+  }
+
+  memwipe(keystream,  0, sizeof(keystream));
+  memwipe(kdf_input,  0, sizeof(kdf_input));
+}
+
+/** Helper function: Calculate the 'intro_secret_hs_input' element used by the
+ * HS ntor handshake and place it in <b>secret_input_out</b>. This function is
+ * used by both client and service code.
+ *
+ * For the client-side it looks like this:
+ *
+ *         intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+ *
+ * whereas for the service-side it looks like this:
+ *
+ *         intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+ *
+ * In this function, <b>dh_result</b> carries the EXP() result (and has size
+ * CURVE25519_OUTPUT_LEN) <b>intro_auth_pubkey</b> is AUTH_KEY,
+ * <b>client_ephemeral_enc_pubkey</b> is X, and <b>intro_enc_pubkey</b> is B.
+ */
+static void
+get_intro_secret_hs_input(const uint8_t *dh_result,
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    const curve25519_public_key_t *intro_enc_pubkey,
+                    uint8_t *secret_input_out)
+{
+  uint8_t *ptr;
+
+  /* Append EXP() */
+  ptr = secret_input_out;
+  APPEND(ptr, dh_result, CURVE25519_OUTPUT_LEN);
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr, client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  tor_assert(ptr == secret_input_out + INTRO_SECRET_HS_INPUT_LEN);
+}
+
+/** Calculate the 'rend_secret_hs_input' element used by the HS ntor handshake
+ *  and place it in <b>rend_secret_hs_input_out</b>. This function is used by
+ *  both client and service code.
+ *
+ * The computation on the client side is:
+ *  rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+ * whereas on the service side it is:
+ *  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+ *
+ * where:
+ * <b>dh_result1</b> and <b>dh_result2</b> carry the two EXP() results (of size
+ * CURVE25519_OUTPUT_LEN)
+ * <b>intro_auth_pubkey</b> is AUTH_KEY,
+ * <b>intro_enc_pubkey</b> is B,
+ * <b>client_ephemeral_enc_pubkey</b> is X, and
+ * <b>service_ephemeral_rend_pubkey</b> is Y.
+ */
+static void
+get_rend_secret_hs_input(const uint8_t *dh_result1, const uint8_t *dh_result2,
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  uint8_t *rend_secret_hs_input_out)
+{
+  uint8_t *ptr;
+
+  ptr = rend_secret_hs_input_out;
+  /* Append the first EXP() */
+  APPEND(ptr, dh_result1, CURVE25519_OUTPUT_LEN);
+  /* Append the other EXP() */
+  APPEND(ptr, dh_result2, CURVE25519_OUTPUT_LEN);
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr,
+         client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append Y */
+  APPEND(ptr,
+         service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  tor_assert(ptr == rend_secret_hs_input_out + REND_SECRET_HS_INPUT_LEN);
+}
+
+/************************* Public functions: *******************************/
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to encrypt and authenticate INTRODUCE1 cells. Return 0 and place the
+ * final key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went
+ * well, otherwise return -1;
+ *
+ * The relevant calculations are as follows:
+ *
+ *     intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+ *     info = m_hsexpand | subcredential
+ *     hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *     ENC_KEY = hs_keys[0:S_KEY_LEN]
+ *     MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor),
+ * <b>intro_enc_pubkey</b> is B (also found in HS descriptor),
+ * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X)
+ * <b>subcredential</b> is the hidden service subcredential (of size
+ * DIGEST256_LEN). */
+int
+hs_ntor_client_get_introduce1_keys(
+                      const ed25519_public_key_t *intro_auth_pubkey,
+                      const curve25519_public_key_t *intro_enc_pubkey,
+                      const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                      const uint8_t *subcredential,
+                      hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_pubkey);
+  tor_assert(client_ephemeral_enc_keypair);
+  tor_assert(subcredential);
+  tor_assert(hs_ntor_intro_cell_keys_out);
+
+  /* Calculate EXP(B,x) */
+  curve25519_handshake(dh_result,
+                       &client_ephemeral_enc_keypair->seckey,
+                       intro_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN);
+
+  /* Get intro_secret_hs_input */
+  get_intro_secret_hs_input(dh_result, intro_auth_pubkey,
+                            &client_ephemeral_enc_keypair->pubkey,
+                            intro_enc_pubkey, secret_input);
+  bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
+
+  /* Get ENC_KEY and MAC_KEY! */
+  get_introduce1_key_material(secret_input, subcredential,
+                              hs_ntor_intro_cell_keys_out);
+
+  /* Cleanup */
+  memwipe(secret_input,  0, sizeof(secret_input));
+  if (bad) {
+    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to verify RENDEZVOUS1 cells and encrypt further rendezvous
+ * traffic. Return 0 and place the final key material in
+ * <b>hs_ntor_rend_cell_keys_out</b> if everything went well, else return -1.
+ *
+ * The relevant calculations are as follows:
+ *
+ *  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+ *  NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *  verify = MAC(rend_secret_hs_input, t_hsverify)
+ *  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *  auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor),
+ * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X)
+ * <b>intro_enc_pubkey</b> is B (also found in HS descriptor),
+ * <b>service_ephemeral_rend_pubkey</b> is Y (SERVER_PK in RENDEZVOUS1 cell) */
+int
+hs_ntor_client_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result1[CURVE25519_OUTPUT_LEN];
+  uint8_t dh_result2[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(client_ephemeral_enc_keypair);
+  tor_assert(intro_enc_pubkey);
+  tor_assert(service_ephemeral_rend_pubkey);
+  tor_assert(hs_ntor_rend_cell_keys_out);
+
+  /* Compute EXP(Y, x) */
+  curve25519_handshake(dh_result1,
+                       &client_ephemeral_enc_keypair->seckey,
+                       service_ephemeral_rend_pubkey);
+  bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN);
+
+  /* Compute EXP(B, x) */
+  curve25519_handshake(dh_result2,
+                       &client_ephemeral_enc_keypair->seckey,
+                       intro_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN);
+
+  /* Get rend_secret_hs_input */
+  get_rend_secret_hs_input(dh_result1, dh_result2,
+                           intro_auth_pubkey, intro_enc_pubkey,
+                           &client_ephemeral_enc_keypair->pubkey,
+                           service_ephemeral_rend_pubkey,
+                           rend_secret_hs_input);
+
+  /* Get NTOR_KEY_SEED and the auth_input MAC */
+  bad |= get_rendezvous1_key_material(rend_secret_hs_input,
+                                      intro_auth_pubkey,
+                                      intro_enc_pubkey,
+                                      service_ephemeral_rend_pubkey,
+                                      &client_ephemeral_enc_keypair->pubkey,
+                                      hs_ntor_rend_cell_keys_out);
+
+  memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input));
+  if (bad) {
+    memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to decrypt and verify INTRODUCE1 cells. Return 0 and place the final
+ * key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went well,
+ * otherwise return -1;
+ *
+ * The relevant calculations are as follows:
+ *
+ *    intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+ *    info = m_hsexpand | subcredential
+ *    hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *    HS_DEC_KEY = hs_keys[0:S_KEY_LEN]
+ *    HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (introduction point auth key),
+ * <b>intro_enc_keypair</b> is (b,B) (introduction point encryption keypair),
+ * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell),
+ * <b>subcredential</b> is the HS subcredential (of size DIGEST256_LEN) */
+int
+hs_ntor_service_get_introduce1_keys(
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_keypair_t *intro_enc_keypair,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    const uint8_t *subcredential,
+                    hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_keypair);
+  tor_assert(client_ephemeral_enc_pubkey);
+  tor_assert(subcredential);
+  tor_assert(hs_ntor_intro_cell_keys_out);
+
+  /* Compute EXP(X, b) */
+  curve25519_handshake(dh_result,
+                       &intro_enc_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN);
+
+  /* Get intro_secret_hs_input */
+  get_intro_secret_hs_input(dh_result, intro_auth_pubkey,
+                            client_ephemeral_enc_pubkey,
+                            &intro_enc_keypair->pubkey,
+                            secret_input);
+  bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
+
+  /* Get ENC_KEY and MAC_KEY! */
+  get_introduce1_key_material(secret_input, subcredential,
+                              hs_ntor_intro_cell_keys_out);
+
+  memwipe(secret_input,  0, sizeof(secret_input));
+  if (bad) {
+    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to create and authenticate RENDEZVOUS1 cells. Return 0 and place the
+ * final key material in <b>hs_ntor_rend_cell_keys_out</b> if all went fine,
+ * return -1 if error happened.
+ *
+ * The relevant calculations are as follows:
+ *
+ *  rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+ *  NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *  verify = MAC(rend_secret_hs_input, t_hsverify)
+ *  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *  auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (intro point auth key),
+ * <b>intro_enc_keypair</b> is (b,B) (intro point enc keypair)
+ * <b>service_ephemeral_rend_keypair</b> is a fresh (y,Y) keypair
+ * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell) */
+int
+hs_ntor_service_get_rendezvous1_keys(
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_keypair_t *intro_enc_keypair,
+                    const curve25519_keypair_t *service_ephemeral_rend_keypair,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result1[CURVE25519_OUTPUT_LEN];
+  uint8_t dh_result2[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_keypair);
+  tor_assert(service_ephemeral_rend_keypair);
+  tor_assert(client_ephemeral_enc_pubkey);
+  tor_assert(hs_ntor_rend_cell_keys_out);
+
+  /* Compute EXP(X, y) */
+  curve25519_handshake(dh_result1,
+                       &service_ephemeral_rend_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN);
+
+  /* Compute EXP(X, b) */
+  curve25519_handshake(dh_result2,
+                       &intro_enc_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN);
+
+  /* Get rend_secret_hs_input */
+  get_rend_secret_hs_input(dh_result1, dh_result2,
+                           intro_auth_pubkey,
+                           &intro_enc_keypair->pubkey,
+                           client_ephemeral_enc_pubkey,
+                           &service_ephemeral_rend_keypair->pubkey,
+                           rend_secret_hs_input);
+
+  /* Get NTOR_KEY_SEED and AUTH_INPUT_MAC! */
+  bad |= get_rendezvous1_key_material(rend_secret_hs_input,
+                                      intro_auth_pubkey,
+                                      &intro_enc_keypair->pubkey,
+                                      &service_ephemeral_rend_keypair->pubkey,
+                                      client_ephemeral_enc_pubkey,
+                                      hs_ntor_rend_cell_keys_out);
+
+  memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input));
+  if (bad) {
+    memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/** Given a received RENDEZVOUS2 MAC in <b>mac</b> (of length DIGEST256_LEN),
+ *  and the RENDEZVOUS1 key material in <b>hs_ntor_rend_cell_keys</b>, return 1
+ *  if the MAC is good, otherwise return 0. */
+int
+hs_ntor_client_rendezvous2_mac_is_good(
+                        const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys,
+                        const uint8_t *rcvd_mac)
+{
+  tor_assert(rcvd_mac);
+  tor_assert(hs_ntor_rend_cell_keys);
+
+  return tor_memeq(hs_ntor_rend_cell_keys->rend_cell_auth_mac,
+                   rcvd_mac, DIGEST256_LEN);
+}
+
+/* Input length to KDF for key expansion */
+#define NTOR_KEY_EXPANSION_KDF_INPUT_LEN (DIGEST256_LEN + M_HSEXPAND_LEN)
+
+/** Given the rendezvous key seed in <b>ntor_key_seed</b> (of size
+ *  DIGEST256_LEN), do the circuit key expansion as specified by section
+ *  '4.2.1. Key expansion' and place the keys in <b>keys_out</b> (which must be
+ *  of size HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN).
+ *
+ * Return 0 if things went well, else return -1. */
+int
+hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
+                              uint8_t *keys_out, size_t keys_out_len)
+{
+  uint8_t *ptr;
+  uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
+  crypto_xof_t *xof;
+
+  /* Sanity checks on lengths to make sure we are good */
+  if (BUG(seed_len != DIGEST256_LEN)) {
+    return -1;
+  }
+  if (BUG(keys_out_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) {
+    return -1;
+  }
+
+  /* Let's build the input to the KDF */
+  ptr = kdf_input;
+  APPEND(ptr, ntor_key_seed, DIGEST256_LEN);
+  APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
+  tor_assert(ptr == kdf_input + sizeof(kdf_input));
+
+  /* Generate the keys */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
+  crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN);
+  crypto_xof_free(xof);
+
+  return 0;
+}
+
diff --git a/src/or/hs_ntor.h b/src/or/hs_ntor.h
new file mode 100644
index 0000000000..77e544a130
--- /dev/null
+++ b/src/or/hs_ntor.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_HS_NTOR_H
+#define TOR_HS_NTOR_H
+
+#include "or.h"
+
+/* Output length of KDF for key expansion */
+#define HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN \
+  (DIGEST256_LEN*2 + CIPHER256_KEY_LEN*2)
+
+/* Key material needed to encode/decode INTRODUCE1 cells */
+typedef struct {
+  /* Key used for encryption of encrypted INTRODUCE1 blob */
+  uint8_t enc_key[CIPHER256_KEY_LEN];
+  /* MAC key used to protect encrypted INTRODUCE1 blob */
+  uint8_t mac_key[DIGEST256_LEN];
+} hs_ntor_intro_cell_keys_t;
+
+/* Key material needed to encode/decode RENDEZVOUS1 cells */
+typedef struct {
+  /* This is the MAC of the HANDSHAKE_INFO field */
+  uint8_t rend_cell_auth_mac[DIGEST256_LEN];
+  /* This is the key seed used to derive further rendezvous crypto keys as
+   * detailed in section 4.2.1 of rend-spec-ng.txt. */
+  uint8_t ntor_key_seed[DIGEST256_LEN];
+} hs_ntor_rend_cell_keys_t;
+
+int hs_ntor_client_get_introduce1_keys(
+                      const ed25519_public_key_t *intro_auth_pubkey,
+                      const curve25519_public_key_t *intro_enc_pubkey,
+                      const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                      const uint8_t *subcredential,
+                      hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
+int hs_ntor_client_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
+
+int hs_ntor_service_get_introduce1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *intro_enc_keypair,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  const uint8_t *subcredential,
+                  hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
+int hs_ntor_service_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *intro_enc_keypair,
+                  const curve25519_keypair_t *service_ephemeral_rend_keypair,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
+
+int hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed,
+                                  size_t seed_len,
+                                  uint8_t *keys_out, size_t keys_out_len);
+
+int hs_ntor_client_rendezvous2_mac_is_good(
+                        const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys,
+                        const uint8_t *rcvd_mac);
+
+#endif /* !defined(TOR_HS_NTOR_H) */
+
diff --git a/src/or/hs_service.c b/src/or/hs_service.c
new file mode 100644
index 0000000000..54ceb9dcad
--- /dev/null
+++ b/src/or/hs_service.c
@@ -0,0 +1,3640 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.c
+ * \brief Implement next generation hidden service functionality
+ **/
+
+#define HS_SERVICE_PRIVATE
+
+#include "or.h"
+#include "circpathbias.h"
+#include "circuitbuild.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "config.h"
+#include "connection.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "directory.h"
+#include "main.h"
+#include "networkstatus.h"
+#include "nodelist.h"
+#include "relay.h"
+#include "rendservice.h"
+#include "router.h"
+#include "routerkeys.h"
+#include "routerlist.h"
+#include "shared_random_client.h"
+#include "statefile.h"
+
+#include "hs_circuit.h"
+#include "hs_common.h"
+#include "hs_config.h"
+#include "hs_control.h"
+#include "hs_descriptor.h"
+#include "hs_ident.h"
+#include "hs_intropoint.h"
+#include "hs_service.h"
+#include "hs_stats.h"
+
+/* Trunnel */
+#include "ed25519_cert.h"
+#include "hs/cell_common.h"
+#include "hs/cell_establish_intro.h"
+
+/* Helper macro. Iterate over every service in the global map. The var is the
+ * name of the service pointer. */
+#define FOR_EACH_SERVICE_BEGIN(var)                          \
+    STMT_BEGIN                                               \
+    hs_service_t **var##_iter, *var;                         \
+    HT_FOREACH(var##_iter, hs_service_ht, hs_service_map) {  \
+      var = *var##_iter;
+#define FOR_EACH_SERVICE_END } STMT_END ;
+
+/* Helper macro. Iterate over both current and previous descriptor of a
+ * service. The var is the name of the descriptor pointer. This macro skips
+ * any descriptor object of the service that is NULL. */
+#define FOR_EACH_DESCRIPTOR_BEGIN(service, var)                  \
+  STMT_BEGIN                                                     \
+    hs_service_descriptor_t *var;                                \
+    for (int var ## _loop_idx = 0; var ## _loop_idx < 2;         \
+         ++var ## _loop_idx) {                                   \
+      (var ## _loop_idx == 0) ? (var = service->desc_current) :  \
+                                (var = service->desc_next);      \
+      if (var == NULL) continue;
+#define FOR_EACH_DESCRIPTOR_END } STMT_END ;
+
+/* Onion service directory file names. */
+static const char fname_keyfile_prefix[] = "hs_ed25519";
+static const char fname_hostname[] = "hostname";
+static const char address_tld[] = "onion";
+
+/* Staging list of service object. When configuring service, we add them to
+ * this list considered a staging area and they will get added to our global
+ * map once the keys have been loaded. These two steps are separated because
+ * loading keys requires that we are an actual running tor process. */
+static smartlist_t *hs_service_staging_list;
+
+/** True if the list of available router descriptors might have changed which
+ *  might result in an altered hash ring. Check if the hash ring changed and
+ *  reupload if needed */
+static int consider_republishing_hs_descriptors = 0;
+
+/* Static declaration. */
+static void set_descriptor_revision_counter(hs_descriptor_t *hs_desc);
+static void move_descriptors(hs_service_t *src, hs_service_t *dst);
+
+/* Helper: Function to compare two objects in the service map. Return 1 if the
+ * two service have the same master public identity key. */
+static inline int
+hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
+{
+  tor_assert(first);
+  tor_assert(second);
+  /* Simple key compare. */
+  return ed25519_pubkey_eq(&first->keys.identity_pk,
+                           &second->keys.identity_pk);
+}
+
+/* Helper: Function for the service hash table code below. The key used is the
+ * master public identity key which is ultimately the onion address. */
+static inline unsigned int
+hs_service_ht_hash(const hs_service_t *service)
+{
+  tor_assert(service);
+  return (unsigned int) siphash24g(service->keys.identity_pk.pubkey,
+                                   sizeof(service->keys.identity_pk.pubkey));
+}
+
+/* This is _the_ global hash map of hidden services which indexed the service
+ * contained in it by master public identity key which is roughly the onion
+ * address of the service. */
+static struct hs_service_ht *hs_service_map;
+
+/* Register the service hash table. */
+HT_PROTOTYPE(hs_service_ht,      /* Name of hashtable. */
+             hs_service_t,       /* Object contained in the map. */
+             hs_service_node,    /* The name of the HT_ENTRY member. */
+             hs_service_ht_hash, /* Hashing function. */
+             hs_service_ht_eq)   /* Compare function for objects. */
+
+HT_GENERATE2(hs_service_ht, hs_service_t, hs_service_node,
+             hs_service_ht_hash, hs_service_ht_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+/* Query the given service map with a public key and return a service object
+ * if found else NULL. It is also possible to set a directory path in the
+ * search query. If pk is NULL, then it will be set to zero indicating the
+ * hash table to compare the directory path instead. */
+STATIC hs_service_t *
+find_service(hs_service_ht *map, const ed25519_public_key_t *pk)
+{
+  hs_service_t dummy_service;
+  tor_assert(map);
+  tor_assert(pk);
+  memset(&dummy_service, 0, sizeof(dummy_service));
+  ed25519_pubkey_copy(&dummy_service.keys.identity_pk, pk);
+  return HT_FIND(hs_service_ht, map, &dummy_service);
+}
+
+/* Register the given service in the given map. If the service already exists
+ * in the map, -1 is returned. On success, 0 is returned and the service
+ * ownership has been transferred to the global map. */
+STATIC int
+register_service(hs_service_ht *map, hs_service_t *service)
+{
+  tor_assert(map);
+  tor_assert(service);
+  tor_assert(!ed25519_public_key_is_zero(&service->keys.identity_pk));
+
+  if (find_service(map, &service->keys.identity_pk)) {
+    /* Existing service with the same key. Do not register it. */
+    return -1;
+  }
+  /* Taking ownership of the object at this point. */
+  HT_INSERT(hs_service_ht, map, service);
+
+  /* If we just modified the global map, we notify. */
+  if (map == hs_service_map) {
+    hs_service_map_has_changed();
+  }
+
+  return 0;
+}
+
+/* Remove a given service from the given map. If service is NULL or the
+ * service key is unset, return gracefully. */
+STATIC void
+remove_service(hs_service_ht *map, hs_service_t *service)
+{
+  hs_service_t *elm;
+
+  tor_assert(map);
+
+  /* Ignore if no service or key is zero. */
+  if (BUG(service == NULL) ||
+      BUG(ed25519_public_key_is_zero(&service->keys.identity_pk))) {
+    return;
+  }
+
+  elm = HT_REMOVE(hs_service_ht, map, service);
+  if (elm) {
+    tor_assert(elm == service);
+  } else {
+    log_warn(LD_BUG, "Could not find service in the global map "
+                     "while removing service %s",
+             escaped(service->config.directory_path));
+  }
+
+  /* If we just modified the global map, we notify. */
+  if (map == hs_service_map) {
+    hs_service_map_has_changed();
+  }
+}
+
+/* Set the default values for a service configuration object <b>c</b>. */
+static void
+set_service_default_config(hs_service_config_t *c,
+                           const or_options_t *options)
+{
+  (void) options;
+  tor_assert(c);
+  c->ports = smartlist_new();
+  c->directory_path = NULL;
+  c->max_streams_per_rdv_circuit = 0;
+  c->max_streams_close_circuit = 0;
+  c->num_intro_points = NUM_INTRO_POINTS_DEFAULT;
+  c->allow_unknown_ports = 0;
+  c->is_single_onion = 0;
+  c->dir_group_readable = 0;
+  c->is_ephemeral = 0;
+}
+
+/* From a service configuration object config, clear everything from it
+ * meaning free allocated pointers and reset the values. */
+static void
+service_clear_config(hs_service_config_t *config)
+{
+  if (config == NULL) {
+    return;
+  }
+  tor_free(config->directory_path);
+  if (config->ports) {
+    SMARTLIST_FOREACH(config->ports, rend_service_port_config_t *, p,
+                      rend_service_port_config_free(p););
+    smartlist_free(config->ports);
+  }
+  memset(config, 0, sizeof(*config));
+}
+
+/* Helper function to return a human readable description of the given intro
+ * point object.
+ *
+ * This function is not thread-safe. Each call to this invalidates the
+ * previous values returned by it. */
+static const char *
+describe_intro_point(const hs_service_intro_point_t *ip)
+{
+  /* Hex identity digest of the IP prefixed by the $ sign and ends with NUL
+   * byte hence the plus two. */
+  static char buf[HEX_DIGEST_LEN + 2];
+  const char *legacy_id = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          const hs_desc_link_specifier_t *, lspec) {
+    if (lspec->type == LS_LEGACY_ID) {
+      legacy_id = (const char *) lspec->u.legacy_id;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(lspec);
+
+  /* For now, we only print the identity digest but we could improve this with
+   * much more information such as the ed25519 identity has well. */
+  buf[0] = '$';
+  if (legacy_id) {
+    base16_encode(buf + 1, HEX_DIGEST_LEN + 1, legacy_id, DIGEST_LEN);
+  }
+
+  return buf;
+}
+
+/* Return the lower bound of maximum INTRODUCE2 cells per circuit before we
+ * rotate intro point (defined by a consensus parameter or the default
+ * value). */
+static int32_t
+get_intro_point_min_introduce2(void)
+{
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_min_introduce2",
+                                 INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS,
+                                 0, INT32_MAX);
+}
+
+/* Return the upper bound of maximum INTRODUCE2 cells per circuit before we
+ * rotate intro point (defined by a consensus parameter or the default
+ * value). */
+static int32_t
+get_intro_point_max_introduce2(void)
+{
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_max_introduce2",
+                                 INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS,
+                                 0, INT32_MAX);
+}
+
+/* Return the minimum lifetime in seconds of an introduction point defined by a
+ * consensus parameter or the default value. */
+static int32_t
+get_intro_point_min_lifetime(void)
+{
+#define MIN_INTRO_POINT_LIFETIME_TESTING 10
+  if (get_options()->TestingTorNetwork) {
+    return MIN_INTRO_POINT_LIFETIME_TESTING;
+  }
+
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_min_lifetime",
+                                 INTRO_POINT_LIFETIME_MIN_SECONDS,
+                                 0, INT32_MAX);
+}
+
+/* Return the maximum lifetime in seconds of an introduction point defined by a
+ * consensus parameter or the default value. */
+static int32_t
+get_intro_point_max_lifetime(void)
+{
+#define MAX_INTRO_POINT_LIFETIME_TESTING 30
+  if (get_options()->TestingTorNetwork) {
+    return MAX_INTRO_POINT_LIFETIME_TESTING;
+  }
+
+  /* The [0, 2147483647] range is quite large to accommodate anything we decide
+   * in the future. */
+  return networkstatus_get_param(NULL, "hs_intro_max_lifetime",
+                                 INTRO_POINT_LIFETIME_MAX_SECONDS,
+                                 0, INT32_MAX);
+}
+
+/* Return the number of extra introduction point defined by a consensus
+ * parameter or the default value. */
+static int32_t
+get_intro_point_num_extra(void)
+{
+  /* The [0, 128] range bounds the number of extra introduction point allowed.
+   * Above 128 intro points, it's getting a bit crazy. */
+  return networkstatus_get_param(NULL, "hs_intro_num_extra",
+                                 NUM_INTRO_POINTS_EXTRA, 0, 128);
+}
+
+/* Helper: Function that needs to return 1 for the HT for each loop which
+ * frees every service in an hash map. */
+static int
+ht_free_service_(struct hs_service_t *service, void *data)
+{
+  (void) data;
+  hs_service_free(service);
+  /* This function MUST return 1 so the given object is then removed from the
+   * service map leading to this free of the object being safe. */
+  return 1;
+}
+
+/* Free every service that can be found in the global map. Once done, clear
+ * and free the global map. */
+static void
+service_free_all(void)
+{
+  if (hs_service_map) {
+    /* The free helper function returns 1 so this is safe. */
+    hs_service_ht_HT_FOREACH_FN(hs_service_map, ht_free_service_, NULL);
+    HT_CLEAR(hs_service_ht, hs_service_map);
+    tor_free(hs_service_map);
+    hs_service_map = NULL;
+  }
+
+  if (hs_service_staging_list) {
+    /* Cleanup staging list. */
+    SMARTLIST_FOREACH(hs_service_staging_list, hs_service_t *, s,
+                      hs_service_free(s));
+    smartlist_free(hs_service_staging_list);
+    hs_service_staging_list = NULL;
+  }
+}
+
+/* Free a given service intro point object. */
+STATIC void
+service_intro_point_free_(hs_service_intro_point_t *ip)
+{
+  if (!ip) {
+    return;
+  }
+  memwipe(&ip->auth_key_kp, 0, sizeof(ip->auth_key_kp));
+  memwipe(&ip->enc_key_kp, 0, sizeof(ip->enc_key_kp));
+  crypto_pk_free(ip->legacy_key);
+  replaycache_free(ip->replay_cache);
+  hs_intropoint_clear(&ip->base);
+  tor_free(ip);
+}
+
+/* Helper: free an hs_service_intro_point_t object. This function is used by
+ * digest256map_free() which requires a void * pointer. */
+static void
+service_intro_point_free_void(void *obj)
+{
+  service_intro_point_free_(obj);
+}
+
+/* Return a newly allocated service intro point and fully initialized from the
+ * given extend_info_t ei if non NULL.
+ * If is_legacy is true, we also generate the legacy key.
+ * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519
+ * key to the link specifiers.
+ *
+ * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * specifier list and no onion key. (This is used for testing.)
+ * On any other error, NULL is returned.
+ *
+ * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
+ * for IPv6 in a later release.) When calling extend_info_from_node(), pass
+ * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+STATIC hs_service_intro_point_t *
+service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
+                        unsigned int supports_ed25519_link_handshake_any)
+{
+  hs_desc_link_specifier_t *ls;
+  hs_service_intro_point_t *ip;
+
+  ip = tor_malloc_zero(sizeof(*ip));
+  /* We'll create the key material. No need for extra strong, those are short
+   * term keys. */
+  ed25519_keypair_generate(&ip->auth_key_kp, 0);
+
+  { /* Set introduce2 max cells limit */
+    int32_t min_introduce2_cells = get_intro_point_min_introduce2();
+    int32_t max_introduce2_cells = get_intro_point_max_introduce2();
+    if (BUG(max_introduce2_cells < min_introduce2_cells)) {
+      goto err;
+    }
+    ip->introduce2_max = crypto_rand_int_range(min_introduce2_cells,
+                                               max_introduce2_cells);
+  }
+  { /* Set intro point lifetime */
+    int32_t intro_point_min_lifetime = get_intro_point_min_lifetime();
+    int32_t intro_point_max_lifetime = get_intro_point_max_lifetime();
+    if (BUG(intro_point_max_lifetime < intro_point_min_lifetime)) {
+      goto err;
+    }
+    ip->time_to_expire = time(NULL) +
+      crypto_rand_int_range(intro_point_min_lifetime,intro_point_max_lifetime);
+  }
+
+  ip->replay_cache = replaycache_new(0, 0);
+
+  /* Initialize the base object. We don't need the certificate object. */
+  ip->base.link_specifiers = smartlist_new();
+
+  /* Generate the encryption key for this intro point. */
+  curve25519_keypair_generate(&ip->enc_key_kp, 0);
+  /* Figure out if this chosen node supports v3 or is legacy only. */
+  if (is_legacy) {
+    ip->base.is_only_legacy = 1;
+    /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
+    ip->legacy_key = crypto_pk_new();
+    if (crypto_pk_generate_key(ip->legacy_key) < 0) {
+      goto err;
+    }
+    if (crypto_pk_get_digest(ip->legacy_key,
+                             (char *) ip->legacy_key_digest) < 0) {
+      goto err;
+    }
+  }
+
+  if (ei == NULL) {
+    goto done;
+  }
+
+  /* We'll try to add all link specifiers. Legacy is mandatory.
+   * IPv4 or IPv6 is required, and we always send IPv4. */
+  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
+  /* It is impossible to have an extend info object without a v4. */
+  if (BUG(!ls)) {
+    goto err;
+  }
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
+  /* It is impossible to have an extend info object without an identity
+   * digest. */
+  if (BUG(!ls)) {
+    goto err;
+  }
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  /* ed25519 identity key is optional for intro points. If the node supports
+   * ed25519 link authentication, we include it. */
+  if (supports_ed25519_link_handshake_any) {
+    ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
+    if (ls) {
+      smartlist_add(ip->base.link_specifiers, ls);
+    }
+  }
+
+  /* IPv6 is not supported in this release. */
+
+  /* Finally, copy onion key from the extend_info_t object. */
+  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+
+ done:
+  return ip;
+ err:
+  service_intro_point_free(ip);
+  return NULL;
+}
+
+/* Add the given intro point object to the given intro point map. The intro
+ * point MUST have its RSA encryption key set if this is a legacy type or the
+ * authentication key set otherwise. */
+STATIC void
+service_intro_point_add(digest256map_t *map, hs_service_intro_point_t *ip)
+{
+  hs_service_intro_point_t *old_ip_entry;
+
+  tor_assert(map);
+  tor_assert(ip);
+
+  old_ip_entry = digest256map_set(map, ip->auth_key_kp.pubkey.pubkey, ip);
+  /* Make sure we didn't just try to double-add an intro point */
+  tor_assert_nonfatal(!old_ip_entry);
+}
+
+/* For a given service, remove the intro point from that service's descriptors
+ * (check both current and next descriptor) */
+STATIC void
+service_intro_point_remove(const hs_service_t *service,
+                           const hs_service_intro_point_t *ip)
+{
+  tor_assert(service);
+  tor_assert(ip);
+
+  /* Trying all descriptors. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* We'll try to remove the descriptor on both descriptors which is not
+     * very expensive to do instead of doing loopup + remove. */
+    digest256map_remove(desc->intro_points.map,
+                        ip->auth_key_kp.pubkey.pubkey);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* For a given service and authentication key, return the intro point or NULL
+ * if not found. This will check both descriptors in the service. */
+STATIC hs_service_intro_point_t *
+service_intro_point_find(const hs_service_t *service,
+                         const ed25519_public_key_t *auth_key)
+{
+  hs_service_intro_point_t *ip = NULL;
+
+  tor_assert(service);
+  tor_assert(auth_key);
+
+  /* Trying all descriptors to find the right intro point.
+   *
+   * Even if we use the same node as intro point in both descriptors, the node
+   * will have a different intro auth key for each descriptor since we generate
+   * a new one everytime we pick an intro point.
+   *
+   * After #22893 gets implemented, intro points will be moved to be
+   * per-service instead of per-descriptor so this function will need to
+   * change.
+   */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    if ((ip = digest256map_get(desc->intro_points.map,
+                               auth_key->pubkey)) != NULL) {
+      break;
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+
+  return ip;
+}
+
+/* For a given service and intro point, return the descriptor for which the
+ * intro point is assigned to. NULL is returned if not found. */
+STATIC hs_service_descriptor_t *
+service_desc_find_by_intro(const hs_service_t *service,
+                           const hs_service_intro_point_t *ip)
+{
+  hs_service_descriptor_t *descp = NULL;
+
+  tor_assert(service);
+  tor_assert(ip);
+
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    if (digest256map_get(desc->intro_points.map,
+                         ip->auth_key_kp.pubkey.pubkey)) {
+      descp = desc;
+      break;
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+
+  return descp;
+}
+
+/* From a circuit identifier, get all the possible objects associated with the
+ * ident. If not NULL, service, ip or desc are set if the object can be found.
+ * They are untouched if they can't be found.
+ *
+ * This is an helper function because we do those lookups often so it's more
+ * convenient to simply call this functions to get all the things at once. */
+STATIC void
+get_objects_from_ident(const hs_ident_circuit_t *ident,
+                       hs_service_t **service, hs_service_intro_point_t **ip,
+                       hs_service_descriptor_t **desc)
+{
+  hs_service_t *s;
+
+  tor_assert(ident);
+
+  /* Get service object from the circuit identifier. */
+  s = find_service(hs_service_map, &ident->identity_pk);
+  if (s && service) {
+    *service = s;
+  }
+
+  /* From the service object, get the intro point object of that circuit. The
+   * following will query both descriptors intro points list. */
+  if (s && ip) {
+    *ip = service_intro_point_find(s, &ident->intro_auth_pk);
+  }
+
+  /* Get the descriptor for this introduction point and service. */
+  if (s && ip && *ip && desc) {
+    *desc = service_desc_find_by_intro(s, *ip);
+  }
+}
+
+/* From a given intro point, return the first link specifier of type
+ * encountered in the link specifier list. Return NULL if it can't be found.
+ *
+ * The caller does NOT have ownership of the object, the intro point does. */
+static hs_desc_link_specifier_t *
+get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
+{
+  hs_desc_link_specifier_t *lnk_spec = NULL;
+
+  tor_assert(ip);
+
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          hs_desc_link_specifier_t *, ls) {
+    if (ls->type == type) {
+      lnk_spec = ls;
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+ end:
+  return lnk_spec;
+}
+
+/* Given a service intro point, return the node_t associated to it. This can
+ * return NULL if the given intro point has no legacy ID or if the node can't
+ * be found in the consensus. */
+STATIC const node_t *
+get_node_from_intro_point(const hs_service_intro_point_t *ip)
+{
+  const hs_desc_link_specifier_t *ls;
+
+  tor_assert(ip);
+
+  ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
+  if (BUG(!ls)) {
+    return NULL;
+  }
+  /* XXX In the future, we want to only use the ed25519 ID (#22173). */
+  return node_get_by_id((const char *) ls->u.legacy_id);
+}
+
+/* Given a service intro point, return the extend_info_t for it. This can
+ * return NULL if the node can't be found for the intro point or the extend
+ * info can't be created for the found node. If direct_conn is set, the extend
+ * info is validated on if we can connect directly. */
+static extend_info_t *
+get_extend_info_from_intro_point(const hs_service_intro_point_t *ip,
+                                 unsigned int direct_conn)
+{
+  extend_info_t *info = NULL;
+  const node_t *node;
+
+  tor_assert(ip);
+
+  node = get_node_from_intro_point(ip);
+  if (node == NULL) {
+    /* This can happen if the relay serving as intro point has been removed
+     * from the consensus. In that case, the intro point will be removed from
+     * the descriptor during the scheduled events. */
+    goto end;
+  }
+
+  /* In the case of a direct connection (single onion service), it is possible
+   * our firewall policy won't allow it so this can return a NULL value. */
+  info = extend_info_from_node(node, direct_conn);
+
+ end:
+  return info;
+}
+
+/* Return the number of introduction points that are established for the
+ * given descriptor. */
+static unsigned int
+count_desc_circuit_established(const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+
+  tor_assert(desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    count += ip->circuit_established;
+  } DIGEST256MAP_FOREACH_END;
+
+  return count;
+}
+
+/* For a given service and descriptor of that service, close all active
+ * directory connections. */
+static void
+close_directory_connections(const hs_service_t *service,
+                            const hs_service_descriptor_t *desc)
+{
+  unsigned int count = 0;
+  smartlist_t *dir_conns;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Close pending HS desc upload connections for the blinded key of 'desc'. */
+  dir_conns = connection_list_by_type_purpose(CONN_TYPE_DIR,
+                                              DIR_PURPOSE_UPLOAD_HSDESC);
+  SMARTLIST_FOREACH_BEGIN(dir_conns, connection_t *, conn) {
+    dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+    if (ed25519_pubkey_eq(&dir_conn->hs_ident->identity_pk,
+                          &service->keys.identity_pk) &&
+        ed25519_pubkey_eq(&dir_conn->hs_ident->blinded_pk,
+                          &desc->blinded_kp.pubkey)) {
+      connection_mark_for_close(conn);
+      count++;
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  log_info(LD_REND, "Closed %u active service directory connections for "
+                    "descriptor %s of service %s",
+           count, safe_str_client(ed25519_fmt(&desc->blinded_kp.pubkey)),
+           safe_str_client(service->onion_address));
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(dir_conns);
+}
+
+/* Close all rendezvous circuits for the given service. */
+static void
+close_service_rp_circuits(hs_service_t *service)
+{
+  origin_circuit_t *ocirc = NULL;
+
+  tor_assert(service);
+
+  /* The reason we go over all circuit instead of using the circuitmap API is
+   * because most hidden service circuits are rendezvous circuits so there is
+   * no real improvement at getting all rendezvous circuits from the
+   * circuitmap and then going over them all to find the right ones.
+   * Furthermore, another option would have been to keep a list of RP cookies
+   * for a service but it creates an engineering complexity since we don't
+   * have a "RP circuit closed" event to clean it up properly so we avoid a
+   * memory DoS possibility. */
+
+  while ((ocirc = circuit_get_next_service_rp_circ(ocirc))) {
+    /* Only close circuits that are v3 and for this service. */
+    if (ocirc->hs_ident != NULL &&
+        ed25519_pubkey_eq(&ocirc->hs_ident->identity_pk,
+                          &service->keys.identity_pk)) {
+      /* Reason is FINISHED because service has been removed and thus the
+       * circuit is considered old/uneeded. When freed, it is removed from the
+       * hs circuitmap. */
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+  }
+}
+
+/* Close the circuit(s) for the given map of introduction points. */
+static void
+close_intro_circuits(hs_service_intropoints_t *intro_points)
+{
+  tor_assert(intro_points);
+
+  DIGEST256MAP_FOREACH(intro_points->map, key,
+                       const hs_service_intro_point_t *, ip) {
+    origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc) {
+      /* Reason is FINISHED because service has been removed and thus the
+       * circuit is considered old/uneeded. When freed, the circuit is removed
+       * from the HS circuitmap. */
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Close all introduction circuits for the given service. */
+static void
+close_service_intro_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    close_intro_circuits(&desc->intro_points);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Close any circuits related to the given service. */
+static void
+close_service_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  /* Only support for version >= 3. */
+  if (BUG(service->config.version < HS_VERSION_THREE)) {
+    return;
+  }
+  /* Close intro points. */
+  close_service_intro_circuits(service);
+  /* Close rendezvous points. */
+  close_service_rp_circuits(service);
+}
+
+/* Move every ephemeral services from the src service map to the dst service
+ * map. It is possible that a service can't be register to the dst map which
+ * won't stop the process of moving them all but will trigger a log warn. */
+static void
+move_ephemeral_services(hs_service_ht *src, hs_service_ht *dst)
+{
+  hs_service_t **iter, **next;
+
+  tor_assert(src);
+  tor_assert(dst);
+
+  /* Iterate over the map to find ephemeral service and move them to the other
+   * map. We loop using this method to have a safe removal process. */
+  for (iter = HT_START(hs_service_ht, src); iter != NULL; iter = next) {
+    hs_service_t *s = *iter;
+    if (!s->config.is_ephemeral) {
+      /* Yeah, we are in a very manual loop :). */
+      next = HT_NEXT(hs_service_ht, src, iter);
+      continue;
+    }
+    /* Remove service from map and then register to it to the other map.
+     * Reminder that "*iter" and "s" are the same thing. */
+    next = HT_NEXT_RMV(hs_service_ht, src, iter);
+    if (register_service(dst, s) < 0) {
+      log_warn(LD_BUG, "Ephemeral service key is already being used. "
+                       "Skipping.");
+    }
+  }
+}
+
+/* Return a const string of the directory path escaped. If this is an
+ * ephemeral service, it returns "[EPHEMERAL]". This can only be called from
+ * the main thread because escaped() uses a static variable. */
+static const char *
+service_escaped_dir(const hs_service_t *s)
+{
+  return (s->config.is_ephemeral) ? "[EPHEMERAL]" :
+                                    escaped(s->config.directory_path);
+}
+
+/** Move the hidden service state from <b>src</b> to <b>dst</b>. We do this
+ *  when we receive a SIGHUP: <b>dst</b> is the post-HUP service */
+static void
+move_hs_state(hs_service_t *src_service, hs_service_t *dst_service)
+{
+  tor_assert(src_service);
+  tor_assert(dst_service);
+
+  hs_service_state_t *src = &src_service->state;
+  hs_service_state_t *dst = &dst_service->state;
+
+  /* Let's do a shallow copy */
+  dst->intro_circ_retry_started_time = src->intro_circ_retry_started_time;
+  dst->num_intro_circ_launched = src->num_intro_circ_launched;
+  /* Freeing a NULL replaycache triggers an info LD_BUG. */
+  if (dst->replay_cache_rend_cookie != NULL) {
+    replaycache_free(dst->replay_cache_rend_cookie);
+  }
+  dst->replay_cache_rend_cookie = src->replay_cache_rend_cookie;
+  dst->next_rotation_time = src->next_rotation_time;
+
+  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+}
+
+/* Register services that are in the staging list. Once this function returns,
+ * the global service map will be set with the right content and all non
+ * surviving services will be cleaned up. */
+static void
+register_all_services(void)
+{
+  struct hs_service_ht *new_service_map;
+
+  tor_assert(hs_service_staging_list);
+
+  /* Allocate a new map that will replace the current one. */
+  new_service_map = tor_malloc_zero(sizeof(*new_service_map));
+  HT_INIT(hs_service_ht, new_service_map);
+
+  /* First step is to transfer all ephemeral services from the current global
+   * map to the new one we are constructing. We do not prune ephemeral
+   * services as the only way to kill them is by deleting it from the control
+   * port or stopping the tor daemon. */
+  move_ephemeral_services(hs_service_map, new_service_map);
+
+  SMARTLIST_FOREACH_BEGIN(hs_service_staging_list, hs_service_t *, snew) {
+    hs_service_t *s;
+
+    /* Check if that service is already in our global map and if so, we'll
+     * transfer the intro points to it. */
+    s = find_service(hs_service_map, &snew->keys.identity_pk);
+    if (s) {
+      /* Pass ownership of the descriptors from s (the current service) to
+       * snew (the newly configured one). */
+      move_descriptors(s, snew);
+      move_hs_state(s, snew);
+      /* Remove the service from the global map because after this, we need to
+       * go over the remaining service in that map that aren't surviving the
+       * reload to close their circuits. */
+      remove_service(hs_service_map, s);
+      hs_service_free(s);
+    }
+    /* Great, this service is now ready to be added to our new map. */
+    if (BUG(register_service(new_service_map, snew) < 0)) {
+      /* This should never happen because prior to registration, we validate
+       * every service against the entire set. Not being able to register a
+       * service means we failed to validate correctly. In that case, don't
+       * break tor and ignore the service but tell user. */
+      log_warn(LD_BUG, "Unable to register service with directory %s",
+               service_escaped_dir(snew));
+      SMARTLIST_DEL_CURRENT(hs_service_staging_list, snew);
+      hs_service_free(snew);
+    }
+  } SMARTLIST_FOREACH_END(snew);
+
+  /* Close any circuits associated with the non surviving services. Every
+   * service in the current global map are roaming. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    close_service_circuits(service);
+  } FOR_EACH_SERVICE_END;
+
+  /* Time to make the switch. We'll clear the staging list because its content
+   * has now changed ownership to the map. */
+  smartlist_clear(hs_service_staging_list);
+  service_free_all();
+  hs_service_map = new_service_map;
+  /* We've just register services into the new map and now we've replaced the
+   * global map with it so we have to notify that the change happened. When
+   * registering a service, the notify is only triggered if the destination
+   * map is the global map for which in here it was not. */
+  hs_service_map_has_changed();
+}
+
+/* Write the onion address of a given service to the given filename fname_ in
+ * the service directory. Return 0 on success else -1 on error. */
+STATIC int
+write_address_to_file(const hs_service_t *service, const char *fname_)
+{
+  int ret = -1;
+  char *fname = NULL;
+  char *addr_buf = NULL;
+
+  tor_assert(service);
+  tor_assert(fname_);
+
+  /* Construct the full address with the onion tld and write the hostname file
+   * to disk. */
+  tor_asprintf(&addr_buf, "%s.%s\n", service->onion_address, address_tld);
+  /* Notice here that we use the given "fname_". */
+  fname = hs_path_from_filename(service->config.directory_path, fname_);
+  if (write_str_to_file(fname, addr_buf, 0) < 0) {
+    log_warn(LD_REND, "Could not write onion address to hostname file %s",
+             escaped(fname));
+    goto end;
+  }
+
+#ifndef _WIN32
+  if (service->config.dir_group_readable) {
+    /* Mode to 0640. */
+    if (chmod(fname, S_IRUSR | S_IWUSR | S_IRGRP) < 0) {
+      log_warn(LD_FS, "Unable to make onion service hostname file %s "
+                      "group-readable.", escaped(fname));
+    }
+  }
+#endif /* !defined(_WIN32) */
+
+  /* Success. */
+  ret = 0;
+ end:
+  tor_free(fname);
+  tor_free(addr_buf);
+  return ret;
+}
+
+/* Load and/or generate private keys for the given service. On success, the
+ * hostname file will be written to disk along with the master private key iff
+ * the service is not configured for offline keys. Return 0 on success else -1
+ * on failure. */
+static int
+load_service_keys(hs_service_t *service)
+{
+  int ret = -1;
+  char *fname = NULL;
+  ed25519_keypair_t *kp;
+  const hs_service_config_t *config;
+
+  tor_assert(service);
+
+  config = &service->config;
+
+  /* Create and fix permission on service directory. We are about to write
+   * files to that directory so make sure it exists and has the right
+   * permissions. We do this here because at this stage we know that Tor is
+   * actually running and the service we have has been validated. */
+  if (hs_check_service_private_dir(get_options()->User,
+                                   config->directory_path,
+                                   config->dir_group_readable, 1) < 0) {
+    goto end;
+  }
+
+  /* Try to load the keys from file or generate it if not found. */
+  fname = hs_path_from_filename(config->directory_path, fname_keyfile_prefix);
+  /* Don't ask for key creation, we want to know if we were able to load it or
+   * we had to generate it. Better logging! */
+  kp = ed_key_init_from_file(fname, INIT_ED_KEY_SPLIT, LOG_INFO, NULL, 0, 0,
+                             0, NULL);
+  if (!kp) {
+    log_info(LD_REND, "Unable to load keys from %s. Generating it...", fname);
+    /* We'll now try to generate the keys and for it we want the strongest
+     * randomness for it. The keypair will be written in different files. */
+    uint32_t key_flags = INIT_ED_KEY_CREATE | INIT_ED_KEY_EXTRA_STRONG |
+                         INIT_ED_KEY_SPLIT;
+    kp = ed_key_init_from_file(fname, key_flags, LOG_WARN, NULL, 0, 0, 0,
+                               NULL);
+    if (!kp) {
+      log_warn(LD_REND, "Unable to generate keys and save in %s.", fname);
+      goto end;
+    }
+  }
+
+  /* Copy loaded or generated keys to service object. */
+  ed25519_pubkey_copy(&service->keys.identity_pk, &kp->pubkey);
+  memcpy(&service->keys.identity_sk, &kp->seckey,
+         sizeof(service->keys.identity_sk));
+  /* This does a proper memory wipe. */
+  ed25519_keypair_free(kp);
+
+  /* Build onion address from the newly loaded keys. */
+  tor_assert(service->config.version <= UINT8_MAX);
+  hs_build_address(&service->keys.identity_pk,
+                   (uint8_t) service->config.version,
+                   service->onion_address);
+
+  /* Write onion address to hostname file. */
+  if (write_address_to_file(service, fname_hostname) < 0) {
+    goto end;
+  }
+
+  /* Succes. */
+  ret = 0;
+ end:
+  tor_free(fname);
+  return ret;
+}
+
+/* Free a given service descriptor object and all key material is wiped. */
+STATIC void
+service_descriptor_free_(hs_service_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+  hs_descriptor_free(desc->desc);
+  memwipe(&desc->signing_kp, 0, sizeof(desc->signing_kp));
+  memwipe(&desc->blinded_kp, 0, sizeof(desc->blinded_kp));
+  /* Cleanup all intro points. */
+  digest256map_free(desc->intro_points.map, service_intro_point_free_void);
+  digestmap_free(desc->intro_points.failed_id, tor_free_);
+  if (desc->previous_hsdirs) {
+    SMARTLIST_FOREACH(desc->previous_hsdirs, char *, s, tor_free(s));
+    smartlist_free(desc->previous_hsdirs);
+  }
+  tor_free(desc);
+}
+
+/* Return a newly allocated service descriptor object. */
+STATIC hs_service_descriptor_t *
+service_descriptor_new(void)
+{
+  hs_service_descriptor_t *sdesc = tor_malloc_zero(sizeof(*sdesc));
+  sdesc->desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+  /* Initialize the intro points map. */
+  sdesc->intro_points.map = digest256map_new();
+  sdesc->intro_points.failed_id = digestmap_new();
+  sdesc->previous_hsdirs = smartlist_new();
+  return sdesc;
+}
+
+/* Move descriptor(s) from the src service to the dst service. We do this
+ * during SIGHUP when we re-create our hidden services. */
+static void
+move_descriptors(hs_service_t *src, hs_service_t *dst)
+{
+  tor_assert(src);
+  tor_assert(dst);
+
+  if (src->desc_current) {
+    /* Nothing should be there, but clean it up just in case */
+    if (BUG(dst->desc_current)) {
+      service_descriptor_free(dst->desc_current);
+    }
+    dst->desc_current = src->desc_current;
+    src->desc_current = NULL;
+  }
+
+  if (src->desc_next) {
+    /* Nothing should be there, but clean it up just in case */
+    if (BUG(dst->desc_next)) {
+      service_descriptor_free(dst->desc_next);
+    }
+    dst->desc_next = src->desc_next;
+    src->desc_next = NULL;
+  }
+}
+
+/* From the given service, remove all expired failing intro points for each
+ * descriptor. */
+static void
+remove_expired_failing_intro(hs_service_t *service, time_t now)
+{
+  tor_assert(service);
+
+  /* For both descriptors, cleanup the failing intro points list. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    DIGESTMAP_FOREACH_MODIFY(desc->intro_points.failed_id, key, time_t *, t) {
+      time_t failure_time = *t;
+      if ((failure_time + INTRO_CIRC_RETRY_PERIOD) <= now) {
+        MAP_DEL_CURRENT(key);
+        tor_free(t);
+      }
+    } DIGESTMAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* For the given descriptor desc, put all node_t object found from its failing
+ * intro point list and put them in the given node_list. */
+static void
+setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
+                               smartlist_t *node_list)
+{
+  tor_assert(desc);
+  tor_assert(node_list);
+
+  DIGESTMAP_FOREACH(desc->intro_points.failed_id, key, time_t *, t) {
+    (void) t; /* Make gcc happy. */
+    const node_t *node = node_get_by_id(key);
+    if (node) {
+      smartlist_add(node_list, (void *) node);
+    }
+  } DIGESTMAP_FOREACH_END;
+}
+
+/* For the given failing intro point ip, we add its time of failure to the
+ * failed map and index it by identity digest (legacy ID) in the descriptor
+ * desc failed id map. */
+static void
+remember_failing_intro_point(const hs_service_intro_point_t *ip,
+                             hs_service_descriptor_t *desc, time_t now)
+{
+  time_t *time_of_failure, *prev_ptr;
+  const hs_desc_link_specifier_t *legacy_ls;
+
+  tor_assert(ip);
+  tor_assert(desc);
+
+  time_of_failure = tor_malloc_zero(sizeof(time_t));
+  *time_of_failure = now;
+  legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
+  tor_assert(legacy_ls);
+  prev_ptr = digestmap_set(desc->intro_points.failed_id,
+                           (const char *) legacy_ls->u.legacy_id,
+                           time_of_failure);
+  tor_free(prev_ptr);
+}
+
+/* Copy the descriptor link specifier object from src to dst. */
+static void
+link_specifier_copy(hs_desc_link_specifier_t *dst,
+                    const hs_desc_link_specifier_t *src)
+{
+  tor_assert(dst);
+  tor_assert(src);
+  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
+}
+
+/* Using a given descriptor signing keypair signing_kp, a service intro point
+ * object ip and the time now, setup the content of an already allocated
+ * descriptor intro desc_ip.
+ *
+ * Return 0 on success else a negative value. */
+static int
+setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
+                       const hs_service_intro_point_t *ip,
+                       time_t now, hs_desc_intro_point_t *desc_ip)
+{
+  int ret = -1;
+  time_t nearest_hour = now - (now % 3600);
+
+  tor_assert(signing_kp);
+  tor_assert(ip);
+  tor_assert(desc_ip);
+
+  /* Copy the onion key. */
+  memcpy(&desc_ip->onion_key, &ip->onion_key, sizeof(desc_ip->onion_key));
+
+  /* Key and certificate material. */
+  desc_ip->auth_key_cert = tor_cert_create(signing_kp,
+                                           CERT_TYPE_AUTH_HS_IP_KEY,
+                                           &ip->auth_key_kp.pubkey,
+                                           nearest_hour,
+                                           HS_DESC_CERT_LIFETIME,
+                                           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  if (desc_ip->auth_key_cert == NULL) {
+    log_warn(LD_REND, "Unable to create intro point auth-key certificate");
+    goto done;
+  }
+
+  /* Copy link specifier(s). */
+  SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                          const hs_desc_link_specifier_t *, ls) {
+    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
+    link_specifier_copy(copy, ls);
+    smartlist_add(desc_ip->link_specifiers, copy);
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* For a legacy intro point, we'll use an RSA/ed cross certificate. */
+  if (ip->base.is_only_legacy) {
+    desc_ip->legacy.key = crypto_pk_dup_key(ip->legacy_key);
+    /* Create cross certification cert. */
+    ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
+                                    &signing_kp->pubkey,
+                                    desc_ip->legacy.key,
+                                    nearest_hour + HS_DESC_CERT_LIFETIME,
+                                    &desc_ip->legacy.cert.encoded);
+    if (cert_len < 0) {
+      log_warn(LD_REND, "Unable to create enc key legacy cross cert.");
+      goto done;
+    }
+    desc_ip->legacy.cert.len = cert_len;
+  }
+
+  /* Encryption key and its cross certificate. */
+  {
+    ed25519_public_key_t ed25519_pubkey;
+
+    /* Use the public curve25519 key. */
+    memcpy(&desc_ip->enc_key, &ip->enc_key_kp.pubkey,
+           sizeof(desc_ip->enc_key));
+    /* The following can't fail. */
+    ed25519_public_key_from_curve25519_public_key(&ed25519_pubkey,
+                                                  &ip->enc_key_kp.pubkey,
+                                                  0);
+    desc_ip->enc_key_cert = tor_cert_create(signing_kp,
+                                            CERT_TYPE_CROSS_HS_IP_KEYS,
+                                            &ed25519_pubkey, nearest_hour,
+                                            HS_DESC_CERT_LIFETIME,
+                                            CERT_FLAG_INCLUDE_SIGNING_KEY);
+    if (desc_ip->enc_key_cert == NULL) {
+      log_warn(LD_REND, "Unable to create enc key curve25519 cross cert.");
+      goto done;
+    }
+  }
+  /* Success. */
+  ret = 0;
+
+ done:
+  return ret;
+}
+
+/* Using the given descriptor from the given service, build the descriptor
+ * intro point list so we can then encode the descriptor for publication. This
+ * function does not pick intro points, they have to be in the descriptor
+ * current map. Cryptographic material (keys) must be initialized in the
+ * descriptor for this function to make sense. */
+static void
+build_desc_intro_points(const hs_service_t *service,
+                        hs_service_descriptor_t *desc, time_t now)
+{
+  hs_desc_encrypted_data_t *encrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Ease our life. */
+  encrypted = &desc->desc->encrypted_data;
+  /* Cleanup intro points, we are about to set them from scratch. */
+  hs_descriptor_clear_intro_points(desc->desc);
+
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       const hs_service_intro_point_t *, ip) {
+    hs_desc_intro_point_t *desc_ip = hs_desc_intro_point_new();
+    if (setup_desc_intro_point(&desc->signing_kp, ip, now, desc_ip) < 0) {
+      hs_desc_intro_point_free(desc_ip);
+      continue;
+    }
+    /* We have a valid descriptor intro point. Add it to the list. */
+    smartlist_add(encrypted->intro_points, desc_ip);
+  } DIGEST256MAP_FOREACH_END;
+}
+
+/* Populate the descriptor encrypted section from the given service object.
+ * This will generate a valid list of introduction points that can be used
+ * after for circuit creation. Return 0 on success else -1 on error. */
+static int
+build_service_desc_encrypted(const hs_service_t *service,
+                             hs_service_descriptor_t *desc)
+{
+  hs_desc_encrypted_data_t *encrypted;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  encrypted = &desc->desc->encrypted_data;
+
+  encrypted->create2_ntor = 1;
+  encrypted->single_onion_service = service->config.is_single_onion;
+
+  /* Setup introduction points from what we have in the service. */
+  if (encrypted->intro_points == NULL) {
+    encrypted->intro_points = smartlist_new();
+  }
+  /* We do NOT build introduction point yet, we only do that once the circuit
+   * have been opened. Until we have the right number of introduction points,
+   * we do not encode anything in the descriptor. */
+
+  /* XXX: Support client authorization (#20700). */
+  encrypted->intro_auth_types = NULL;
+  return 0;
+}
+
+/* Populate the descriptor plaintext section from the given service object.
+ * The caller must make sure that the keys in the descriptors are valid that
+ * is are non-zero. Return 0 on success else -1 on error. */
+static int
+build_service_desc_plaintext(const hs_service_t *service,
+                             hs_service_descriptor_t *desc, time_t now)
+{
+  int ret = -1;
+  hs_desc_plaintext_data_t *plaintext;
+
+  tor_assert(service);
+  tor_assert(desc);
+  /* XXX: Use a "assert_desc_ok()" ? */
+  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+                              sizeof(desc->blinded_kp)));
+  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+                              sizeof(desc->signing_kp)));
+
+  /* Set the subcredential. */
+  hs_get_subcredential(&service->keys.identity_pk, &desc->blinded_kp.pubkey,
+                       desc->desc->subcredential);
+
+  plaintext = &desc->desc->plaintext_data;
+
+  plaintext->version = service->config.version;
+  plaintext->lifetime_sec = HS_DESC_DEFAULT_LIFETIME;
+  plaintext->signing_key_cert =
+    tor_cert_create(&desc->blinded_kp, CERT_TYPE_SIGNING_HS_DESC,
+                    &desc->signing_kp.pubkey, now, HS_DESC_CERT_LIFETIME,
+                    CERT_FLAG_INCLUDE_SIGNING_KEY);
+  if (plaintext->signing_key_cert == NULL) {
+    log_warn(LD_REND, "Unable to create descriptor signing certificate for "
+                      "service %s",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+  /* Copy public key material to go in the descriptor. */
+  ed25519_pubkey_copy(&plaintext->signing_pubkey, &desc->signing_kp.pubkey);
+  ed25519_pubkey_copy(&plaintext->blinded_pubkey, &desc->blinded_kp.pubkey);
+  /* Success. */
+  ret = 0;
+
+ end:
+  return ret;
+}
+
+/* For the given service and descriptor object, create the key material which
+ * is the blinded keypair and the descriptor signing keypair. Return 0 on
+ * success else -1 on error where the generated keys MUST be ignored. */
+static int
+build_service_desc_keys(const hs_service_t *service,
+                        hs_service_descriptor_t *desc,
+                        uint64_t time_period_num)
+{
+  int ret = 0;
+  ed25519_keypair_t kp;
+
+  tor_assert(desc);
+  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+             ED25519_PUBKEY_LEN));
+
+  /* XXX: Support offline key feature (#18098). */
+
+  /* Copy the identity keys to the keypair so we can use it to create the
+   * blinded key. */
+  memcpy(&kp.pubkey, &service->keys.identity_pk, sizeof(kp.pubkey));
+  memcpy(&kp.seckey, &service->keys.identity_sk, sizeof(kp.seckey));
+  /* Build blinded keypair for this time period. */
+  hs_build_blinded_keypair(&kp, NULL, 0, time_period_num, &desc->blinded_kp);
+  /* Let's not keep too much traces of our keys in memory. */
+  memwipe(&kp, 0, sizeof(kp));
+
+  /* No need for extra strong, this is a temporary key only for this
+   * descriptor. Nothing long term. */
+  if (ed25519_keypair_generate(&desc->signing_kp, 0) < 0) {
+    log_warn(LD_REND, "Can't generate descriptor signing keypair for "
+                      "service %s",
+             safe_str_client(service->onion_address));
+    ret = -1;
+  }
+
+  return ret;
+}
+
+/* Given a service and the current time, build a descriptor for the service.
+ * This function does not pick introduction point, this needs to be done by
+ * the update function. On success, desc_out will point to the newly allocated
+ * descriptor object.
+ *
+ * This can error if we are unable to create keys or certificate. */
+static void
+build_service_descriptor(hs_service_t *service, time_t now,
+                         uint64_t time_period_num,
+                         hs_service_descriptor_t **desc_out)
+{
+  char *encoded_desc;
+  hs_service_descriptor_t *desc;
+
+  tor_assert(service);
+  tor_assert(desc_out);
+
+  desc = service_descriptor_new();
+  desc->time_period_num = time_period_num;
+
+  /* Create the needed keys so we can setup the descriptor content. */
+  if (build_service_desc_keys(service, desc, time_period_num) < 0) {
+    goto err;
+  }
+  /* Setup plaintext descriptor content. */
+  if (build_service_desc_plaintext(service, desc, now) < 0) {
+    goto err;
+  }
+  /* Setup encrypted descriptor content. */
+  if (build_service_desc_encrypted(service, desc) < 0) {
+    goto err;
+  }
+
+  /* Set the revision counter for this descriptor */
+  set_descriptor_revision_counter(desc->desc);
+
+  /* Let's make sure that we've created a descriptor that can actually be
+   * encoded properly. This function also checks if the encoded output is
+   * decodable after. */
+  if (BUG(hs_desc_encode_descriptor(desc->desc, &desc->signing_kp,
+                                    &encoded_desc) < 0)) {
+    goto err;
+  }
+  tor_free(encoded_desc);
+
+  /* Assign newly built descriptor to the next slot. */
+  *desc_out = desc;
+  /* Fire a CREATED control port event. */
+  hs_control_desc_event_created(service->onion_address,
+                                &desc->blinded_kp.pubkey);
+  return;
+
+ err:
+  service_descriptor_free(desc);
+}
+
+/* Build both descriptors for the given service that has just booted up.
+ * Because it's a special case, it deserves its special function ;). */
+static void
+build_descriptors_for_new_service(hs_service_t *service, time_t now)
+{
+  uint64_t current_desc_tp, next_desc_tp;
+
+  tor_assert(service);
+  /* These are the conditions for a new service. */
+  tor_assert(!service->desc_current);
+  tor_assert(!service->desc_next);
+
+  /*
+   * +------------------------------------------------------------------+
+   * |                                                                  |
+   * | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   * | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   * |                                                                  |
+   * |  $==========|-----------$===========|-----------$===========|    |
+   * |                             ^         ^                          |
+   * |                             A         B                          |
+   * +------------------------------------------------------------------+
+   *
+   * Case A: The service boots up before a new time period, the current time
+   * period is thus TP#1 and the next is TP#2 which for both we have access to
+   * their SRVs.
+   *
+   * Case B: The service boots up inside TP#2, we can't use the TP#3 for the
+   * next descriptor because we don't have the SRV#3 so the current should be
+   * TP#1 and next TP#2.
+   */
+
+  if (hs_in_period_between_tp_and_srv(NULL, now)) {
+    /* Case B from the above, inside of the new time period. */
+    current_desc_tp = hs_get_previous_time_period_num(0); /* TP#1 */
+    next_desc_tp = hs_get_time_period_num(0);             /* TP#2 */
+  } else {
+    /* Case A from the above, outside of the new time period. */
+    current_desc_tp = hs_get_time_period_num(0);    /* TP#1 */
+    next_desc_tp = hs_get_next_time_period_num(0);  /* TP#2 */
+  }
+
+  /* Build descriptors. */
+  build_service_descriptor(service, now, current_desc_tp,
+                           &service->desc_current);
+  build_service_descriptor(service, now, next_desc_tp,
+                           &service->desc_next);
+  log_info(LD_REND, "Hidden service %s has just started. Both descriptors "
+                    "built. Now scheduled for upload.",
+           safe_str_client(service->onion_address));
+}
+
+/* Build descriptors for each service if needed. There are conditions to build
+ * a descriptor which are details in the function. */
+STATIC void
+build_all_descriptors(time_t now)
+{
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* A service booting up will have both descriptors to NULL. No other cases
+     * makes both descriptor non existent. */
+    if (service->desc_current == NULL && service->desc_next == NULL) {
+      build_descriptors_for_new_service(service, now);
+      continue;
+    }
+
+    /* Reaching this point means we are pass bootup so at runtime. We should
+     * *never* have an empty current descriptor. If the next descriptor is
+     * empty, we'll try to build it for the next time period. This only
+     * happens when we rotate meaning that we are guaranteed to have a new SRV
+     * at that point for the next time period. */
+    if (BUG(service->desc_current == NULL)) {
+      continue;
+    }
+
+    if (service->desc_next == NULL) {
+      build_service_descriptor(service, now, hs_get_next_time_period_num(0),
+                               &service->desc_next);
+      log_info(LD_REND, "Hidden service %s next descriptor successfully "
+                        "built. Now scheduled for upload.",
+               safe_str_client(service->onion_address));
+    }
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Randomly pick a node to become an introduction point but not present in the
+ * given exclude_nodes list. The chosen node is put in the exclude list
+ * regardless of success or not because in case of failure, the node is simply
+ * unsusable from that point on.
+ *
+ * If direct_conn is set, try to pick a node that our local firewall/policy
+ * allows us to connect to directly. If we can't find any, return NULL.
+ * This function supports selecting dual-stack nodes for direct single onion
+ * service IPv6 connections. But it does not send IPv6 addresses in link
+ * specifiers. (Current clients don't use IPv6 addresses to extend, and
+ * direct client connections to intro points are not supported.)
+ *
+ * Return a newly allocated service intro point ready to be used for encoding.
+ * Return NULL on error. */
+static hs_service_intro_point_t *
+pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
+{
+  const node_t *node;
+  extend_info_t *info = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  /* Normal 3-hop introduction point flags. */
+  router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
+  /* Single onion flags. */
+  router_crn_flags_t direct_flags = flags | CRN_PREF_ADDR | CRN_DIRECT_CONN;
+
+  node = router_choose_random_node(exclude_nodes, get_options()->ExcludeNodes,
+                                   direct_conn ? direct_flags : flags);
+  /* Unable to find a node. When looking for a node for a direct connection,
+   * we could try a 3-hop path instead. We'll add support for this in a later
+   * release. */
+  if (!node) {
+    goto err;
+  }
+
+  /* We have a suitable node, add it to the exclude list. We do this *before*
+   * we can validate the extend information because even in case of failure,
+   * we don't want to use that node anymore. */
+  smartlist_add(exclude_nodes, (void *) node);
+
+  /* We do this to ease our life but also this call makes appropriate checks
+   * of the node object such as validating ntor support for instance.
+   *
+   * We must provide an extend_info for clients to connect over a 3-hop path,
+   * so we don't pass direct_conn here. */
+  info = extend_info_from_node(node, 0);
+  if (BUG(info == NULL)) {
+    goto err;
+  }
+
+  /* Let's do a basic sanity check here so that we don't end up advertising the
+   * ed25519 identity key of relays that don't actually support the link
+   * protocol */
+  if (!node_supports_ed25519_link_authentication(node, 0)) {
+    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
+  } else {
+    /* Make sure we *do* have an ed key if we support the link authentication.
+     * Sending an empty key would result in a failure to extend. */
+    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
+  }
+
+  /* Create our objects and populate them with the node information.
+   * We don't care if the intro's link auth is compatible with us, because
+   * we are sending the ed25519 key to a remote client via the descriptor. */
+  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node),
+                               node_supports_ed25519_link_authentication(node,
+                                                                         0));
+  if (ip == NULL) {
+    goto err;
+  }
+
+  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
+  extend_info_free(info);
+  return ip;
+ err:
+  service_intro_point_free(ip);
+  extend_info_free(info);
+  return NULL;
+}
+
+/* For a given descriptor from the given service, pick any needed intro points
+ * and update the current map with those newly picked intro points. Return the
+ * number node that might have been added to the descriptor current map. */
+static unsigned int
+pick_needed_intro_points(hs_service_t *service,
+                         hs_service_descriptor_t *desc)
+{
+  int i = 0, num_needed_ip;
+  smartlist_t *exclude_nodes = smartlist_new();
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* Compute how many intro points we actually need to open. */
+  num_needed_ip = service->config.num_intro_points -
+                  digest256map_size(desc->intro_points.map);
+  if (BUG(num_needed_ip < 0)) {
+    /* Let's not make tor freak out here and just skip this. */
+    goto done;
+  }
+
+  /* We want to end up with config.num_intro_points intro points, but if we
+   * have no intro points at all (chances are they all cycled or we are
+   * starting up), we launch get_intro_point_num_extra() extra circuits and
+   * use the first config.num_intro_points that complete. See proposal #155,
+   * section 4 for the rationale of this which is purely for performance.
+   *
+   * The ones after the first config.num_intro_points will be converted to
+   * 'General' internal circuits and then we'll drop them from the list of
+   * intro points. */
+  if (digest256map_size(desc->intro_points.map) == 0) {
+    num_needed_ip += get_intro_point_num_extra();
+  }
+
+  /* Build an exclude list of nodes of our intro point(s). The expiring intro
+   * points are OK to pick again because this is afterall a concept of round
+   * robin so they are considered valid nodes to pick again. */
+  DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                       hs_service_intro_point_t *, ip) {
+    const node_t *intro_node = get_node_from_intro_point(ip);
+    if (intro_node) {
+      smartlist_add(exclude_nodes, (void*)intro_node);
+    }
+  } DIGEST256MAP_FOREACH_END;
+  /* Also, add the failing intro points that our descriptor encounteered in
+   * the exclude node list. */
+  setup_intro_point_exclude_list(desc, exclude_nodes);
+
+  for (i = 0; i < num_needed_ip; i++) {
+    hs_service_intro_point_t *ip;
+
+    /* This function will add the picked intro point node to the exclude nodes
+     * list so we don't pick the same one at the next iteration. */
+    ip = pick_intro_point(service->config.is_single_onion, exclude_nodes);
+    if (ip == NULL) {
+      /* If we end up unable to pick an introduction point it is because we
+       * can't find suitable node and calling this again is highly unlikely to
+       * give us a valid node all of the sudden. */
+      log_info(LD_REND, "Unable to find a suitable node to be an "
+                        "introduction point for service %s.",
+               safe_str_client(service->onion_address));
+      goto done;
+    }
+    /* Valid intro point object, add it to the descriptor current map. */
+    service_intro_point_add(desc->intro_points.map, ip);
+  }
+  /* We've successfully picked all our needed intro points thus none are
+   * missing which will tell our upload process to expect the number of
+   * circuits to be the number of configured intro points circuits and not the
+   * number of intro points object that we have. */
+  desc->missing_intro_points = 0;
+
+  /* Success. */
+ done:
+  /* We don't have ownership of the node_t object in this list. */
+  smartlist_free(exclude_nodes);
+  return i;
+}
+
+/** Clear previous cached HSDirs in <b>desc</b>. */
+static void
+service_desc_clear_previous_hsdirs(hs_service_descriptor_t *desc)
+{
+  if (BUG(!desc->previous_hsdirs)) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(desc->previous_hsdirs, char*, s, tor_free(s));
+  smartlist_clear(desc->previous_hsdirs);
+}
+
+/** Note that we attempted to upload <b>desc</b> to <b>hsdir</b>. */
+static void
+service_desc_note_upload(hs_service_descriptor_t *desc, const node_t *hsdir)
+{
+  char b64_digest[BASE64_DIGEST_LEN+1] = {0};
+  digest_to_base64(b64_digest, hsdir->identity);
+
+  if (BUG(!desc->previous_hsdirs)) {
+    return;
+  }
+
+  if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) {
+    smartlist_add_strdup(desc->previous_hsdirs, b64_digest);
+  }
+}
+
+/** Schedule an upload of <b>desc</b>. If <b>descriptor_changed</b> is set, it
+ *  means that this descriptor is dirty. */
+STATIC void
+service_desc_schedule_upload(hs_service_descriptor_t *desc,
+                             time_t now,
+                             int descriptor_changed)
+
+{
+  desc->next_upload_time = now;
+
+  /* If the descriptor changed, clean up the old HSDirs list. We want to
+   * re-upload no matter what. */
+  if (descriptor_changed) {
+    service_desc_clear_previous_hsdirs(desc);
+  }
+}
+
+/* Update the given descriptor from the given service. The possible update
+ * actions includes:
+ *    - Picking missing intro points if needed.
+ *    - Incrementing the revision counter if needed.
+ */
+static void
+update_service_descriptor(hs_service_t *service,
+                          hs_service_descriptor_t *desc, time_t now)
+{
+  unsigned int num_intro_points;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(desc->desc);
+
+  num_intro_points = digest256map_size(desc->intro_points.map);
+
+  /* Pick any missing introduction point(s). */
+  if (num_intro_points < service->config.num_intro_points) {
+    unsigned int num_new_intro_points = pick_needed_intro_points(service,
+                                                                 desc);
+    if (num_new_intro_points != 0) {
+      log_info(LD_REND, "Service %s just picked %u intro points and wanted "
+                        "%u for %s descriptor. It currently has %d intro "
+                        "points. Launching ESTABLISH_INTRO circuit shortly.",
+               safe_str_client(service->onion_address),
+               num_new_intro_points,
+               service->config.num_intro_points - num_intro_points,
+               (desc == service->desc_current) ? "current" : "next",
+               num_intro_points);
+      /* We'll build those introduction point into the descriptor once we have
+       * confirmation that the circuits are opened and ready. However,
+       * indicate that this descriptor should be uploaded from now on. */
+      service_desc_schedule_upload(desc, now, 1);
+    }
+    /* Were we able to pick all the intro points we needed? If not, we'll
+     * flag the descriptor that it's missing intro points because it
+     * couldn't pick enough which will trigger a descriptor upload. */
+    if ((num_new_intro_points + num_intro_points) <
+        service->config.num_intro_points) {
+      desc->missing_intro_points = 1;
+    }
+  }
+}
+
+/* Update descriptors for each service if needed. */
+STATIC void
+update_all_descriptors(time_t now)
+{
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* We'll try to update each descriptor that is if certain conditions apply
+     * in order for the descriptor to be updated. */
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      update_service_descriptor(service, desc, now);
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Return true iff the given intro point has expired that is it has been used
+ * for too long or we've reached our max seen INTRODUCE2 cell. */
+STATIC int
+intro_point_should_expire(const hs_service_intro_point_t *ip,
+                          time_t now)
+{
+  tor_assert(ip);
+
+  if (ip->introduce2_count >= ip->introduce2_max) {
+    goto expired;
+  }
+
+  if (ip->time_to_expire <= now) {
+    goto expired;
+  }
+
+  /* Not expiring. */
+  return 0;
+ expired:
+  return 1;
+}
+
+/* Go over the given set of intro points for each service and remove any
+ * invalid ones. The conditions for removal are:
+ *
+ *    - The node doesn't exists anymore (not in consensus)
+ *                          OR
+ *    - The intro point maximum circuit retry count has been reached and no
+ *      circuit can be found associated with it.
+ *                          OR
+ *    - The intro point has expired and we should pick a new one.
+ *
+ * If an intro point is removed, the circuit (if any) is immediately close.
+ * If a circuit can't be found, the intro point is kept if it hasn't reached
+ * its maximum circuit retry value and thus should be retried.  */
+static void
+cleanup_intro_points(hs_service_t *service, time_t now)
+{
+  /* List of intro points to close. We can't mark the intro circuits for close
+   * in the modify loop because doing so calls
+   * hs_service_intro_circ_has_closed() which does a digest256map_get() on the
+   * intro points map (that we are iterating over). This can't be done in a
+   * single iteration after a MAP_DEL_CURRENT, the object will still be
+   * returned leading to a use-after-free. So, we close the circuits and free
+   * the intro points after the loop if any. */
+  smartlist_t *ips_to_free = smartlist_new();
+
+  tor_assert(service);
+
+  /* For both descriptors, cleanup the intro points. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* Go over the current intro points we have, make sure they are still
+     * valid and remove any of them that aren't. */
+    DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
+                                hs_service_intro_point_t *, ip) {
+      const node_t *node = get_node_from_intro_point(ip);
+      int has_expired = intro_point_should_expire(ip, now);
+
+      /* We cleanup an intro point if it has expired or if we do not know the
+       * node_t anymore (removed from our latest consensus) or if we've
+       * reached the maximum number of retry with a non existing circuit. */
+      if (has_expired || node == NULL ||
+          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                          "Removing it.",
+                 describe_intro_point(ip),
+                 has_expired ? " has expired" :
+                    (node == NULL) ?  " fell off the consensus" : "",
+                 ip->circuit_retries);
+
+        /* We've retried too many times, remember it as a failed intro point
+         * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
+        if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
+          remember_failing_intro_point(ip, desc, approx_time());
+        }
+
+        /* Remove intro point from descriptor map and add it to the list of
+         * ips to free for which we'll also try to close the intro circuit. */
+        MAP_DEL_CURRENT(key);
+        smartlist_add(ips_to_free, ip);
+      }
+    } DIGEST256MAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+
+  /* Go over the intro points to free and close their circuit if any. */
+  SMARTLIST_FOREACH_BEGIN(ips_to_free, hs_service_intro_point_t *, ip) {
+    /* See if we need to close the intro point circuit as well */
+
+    /* XXX: Legacy code does NOT close circuits like this: it keeps the circuit
+     * open until a new descriptor is uploaded and then closed all expiring
+     * intro point circuit. Here, we close immediately and because we just
+     * discarded the intro point, a new one will be selected, a new descriptor
+     * created and uploaded. There is no difference to an attacker between the
+     * timing of a new consensus and intro point rotation (possibly?). */
+    origin_circuit_t *ocirc = hs_circ_service_get_intro_circ(ip);
+    if (ocirc && !TO_CIRCUIT(ocirc)->marked_for_close) {
+      circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+    }
+
+    /* Cleanup the intro point */
+    service_intro_point_free(ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  smartlist_free(ips_to_free);
+}
+
+/* Set the next rotation time of the descriptors for the given service for the
+ * time now. */
+static void
+set_rotation_time(hs_service_t *service, time_t now)
+{
+  time_t valid_after;
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  if (ns) {
+    valid_after = ns->valid_after;
+  } else {
+    valid_after = now;
+  }
+
+  tor_assert(service);
+  service->state.next_rotation_time =
+    sr_state_get_start_time_of_current_protocol_run(valid_after) +
+    sr_state_get_protocol_run_duration();
+
+  {
+    char fmt_time[ISO_TIME_LEN + 1];
+    format_local_iso_time(fmt_time, service->state.next_rotation_time);
+    log_info(LD_REND, "Next descriptor rotation time set to %s for %s",
+             fmt_time, safe_str_client(service->onion_address));
+  }
+}
+
+/* Return true iff the service should rotate its descriptor. The time now is
+ * only used to fetch the live consensus and if none can be found, this
+ * returns false. */
+static unsigned int
+should_rotate_descriptors(hs_service_t *service, time_t now)
+{
+  const networkstatus_t *ns;
+
+  tor_assert(service);
+
+  ns = networkstatus_get_live_consensus(now);
+  if (ns == NULL) {
+    goto no_rotation;
+  }
+
+  if (ns->valid_after >= service->state.next_rotation_time) {
+    /* In theory, we should never get here with no descriptors. We can never
+     * have a NULL current descriptor except when tor starts up. The next
+     * descriptor can be NULL after a rotation but we build a new one right
+     * after.
+     *
+     * So, when tor starts, the next rotation time is set to the start of the
+     * next SRV period using the consensus valid after time so it should
+     * always be set to a future time value. This means that we should never
+     * reach this point at bootup that is this check safeguards tor in never
+     * allowing a rotation if the valid after time is smaller than the next
+     * rotation time.
+     *
+     * This is all good in theory but we've had a NULL descriptor issue here
+     * so this is why we BUG() on both with extra logging to try to understand
+     * how this can possibly happens. We'll simply ignore and tor should
+     * recover from this by skipping rotation and building the missing
+     * descriptors just after this. */
+    if (BUG(service->desc_current == NULL || service->desc_next == NULL)) {
+      log_warn(LD_BUG, "Service descriptor is NULL (%p/%p). Next rotation "
+                       "time is %ld (now: %ld). Valid after time from "
+                       "consensus is %ld",
+               service->desc_current, service->desc_next,
+               (long)service->state.next_rotation_time,
+               (long)now,
+               (long)ns->valid_after);
+      goto no_rotation;
+    }
+    goto rotation;
+  }
+
+ no_rotation:
+  return 0;
+ rotation:
+  return 1;
+}
+
+/* Rotate the service descriptors of the given service. The current descriptor
+ * will be freed, the next one put in as the current and finally the next
+ * descriptor pointer is NULLified. */
+static void
+rotate_service_descriptors(hs_service_t *service, time_t now)
+{
+  if (service->desc_current) {
+    /* Close all IP circuits for the descriptor. */
+    close_intro_circuits(&service->desc_current->intro_points);
+    /* We don't need this one anymore, we won't serve any clients coming with
+     * this service descriptor. */
+    service_descriptor_free(service->desc_current);
+  }
+  /* The next one become the current one and emptying the next will trigger
+   * a descriptor creation for it. */
+  service->desc_current = service->desc_next;
+  service->desc_next = NULL;
+
+  /* We've just rotated, set the next time for the rotation. */
+  set_rotation_time(service, now);
+}
+
+/* Rotate descriptors for each service if needed. A non existing current
+ * descriptor will trigger a descriptor build for the next time period. */
+STATIC void
+rotate_all_descriptors(time_t now)
+{
+  /* XXX We rotate all our service descriptors at once. In the future it might
+   *     be wise, to rotate service descriptors independently to hide that all
+   *     those descriptors are on the same tor instance */
+
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* Note for a service booting up: Both descriptors are NULL in that case
+     * so this function might return true if we are in the timeframe for a
+     * rotation leading to basically swapping two NULL pointers which is
+     * harmless. However, the side effect is that triggering a rotation will
+     * update the service state and avoid doing anymore rotations after the
+     * two descriptors have been built. */
+    if (!should_rotate_descriptors(service, now)) {
+      continue;
+    }
+
+    log_info(LD_REND, "Time to rotate our descriptors (%p / %p) for %s",
+             service->desc_current, service->desc_next,
+             safe_str_client(service->onion_address));
+
+    rotate_service_descriptors(service, now);
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Scheduled event run from the main loop. Make sure all our services are up
+ * to date and ready for the other scheduled events. This includes looking at
+ * the introduction points status and descriptor rotation time. */
+STATIC void
+run_housekeeping_event(time_t now)
+{
+  /* Note that nothing here opens circuit(s) nor uploads descriptor(s). We are
+   * simply moving things around or removing unneeded elements. */
+
+  FOR_EACH_SERVICE_BEGIN(service) {
+
+    /* If the service is starting off, set the rotation time. We can't do that
+     * at configure time because the get_options() needs to be set for setting
+     * that time that uses the voting interval. */
+    if (service->state.next_rotation_time == 0) {
+      /* Set the next rotation time of the descriptors. If it's Oct 25th
+       * 23:47:00, the next rotation time is when the next SRV is computed
+       * which is at Oct 26th 00:00:00 that is in 13 minutes. */
+      set_rotation_time(service, now);
+    }
+
+    /* Cleanup invalid intro points from the service descriptor. */
+    cleanup_intro_points(service, now);
+
+    /* Remove expired failing intro point from the descriptor failed list. We
+     * reset them at each INTRO_CIRC_RETRY_PERIOD. */
+    remove_expired_failing_intro(service, now);
+
+    /* At this point, the service is now ready to go through the scheduled
+     * events guaranteeing a valid state. Intro points might be missing from
+     * the descriptors after the cleanup but the update/build process will
+     * make sure we pick those missing ones. */
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Scheduled event run from the main loop. Make sure all descriptors are up to
+ * date. Once this returns, each service descriptor needs to be considered for
+ * new introduction circuits and then for upload. */
+static void
+run_build_descriptor_event(time_t now)
+{
+  /* For v2 services, this step happens in the upload event. */
+
+  /* Run v3+ events. */
+  /* We start by rotating the descriptors only if needed. */
+  rotate_all_descriptors(now);
+
+  /* Then, we'll try to build  new descriptors that we might need. The
+   * condition is that the next descriptor is non existing because it has
+   * been rotated or we just started up. */
+  build_all_descriptors(now);
+
+  /* Finally, we'll check if we should update the descriptors. Missing
+   * introduction points will be picked in this function which is useful for
+   * newly built descriptors. */
+  update_all_descriptors(now);
+}
+
+/* For the given service, launch any intro point circuits that could be
+ * needed. This considers every descriptor of the service. */
+static void
+launch_intro_point_circuits(hs_service_t *service)
+{
+  tor_assert(service);
+
+  /* For both descriptors, try to launch any missing introduction point
+   * circuits using the current map. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    /* Keep a ref on if we need a direct connection. We use this often. */
+    unsigned int direct_conn = service->config.is_single_onion;
+
+    DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
+                                hs_service_intro_point_t *, ip) {
+      extend_info_t *ei;
+
+      /* Skip the intro point that already has an existing circuit
+       * (established or not). */
+      if (hs_circ_service_get_intro_circ(ip)) {
+        continue;
+      }
+
+      ei = get_extend_info_from_intro_point(ip, direct_conn);
+      if (ei == NULL) {
+        /* This is possible if we can get a node_t but not the extend info out
+         * of it. In this case, we remove the intro point and a new one will
+         * be picked at the next main loop callback. */
+        MAP_DEL_CURRENT(key);
+        service_intro_point_free(ip);
+        continue;
+      }
+
+      /* Launch a circuit to the intro point. */
+      ip->circuit_retries++;
+      if (hs_circ_launch_intro_point(service, ip, ei) < 0) {
+        log_info(LD_REND, "Unable to launch intro circuit to node %s "
+                          "for service %s.",
+                 safe_str_client(extend_info_describe(ei)),
+                 safe_str_client(service->onion_address));
+        /* Intro point will be retried if possible after this. */
+      }
+      extend_info_free(ei);
+    } DIGEST256MAP_FOREACH_END;
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Don't try to build more than this many circuits before giving up for a
+ * while. Dynamically calculated based on the configured number of intro
+ * points for the given service and how many descriptor exists. The default
+ * use case of 3 introduction points and two descriptors will allow 28
+ * circuits for a retry period (((3 + 2) + (3 * 3)) * 2). */
+static unsigned int
+get_max_intro_circ_per_period(const hs_service_t *service)
+{
+  unsigned int count = 0;
+  unsigned int multiplier = 0;
+  unsigned int num_wanted_ip;
+
+  tor_assert(service);
+  tor_assert(service->config.num_intro_points <=
+             HS_CONFIG_V3_MAX_INTRO_POINTS);
+
+/* For a testing network, allow to do it for the maximum amount so circuit
+ * creation and rotation and so on can actually be tested without limit. */
+#define MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING -1
+  if (get_options()->TestingTorNetwork) {
+    return MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING;
+  }
+
+  num_wanted_ip = service->config.num_intro_points;
+
+  /* The calculation is as follow. We have a number of intro points that we
+   * want configured as a torrc option (num_intro_points). We then add an
+   * extra value so we can launch multiple circuits at once and pick the
+   * quickest ones. For instance, we want 3 intros, we add 2 extra so we'll
+   * pick 5 intros and launch 5 circuits. */
+  count += (num_wanted_ip + get_intro_point_num_extra());
+
+  /* Then we add the number of retries that is possible to do for each intro
+   * point. If we want 3 intros, we'll allow 3 times the number of possible
+   * retry. */
+  count += (num_wanted_ip * MAX_INTRO_POINT_CIRCUIT_RETRIES);
+
+  /* Then, we multiply by a factor of 2 if we have both descriptor or 0 if we
+   * have none.  */
+  multiplier += (service->desc_current) ? 1 : 0;
+  multiplier += (service->desc_next) ? 1 : 0;
+
+  return (count * multiplier);
+}
+
+/* For the given service, return 1 if the service is allowed to launch more
+ * introduction circuits else 0 if the maximum has been reached for the retry
+ * period of INTRO_CIRC_RETRY_PERIOD. */
+STATIC int
+can_service_launch_intro_circuit(hs_service_t *service, time_t now)
+{
+  tor_assert(service);
+
+  /* Consider the intro circuit retry period of the service. */
+  if (now > (service->state.intro_circ_retry_started_time +
+             INTRO_CIRC_RETRY_PERIOD)) {
+    service->state.intro_circ_retry_started_time = now;
+    service->state.num_intro_circ_launched = 0;
+    goto allow;
+  }
+  /* Check if we can still launch more circuits in this period. */
+  if (service->state.num_intro_circ_launched <=
+      get_max_intro_circ_per_period(service)) {
+    goto allow;
+  }
+
+  /* Rate limit log that we've reached our circuit creation limit. */
+  {
+    char *msg;
+    time_t elapsed_time = now - service->state.intro_circ_retry_started_time;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, now))) {
+      log_info(LD_REND, "Hidden service %s exceeded its circuit launch limit "
+                        "of %u per %d seconds. It launched %u circuits in "
+                        "the last %ld seconds. Will retry in %ld seconds.",
+               safe_str_client(service->onion_address),
+               get_max_intro_circ_per_period(service),
+               INTRO_CIRC_RETRY_PERIOD,
+               service->state.num_intro_circ_launched,
+               (long int) elapsed_time,
+               (long int) (INTRO_CIRC_RETRY_PERIOD - elapsed_time));
+      tor_free(msg);
+    }
+  }
+
+  /* Not allow. */
+  return 0;
+ allow:
+  return 1;
+}
+
+/* Scheduled event run from the main loop. Make sure we have all the circuits
+ * we need for each service. */
+static void
+run_build_circuit_event(time_t now)
+{
+  /* Make sure we can actually have enough information or able to build
+   * internal circuits as required by services. */
+  if (router_have_consensus_path() == CONSENSUS_PATH_UNKNOWN ||
+      !have_completed_a_circuit()) {
+    return;
+  }
+
+  /* Run v2 check. */
+  if (rend_num_services() > 0) {
+    rend_consider_services_intro_points(now);
+  }
+
+  /* Run v3+ check. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* For introduction circuit, we need to make sure we don't stress too much
+     * circuit creation so make sure this service is respecting that limit. */
+    if (can_service_launch_intro_circuit(service, now)) {
+      /* Launch intro point circuits if needed. */
+      launch_intro_point_circuits(service);
+      /* Once the circuits have opened, we'll make sure to update the
+       * descriptor intro point list and cleanup any extraneous. */
+    }
+  } FOR_EACH_SERVICE_END;
+}
+
+/* Encode and sign the service descriptor desc and upload it to the given
+ * hidden service directory.  This does nothing if PublishHidServDescriptors
+ * is false. */
+static void
+upload_descriptor_to_hsdir(const hs_service_t *service,
+                           hs_service_descriptor_t *desc, const node_t *hsdir)
+{
+  char *encoded_desc = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(hsdir);
+
+  /* Let's avoid doing that if tor is configured to not publish. */
+  if (!get_options()->PublishHidServDescriptors) {
+    log_info(LD_REND, "Service %s not publishing descriptor. "
+                      "PublishHidServDescriptors is set to 1.",
+             safe_str_client(service->onion_address));
+    goto end;
+  }
+
+  /* First of all, we'll encode the descriptor. This should NEVER fail but
+   * just in case, let's make sure we have an actual usable descriptor. */
+  if (BUG(hs_desc_encode_descriptor(desc->desc, &desc->signing_kp,
+                                    &encoded_desc) < 0)) {
+    goto end;
+  }
+
+  /* Time to upload the descriptor to the directory. */
+  hs_service_upload_desc_to_dir(encoded_desc, service->config.version,
+                                &service->keys.identity_pk,
+                                &desc->blinded_kp.pubkey, hsdir->rs);
+
+  /* Add this node to previous_hsdirs list */
+  service_desc_note_upload(desc, hsdir);
+
+  /* Logging so we know where it was sent. */
+  {
+    int is_next_desc = (service->desc_next == desc);
+    const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index.store_second:
+                                          hsdir->hsdir_index.store_first;
+    log_info(LD_REND, "Service %s %s descriptor of revision %" PRIu64
+                      " initiated upload request to %s with index %s",
+             safe_str_client(service->onion_address),
+             (is_next_desc) ? "next" : "current",
+             desc->desc->plaintext_data.revision_counter,
+             safe_str_client(node_describe(hsdir)),
+             safe_str_client(hex_str((const char *) idx, 32)));
+
+    /* Fire a UPLOAD control port event. */
+    hs_control_desc_event_upload(service->onion_address, hsdir->identity,
+                                 &desc->blinded_kp.pubkey, idx);
+  }
+
+ end:
+  tor_free(encoded_desc);
+  return;
+}
+
+/** Return a newly-allocated string for our state file which contains revision
+ *  counter information for <b>desc</b>. The format is:
+ *
+ *     HidServRevCounter <blinded_pubkey> <rev_counter>
+ */
+STATIC char *
+encode_desc_rev_counter_for_state(const hs_service_descriptor_t *desc)
+{
+  char *state_str = NULL;
+  char blinded_pubkey_b64[ED25519_BASE64_LEN+1];
+  uint64_t rev_counter = desc->desc->plaintext_data.revision_counter;
+  const ed25519_public_key_t *blinded_pubkey = &desc->blinded_kp.pubkey;
+
+  /* Turn the blinded key into b64 so that we save it on state */
+  tor_assert(blinded_pubkey);
+  if (ed25519_public_to_base64(blinded_pubkey_b64, blinded_pubkey) < 0) {
+    goto done;
+  }
+
+  /* Format is: <blinded key> <rev counter> */
+  tor_asprintf(&state_str, "%s %" PRIu64, blinded_pubkey_b64, rev_counter);
+
+  log_info(LD_GENERAL, "[!] Adding rev counter %" PRIu64 " for %s!",
+           rev_counter, blinded_pubkey_b64);
+
+ done:
+  return state_str;
+}
+
+/** Update HS descriptor revision counters in our state by removing the old
+ *  ones and writing down the ones that are currently active. */
+static void
+update_revision_counters_in_state(void)
+{
+  config_line_t *lines = NULL;
+  config_line_t **nextline = &lines;
+  or_state_t *state = get_or_state();
+
+  /* Prepare our state structure with the rev counters */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      /* We don't want to save zero counters */
+      if (desc->desc->plaintext_data.revision_counter == 0) {
+        continue;
+      }
+
+      *nextline = tor_malloc_zero(sizeof(config_line_t));
+      (*nextline)->key = tor_strdup("HidServRevCounter");
+      (*nextline)->value = encode_desc_rev_counter_for_state(desc);
+      nextline = &(*nextline)->next;
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+
+  /* Remove the old rev counters, and replace them with the new ones */
+  config_free_lines(state->HidServRevCounter);
+  state->HidServRevCounter = lines;
+
+  /* Set the state as dirty since we just edited it */
+  if (!get_options()->AvoidDiskWrites) {
+    or_state_mark_dirty(state, 0);
+  }
+}
+
+/** Scan the string <b>state_line</b> for the revision counter of the service
+ *  with <b>blinded_pubkey</b>. Set <b>service_found_out</b> to True if the
+ *  line is relevant to this service, and return the cached revision
+ *  counter. Else set <b>service_found_out</b> to False. */
+STATIC uint64_t
+check_state_line_for_service_rev_counter(const char *state_line,
+                                    const ed25519_public_key_t *blinded_pubkey,
+                                    int *service_found_out)
+{
+  smartlist_t *items = NULL;
+  int ok;
+  ed25519_public_key_t pubkey_in_state;
+  uint64_t rev_counter = 0;
+
+  tor_assert(service_found_out);
+  tor_assert(state_line);
+  tor_assert(blinded_pubkey);
+
+  /* Assume that the line is not for this service */
+  *service_found_out = 0;
+
+  /* Start parsing the state line */
+  items = smartlist_new();
+  smartlist_split_string(items, state_line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  if (smartlist_len(items) < 2) {
+    log_warn(LD_GENERAL, "Incomplete rev counter line. Ignoring.");
+    goto done;
+  }
+
+  char *b64_key_str = smartlist_get(items, 0);
+  char *saved_rev_counter_str = smartlist_get(items, 1);
+
+  /* Parse blinded key to check if it's for this hidden service */
+  if (ed25519_public_from_base64(&pubkey_in_state, b64_key_str) < 0) {
+    log_warn(LD_GENERAL, "Unable to base64 key in revcount line. Ignoring.");
+    goto done;
+  }
+  /* State line not for this hidden service */
+  if (!ed25519_pubkey_eq(&pubkey_in_state, blinded_pubkey)) {
+    goto done;
+  }
+
+  rev_counter = tor_parse_uint64(saved_rev_counter_str,
+                                 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_GENERAL, "Unable to parse rev counter. Ignoring.");
+    goto done;
+  }
+
+  /* Since we got this far, the line was for this service */
+  *service_found_out = 1;
+
+  log_info(LD_GENERAL, "Found rev counter for %s: %" PRIu64,
+           b64_key_str, rev_counter);
+
+ done:
+  tor_assert(items);
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+
+  return rev_counter;
+}
+
+/** Dig into our state file and find the current revision counter for the
+ *  service with blinded key <b>blinded_pubkey</b>. If no revision counter is
+ *  found, return 0. */
+static uint64_t
+get_rev_counter_for_service(const ed25519_public_key_t *blinded_pubkey)
+{
+  or_state_t *state = get_or_state();
+  config_line_t *line;
+
+  /* Set default value for rev counters (if not found) to 0 */
+  uint64_t final_rev_counter = 0;
+
+  for (line = state->HidServRevCounter ; line ; line = line->next) {
+    int service_found = 0;
+    uint64_t rev_counter = 0;
+
+    tor_assert(!strcmp(line->key, "HidServRevCounter"));
+
+    /* Scan all the HidServRevCounter lines till we find the line for this
+       service: */
+    rev_counter = check_state_line_for_service_rev_counter(line->value,
+                                                           blinded_pubkey,
+                                                           &service_found);
+    if (service_found) {
+      final_rev_counter = rev_counter;
+      goto done;
+    }
+  }
+
+ done:
+  return final_rev_counter;
+}
+
+/** Update the value of the revision counter for <b>hs_desc</b> and save it on
+    our state file. */
+static void
+increment_descriptor_revision_counter(hs_descriptor_t *hs_desc)
+{
+  /* Find stored rev counter if it exists */
+  uint64_t rev_counter =
+    get_rev_counter_for_service(&hs_desc->plaintext_data.blinded_pubkey);
+
+  /* Increment the revision counter of <b>hs_desc</b> so the next update (which
+   * will trigger an upload) will have the right value. We do this at this
+   * stage to only do it once because a descriptor can have many updates before
+   * being uploaded. By doing it at upload, we are sure to only increment by 1
+   * and thus avoid leaking how many operations we made on the descriptor from
+   * the previous one before uploading. */
+  rev_counter++;
+  hs_desc->plaintext_data.revision_counter = rev_counter;
+
+  update_revision_counters_in_state();
+}
+
+/** Set the revision counter in <b>hs_desc</b>, using the state file to find
+ *  the current counter value if it exists. */
+static void
+set_descriptor_revision_counter(hs_descriptor_t *hs_desc)
+{
+  /* Find stored rev counter if it exists */
+  uint64_t rev_counter =
+    get_rev_counter_for_service(&hs_desc->plaintext_data.blinded_pubkey);
+
+  hs_desc->plaintext_data.revision_counter = rev_counter;
+}
+
+/* Encode and sign the service descriptor desc and upload it to the
+ * responsible hidden service directories. If for_next_period is true, the set
+ * of directories are selected using the next hsdir_index. This does nothing
+ * if PublishHidServDescriptors is false. */
+STATIC void
+upload_descriptor_to_all(const hs_service_t *service,
+                         hs_service_descriptor_t *desc)
+{
+  smartlist_t *responsible_dirs = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* We'll first cancel any directory request that are ongoing for this
+   * descriptor. It is possible that we can trigger multiple uploads in a
+   * short time frame which can lead to a race where the second upload arrives
+   * before the first one leading to a 400 malformed descriptor response from
+   * the directory. Closing all pending requests avoids that. */
+  close_directory_connections(service, desc);
+
+  /* Get our list of responsible HSDir. */
+  responsible_dirs = smartlist_new();
+  /* The parameter 0 means that we aren't a client so tell the function to use
+   * the spread store consensus paremeter. */
+  hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num,
+                            service->desc_next == desc, 0, responsible_dirs);
+
+  /** Clear list of previous hsdirs since we are about to upload to a new
+   *  list. Let's keep it up to date. */
+  service_desc_clear_previous_hsdirs(desc);
+
+  /* For each responsible HSDir we have, initiate an upload command. */
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *,
+                          hsdir_rs) {
+    const node_t *hsdir_node = node_get_by_id(hsdir_rs->identity_digest);
+    /* Getting responsible hsdir implies that the node_t object exists for the
+     * routerstatus_t found in the consensus else we have a problem. */
+    tor_assert(hsdir_node);
+    /* Upload this descriptor to the chosen directory. */
+    upload_descriptor_to_hsdir(service, desc, hsdir_node);
+  } SMARTLIST_FOREACH_END(hsdir_rs);
+
+  /* Set the next upload time for this descriptor. Even if we are configured
+   * to not upload, we still want to follow the right cycle of life for this
+   * descriptor. */
+  desc->next_upload_time =
+    (time(NULL) + crypto_rand_int_range(HS_SERVICE_NEXT_UPLOAD_TIME_MIN,
+                                        HS_SERVICE_NEXT_UPLOAD_TIME_MAX));
+  {
+    char fmt_next_time[ISO_TIME_LEN+1];
+    format_local_iso_time(fmt_next_time, desc->next_upload_time);
+    log_debug(LD_REND, "Service %s set to upload a descriptor at %s",
+              safe_str_client(service->onion_address), fmt_next_time);
+  }
+
+  /* Update the revision counter of this descriptor */
+  increment_descriptor_revision_counter(desc->desc);
+
+  smartlist_free(responsible_dirs);
+  return;
+}
+
+/** The set of HSDirs have changed: check if the change affects our descriptor
+ *  HSDir placement, and if it does, reupload the desc. */
+STATIC int
+service_desc_hsdirs_changed(const hs_service_t *service,
+                            const hs_service_descriptor_t *desc)
+{
+  int should_reupload = 0;
+  smartlist_t *responsible_dirs = smartlist_new();
+
+  /* No desc upload has happened yet: it will happen eventually */
+  if (!desc->previous_hsdirs || !smartlist_len(desc->previous_hsdirs)) {
+    goto done;
+  }
+
+  /* Get list of responsible hsdirs */
+  hs_get_responsible_hsdirs(&desc->blinded_kp.pubkey, desc->time_period_num,
+                            service->desc_next == desc, 0, responsible_dirs);
+
+  /* Check if any new hsdirs have been added to the responsible hsdirs set:
+   * Iterate over the list of new hsdirs, and reupload if any of them is not
+   * present in the list of previous hsdirs.
+   */
+  SMARTLIST_FOREACH_BEGIN(responsible_dirs, const routerstatus_t *, hsdir_rs) {
+    char b64_digest[BASE64_DIGEST_LEN+1] = {0};
+    digest_to_base64(b64_digest, hsdir_rs->identity_digest);
+
+    if (!smartlist_contains_string(desc->previous_hsdirs, b64_digest)) {
+      should_reupload = 1;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(hsdir_rs);
+
+ done:
+  smartlist_free(responsible_dirs);
+
+  return should_reupload;
+}
+
+/* Return 1 if the given descriptor from the given service can be uploaded
+ * else return 0 if it can not. */
+static int
+should_service_upload_descriptor(const hs_service_t *service,
+                              const hs_service_descriptor_t *desc, time_t now)
+{
+  unsigned int num_intro_points;
+
+  tor_assert(service);
+  tor_assert(desc);
+
+  /* If this descriptors has missing intro points that is that it couldn't get
+   * them all when it was time to pick them, it means that we should upload
+   * instead of waiting an arbitrary amount of time breaking the service.
+   * Else, if we have no missing intro points, we use the value taken from the
+   * service configuration. */
+  if (desc->missing_intro_points) {
+    num_intro_points = digest256map_size(desc->intro_points.map);
+  } else {
+    num_intro_points = service->config.num_intro_points;
+  }
+
+  /* This means we tried to pick intro points but couldn't get any so do not
+   * upload descriptor in this case. We need at least one for the service to
+   * be reachable. */
+  if (desc->missing_intro_points && num_intro_points == 0) {
+    goto cannot;
+  }
+
+  /* Check if all our introduction circuit have been established for all the
+   * intro points we have selected. */
+  if (count_desc_circuit_established(desc) != num_intro_points) {
+    goto cannot;
+  }
+
+  /* Is it the right time to upload? */
+  if (desc->next_upload_time > now) {
+    goto cannot;
+  }
+
+  /* Don't upload desc if we don't have a live consensus */
+  if (!networkstatus_get_live_consensus(now)) {
+    goto cannot;
+  }
+
+  /* Do we know enough router descriptors to have adequate vision of the HSDir
+     hash ring? */
+  if (!router_have_minimum_dir_info()) {
+    goto cannot;
+  }
+
+  /* Can upload! */
+  return 1;
+ cannot:
+  return 0;
+}
+
+/* Scheduled event run from the main loop. Try to upload the descriptor for
+ * each service. */
+STATIC void
+run_upload_descriptor_event(time_t now)
+{
+  /* v2 services use the same function for descriptor creation and upload so
+   * we do everything here because the intro circuits were checked before. */
+  if (rend_num_services() > 0) {
+    rend_consider_services_upload(now);
+    rend_consider_descriptor_republication();
+  }
+
+  /* Run v3+ check. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+      /* If we were asked to re-examine the hash ring, and it changed, then
+         schedule an upload */
+      if (consider_republishing_hs_descriptors &&
+          service_desc_hsdirs_changed(service, desc)) {
+        service_desc_schedule_upload(desc, now, 0);
+      }
+
+      /* Can this descriptor be uploaded? */
+      if (!should_service_upload_descriptor(service, desc, now)) {
+        continue;
+      }
+
+      log_info(LD_REND, "Initiating upload for hidden service %s descriptor "
+                        "for service %s with %u/%u introduction points%s.",
+               (desc == service->desc_current) ? "current" : "next",
+               safe_str_client(service->onion_address),
+               digest256map_size(desc->intro_points.map),
+               service->config.num_intro_points,
+               (desc->missing_intro_points) ? " (couldn't pick more)" : "");
+
+      /* At this point, we have to upload the descriptor so start by building
+       * the intro points descriptor section which we are now sure to be
+       * accurate because all circuits have been established. */
+      build_desc_intro_points(service, desc, now);
+
+      upload_descriptor_to_all(service, desc);
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+
+  /* We are done considering whether to republish rend descriptors */
+  consider_republishing_hs_descriptors = 0;
+}
+
+/* Called when the introduction point circuit is done building and ready to be
+ * used. */
+static void
+service_intro_circ_has_opened(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+
+  /* Let's do some basic sanity checking of the circ state */
+  if (BUG(!circ->cpath)) {
+    return;
+  }
+  if (BUG(TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)) {
+    return;
+  }
+  if (BUG(!circ->hs_ident)) {
+    return;
+  }
+
+  /* Get the corresponding service and intro point. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the introduction "
+                      "circuit %u. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    log_warn(LD_REND, "Unknown introduction point auth key on circuit %u "
+                      "for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* We can't have an IP object without a descriptor. */
+  tor_assert(desc);
+
+  if (hs_circ_service_intro_has_opened(service, ip, desc, circ)) {
+    /* Getting here means that the circuit has been re-purposed because we
+     * have enough intro circuit opened. Remove the IP from the service. */
+    service_intro_point_remove(service, ip);
+    service_intro_point_free(ip);
+  }
+
+  goto done;
+
+ err:
+  /* Close circuit, we can't use it. */
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOSUCHSERVICE);
+ done:
+  return;
+}
+
+/* Called when a rendezvous circuit is done building and ready to be used. */
+static void
+service_rendezvous_circ_has_opened(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+
+  tor_assert(circ);
+  tor_assert(circ->cpath);
+  /* Getting here means this is a v3 rendezvous circuit. */
+  tor_assert(circ->hs_ident);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
+
+  /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the
+   * timestamp regardless of its content because that circuit could have been
+   * cannibalized so in any cases, we are about to use that circuit more. */
+  TO_CIRCUIT(circ)->timestamp_dirty = time(NULL);
+  pathbias_count_use_attempt(circ);
+
+  /* Get the corresponding service and intro point. */
+  get_objects_from_ident(circ->hs_ident, &service, NULL, NULL);
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the rendezvous "
+                      "circuit %u with cookie %s. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id,
+             hex_str((const char *) circ->hs_ident->rendezvous_cookie,
+                     REND_COOKIE_LEN));
+    goto err;
+  }
+
+  /* If the cell can't be sent, the circuit will be closed within this
+   * function. */
+  hs_circ_service_rp_has_opened(service, circ);
+  goto done;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NOSUCHSERVICE);
+ done:
+  return;
+}
+
+/* We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
+ * arrived. Handle the INTRO_ESTABLISHED cell arriving on the given
+ * introduction circuit. Return 0 on success else a negative value. */
+static int
+service_handle_intro_established(origin_circuit_t *circ,
+                                 const uint8_t *payload,
+                                 size_t payload_len)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  /* We need the service and intro point for this cell. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, NULL);
+
+  /* Get service object from the circuit identifier. */
+  if (service == NULL) {
+    log_warn(LD_REND, "Unknown service identity key %s on the introduction "
+                      "circuit %u. Can't find onion service.",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    /* We don't recognize the key. */
+    log_warn(LD_REND, "Introduction circuit established without an intro "
+                      "point object on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+
+  /* Try to parse the payload into a cell making sure we do actually have a
+   * valid cell. On success, the ip object and circuit purpose is updated to
+   * reflect the fact that the introduction circuit is established. */
+  if (hs_circ_handle_intro_established(service, ip, circ, payload,
+                                       payload_len) < 0) {
+    goto err;
+  }
+
+  /* Flag that we have an established circuit for this intro point. This value
+   * is what indicates the upload scheduled event if we are ready to build the
+   * intro point into the descriptor and upload. */
+  ip->circuit_established = 1;
+
+  log_info(LD_REND, "Successfully received an INTRO_ESTABLISHED cell "
+                    "on circuit %u for service %s",
+           TO_CIRCUIT(circ)->n_circ_id,
+           safe_str_client(service->onion_address));
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* We just received an INTRODUCE2 cell on the established introduction circuit
+ * circ. Handle the cell and return 0 on success else a negative value. */
+static int
+service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
+                          size_t payload_len)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+  tor_assert(payload);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
+
+  /* We'll need every object associated with this circuit. */
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+
+  /* Get service object from the circuit identifier. */
+  if (service == NULL) {
+    log_warn(LD_BUG, "Unknown service identity key %s when handling "
+                     "an INTRODUCE2 cell on circuit %u",
+             safe_str_client(ed25519_fmt(&circ->hs_ident->identity_pk)),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto err;
+  }
+  if (ip == NULL) {
+    /* We don't recognize the key. */
+    log_warn(LD_BUG, "Unknown introduction auth key when handling "
+                     "an INTRODUCE2 cell on circuit %u for service %s",
+             TO_CIRCUIT(circ)->n_circ_id,
+             safe_str_client(service->onion_address));
+    goto err;
+  }
+  /* If we have an IP object, we MUST have a descriptor object. */
+  tor_assert(desc);
+
+  /* The following will parse, decode and launch the rendezvous point circuit.
+   * Both current and legacy cells are handled. */
+  if (hs_circ_handle_introduce2(service, circ, ip, desc->desc->subcredential,
+                                payload, payload_len) < 0) {
+    goto err;
+  }
+
+  return 0;
+ err:
+  return -1;
+}
+
+/* Add to list every filename used by service. This is used by the sandbox
+ * subsystem. */
+static void
+service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
+{
+  const char *s_dir;
+  char fname[128] = {0};
+
+  tor_assert(service);
+  tor_assert(list);
+
+  /* Ease our life. */
+  s_dir = service->config.directory_path;
+  /* The hostname file. */
+  smartlist_add(list, hs_path_from_filename(s_dir, fname_hostname));
+  /* The key files splitted in two. */
+  tor_snprintf(fname, sizeof(fname), "%s_secret_key", fname_keyfile_prefix);
+  smartlist_add(list, hs_path_from_filename(s_dir, fname));
+  tor_snprintf(fname, sizeof(fname), "%s_public_key", fname_keyfile_prefix);
+  smartlist_add(list, hs_path_from_filename(s_dir, fname));
+}
+
+/* ========== */
+/* Public API */
+/* ========== */
+
+/* This is called everytime the service map (v2 or v3) changes that is if an
+ * element is added or removed. */
+void
+hs_service_map_has_changed(void)
+{
+  /* If we now have services where previously we had not, we need to enable
+   * the HS service main loop event. If we changed to having no services, we
+   * need to disable the event. */
+  rescan_periodic_events(get_options());
+}
+
+/* Upload an encoded descriptor in encoded_desc of the given version. This
+ * descriptor is for the service identity_pk and blinded_pk used to setup the
+ * directory connection identifier. It is uploaded to the directory hsdir_rs
+ * routerstatus_t object.
+ *
+ * NOTE: This function does NOT check for PublishHidServDescriptors because it
+ * is only used by the control port command HSPOST outside of this subsystem.
+ * Inside this code, upload_descriptor_to_hsdir() should be used. */
+void
+hs_service_upload_desc_to_dir(const char *encoded_desc,
+                              const uint8_t version,
+                              const ed25519_public_key_t *identity_pk,
+                              const ed25519_public_key_t *blinded_pk,
+                              const routerstatus_t *hsdir_rs)
+{
+  char version_str[4] = {0};
+  directory_request_t *dir_req;
+  hs_ident_dir_conn_t ident;
+
+  tor_assert(encoded_desc);
+  tor_assert(identity_pk);
+  tor_assert(blinded_pk);
+  tor_assert(hsdir_rs);
+
+  /* Setup the connection identifier. */
+  memset(&ident, 0, sizeof(ident));
+  hs_ident_dir_conn_init(identity_pk, blinded_pk, &ident);
+
+  /* This is our resource when uploading which is used to construct the URL
+   * with the version number: "/tor/hs/<version>/publish". */
+  tor_snprintf(version_str, sizeof(version_str), "%u", version);
+
+  /* Build the directory request for this HSDir. */
+  dir_req = directory_request_new(DIR_PURPOSE_UPLOAD_HSDESC);
+  directory_request_set_routerstatus(dir_req, hsdir_rs);
+  directory_request_set_indirection(dir_req, DIRIND_ANONYMOUS);
+  directory_request_set_resource(dir_req, version_str);
+  directory_request_set_payload(dir_req, encoded_desc,
+                                strlen(encoded_desc));
+  /* The ident object is copied over the directory connection object once
+   * the directory request is initiated. */
+  directory_request_upload_set_hs_ident(dir_req, &ident);
+
+  /* Initiate the directory request to the hsdir.*/
+  directory_initiate_request(dir_req);
+  directory_request_free(dir_req);
+}
+
+/* Add the ephemeral service using the secret key sk and ports. Both max
+ * streams parameter will be set in the newly created service.
+ *
+ * Ownership of sk and ports is passed to this routine.  Regardless of
+ * success/failure, callers should not touch these values after calling this
+ * routine, and may assume that correct cleanup has been done on failure.
+ *
+ * Return an appropriate hs_service_add_ephemeral_status_t. */
+hs_service_add_ephemeral_status_t
+hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
+                         int max_streams_per_rdv_circuit,
+                         int max_streams_close_circuit, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+  hs_service_t *service = NULL;
+
+  tor_assert(sk);
+  tor_assert(ports);
+  tor_assert(address_out);
+
+  service = hs_service_new(get_options());
+
+  /* Setup the service configuration with specifics. A default service is
+   * HS_VERSION_TWO so explicitly set it. */
+  service->config.version = HS_VERSION_THREE;
+  service->config.max_streams_per_rdv_circuit = max_streams_per_rdv_circuit;
+  service->config.max_streams_close_circuit = !!max_streams_close_circuit;
+  service->config.is_ephemeral = 1;
+  smartlist_free(service->config.ports);
+  service->config.ports = ports;
+
+  /* Handle the keys. */
+  memcpy(&service->keys.identity_sk, sk, sizeof(service->keys.identity_sk));
+  if (ed25519_public_key_generate(&service->keys.identity_pk,
+                                  &service->keys.identity_sk) < 0) {
+    log_warn(LD_CONFIG, "Unable to generate ed25519 public key"
+                        "for v3 service.");
+    ret = RSAE_BADPRIVKEY;
+    goto err;
+  }
+
+  /* Make sure we have at least one port. */
+  if (smartlist_len(service->config.ports) == 0) {
+    log_warn(LD_CONFIG, "At least one VIRTPORT/TARGET must be specified "
+                        "for v3 service.");
+    ret = RSAE_BADVIRTPORT;
+    goto err;
+  }
+
+  /* Build the onion address for logging purposes but also the control port
+   * uses it for the HS_DESC event. */
+  hs_build_address(&service->keys.identity_pk,
+                   (uint8_t) service->config.version,
+                   service->onion_address);
+
+  /* The only way the registration can fail is if the service public key
+   * already exists. */
+  if (BUG(register_service(hs_service_map, service) < 0)) {
+    log_warn(LD_CONFIG, "Onion Service private key collides with an "
+                        "existing v3 service.");
+    ret = RSAE_ADDREXISTS;
+    goto err;
+  }
+
+  log_info(LD_CONFIG, "Added ephemeral v3 onion service: %s",
+           safe_str_client(service->onion_address));
+
+  *address_out = tor_strdup(service->onion_address);
+  ret = RSAE_OKAY;
+  goto end;
+
+ err:
+  hs_service_free(service);
+
+ end:
+  memwipe(sk, 0, sizeof(ed25519_secret_key_t));
+  tor_free(sk);
+  return ret;
+}
+
+/* For the given onion address, delete the ephemeral service. Return 0 on
+ * success else -1 on error. */
+int
+hs_service_del_ephemeral(const char *address)
+{
+  uint8_t version;
+  ed25519_public_key_t pk;
+  hs_service_t *service = NULL;
+
+  tor_assert(address);
+
+  if (hs_parse_address(address, &pk, NULL, &version) < 0) {
+    log_warn(LD_CONFIG, "Requested malformed v3 onion address for removal.");
+    goto err;
+  }
+
+  if (version != HS_VERSION_THREE) {
+    log_warn(LD_CONFIG, "Requested version of onion address for removal "
+                        "is not supported.");
+    goto err;
+  }
+
+  service = find_service(hs_service_map, &pk);
+  if (service == NULL) {
+    log_warn(LD_CONFIG, "Requested non-existent v3 hidden service for "
+                        "removal.");
+    goto err;
+  }
+
+  if (!service->config.is_ephemeral) {
+    log_warn(LD_CONFIG, "Requested non-ephemeral v3 hidden service for "
+                        "removal.");
+    goto err;
+  }
+
+  /* Close introduction circuits, remove from map and finally free. Notice
+   * that the rendezvous circuits aren't closed in order for any existing
+   * connections to finish. We let the application terminate them. */
+  close_service_intro_circuits(service);
+  remove_service(hs_service_map, service);
+  hs_service_free(service);
+
+  log_info(LD_CONFIG, "Removed ephemeral v3 hidden service: %s",
+           safe_str_client(address));
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the ed25519 public key pk, find a service for that key and return the
+ * current encoded descriptor as a newly allocated string or NULL if not
+ * found. This is used by the control port subsystem. */
+char *
+hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
+{
+  const hs_service_t *service;
+
+  tor_assert(pk);
+
+  service = find_service(hs_service_map, pk);
+  if (service && service->desc_current) {
+    char *encoded_desc = NULL;
+    /* No matter what is the result (which should never be a failure), return
+     * the encoded variable, if success it will contain the right thing else
+     * it will be NULL. */
+    hs_desc_encode_descriptor(service->desc_current->desc,
+                              &service->desc_current->signing_kp,
+                              &encoded_desc);
+    return encoded_desc;
+  }
+
+  return NULL;
+}
+
+/* Return the number of service we have configured and usable. */
+unsigned int
+hs_service_get_num_services(void)
+{
+  if (hs_service_map == NULL) {
+    return 0;
+  }
+  return HT_SIZE(hs_service_map);
+}
+
+/* Called once an introduction circuit is closed. If the circuit doesn't have
+ * a v3 identifier, it is ignored. */
+void
+hs_service_intro_circ_has_closed(origin_circuit_t *circ)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_descriptor_t *desc = NULL;
+
+  tor_assert(circ);
+
+  if (circ->hs_ident == NULL) {
+    /* This is not a v3 circuit, ignore. */
+    goto end;
+  }
+
+  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
+  if (service == NULL) {
+    /* This is possible if the circuits are closed and the service is
+     * immediately deleted. */
+    log_info(LD_REND, "Unable to find any hidden service associated "
+                      "identity key %s on intro circuit %u.",
+             ed25519_fmt(&circ->hs_ident->identity_pk),
+             TO_CIRCUIT(circ)->n_circ_id);
+    goto end;
+  }
+  if (ip == NULL) {
+    /* The introduction point object has already been removed probably by our
+     * cleanup process so ignore. */
+    goto end;
+  }
+  /* Can't have an intro point object without a descriptor. */
+  tor_assert(desc);
+
+  /* Circuit disappeared so make sure the intro point is updated. By
+   * keeping the object in the descriptor, we'll be able to retry. */
+  ip->circuit_established = 0;
+
+ end:
+  return;
+}
+
+/* Given conn, a rendezvous edge connection acting as an exit stream, look up
+ * the hidden service for the circuit circ, and look up the port and address
+ * based on the connection port. Assign the actual connection address.
+ *
+ * Return 0 on success. Return -1 on failure and the caller should NOT close
+ * the circuit. Return -2 on failure and the caller MUST close the circuit for
+ * security reasons. */
+int
+hs_service_set_conn_addr_port(const origin_circuit_t *circ,
+                              edge_connection_t *conn)
+{
+  hs_service_t *service = NULL;
+
+  tor_assert(circ);
+  tor_assert(conn);
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+  tor_assert(circ->hs_ident);
+
+  get_objects_from_ident(circ->hs_ident, &service, NULL, NULL);
+
+  if (service == NULL) {
+    log_warn(LD_REND, "Unable to find any hidden service associated "
+                      "identity key %s on rendezvous circuit %u.",
+             ed25519_fmt(&circ->hs_ident->identity_pk),
+             TO_CIRCUIT(circ)->n_circ_id);
+    /* We want the caller to close the circuit because it's not a valid
+     * service so no danger. Attempting to bruteforce the entire key space by
+     * opening circuits to learn which service is being hosted here is
+     * impractical. */
+    goto err_close;
+  }
+
+  /* Enforce the streams-per-circuit limit, and refuse to provide a mapping if
+   * this circuit will exceed the limit. */
+  if (service->config.max_streams_per_rdv_circuit > 0 &&
+      (circ->hs_ident->num_rdv_streams >=
+       service->config.max_streams_per_rdv_circuit)) {
+#define MAX_STREAM_WARN_INTERVAL 600
+    static struct ratelim_t stream_ratelim =
+      RATELIM_INIT(MAX_STREAM_WARN_INTERVAL);
+    log_fn_ratelim(&stream_ratelim, LOG_WARN, LD_REND,
+                   "Maximum streams per circuit limit reached on "
+                   "rendezvous circuit %u for service %s. Circuit has "
+                   "%" PRIu64 " out of %" PRIu64 " streams. %s.",
+                   TO_CIRCUIT(circ)->n_circ_id,
+                   service->onion_address,
+                   circ->hs_ident->num_rdv_streams,
+                   service->config.max_streams_per_rdv_circuit,
+                   service->config.max_streams_close_circuit ?
+                    "Closing circuit" : "Ignoring open stream request");
+    if (service->config.max_streams_close_circuit) {
+      /* Service explicitly configured to close immediately. */
+      goto err_close;
+    }
+    /* Exceeding the limit makes tor silently ignore the stream creation
+     * request and keep the circuit open. */
+    goto err_no_close;
+  }
+
+  /* Find a virtual port of that service mathcing the one in the connection if
+   * successful, set the address in the connection. */
+  if (hs_set_conn_addr_port(service->config.ports, conn) < 0) {
+    log_info(LD_REND, "No virtual port mapping exists for port %d for "
+                      "hidden service %s.",
+             TO_CONN(conn)->port, service->onion_address);
+    if (service->config.allow_unknown_ports) {
+      /* Service explicitly allow connection to unknown ports so close right
+       * away because we do not care about port mapping. */
+      goto err_close;
+    }
+    /* If the service didn't explicitly allow it, we do NOT close the circuit
+     * here to raise the bar in terms of performance for port mapping. */
+    goto err_no_close;
+  }
+
+  /* Success. */
+  return 0;
+ err_close:
+  /* Indicate the caller that the circuit should be closed. */
+  return -2;
+ err_no_close:
+  /* Indicate the caller to NOT close the circuit. */
+  return -1;
+}
+
+/* Add to file_list every filename used by a configured hidden service, and to
+ * dir_list every directory path used by a configured hidden service. This is
+ * used by the sandbox subsystem to whitelist those. */
+void
+hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
+                                    smartlist_t *dir_list)
+{
+  tor_assert(file_list);
+  tor_assert(dir_list);
+
+  /* Add files and dirs for legacy services. */
+  rend_services_add_filenames_to_lists(file_list, dir_list);
+
+  /* Add files and dirs for v3+. */
+  FOR_EACH_SERVICE_BEGIN(service) {
+    /* Skip ephemeral service, they don't touch the disk. */
+    if (service->config.is_ephemeral) {
+      continue;
+    }
+    service_add_fnames_to_list(service, file_list);
+    smartlist_add_strdup(dir_list, service->config.directory_path);
+  } FOR_EACH_DESCRIPTOR_END;
+}
+
+/* Called when our internal view of the directory has changed. We might have
+ * received a new batch of descriptors which might affect the shape of the
+ * HSDir hash ring. Signal that we should reexamine the hash ring and
+ * re-upload our HS descriptors if needed. */
+void
+hs_service_dir_info_changed(void)
+{
+  if (hs_service_get_num_services() > 0) {
+    /* New directory information usually goes every consensus so rate limit
+     * every 30 minutes to not be too conservative. */
+    static struct ratelim_t dir_info_changed_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&dir_info_changed_ratelim, LOG_INFO, LD_REND,
+                   "New dirinfo arrived: consider reuploading descriptor");
+    consider_republishing_hs_descriptors = 1;
+  }
+}
+
+/* Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
+ * launch a circuit to the rendezvous point. */
+int
+hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
+                              size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  /* Do some initial validation and logging before we parse the cell */
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_INTRO) {
+    log_warn(LD_PROTOCOL, "Received an INTRODUCE2 cell on a "
+                          "non introduction circuit of purpose %d",
+             TO_CIRCUIT(circ)->purpose);
+    goto done;
+  }
+
+  if (circ->hs_ident) {
+    ret = service_handle_introduce2(circ, payload, payload_len);
+    hs_stats_note_introduce2_cell(1);
+  } else {
+    ret = rend_service_receive_introduction(circ, payload, payload_len);
+    hs_stats_note_introduce2_cell(0);
+  }
+
+ done:
+  return ret;
+}
+
+/* Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
+ * established introduction point. Return 0 on success else a negative value
+ * and the circuit is closed. */
+int
+hs_service_receive_intro_established(origin_circuit_t *circ,
+                                     const uint8_t *payload,
+                                     size_t payload_len)
+{
+  int ret = -1;
+
+  tor_assert(circ);
+  tor_assert(payload);
+
+  if (TO_CIRCUIT(circ)->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
+    log_warn(LD_PROTOCOL, "Received an INTRO_ESTABLISHED cell on a "
+                          "non introduction circuit of purpose %d",
+             TO_CIRCUIT(circ)->purpose);
+    goto err;
+  }
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  if (circ->hs_ident) {
+    ret = service_handle_intro_established(circ, payload, payload_len);
+  } else {
+    ret = rend_service_intro_established(circ, payload, payload_len);
+  }
+
+  if (ret < 0) {
+    goto err;
+  }
+  return 0;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Called when any kind of hidden service circuit is done building thus
+ * opened. This is the entry point from the circuit subsystem. */
+void
+hs_service_circuit_has_opened(origin_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Handle both version. v2 uses rend_data and v3 uses the hs circuit
+   * identifier hs_ident. Can't be both. */
+  switch (TO_CIRCUIT(circ)->purpose) {
+  case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
+    if (circ->hs_ident) {
+      service_intro_circ_has_opened(circ);
+    } else {
+      rend_service_intro_has_opened(circ);
+    }
+    break;
+  case CIRCUIT_PURPOSE_S_CONNECT_REND:
+    if (circ->hs_ident) {
+      service_rendezvous_circ_has_opened(circ);
+    } else {
+      rend_service_rendezvous_has_opened(circ);
+    }
+    break;
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Load and/or generate keys for all onion services including the client
+ * authorization if any. Return 0 on success, -1 on failure. */
+int
+hs_service_load_all_keys(void)
+{
+  /* Load v2 service keys if we have v2. */
+  if (rend_num_services() != 0) {
+    if (rend_service_load_all_keys(NULL) < 0) {
+      goto err;
+    }
+  }
+
+  /* Load or/and generate them for v3+. */
+  SMARTLIST_FOREACH_BEGIN(hs_service_staging_list, hs_service_t *, service) {
+    /* Ignore ephemeral service, they already have their keys set. */
+    if (service->config.is_ephemeral) {
+      continue;
+    }
+    log_info(LD_REND, "Loading v3 onion service keys from %s",
+             service_escaped_dir(service));
+    if (load_service_keys(service) < 0) {
+      goto err;
+    }
+    /* XXX: Load/Generate client authorization keys. (#20700) */
+  } SMARTLIST_FOREACH_END(service);
+
+  /* Final step, the staging list contains service in a quiescent state that
+   * is ready to be used. Register them to the global map. Once this is over,
+   * the staging list will be cleaned up. */
+  register_all_services();
+
+  /* All keys have been loaded successfully. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Put all service object in the given service list. After this, the caller
+ * looses ownership of every elements in the list and responsible to free the
+ * list pointer. */
+void
+hs_service_stage_services(const smartlist_t *service_list)
+{
+  tor_assert(service_list);
+  /* This list is freed at registration time but this function can be called
+   * multiple time. */
+  if (hs_service_staging_list == NULL) {
+    hs_service_staging_list = smartlist_new();
+  }
+  /* Add all service object to our staging list. Caller is responsible for
+   * freeing the service_list. */
+  smartlist_add_all(hs_service_staging_list, service_list);
+}
+
+/* Allocate and initilize a service object. The service configuration will
+ * contain the default values. Return the newly allocated object pointer. This
+ * function can't fail. */
+hs_service_t *
+hs_service_new(const or_options_t *options)
+{
+  hs_service_t *service = tor_malloc_zero(sizeof(hs_service_t));
+  /* Set default configuration value. */
+  set_service_default_config(&service->config, options);
+  /* Set the default service version. */
+  service->config.version = HS_SERVICE_DEFAULT_VERSION;
+  /* Allocate the CLIENT_PK replay cache in service state. */
+  service->state.replay_cache_rend_cookie =
+    replaycache_new(REND_REPLAY_TIME_INTERVAL, REND_REPLAY_TIME_INTERVAL);
+
+  return service;
+}
+
+/* Free the given <b>service</b> object and all its content. This function
+ * also takes care of wiping service keys from memory. It is safe to pass a
+ * NULL pointer. */
+void
+hs_service_free_(hs_service_t *service)
+{
+  if (service == NULL) {
+    return;
+  }
+
+  /* Free descriptors. Go over both descriptor with this loop. */
+  FOR_EACH_DESCRIPTOR_BEGIN(service, desc) {
+    service_descriptor_free(desc);
+  } FOR_EACH_DESCRIPTOR_END;
+
+  /* Free service configuration. */
+  service_clear_config(&service->config);
+
+  /* Free replay cache from state. */
+  if (service->state.replay_cache_rend_cookie) {
+    replaycache_free(service->state.replay_cache_rend_cookie);
+  }
+
+  /* Wipe service keys. */
+  memwipe(&service->keys.identity_sk, 0, sizeof(service->keys.identity_sk));
+
+  tor_free(service);
+}
+
+/* Periodic callback. Entry point from the main loop to the HS service
+ * subsystem. This is call every second. This is skipped if tor can't build a
+ * circuit or the network is disabled. */
+void
+hs_service_run_scheduled_events(time_t now)
+{
+  /* First thing we'll do here is to make sure our services are in a
+   * quiescent state for the scheduled events. */
+  run_housekeeping_event(now);
+
+  /* Order matters here. We first make sure the descriptor object for each
+   * service contains the latest data. Once done, we check if we need to open
+   * new introduction circuit. Finally, we try to upload the descriptor for
+   * each service. */
+
+  /* Make sure descriptors are up to date. */
+  run_build_descriptor_event(now);
+  /* Make sure services have enough circuits. */
+  run_build_circuit_event(now);
+  /* Upload the descriptors if needed/possible. */
+  run_upload_descriptor_event(now);
+}
+
+/* Initialize the service HS subsystem. */
+void
+hs_service_init(void)
+{
+  /* Should never be called twice. */
+  tor_assert(!hs_service_map);
+  tor_assert(!hs_service_staging_list);
+
+  /* v2 specific. */
+  rend_service_init();
+
+  hs_service_map = tor_malloc_zero(sizeof(struct hs_service_ht));
+  HT_INIT(hs_service_ht, hs_service_map);
+
+  hs_service_staging_list = smartlist_new();
+}
+
+/* Release all global storage of the hidden service subsystem. */
+void
+hs_service_free_all(void)
+{
+  rend_service_free_all();
+  service_free_all();
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global service map size. Only used by unit test. */
+STATIC unsigned int
+get_hs_service_map_size(void)
+{
+  return HT_SIZE(hs_service_map);
+}
+
+/* Return the staging list size. Only used by unit test. */
+STATIC int
+get_hs_service_staging_list_size(void)
+{
+  return smartlist_len(hs_service_staging_list);
+}
+
+STATIC hs_service_ht *
+get_hs_service_map(void)
+{
+  return hs_service_map;
+}
+
+STATIC hs_service_t *
+get_first_service(void)
+{
+  hs_service_t **obj = HT_START(hs_service_ht, hs_service_map);
+  if (obj == NULL) {
+    return NULL;
+  }
+  return *obj;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/or/hs_service.h b/src/or/hs_service.h
new file mode 100644
index 0000000000..911e177702
--- /dev/null
+++ b/src/or/hs_service.h
@@ -0,0 +1,379 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.h
+ * \brief Header file containing service data for the HS subsytem.
+ **/
+
+#ifndef TOR_HS_SERVICE_H
+#define TOR_HS_SERVICE_H
+
+#include "crypto_curve25519.h"
+#include "crypto_ed25519.h"
+#include "replaycache.h"
+
+#include "hs_common.h"
+#include "hs_descriptor.h"
+#include "hs_ident.h"
+#include "hs_intropoint.h"
+
+/* Trunnel */
+#include "hs/cell_establish_intro.h"
+
+/* When loading and configuring a service, this is the default version it will
+ * be configured for as it is possible that no HiddenServiceVersion is
+ * present. */
+#define HS_SERVICE_DEFAULT_VERSION HS_VERSION_TWO
+
+/* As described in the specification, service publishes their next descriptor
+ * at a random time between those two values (in seconds). */
+#define HS_SERVICE_NEXT_UPLOAD_TIME_MIN (60 * 60)
+#define HS_SERVICE_NEXT_UPLOAD_TIME_MAX (120 * 60)
+
+/* Service side introduction point. */
+typedef struct hs_service_intro_point_t {
+  /* Top level intropoint "shared" data between client/service. */
+  hs_intropoint_t base;
+
+  /* Onion key of the introduction point used to extend to it for the ntor
+   * handshake. */
+  curve25519_public_key_t onion_key;
+
+  /* Authentication keypair used to create the authentication certificate
+   * which is published in the descriptor. */
+  ed25519_keypair_t auth_key_kp;
+
+  /* Encryption keypair for the "ntor" type. */
+  curve25519_keypair_t enc_key_kp;
+
+  /* Legacy key if that intro point doesn't support v3. This should be used if
+   * the base object legacy flag is set. */
+  crypto_pk_t *legacy_key;
+  /* Legacy key SHA1 public key digest. This should be used only if the base
+   * object legacy flag is set. */
+  uint8_t legacy_key_digest[DIGEST_LEN];
+
+  /* Amount of INTRODUCE2 cell accepted from this intro point. */
+  uint64_t introduce2_count;
+
+  /* Maximum number of INTRODUCE2 cell this intro point should accept. */
+  uint64_t introduce2_max;
+
+  /* The time at which this intro point should expire and stop being used. */
+  time_t time_to_expire;
+
+  /* The amount of circuit creation we've made to this intro point. This is
+   * incremented every time we do a circuit relaunch on this intro point which
+   * is triggered when the circuit dies but the node is still in the
+   * consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give up on it. */
+  uint32_t circuit_retries;
+
+  /* Set if this intro point has an established circuit. */
+  unsigned int circuit_established : 1;
+
+  /* Replay cache recording the encrypted part of an INTRODUCE2 cell that the
+   * circuit associated with this intro point has received. This is used to
+   * prevent replay attacks. */
+  replaycache_t *replay_cache;
+} hs_service_intro_point_t;
+
+/* Object handling introduction points of a service. */
+typedef struct hs_service_intropoints_t {
+  /* The time at which we've started our retry period to build circuits. We
+   * don't want to stress circuit creation so we can only retry for a certain
+   * time and then after we stop and wait. */
+  time_t retry_period_started;
+
+  /* Number of circuit we've launched during a single retry period. */
+  unsigned int num_circuits_launched;
+
+  /* Contains the current hs_service_intro_point_t objects indexed by
+   * authentication public key. */
+  digest256map_t *map;
+
+  /* Contains node's identity key digest that were introduction point for this
+   * descriptor but were retried to many times. We keep those so we avoid
+   * re-picking them over and over for a circuit retry period.
+   * XXX: Once we have #22173, change this to only use ed25519 identity. */
+  digestmap_t *failed_id;
+} hs_service_intropoints_t;
+
+/* Representation of a service descriptor. */
+typedef struct hs_service_descriptor_t {
+  /* Decoded descriptor. This object is used for encoding when the service
+   * publishes the descriptor. */
+  hs_descriptor_t *desc;
+
+  /* Descriptor signing keypair. */
+  ed25519_keypair_t signing_kp;
+
+  /* Blinded keypair derived from the master identity public key. */
+  ed25519_keypair_t blinded_kp;
+
+  /* When is the next time when we should upload the descriptor. */
+  time_t next_upload_time;
+
+  /* Introduction points assign to this descriptor which contains
+   * hs_service_intropoints_t object indexed by authentication key (the RSA
+   * key if the node is legacy). */
+  hs_service_intropoints_t intro_points;
+
+  /* The time period number this descriptor has been created for. */
+  uint64_t time_period_num;
+
+  /* True iff we have missing intro points for this descriptor because we
+   * couldn't pick any nodes. */
+  unsigned int missing_intro_points : 1;
+
+  /** List of the responsible HSDirs (their b64ed identity digest) last time we
+   *  uploaded this descriptor. If the set of responsible HSDirs is different
+   *  from this list, this means we received new dirinfo and we need to
+   *  reupload our descriptor. */
+  smartlist_t *previous_hsdirs;
+} hs_service_descriptor_t;
+
+/* Service key material. */
+typedef struct hs_service_keys_t {
+  /* Master identify public key. */
+  ed25519_public_key_t identity_pk;
+  /* Master identity private key. */
+  ed25519_secret_key_t identity_sk;
+  /* True iff the key is kept offline which means the identity_sk MUST not be
+   * used in that case. */
+  unsigned int is_identify_key_offline : 1;
+} hs_service_keys_t;
+
+/* Service configuration. The following are set from the torrc options either
+ * set by the configuration file or by the control port. Nothing else should
+ * change those values. */
+typedef struct hs_service_config_t {
+  /* Protocol version of the service. Specified by HiddenServiceVersion
+   * option. */
+  uint32_t version;
+
+  /* List of rend_service_port_config_t */
+  smartlist_t *ports;
+
+  /* Path on the filesystem where the service persistent data is stored. NULL
+   * if the service is ephemeral. Specified by HiddenServiceDir option. */
+  char *directory_path;
+
+  /* The maximum number of simultaneous streams per rendezvous circuit that
+   * are allowed to be created. No limit if 0. Specified by
+   * HiddenServiceMaxStreams option. */
+  uint64_t max_streams_per_rdv_circuit;
+
+  /* If true, we close circuits that exceed the max_streams_per_rdv_circuit
+   * limit. Specified by HiddenServiceMaxStreamsCloseCircuit option. */
+  unsigned int max_streams_close_circuit : 1;
+
+  /* How many introduction points this service has. Specified by
+   * HiddenServiceNumIntroductionPoints option. */
+  unsigned int num_intro_points;
+
+  /* True iff we allow request made on unknown ports. Specified by
+   * HiddenServiceAllowUnknownPorts option. */
+  unsigned int allow_unknown_ports : 1;
+
+  /* If true, this service is a Single Onion Service. Specified by
+   * HiddenServiceSingleHopMode and HiddenServiceNonAnonymousMode options. */
+  unsigned int is_single_onion : 1;
+
+  /* If true, allow group read permissions on the directory_path. Specified by
+   * HiddenServiceDirGroupReadable option. */
+  unsigned int dir_group_readable : 1;
+
+  /* Is this service ephemeral? */
+  unsigned int is_ephemeral : 1;
+} hs_service_config_t;
+
+/* Service state. */
+typedef struct hs_service_state_t {
+  /* The time at which we've started our retry period to build circuits. We
+   * don't want to stress circuit creation so we can only retry for a certain
+   * time and then after we stop and wait. */
+  time_t intro_circ_retry_started_time;
+
+  /* Number of circuit we've launched during a single retry period. This
+   * should never go over MAX_INTRO_CIRCS_PER_PERIOD. */
+  unsigned int num_intro_circ_launched;
+
+  /* Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
+   * repeats. Clients may send INTRODUCE1 cells for the same rendezvous point
+   * through two or more different introduction points; when they do, this
+   * keeps us from launching multiple simultaneous attempts to connect to the
+   * same rend point. */
+  replaycache_t *replay_cache_rend_cookie;
+
+  /* When is the next time we should rotate our descriptors. This is has to be
+   * done at the start time of the next SRV protocol run. */
+  time_t next_rotation_time;
+} hs_service_state_t;
+
+/* Representation of a service running on this tor instance. */
+typedef struct hs_service_t {
+  /* Onion address base32 encoded and NUL terminated. We keep it for logging
+   * purposes so we don't have to build it everytime. */
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  /* Hashtable node: use to look up the service by its master public identity
+   * key in the service global map. */
+  HT_ENTRY(hs_service_t) hs_service_node;
+
+  /* Service state which contains various flags and counters. */
+  hs_service_state_t state;
+
+  /* Key material of the service. */
+  hs_service_keys_t keys;
+
+  /* Configuration of the service. */
+  hs_service_config_t config;
+
+  /* Current descriptor. */
+  hs_service_descriptor_t *desc_current;
+  /* Next descriptor. */
+  hs_service_descriptor_t *desc_next;
+
+  /* XXX: Credential (client auth.) #20700. */
+
+} hs_service_t;
+
+/* For the service global hash map, we define a specific type for it which
+ * will make it safe to use and specific to some controlled parameters such as
+ * the hashing function and how to compare services. */
+typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht;
+
+/* API */
+
+/* Global initializer and cleanup function. */
+void hs_service_init(void);
+void hs_service_free_all(void);
+
+/* Service new/free functions. */
+hs_service_t *hs_service_new(const or_options_t *options);
+void hs_service_free_(hs_service_t *service);
+#define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
+
+unsigned int hs_service_get_num_services(void);
+void hs_service_stage_services(const smartlist_t *service_list);
+int hs_service_load_all_keys(void);
+void hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
+                                         smartlist_t *dir_list);
+int hs_service_set_conn_addr_port(const origin_circuit_t *circ,
+                                  edge_connection_t *conn);
+
+void hs_service_map_has_changed(void);
+void hs_service_dir_info_changed(void);
+void hs_service_run_scheduled_events(time_t now);
+void hs_service_circuit_has_opened(origin_circuit_t *circ);
+int hs_service_receive_intro_established(origin_circuit_t *circ,
+                                         const uint8_t *payload,
+                                         size_t payload_len);
+int hs_service_receive_introduce2(origin_circuit_t *circ,
+                                  const uint8_t *payload,
+                                  size_t payload_len);
+
+void hs_service_intro_circ_has_closed(origin_circuit_t *circ);
+
+char *hs_service_lookup_current_desc(const ed25519_public_key_t *pk);
+
+hs_service_add_ephemeral_status_t
+hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
+                         int max_streams_per_rdv_circuit,
+                         int max_streams_close_circuit, char **address_out);
+int hs_service_del_ephemeral(const char *address);
+
+/* Used outside of the HS subsystem by the control port command HSPOST. */
+void hs_service_upload_desc_to_dir(const char *encoded_desc,
+                                   const uint8_t version,
+                                   const ed25519_public_key_t *identity_pk,
+                                   const ed25519_public_key_t *blinded_pk,
+                                   const routerstatus_t *hsdir_rs);
+
+#ifdef HS_SERVICE_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+/* Useful getters for unit tests. */
+STATIC unsigned int get_hs_service_map_size(void);
+STATIC int get_hs_service_staging_list_size(void);
+STATIC hs_service_ht *get_hs_service_map(void);
+STATIC hs_service_t *get_first_service(void);
+STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
+                                         const hs_service_t *service,
+                                         const hs_ident_circuit_t *ident);
+#endif
+
+/* Service accessors. */
+STATIC hs_service_t *find_service(hs_service_ht *map,
+                                  const ed25519_public_key_t *pk);
+STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
+STATIC int register_service(hs_service_ht *map, hs_service_t *service);
+/* Service introduction point functions. */
+STATIC hs_service_intro_point_t *service_intro_point_new(
+                            const extend_info_t *ei,
+                            unsigned int is_legacy,
+                            unsigned int supports_ed25519_link_handshake_any);
+STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
+#define service_intro_point_free(ip)                            \
+  FREE_AND_NULL(hs_service_intro_point_t,             \
+                          service_intro_point_free_, (ip))
+STATIC void service_intro_point_add(digest256map_t *map,
+                                    hs_service_intro_point_t *ip);
+STATIC void service_intro_point_remove(const hs_service_t *service,
+                                       const hs_service_intro_point_t *ip);
+STATIC hs_service_intro_point_t *service_intro_point_find(
+                                 const hs_service_t *service,
+                                 const ed25519_public_key_t *auth_key);
+/* Service descriptor functions. */
+STATIC hs_service_descriptor_t *service_descriptor_new(void);
+STATIC hs_service_descriptor_t *service_desc_find_by_intro(
+                                         const hs_service_t *service,
+                                         const hs_service_intro_point_t *ip);
+/* Helper functions. */
+STATIC void get_objects_from_ident(const hs_ident_circuit_t *ident,
+                                   hs_service_t **service,
+                                   hs_service_intro_point_t **ip,
+                                   hs_service_descriptor_t **desc);
+STATIC const node_t *
+get_node_from_intro_point(const hs_service_intro_point_t *ip);
+STATIC int can_service_launch_intro_circuit(hs_service_t *service,
+                                            time_t now);
+STATIC int intro_point_should_expire(const hs_service_intro_point_t *ip,
+                                     time_t now);
+STATIC void run_housekeeping_event(time_t now);
+STATIC void rotate_all_descriptors(time_t now);
+STATIC void build_all_descriptors(time_t now);
+STATIC void update_all_descriptors(time_t now);
+STATIC void run_upload_descriptor_event(time_t now);
+
+STATIC char *
+encode_desc_rev_counter_for_state(const hs_service_descriptor_t *desc);
+
+STATIC void service_descriptor_free_(hs_service_descriptor_t *desc);
+#define service_descriptor_free(d) \
+  FREE_AND_NULL(hs_service_descriptor_t, \
+                           service_descriptor_free_, (d))
+
+STATIC uint64_t
+check_state_line_for_service_rev_counter(const char *state_line,
+                                    const ed25519_public_key_t *blinded_pubkey,
+                                    int *service_found_out);
+
+STATIC int
+write_address_to_file(const hs_service_t *service, const char *fname_);
+
+STATIC void upload_descriptor_to_all(const hs_service_t *service,
+                                     hs_service_descriptor_t *desc);
+
+STATIC void service_desc_schedule_upload(hs_service_descriptor_t *desc,
+                                         time_t now,
+                                         int descriptor_changed);
+
+STATIC int service_desc_hsdirs_changed(const hs_service_t *service,
+                                const hs_service_descriptor_t *desc);
+
+#endif /* defined(HS_SERVICE_PRIVATE) */
+
+#endif /* !defined(TOR_HS_SERVICE_H) */
+
diff --git a/src/or/hs_stats.c b/src/or/hs_stats.c
new file mode 100644
index 0000000000..1e2a96945b
--- /dev/null
+++ b/src/or/hs_stats.c
@@ -0,0 +1,58 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_stats.c
+ * \brief Keeps stats about the activity of our onion service(s).
+ **/
+
+#include "or.h"
+#include "hs_stats.h"
+#include "hs_service.h"
+
+/** Number of v3 INTRODUCE2 cells received */
+static uint32_t n_introduce2_v3 = 0;
+/** Number of v2 INTRODUCE2 cells received */
+static uint32_t n_introduce2_v2 = 0;
+/** Number of attempts to make a circuit to a rendezvous point */
+static uint32_t n_rendezvous_launches = 0;
+
+/** Note that we received another INTRODUCE2 cell. */
+void
+hs_stats_note_introduce2_cell(int is_hsv3)
+{
+  if (is_hsv3) {
+    n_introduce2_v3++;
+  } else {
+    n_introduce2_v2++;
+  }
+}
+
+/** Return the number of v3 INTRODUCE2 cells we have received. */
+uint32_t
+hs_stats_get_n_introduce2_v3_cells(void)
+{
+  return n_introduce2_v3;
+}
+
+/** Return the number of v2 INTRODUCE2 cells we have received. */
+uint32_t
+hs_stats_get_n_introduce2_v2_cells(void)
+{
+  return n_introduce2_v2;
+}
+
+/** Note that we attempted to launch another circuit to a rendezvous point. */
+void
+hs_stats_note_service_rendezvous_launch(void)
+{
+  n_rendezvous_launches++;
+}
+
+/** Return the number of rendezvous circuits we have attempted to launch. */
+uint32_t
+hs_stats_get_n_rendezvous_launches(void)
+{
+  return n_rendezvous_launches;
+}
+
diff --git a/src/or/hs_stats.h b/src/or/hs_stats.h
new file mode 100644
index 0000000000..a946ad75e5
--- /dev/null
+++ b/src/or/hs_stats.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2016-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_stats.h
+ * \brief Header file for hs_stats.c
+ **/
+
+void hs_stats_note_introduce2_cell(int is_hsv3);
+uint32_t hs_stats_get_n_introduce2_v3_cells(void);
+uint32_t hs_stats_get_n_introduce2_v2_cells(void);
+void hs_stats_note_service_rendezvous_launch(void);
+uint32_t hs_stats_get_n_rendezvous_launches(void);
+
diff --git a/src/or/include.am b/src/or/include.am
index 19cf00264b..59d593a5e9 100644
--- a/src/or/include.am
+++ b/src/or/include.am
@@ -19,8 +19,9 @@ EXTRA_DIST+= src/or/ntmain.c src/or/Makefile.nmake
 
 LIBTOR_A_SOURCES = \
 	src/or/addressmap.c				\
-	src/or/buffers.c				\
+	src/or/bridges.c				\
 	src/or/channel.c				\
+	src/or/channelpadding.c				\
 	src/or/channeltls.c				\
 	src/or/circpathbias.c				\
 	src/or/circuitbuild.c				\
@@ -35,12 +36,13 @@ LIBTOR_A_SOURCES = \
 	src/or/connection.c				\
 	src/or/connection_edge.c			\
 	src/or/connection_or.c				\
+	src/or/conscache.c				\
+	src/or/consdiff.c				\
+	src/or/consdiffmgr.c				\
 	src/or/control.c				\
 	src/or/cpuworker.c				\
-	src/or/dircollate.c				\
 	src/or/directory.c				\
 	src/or/dirserv.c				\
-	src/or/dirvote.c				\
 	src/or/dns.c					\
 	src/or/dnsserv.c				\
 	src/or/dos.c					\
@@ -48,7 +50,22 @@ LIBTOR_A_SOURCES = \
 	src/or/geoip.c					\
 	src/or/entrynodes.c				\
 	src/or/ext_orport.c				\
+	src/or/git_revision.c				\
 	src/or/hibernate.c				\
+	src/or/hs_cache.c				\
+	src/or/hs_cell.c				\
+	src/or/hs_circuit.c				\
+	src/or/hs_circuitmap.c				\
+	src/or/hs_client.c				\
+	src/or/hs_common.c				\
+	src/or/hs_config.c				\
+	src/or/hs_control.c				\
+	src/or/hs_descriptor.c				\
+	src/or/hs_ident.c				\
+	src/or/hs_intropoint.c				\
+	src/or/hs_ntor.c				\
+	src/or/hs_service.c				\
+	src/or/hs_stats.c				\
 	src/or/keypin.c					\
 	src/or/main.c					\
 	src/or/microdesc.c				\
@@ -57,14 +74,20 @@ LIBTOR_A_SOURCES = \
 	src/or/onion.c					\
 	src/or/onion_fast.c				\
 	src/or/onion_tap.c				\
-	src/or/shared_random.c			\
-	src/or/shared_random_state.c		\
 	src/or/transports.c				\
+	src/or/parsecommon.c			\
 	src/or/periodic.c				\
 	src/or/protover.c				\
+	src/or/protover_rust.c				\
+	src/or/proto_cell.c				\
+	src/or/proto_control0.c				\
+	src/or/proto_ext_or.c				\
+	src/or/proto_http.c				\
+	src/or/proto_socks.c				\
 	src/or/policies.c				\
 	src/or/reasons.c				\
 	src/or/relay.c					\
+	src/or/relay_crypto.c				\
 	src/or/rendcache.c				\
 	src/or/rendclient.c				\
 	src/or/rendcommon.c				\
@@ -78,14 +101,45 @@ LIBTOR_A_SOURCES = \
 	src/or/routerparse.c				\
 	src/or/routerset.c				\
 	src/or/scheduler.c				\
+	src/or/scheduler_kist.c				\
+	src/or/scheduler_vanilla.c			\
+	src/or/shared_random_client.c			\
 	src/or/statefile.c				\
 	src/or/status.c					\
 	src/or/torcert.c				\
+	src/or/tor_api.c				\
+	src/or/voting_schedule.c			\
 	src/or/onion_ntor.c				\
 	$(tor_platform_source)
 
+#
+# Modules are conditionnally compiled in tor starting here. We add the C files
+# only if the modules has been enabled at configure time. We always add the
+# source files of every module to libtor-testing.a so we can build the unit
+# tests for everything. See the UNITTESTS_ENABLED branch below.
+#
+LIBTOR_TESTING_A_SOURCES = $(LIBTOR_A_SOURCES)
+
+# The Directory Authority module.
+MODULE_DIRAUTH_SOURCES = \
+	src/or/dirauth/dircollate.c				\
+	src/or/dirauth/dirvote.c				\
+	src/or/dirauth/shared_random.c				\
+	src/or/dirauth/shared_random_state.c
+if BUILD_MODULE_DIRAUTH
+LIBTOR_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
+endif
+
 src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES)
-src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES)
+if UNITTESTS_ENABLED
+
+# Add the sources of the modules that are needed for tests to work here.
+LIBTOR_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
+
+src_or_libtor_testing_a_SOURCES = $(LIBTOR_TESTING_A_SOURCES)
+else
+src_or_libtor_testing_a_SOURCES =
+endif
 
 src_or_tor_SOURCES = src/or/tor_main.c
 AM_CPPFLAGS += -I$(srcdir)/src/or -Isrc/or
@@ -109,8 +163,12 @@ src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libev
 src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \
 	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event.a src/trunnel/libor-trunnel.a \
+	src/trace/libor-trace.a \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 if COVERAGE_ENABLED
 src_or_tor_cov_SOURCES = src/or/tor_main.c
@@ -122,14 +180,16 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \
 	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 endif
 
 ORHEADERS = \
 	src/or/addressmap.h				\
 	src/or/auth_dirs.inc				\
-	src/or/buffers.h				\
+	src/or/bridges.h				\
 	src/or/channel.h				\
+	src/or/channelpadding.h				\
 	src/or/channeltls.h				\
 	src/or/circpathbias.h				\
 	src/or/circuitbuild.h				\
@@ -144,12 +204,13 @@ ORHEADERS = \
 	src/or/connection.h				\
 	src/or/connection_edge.h			\
 	src/or/connection_or.h				\
+	src/or/conscache.h				\
+	src/or/consdiff.h				\
+	src/or/consdiffmgr.h				\
 	src/or/control.h				\
 	src/or/cpuworker.h				\
-	src/or/dircollate.h				\
 	src/or/directory.h				\
 	src/or/dirserv.h				\
-	src/or/dirvote.h				\
 	src/or/dns.h					\
 	src/or/dns_structs.h				\
 	src/or/dnsserv.h				\
@@ -159,7 +220,22 @@ ORHEADERS = \
 	src/or/fp_pair.h				\
 	src/or/geoip.h					\
 	src/or/entrynodes.h				\
+	src/or/git_revision.h				\
 	src/or/hibernate.h				\
+	src/or/hs_cache.h				\
+	src/or/hs_cell.h				\
+	src/or/hs_circuit.h				\
+	src/or/hs_circuitmap.h				\
+	src/or/hs_client.h				\
+	src/or/hs_common.h				\
+	src/or/hs_config.h				\
+	src/or/hs_control.h				\
+	src/or/hs_descriptor.h				\
+	src/or/hs_ident.h				\
+	src/or/hs_intropoint.h				\
+	src/or/hs_ntor.h				\
+	src/or/hs_stats.h				\
+	src/or/hs_service.h				\
 	src/or/keypin.h					\
 	src/or/main.h					\
 	src/or/microdesc.h				\
@@ -171,14 +247,19 @@ ORHEADERS = \
 	src/or/onion_ntor.h				\
 	src/or/onion_tap.h				\
 	src/or/or.h					\
-	src/or/shared_random.h			\
-	src/or/shared_random_state.h		\
 	src/or/transports.h				\
+	src/or/parsecommon.h			\
 	src/or/periodic.h				\
 	src/or/policies.h				\
 	src/or/protover.h				\
+	src/or/proto_cell.h				\
+	src/or/proto_control0.h				\
+	src/or/proto_ext_or.h				\
+	src/or/proto_http.h				\
+	src/or/proto_socks.h				\
 	src/or/reasons.h				\
 	src/or/relay.h					\
+	src/or/relay_crypto.h				\
 	src/or/rendcache.h				\
 	src/or/rendclient.h				\
 	src/or/rendcommon.h				\
@@ -193,11 +274,28 @@ ORHEADERS = \
 	src/or/routerset.h				\
 	src/or/routerparse.h				\
 	src/or/scheduler.h				\
+	src/or/shared_random_client.h			\
 	src/or/statefile.h				\
 	src/or/status.h					\
-	src/or/torcert.h
+	src/or/torcert.h				\
+	src/or/tor_api_internal.h			\
+	src/or/voting_schedule.h
+
+# We add the headers of the modules even though they are disabled so we can
+# properly compiled the entry points stub.
+
+# The Directory Authority module headers.
+ORHEADERS += \
+	src/or/dirauth/dircollate.h				\
+	src/or/dirauth/dirvote.h				\
+	src/or/dirauth/mode.h					\
+	src/or/dirauth/shared_random.h				\
+	src/or/dirauth/shared_random_state.h
+
+# This may someday want to be an installed file?
+noinst_HEADERS += src/or/tor_api.h
 
-noinst_HEADERS+= $(ORHEADERS) micro-revision.i
+noinst_HEADERS += $(ORHEADERS) micro-revision.i
 
 micro-revision.i: FORCE
 	$(AM_V_at)rm -f micro-revision.tmp; \
diff --git a/src/or/keypin.c b/src/or/keypin.c
index 2d4c4e92d2..97e16c1f78 100644
--- a/src/or/keypin.c
+++ b/src/or/keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 
 #include "orconfig.h"
 #include "compat.h"
-#include "crypto.h"
+#include "crypto_digest.h"
 #include "crypto_format.h"
 #include "di_ops.h"
 #include "ht.h"
@@ -289,8 +289,10 @@ static int keypin_journal_fd = -1;
 int
 keypin_open_journal(const char *fname)
 {
-  /* O_SYNC ??*/
-  int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY, 0600);
+#ifndef O_SYNC
+#define O_SYNC 0
+#endif
+  int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY|O_SYNC, 0600);
   if (fd < 0)
     goto err;
 
@@ -417,10 +419,11 @@ keypin_load_journal_impl(const char *data, size_t size)
     ++n_entries;
   }
 
-  int severity = (n_corrupt_lines || n_duplicates) ? LOG_WARN : LOG_INFO;
+  int severity = (n_corrupt_lines || n_duplicates) ? LOG_NOTICE : LOG_INFO;
   tor_log(severity, LD_DIRSERV,
           "Loaded %d entries from keypin journal. "
-          "Found %d corrupt lines, %d duplicates, and %d conflicts.",
+          "Found %d corrupt lines (ignored), %d duplicates (harmless), "
+          "and %d conflicts (resolved in favor or more recent entry).",
           n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
 
   return 0;
diff --git a/src/or/keypin.h b/src/or/keypin.h
index 673f24d9e3..fbb77e5c35 100644
--- a/src/or/keypin.h
+++ b/src/or/keypin.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_KEYPIN_H
@@ -41,7 +41,7 @@ STATIC keypin_ent_t * keypin_parse_journal_line(const char *cp);
 STATIC int keypin_load_journal_impl(const char *data, size_t size);
 
 MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
-#endif
+#endif /* defined(KEYPIN_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_KEYPIN_H) */
 
diff --git a/src/or/main.c b/src/or/main.c
index 55bcb18c4b..c69a5b8b8b 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -1,43 +1,88 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file main.c
  * \brief Toplevel module. Handles signals, multiplexes between
  * connections, implements main loop, and drives scheduled events.
+ *
+ * For the main loop itself; see run_main_loop_once().  It invokes the rest of
+ * Tor mostly through Libevent callbacks.  Libevent callbacks can happen when
+ * a timer elapses, a signal is received, a socket is ready to read or write,
+ * or an event is manually activated.
+ *
+ * Most events in Tor are driven from these callbacks:
+ *  <ul>
+ *   <li>conn_read_callback() and conn_write_callback() here, which are
+ *     invoked when a socket is ready to read or write respectively.
+ *   <li>signal_callback(), which handles incoming signals.
+ *  </ul>
+ * Other events are used for specific purposes, or for building more complex
+ * control structures.  If you search for usage of tor_libevent_new(), you
+ * will find all the events that we construct in Tor.
+ *
+ * Tor has numerous housekeeping operations that need to happen
+ * regularly. They are handled in different ways:
+ * <ul>
+ *   <li>The most frequent operations are handled after every read or write
+ *    event, at the end of connection_handle_read() and
+ *    connection_handle_write().
+ *
+ *   <li>The next most frequent operations happen after each invocation of the
+ *     main loop, in run_main_loop_once().
+ *
+ *   <li>Once per second, we run all of the operations listed in
+ *     second_elapsed_callback(), and in its child, run_scheduled_events().
+ *
+ *   <li>Once-a-second operations are handled in second_elapsed_callback().
+ *
+ *   <li>More infrequent operations take place based on the periodic event
+ *     driver in periodic.c .  These are stored in the periodic_events[]
+ *     table.
+ * </ul>
+ *
  **/
 
 #define MAIN_PRIVATE
 #include "or.h"
 #include "addressmap.h"
 #include "backtrace.h"
+#include "bridges.h"
 #include "buffers.h"
+#include "buffers_tls.h"
 #include "channel.h"
 #include "channeltls.h"
+#include "channelpadding.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
+#include "circuitmux_ewma.h"
 #include "command.h"
+#include "compress.h"
 #include "config.h"
 #include "confparse.h"
 #include "connection.h"
 #include "connection_edge.h"
 #include "connection_or.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "cpuworker.h"
 #include "crypto_s2k.h"
+#include "crypto_rand.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "dns.h"
 #include "dnsserv.h"
 #include "dos.h"
 #include "entrynodes.h"
 #include "geoip.h"
 #include "hibernate.h"
+#include "hs_cache.h"
+#include "hs_circuitmap.h"
+#include "hs_client.h"
 #include "keypin.h"
 #include "main.h"
 #include "microdesc.h"
@@ -59,32 +104,42 @@
 #include "routerlist.h"
 #include "routerparse.h"
 #include "scheduler.h"
-#include "shared_random.h"
 #include "statefile.h"
 #include "status.h"
+#include "tor_api.h"
+#include "tor_api_internal.h"
 #include "util_process.h"
 #include "ext_orport.h"
 #ifdef USE_DMALLOC
 #include <dmalloc.h>
-#include <openssl/crypto.h>
 #endif
 #include "memarea.h"
 #include "sandbox.h"
 
 #include <event2/event.h>
 
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
+#include "dirauth/shared_random.h"
+
 #ifdef HAVE_SYSTEMD
 #   if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__)
 /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-#   endif
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
 
 void evdns_shutdown(int);
 
+#ifdef HAVE_RUST
+// helper function defined in Rust to output a log message indicating if tor is
+// running with Rust enabled. See src/rust/tor_util
+void rust_log_welcome_string(void);
+#endif
+
 /********* PROTOTYPES **********/
 
 static void dumpmemusage(int severity);
@@ -97,26 +152,16 @@ static void connection_start_reading_from_linked_conn(connection_t *conn);
 static int connection_should_read_from_linked_conn(connection_t *conn);
 static int run_main_loop_until_done(void);
 static void process_signal(int sig);
+static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
+                                           void *arg) ATTR_NORETURN;
 
 /********* START VARIABLES **********/
-int global_read_bucket; /**< Max number of bytes I can read this second. */
-int global_write_bucket; /**< Max number of bytes I can write this second. */
-
-/** Max number of relayed (bandwidth class 1) bytes I can read this second. */
-int global_relayed_read_bucket;
-/** Max number of relayed (bandwidth class 1) bytes I can write this second. */
-int global_relayed_write_bucket;
-/** What was the read bucket before the last second_elapsed_callback() call?
- * (used to determine how many bytes we've read). */
-static int stats_prev_global_read_bucket;
-/** What was the write bucket before the last second_elapsed_callback() call?
- * (used to determine how many bytes we've written). */
-static int stats_prev_global_write_bucket;
-
-/* DOCDOC stats_prev_n_read */
-static uint64_t stats_prev_n_read = 0;
-/* DOCDOC stats_prev_n_written */
-static uint64_t stats_prev_n_written = 0;
+
+/* Token bucket for all traffic. */
+token_bucket_rw_t global_bucket;
+
+/* Token bucket for relayed traffic. */
+token_bucket_rw_t global_relayed_bucket;
 
 /* XXX we might want to keep stats about global_relayed_*_bucket too. Or not.*/
 /** How many bytes have we read since we started the process? */
@@ -126,7 +171,13 @@ static uint64_t stats_n_bytes_written = 0;
 /** What time did this process start up? */
 time_t time_of_process_start = 0;
 /** How many seconds have we been running? */
-long stats_n_seconds_working = 0;
+static long stats_n_seconds_working = 0;
+/** How many times have we returned from the main loop successfully? */
+static uint64_t stats_n_main_loop_successes = 0;
+/** How many times have we received an error from the main loop? */
+static uint64_t stats_n_main_loop_errors = 0;
+/** How many times have we returned from the main loop with no events. */
+static uint64_t stats_n_main_loop_idle = 0;
 
 /** How often will we honor SIGNEWNYM requests? */
 #define MAX_SIGNEWNYM_RATE 10
@@ -134,11 +185,13 @@ long stats_n_seconds_working = 0;
 static time_t time_of_last_signewnym = 0;
 /** Is there a signewnym request we're currently waiting to handle? */
 static int signewnym_is_pending = 0;
+/** Mainloop event for the deferred signewnym call. */
+static mainloop_event_t *handle_deferred_signewnym_ev = NULL;
 /** How many times have we called newnym? */
 static unsigned newnym_epoch = 0;
 
 /** Smartlist of all open connections. */
-static smartlist_t *connection_array = NULL;
+STATIC smartlist_t *connection_array = NULL;
 /** List of connections that have been marked for close and need to be freed
  * and removed from connection_array. */
 static smartlist_t *closeable_connection_lst = NULL;
@@ -149,6 +202,14 @@ static smartlist_t *active_linked_connection_lst = NULL;
  * <b>loop_once</b>. If so, there's no need to trigger a loopexit in order
  * to handle linked connections. */
 static int called_loop_once = 0;
+/** Flag: if true, it's time to shut down, so the main loop should exit as
+ * soon as possible.
+ */
+static int main_loop_should_exit = 0;
+/** The return value that the main loop should yield when it exits, if
+ * main_loop_should_exit is true.
+ */
+static int main_loop_exit_value = 0;
 
 /** We set this to 1 when we've opened a circuit, so we can print a log
  * entry to inform the user that Tor is working.  We set it to 0 when
@@ -275,7 +336,7 @@ connection_remove(connection_t *conn)
             smartlist_len(connection_array));
 
   if (conn->type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) {
-    log_info(LD_NET, "Closing SOCKS SocksSocket connection");
+    log_info(LD_NET, "Closing SOCKS Unix socket connection");
   }
 
   control_event_conn_bandwidth(conn);
@@ -326,7 +387,7 @@ connection_unlink(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_OR) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest))
-      connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+      connection_or_clear_identity(TO_OR_CONN(conn));
     /* connection_unlink() can only get called if the connection
      * was already on the closeable list, and it got there by
      * connection_mark_for_close(), which was called from
@@ -343,6 +404,38 @@ connection_unlink(connection_t *conn)
   connection_free(conn);
 }
 
+/** Event that invokes schedule_active_linked_connections_cb. */
+static mainloop_event_t *schedule_active_linked_connections_event = NULL;
+
+/**
+ * Callback: used to activate read events for all linked connections, so
+ * libevent knows to call their read callbacks.  This callback run as a
+ * postloop event, so that the events _it_ activates don't happen until
+ * Libevent has a chance to check for other events.
+ */
+static void
+schedule_active_linked_connections_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+
+  /* All active linked conns should get their read events activated,
+   * so that libevent knows to run their callbacks. */
+  SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn,
+                    event_active(conn->read_event, EV_READ, 1));
+
+  /* Reactivate the event if we still have connections in the active list.
+   *
+   * A linked connection doesn't get woken up by I/O but rather artificially
+   * by this event callback. It has directory data spooled in it and it is
+   * sent incrementally by small chunks unless spool_eagerly is true. For that
+   * to happen, we need to induce the activation of the read event so it can
+   * be flushed. */
+  if (smartlist_len(active_linked_connection_lst)) {
+    mainloop_event_activate(schedule_active_linked_connections_event);
+  }
+}
+
 /** Initialize the global connection list, closeable connection list,
  * and active connection list. */
 STATIC void
@@ -364,6 +457,7 @@ add_connection_to_closeable_list(connection_t *conn)
   tor_assert(conn->marked_for_close);
   assert_connection_ok(conn, time(NULL));
   smartlist_add(closeable_connection_lst, conn);
+  mainloop_schedule_postloop_cleanup();
 }
 
 /** Return 1 if conn is on the closeable list, else return 0. */
@@ -391,21 +485,37 @@ get_connection_array, (void))
   return connection_array;
 }
 
-/** Provides the traffic read and written over the life of the process. */
-
+/**
+ * Return the amount of network traffic read, in bytes, over the life of this
+ * process.
+ */
 MOCK_IMPL(uint64_t,
 get_bytes_read,(void))
 {
   return stats_n_bytes_read;
 }
 
-/* DOCDOC get_bytes_written */
+/**
+ * Return the amount of network traffic read, in bytes, over the life of this
+ * process.
+ */
 MOCK_IMPL(uint64_t,
 get_bytes_written,(void))
 {
   return stats_n_bytes_written;
 }
 
+/**
+ * Increment the amount of network traffic read and written, over the life of
+ * this process.
+ */
+void
+stats_increment_bytes_read_and_written(uint64_t r, uint64_t w)
+{
+  stats_n_bytes_read += r;
+  stats_n_bytes_written += w;
+}
+
 /** Set the event mask on <b>conn</b> to <b>events</b>.  (The event
  * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT)
  */
@@ -433,6 +543,57 @@ connection_is_reading(connection_t *conn)
     (conn->read_event && event_pending(conn->read_event, EV_READ, NULL));
 }
 
+/** Reset our main loop counters. */
+void
+reset_main_loop_counters(void)
+{
+  stats_n_main_loop_successes = 0;
+  stats_n_main_loop_errors = 0;
+  stats_n_main_loop_idle = 0;
+}
+
+/** Increment the main loop success counter. */
+static void
+increment_main_loop_success_count(void)
+{
+  ++stats_n_main_loop_successes;
+}
+
+/** Get the main loop success counter. */
+uint64_t
+get_main_loop_success_count(void)
+{
+  return stats_n_main_loop_successes;
+}
+
+/** Increment the main loop error counter. */
+static void
+increment_main_loop_error_count(void)
+{
+  ++stats_n_main_loop_errors;
+}
+
+/** Get the main loop error counter. */
+uint64_t
+get_main_loop_error_count(void)
+{
+  return stats_n_main_loop_errors;
+}
+
+/** Increment the main loop idle counter. */
+static void
+increment_main_loop_idle_count(void)
+{
+  ++stats_n_main_loop_idle;
+}
+
+/** Get the main loop idle counter. */
+uint64_t
+get_main_loop_idle_count(void)
+{
+  return stats_n_main_loop_idle;
+}
+
 /** Check whether <b>conn</b> is correct in having (or not having) a
  * read/write event (passed in <b>ev</b>). On success, return 0. On failure,
  * log a warning and return -1. */
@@ -448,7 +609,7 @@ connection_check_event(connection_t *conn, struct event *ev)
      */
     bad = ev != NULL;
   } else {
-    /* Everytyhing else should have an underlying socket, or a linked
+    /* Everything else should have an underlying socket, or a linked
      * connection (which is also tracked with a read_event/write_event pair).
      */
     bad = ev == NULL;
@@ -592,17 +753,71 @@ connection_should_read_from_linked_conn(connection_t *conn)
   return 0;
 }
 
-/** If we called event_base_loop() and told it to never stop until it
- * runs out of events, now we've changed our mind: tell it we want it to
- * finish. */
+/** Event to run 'shutdown did not work callback'. */
+static struct event *shutdown_did_not_work_event = NULL;
+
+/** Failsafe measure that should never actually be necessary: If
+ * tor_shutdown_event_loop_and_exit() somehow doesn't successfully exit the
+ * event loop, then this callback will kill Tor with an assertion failure
+ * seconds later
+ */
+static void
+shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg)
+{
+  // LCOV_EXCL_START
+  (void) fd;
+  (void) event;
+  (void) arg;
+  tor_assert_unreached();
+  // LCOV_EXCL_STOP
+}
+
+#ifdef ENABLE_RESTART_DEBUGGING
+static struct event *tor_shutdown_event_loop_for_restart_event = NULL;
+static void
+tor_shutdown_event_loop_for_restart_cb(
+                      evutil_socket_t fd, short event, void *arg)
+{
+  (void)fd;
+  (void)event;
+  (void)arg;
+  tor_event_free(tor_shutdown_event_loop_for_restart_event);
+  tor_shutdown_event_loop_and_exit(0);
+}
+#endif
+
+/**
+ * After finishing the current callback (if any), shut down the main loop,
+ * clean up the process, and exit with <b>exitcode</b>.
+ */
 void
-tell_event_loop_to_finish(void)
+tor_shutdown_event_loop_and_exit(int exitcode)
 {
-  if (!called_loop_once) {
-    struct timeval tv = { 0, 0 };
-    tor_event_base_loopexit(tor_libevent_get_base(), &tv);
-    called_loop_once = 1; /* hack to avoid adding more exit events */
-  }
+  if (main_loop_should_exit)
+    return; /* Ignore multiple calls to this function. */
+
+  main_loop_should_exit = 1;
+  main_loop_exit_value = exitcode;
+
+  /* Die with an assertion failure in ten seconds, if for some reason we don't
+   * exit normally. */
+  /* XXXX We should consider this code if it's never used. */
+  struct timeval ten_seconds = { 10, 0 };
+  shutdown_did_not_work_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  shutdown_did_not_work_callback, NULL);
+  event_add(shutdown_did_not_work_event, &ten_seconds);
+
+  /* Unlike exit_loop_after_delay(), exit_loop_after_callback
+   * prevents other callbacks from running. */
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+}
+
+/** Return true iff tor_shutdown_event_loop_and_exit() has been called. */
+int
+tor_event_loop_shutdown_is_pending(void)
+{
+  return main_loop_should_exit;
 }
 
 /** Helper: Tell the main loop to begin reading bytes into <b>conn</b> from
@@ -617,10 +832,7 @@ connection_start_reading_from_linked_conn(connection_t *conn)
   if (!conn->active_on_link) {
     conn->active_on_link = 1;
     smartlist_add(active_linked_connection_lst, conn);
-    /* make sure that the event_base_loop() function exits at
-     * the end of its run through the current connections, so we can
-     * activate read events for linked connections. */
-    tell_event_loop_to_finish();
+    mainloop_event_activate(schedule_active_linked_connections_event);
   } else {
     tor_assert(smartlist_contains(active_linked_connection_lst, conn));
   }
@@ -699,7 +911,7 @@ conn_read_callback(evutil_socket_t fd, short event, void *_conn)
                "(fd %d); removing",
                conn_type_to_string(conn->type), (int)conn->s);
       tor_fragile_assert();
-#endif
+#endif /* !defined(_WIN32) */
       if (CONN_IS_EDGE(conn))
         connection_edge_end_errno(TO_EDGE_CONN(conn));
       connection_mark_for_close(conn);
@@ -794,7 +1006,7 @@ conn_close_if_marked(int i)
                (int)conn->outbuf_flushlen,
                 conn->marked_for_close_file, conn->marked_for_close);
     if (conn->linked_conn) {
-      retval = move_buf_to_buf(conn->linked_conn->inbuf, conn->outbuf,
+      retval = buf_move_to_buf(conn->linked_conn->inbuf, conn->outbuf,
                                &conn->outbuf_flushlen);
       if (retval >= 0) {
         /* The linked conn will notice that it has data when it notices that
@@ -808,12 +1020,13 @@ conn_close_if_marked(int i)
                 connection_wants_to_flush(conn));
     } else if (connection_speaks_cells(conn)) {
       if (conn->state == OR_CONN_STATE_OPEN) {
-        retval = flush_buf_tls(TO_OR_CONN(conn)->tls, conn->outbuf, sz,
+        retval = buf_flush_to_tls(conn->outbuf, TO_OR_CONN(conn)->tls, sz,
                                &conn->outbuf_flushlen);
       } else
         retval = -1; /* never flush non-open broken tls connections */
     } else {
-      retval = flush_buf(conn->s, conn->outbuf, sz, &conn->outbuf_flushlen);
+      retval = buf_flush_to_socket(conn->outbuf, conn->s, sz,
+                                   &conn->outbuf_flushlen);
     }
     if (retval >= 0 && /* Technically, we could survive things like
                           TLS_WANT_WRITE here. But don't bother for now. */
@@ -822,7 +1035,8 @@ conn_close_if_marked(int i)
         LOG_FN_CONN(conn, (LOG_INFO,LD_NET,
                            "Holding conn (fd %d) open for more flushing.",
                            (int)conn->s));
-        conn->timestamp_lastwritten = now; /* reset so we can flush more */
+        conn->timestamp_last_write_allowed = now; /* reset so we can flush
+                                                   * more */
       } else if (sz == 0) {
         /* Also, retval==0.  If we get here, we didn't want to write anything
          * (because of rate-limiting) and we didn't. */
@@ -833,20 +1047,18 @@ conn_close_if_marked(int i)
          * busy Libevent loops where we keep ending up here and returning
          * 0 until we are no longer blocked on bandwidth.
          */
-        if (connection_is_writing(conn)) {
-          conn->write_blocked_on_bw = 1;
-          connection_stop_writing(conn);
-        }
-        if (connection_is_reading(conn)) {
-          /* XXXX+ We should make this code unreachable; if a connection is
-           * marked for close and flushing, there is no point in reading to it
-           * at all. Further, checking at this point is a bit of a hack: it
-           * would make much more sense to react in
-           * connection_handle_read_impl, or to just stop reading in
-           * mark_and_flush */
-          conn->read_blocked_on_bw = 1;
-          connection_stop_reading(conn);
+        connection_consider_empty_write_buckets(conn);
+        /* Make sure that consider_empty_buckets really disabled the
+         * connection: */
+        if (BUG(connection_is_writing(conn))) {
+          connection_write_bw_exhausted(conn, true);
         }
+
+        /* The connection is being held due to write rate limit and thus will
+         * flush its data later. We need to stop reading because this
+         * connection is about to be closed once flushed. It should not
+         * process anything more coming in at this stage. */
+        connection_stop_reading(conn);
       }
       return 0;
     }
@@ -873,9 +1085,8 @@ conn_close_if_marked(int i)
  * reason.
  */
 static void
-directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg)
+directory_all_unreachable_cb(mainloop_event_t *event, void *arg)
 {
-  (void)fd;
   (void)event;
   (void)arg;
 
@@ -895,7 +1106,7 @@ directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg)
   control_event_general_error("DIR_ALL_UNREACHABLE");
 }
 
-static struct event *directory_all_unreachable_cb_event = NULL;
+static mainloop_event_t *directory_all_unreachable_cb_event = NULL;
 
 /** We've just tried every dirserver we know about, and none of
  * them were reachable. Assume the network is down. Change state
@@ -908,16 +1119,15 @@ directory_all_unreachable(time_t now)
 {
   (void)now;
 
-  stats_n_seconds_working=0; /* reset it */
+  reset_uptime(); /* reset it */
 
   if (!directory_all_unreachable_cb_event) {
     directory_all_unreachable_cb_event =
-      tor_event_new(tor_libevent_get_base(),
-                    -1, EV_READ, directory_all_unreachable_cb, NULL);
+      mainloop_event_new(directory_all_unreachable_cb, NULL);
     tor_assert(directory_all_unreachable_cb_event);
   }
 
-  event_active(directory_all_unreachable_cb_event, EV_READ, 1);
+  mainloop_event_activate(directory_all_unreachable_cb_event);
 }
 
 /** This function is called whenever we successfully pull down some new
@@ -927,6 +1137,15 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 {
   const or_options_t *options = get_options();
 
+  /* if we have enough dir info, then update our guard status with
+   * whatever we just learned. */
+  int invalidate_circs = guards_update_all();
+
+  if (invalidate_circs) {
+    circuit_mark_all_unused_circs();
+    circuit_mark_all_dirty_circs_as_unusable();
+  }
+
   if (!router_have_minimum_dir_info()) {
     int quiet = suppress_logs || from_cache ||
                 directory_too_idle_to_fetch_descriptors(options, now);
@@ -940,9 +1159,6 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
       update_all_descriptor_downloads(now);
     }
 
-    /* if we have enough dir info, then update our guard status with
-     * whatever we just learned. */
-    entry_guards_compute_status(options, now);
     /* Don't even bother trying to get extrainfo until the rest of our
      * directory info is up-to-date */
     if (options->DownloadExtraInfo)
@@ -951,7 +1167,7 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 
   if (server_mode(options) && !net_is_disabled() && !from_cache &&
       (have_completed_a_circuit() || !any_predicted_circuits(now)))
-    consider_testing_reachability(1, 1);
+   router_do_reachability_checks(1, 1);
 }
 
 /** Perform regular maintenance tasks for a single connection.  This
@@ -967,7 +1183,7 @@ run_connection_housekeeping(int i, time_t now)
   channel_t *chan = NULL;
   int have_any_circuits;
   int past_keepalive =
-    now >= conn->timestamp_lastwritten + options->KeepalivePeriod;
+    now >= conn->timestamp_last_write_allowed + options->KeepalivePeriod;
 
   if (conn->outbuf && !connection_get_outbuf_len(conn) &&
       conn->type == CONN_TYPE_OR)
@@ -982,10 +1198,10 @@ run_connection_housekeeping(int i, time_t now)
    * if a server or received if a client) for 5 min */
   if (conn->type == CONN_TYPE_DIR &&
       ((DIR_CONN_IS_SERVER(conn) &&
-        conn->timestamp_lastwritten
+        conn->timestamp_last_write_allowed
             + options->TestingDirConnectionMaxStall < now) ||
        (!DIR_CONN_IS_SERVER(conn) &&
-        conn->timestamp_lastread
+        conn->timestamp_last_read_allowed
             + options->TestingDirConnectionMaxStall < now))) {
     log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)",
              (int)conn->s, conn->purpose);
@@ -1005,7 +1221,7 @@ run_connection_housekeeping(int i, time_t now)
   if (!connection_speaks_cells(conn))
     return; /* we're all done here, the rest is just for OR conns */
 
-  /* If we haven't written to an OR connection for a while, then either nuke
+  /* If we haven't flushed to an OR connection for a while, then either nuke
      the connection or send a keepalive, depending. */
 
   or_conn = TO_OR_CONN(conn);
@@ -1043,7 +1259,8 @@ run_connection_housekeeping(int i, time_t now)
   } else if (we_are_hibernating() &&
              ! have_any_circuits &&
              !connection_get_outbuf_len(conn)) {
-    /* We're hibernating, there's no circuits, and nothing to flush.*/
+    /* We're hibernating or shutting down, there's no circuits, and nothing to
+     * flush.*/
     log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
              "[Hibernating or exiting].",
              (int)conn->s,conn->address, conn->port);
@@ -1051,8 +1268,9 @@ run_connection_housekeeping(int i, time_t now)
   } else if (!have_any_circuits &&
              now - or_conn->idle_timeout >=
                                          chan->timestamp_last_had_circuits) {
-    log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
-             "[no circuits for %d; timeout %d; %scanonical].",
+    log_info(LD_OR,"Expiring non-used OR connection "U64_FORMAT" to fd %d "
+             "(%s:%d) [no circuits for %d; timeout %d; %scanonical].",
+             U64_PRINTF_ARG(chan->global_identifier),
              (int)conn->s, conn->address, conn->port,
              (int)(now - chan->timestamp_last_had_circuits),
              or_conn->idle_timeout,
@@ -1060,13 +1278,14 @@ run_connection_housekeeping(int i, time_t now)
     connection_or_close_normally(TO_OR_CONN(conn), 0);
   } else if (
       now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 &&
-      now >= conn->timestamp_lastwritten + options->KeepalivePeriod*10) {
+      now >=
+          conn->timestamp_last_write_allowed + options->KeepalivePeriod*10) {
     log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL,
            "Expiring stuck OR connection to fd %d (%s:%d). (%d bytes to "
            "flush; %d seconds since last write)",
            (int)conn->s, conn->address, conn->port,
            (int)connection_get_outbuf_len(conn),
-           (int)(now-conn->timestamp_lastwritten));
+           (int)(now-conn->timestamp_last_write_allowed));
     connection_or_close_normally(TO_OR_CONN(conn), 0);
   } else if (past_keepalive && !connection_get_outbuf_len(conn)) {
     /* send a padding cell */
@@ -1075,6 +1294,8 @@ run_connection_housekeeping(int i, time_t now)
     memset(&cell,0,sizeof(cell_t));
     cell.command = CELL_PADDING;
     connection_or_write_cell_to_buf(&cell, or_conn);
+  } else {
+    channelpadding_decide_to_pad_channel(chan);
   }
 }
 
@@ -1092,7 +1313,7 @@ signewnym_impl(time_t now)
 
   circuit_mark_all_dirty_circs_as_unusable();
   addressmap_clear_transient();
-  rend_client_purge_state();
+  hs_client_purge_state();
   time_of_last_signewnym = now;
   signewnym_is_pending = 0;
 
@@ -1101,6 +1322,16 @@ signewnym_impl(time_t now)
   control_event_signal(SIGNEWNYM);
 }
 
+/** Callback: run a deferred signewnym. */
+static void
+handle_deferred_signewnym_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+  log_info(LD_CONTROL, "Honoring delayed NEWNYM request");
+  signewnym_impl(time(NULL));
+}
+
 /** Return the number of times that signewnym has been called. */
 unsigned
 get_signewnym_epoch(void)
@@ -1116,61 +1347,106 @@ static int periodic_events_initialized = 0;
 #undef CALLBACK
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
-CALLBACK(rotate_onion_key);
-CALLBACK(check_ed_keys);
-CALLBACK(launch_descriptor_fetches);
-CALLBACK(rotate_x509_certificate);
 CALLBACK(add_entropy);
-CALLBACK(launch_reachability_tests);
-CALLBACK(downrate_stability);
-CALLBACK(save_stability);
 CALLBACK(check_authority_cert);
+CALLBACK(check_canonical_channels);
+CALLBACK(check_descriptor);
+CALLBACK(check_dns_honesty);
+CALLBACK(check_ed_keys);
 CALLBACK(check_expired_networkstatus);
-CALLBACK(write_stats_file);
-CALLBACK(record_bridge_stats);
+CALLBACK(check_for_reachability_bw);
+CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(clean_caches);
+CALLBACK(clean_consdiffmgr);
+CALLBACK(dirvote);
+CALLBACK(downrate_stability);
+CALLBACK(expire_old_ciruits_serverside);
+CALLBACK(fetch_networkstatus);
+CALLBACK(heartbeat);
+CALLBACK(hs_service);
+CALLBACK(launch_descriptor_fetches);
+CALLBACK(launch_reachability_tests);
+CALLBACK(reachability_warnings);
+CALLBACK(record_bridge_stats);
 CALLBACK(rend_cache_failure_clean);
+CALLBACK(reset_padding_counts);
 CALLBACK(retry_dns);
-CALLBACK(check_descriptor);
-CALLBACK(check_for_reachability_bw);
-CALLBACK(fetch_networkstatus);
 CALLBACK(retry_listeners);
-CALLBACK(expire_old_ciruits_serverside);
-CALLBACK(check_dns_honesty);
+CALLBACK(rotate_onion_key);
+CALLBACK(rotate_x509_certificate);
+CALLBACK(save_stability);
+CALLBACK(save_state);
 CALLBACK(write_bridge_ns);
-CALLBACK(check_fw_helper_app);
-CALLBACK(heartbeat);
+CALLBACK(write_stats_file);
 
 #undef CALLBACK
 
 /* Now we declare an array of periodic_event_item_t for each periodic event */
-#define CALLBACK(name) PERIODIC_EVENT(name)
-
-static periodic_event_item_t periodic_events[] = {
-  CALLBACK(rotate_onion_key),
-  CALLBACK(check_ed_keys),
-  CALLBACK(launch_descriptor_fetches),
-  CALLBACK(rotate_x509_certificate),
-  CALLBACK(add_entropy),
-  CALLBACK(launch_reachability_tests),
-  CALLBACK(downrate_stability),
-  CALLBACK(save_stability),
-  CALLBACK(check_authority_cert),
-  CALLBACK(check_expired_networkstatus),
-  CALLBACK(write_stats_file),
-  CALLBACK(record_bridge_stats),
-  CALLBACK(clean_caches),
-  CALLBACK(rend_cache_failure_clean),
-  CALLBACK(retry_dns),
-  CALLBACK(check_descriptor),
-  CALLBACK(check_for_reachability_bw),
-  CALLBACK(fetch_networkstatus),
-  CALLBACK(retry_listeners),
-  CALLBACK(expire_old_ciruits_serverside),
-  CALLBACK(check_dns_honesty),
-  CALLBACK(write_bridge_ns),
-  CALLBACK(check_fw_helper_app),
-  CALLBACK(heartbeat),
+#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f)
+
+STATIC periodic_event_item_t periodic_events[] = {
+  /* Everyone needs to run those. */
+  CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0),
+  CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0),
+
+  /* Routers (bridge and relay) only. */
+  CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0),
+  CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0),
+  CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0),
+  CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0),
+
+  /* Authorities (bridge and directory) only. */
+  CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
+  CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
+
+  /* Directory authority only. */
+  CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0),
+  CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET),
+
+  /* Relay only. */
+  CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+
+  /* Hidden Service service only. */
+  CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE,
+           PERIODIC_EVENT_FLAG_NEED_NET),
+
+  /* Bridge only. */
+  CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0),
+
+  /* Client only. */
+  CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0),
+
+  /* Bridge Authority only. */
+  CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0),
+
+  /* Directory server only. */
+  CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0),
+
   END_OF_PERIODIC_EVENTS
 };
 #undef CALLBACK
@@ -1180,9 +1456,11 @@ static periodic_event_item_t periodic_events[] = {
  * can access them by name.  We also keep them inside periodic_events[]
  * so that we can implement "reset all timers" in a reasonable way. */
 static periodic_event_item_t *check_descriptor_event=NULL;
+static periodic_event_item_t *dirvote_event=NULL;
 static periodic_event_item_t *fetch_networkstatus_event=NULL;
 static periodic_event_item_t *launch_descriptor_fetches_event=NULL;
 static periodic_event_item_t *check_dns_honesty_event=NULL;
+static periodic_event_item_t *save_state_event=NULL;
 
 /** Reset all the periodic events so we'll do all our actions again as if we
  * just started up.
@@ -1212,6 +1490,42 @@ find_periodic_event(const char *name)
   return NULL;
 }
 
+/** Return a bitmask of the roles this tor instance is configured for using
+ * the given options. */
+STATIC int
+get_my_roles(const or_options_t *options)
+{
+  tor_assert(options);
+
+  int roles = 0;
+  int is_bridge = options->BridgeRelay;
+  int is_relay = server_mode(options);
+  int is_dirauth = authdir_mode_v3(options);
+  int is_bridgeauth = authdir_mode_bridge(options);
+  int is_hidden_service = !!hs_service_get_num_services() ||
+                          !!rend_num_services();
+  int is_dirserver = dir_server_mode(options);
+  /* We also consider tor to have the role of a client if the ControlPort is
+   * set because a lot of things can be done over the control port which
+   * requires tor to have basic functionnalities. */
+  int is_client = options_any_client_port_set(options) ||
+                  options->ControlPort_set ||
+                  options->OwningControllerFD >= 0;
+
+  if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE;
+  if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT;
+  if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY;
+  if (is_dirauth) roles |= PERIODIC_EVENT_ROLE_DIRAUTH;
+  if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH;
+  if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE;
+  if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER;
+
+  return roles;
+}
+
+/** Event to run initialize_periodic_events_cb */
+static struct event *initialize_periodic_events_event = NULL;
+
 /** Helper, run one second after setup:
  * Initializes all members of periodic_events and starts them running.
  *
@@ -1223,10 +1537,10 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
   (void) fd;
   (void) events;
   (void) data;
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_launch(&periodic_events[i]);
-  }
+
+  tor_event_free(initialize_periodic_events_event);
+
+  rescan_periodic_events(get_options());
 }
 
 /** Set up all the members of periodic_events[], and configure them all to be
@@ -1237,6 +1551,7 @@ initialize_periodic_events(void)
   tor_assert(periodic_events_initialized == 0);
   periodic_events_initialized = 1;
 
+  /* Set up all periodic events. We'll launch them by roles. */
   int i;
   for (i = 0; periodic_events[i].name; ++i) {
     periodic_event_setup(&periodic_events[i]);
@@ -1246,14 +1561,17 @@ initialize_periodic_events(void)
   STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END
 
   NAMED_CALLBACK(check_descriptor);
+  NAMED_CALLBACK(dirvote);
   NAMED_CALLBACK(fetch_networkstatus);
   NAMED_CALLBACK(launch_descriptor_fetches);
   NAMED_CALLBACK(check_dns_honesty);
+  NAMED_CALLBACK(save_state);
 
   struct timeval one_second = { 1, 0 };
-  event_base_once(tor_libevent_get_base(), -1, EV_TIMEOUT,
-                  initialize_periodic_events_cb, NULL,
-                  &one_second);
+  initialize_periodic_events_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  initialize_periodic_events_cb, NULL);
+  event_add(initialize_periodic_events_event, &one_second);
 }
 
 STATIC void
@@ -1263,6 +1581,58 @@ teardown_periodic_events(void)
   for (i = 0; periodic_events[i].name; ++i) {
     periodic_event_destroy(&periodic_events[i]);
   }
+  periodic_events_initialized = 0;
+}
+
+/** Do a pass at all our periodic events, disable those we don't need anymore
+ * and enable those we need now using the given options. */
+void
+rescan_periodic_events(const or_options_t *options)
+{
+  tor_assert(options);
+
+  /* Avoid scanning the event list if we haven't initialized it yet. This is
+   * particularly useful for unit tests in order to avoid initializing main
+   * loop events everytime. */
+  if (!periodic_events_initialized) {
+    return;
+  }
+
+  int roles = get_my_roles(options);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+
+    /* Handle the event flags. */
+    if (net_is_disabled() &&
+        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
+      continue;
+    }
+
+    /* Enable the event if needed. It is safe to enable an event that was
+     * already enabled. Same goes for disabling it. */
+    if (item->roles & roles) {
+      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
+      periodic_event_enable(item);
+    } else {
+      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
+      periodic_event_disable(item);
+    }
+  }
+}
+
+/* We just got new options globally set, see if we need to enabled or disable
+ * periodic events. */
+void
+periodic_events_on_new_options(const or_options_t *options)
+{
+  /* Only if we've already initialized the events, rescan the list which will
+   * enable or disable events depending on our roles. This will be called at
+   * bootup and we don't want this function to initialize the events because
+   * they aren't set up at this stage. */
+  if (periodic_events_initialized) {
+    rescan_periodic_events(options);
+  }
 }
 
 /**
@@ -1273,8 +1643,9 @@ teardown_periodic_events(void)
 void
 reschedule_descriptor_update_check(void)
 {
-  tor_assert(check_descriptor_event);
-  periodic_event_reschedule(check_descriptor_event);
+  if (check_descriptor_event) {
+    periodic_event_reschedule(check_descriptor_event);
+  }
 }
 
 /**
@@ -1291,6 +1662,35 @@ reschedule_directory_downloads(void)
   periodic_event_reschedule(launch_descriptor_fetches_event);
 }
 
+/** Mainloop callback: clean up circuits, channels, and connections
+ * that are pending close. */
+static void
+postloop_cleanup_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  circuit_close_all_marked();
+  close_closeable_connections();
+  channel_run_cleanup();
+  channel_listener_run_cleanup();
+}
+
+/** Event to run postloop_cleanup_cb */
+static mainloop_event_t *postloop_cleanup_ev=NULL;
+
+/** Schedule a post-loop event to clean up marked channels, connections, and
+ * circuits. */
+void
+mainloop_schedule_postloop_cleanup(void)
+{
+  if (PREDICT_UNLIKELY(postloop_cleanup_ev == NULL)) {
+    // (It's possible that we can get here if we decide to close a connection
+    // in the earliest stages of our configuration, before we create events.)
+    return;
+  }
+  mainloop_event_activate(postloop_cleanup_ev);
+}
+
 #define LONGEST_TIMER_PERIOD (30 * 86400)
 /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
  * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */
@@ -1328,18 +1728,12 @@ run_scheduled_events(time_t now)
    */
   consider_hibernation(now);
 
-  /* 0b. If we've deferred a signewnym, make sure it gets handled
-   * eventually. */
-  if (signewnym_is_pending &&
-      time_of_last_signewnym + MAX_SIGNEWNYM_RATE <= now) {
-    log_info(LD_CONTROL, "Honoring delayed NEWNYM request");
-    signewnym_impl(now);
-  }
-
-  /* 0c. If we've deferred log messages for the controller, handle them now */
-  flush_pending_log_callbacks();
+  /* Maybe enough time elapsed for us to reconsider a circuit. */
+  circuit_upgrade_circuits_from_guard_wait();
 
-  if (options->UseBridges && !options->DisableNetwork) {
+  if (options->UseBridges && !net_is_disabled()) {
+    /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches()
+     * -- the latter is only for fetching consensus-derived directory info. */
     fetch_bridge_descriptors(options, now);
   }
 
@@ -1347,10 +1741,6 @@ run_scheduled_events(time_t now)
     accounting_run_housekeeping(now);
   }
 
-  if (authdir_mode_v3(options)) {
-    dirvote_act(options, now);
-  }
-
   /* 3a. Every second, we examine pending circuits and prune the
    *    ones which have been pending for more than a few seconds.
    *    We do this before step 4, so it can try building more if
@@ -1359,6 +1749,7 @@ run_scheduled_events(time_t now)
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
   circuit_expire_building();
+  circuit_expire_waiting_for_better_guard();
 
   /* 3b. Also look at pending streams and prune the ones that 'began'
    *     a long time ago but haven't gotten a 'connected' yet.
@@ -1383,58 +1774,30 @@ run_scheduled_events(time_t now)
     circuit_expire_old_circs_as_needed(now);
   }
 
-  if (!net_is_disabled()) {
-    /* This is usually redundant with circuit_build_needed_circs() above,
-     * but it is very fast when there is no work to do. */
-    connection_ap_attach_pending(0);
-  }
-
   /* 5. We do housekeeping for each connection... */
-  connection_or_set_bad_connections(NULL, 0);
+  channel_update_bad_for_new_circs(NULL, 0);
   int i;
   for (i=0;i<smartlist_len(connection_array);i++) {
     run_connection_housekeeping(i, now);
   }
 
-  /* 6. And remove any marked circuits... */
-  circuit_close_all_marked();
-
-  /* 7. And upload service descriptors if necessary. */
-  if (have_completed_a_circuit() && !net_is_disabled()) {
-    rend_consider_services_upload(now);
-    rend_consider_descriptor_republication();
-  }
-
-  /* 8. and blow away any connections that need to die. have to do this now,
-   * because if we marked a conn for close and left its socket -1, then
-   * we'll pass it to poll/select and bad things will happen.
-   */
-  close_closeable_connections();
-
-  /* 8b. And if anything in our state is ready to get flushed to disk, we
-   * flush it. */
-  or_state_save(now);
-
-  /* 8c. Do channel cleanup just like for connections */
-  channel_run_cleanup();
-  channel_listener_run_cleanup();
-
   /* 11b. check pending unconfigured managed proxies */
   if (!net_is_disabled() && pt_proxies_configuration_pending())
     pt_configure_remaining_proxies();
 }
 
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
 static int
 rotate_onion_key_callback(time_t now, const or_options_t *options)
 {
-  /* 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys,
-   *  shut down and restart all cpuworkers, and update the directory if
-   *  necessary.
-   */
   if (server_mode(options)) {
-    time_t rotation_time = get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME;
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
     if (rotation_time > now) {
-      return safe_timer_diff(now, rotation_time);
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
     }
 
     log_info(LD_GENERAL,"Rotating onion key.");
@@ -1443,23 +1806,50 @@ rotate_onion_key_callback(time_t now, const or_options_t *options)
     if (router_rebuild_descriptor(1)<0) {
       log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
     }
-    if (advertised_server_mode() && !options->DisableNetwork)
+    if (advertised_server_mode() && !net_is_disabled())
       router_upload_dir_desc_to_dirservers(0);
-    return MIN_ONION_KEY_LIFETIME;
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
   }
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
 static int
 check_ed_keys_callback(time_t now, const or_options_t *options)
 {
   if (server_mode(options)) {
     if (should_make_new_ed_keys(options, now)) {
-      if (load_ed_keys(options, now) < 0 ||
-          generate_ed_link_cert(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
         log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
-        tor_cleanup();
-        exit(0);
+        tor_shutdown_event_loop_and_exit(1);
       }
     }
     return 30;
@@ -1467,6 +1857,11 @@ check_ed_keys_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: Every {LAZY,GREEDY}_DESCRIPTOR_RETRY_INTERVAL,
+ * see about fetching descriptors, microdescriptors, and extrainfo
+ * documents.
+ */
 static int
 launch_descriptor_fetches_callback(time_t now, const or_options_t *options)
 {
@@ -1481,6 +1876,10 @@ launch_descriptor_fetches_callback(time_t now, const or_options_t *options)
     return GREEDY_DESCRIPTOR_RETRY_INTERVAL;
 }
 
+/**
+ * Periodic event: Rotate our X.509 certificates and TLS keys once every
+ * MAX_SSL_KEY_LIFETIME_INTERNAL.
+ */
 static int
 rotate_x509_certificate_callback(time_t now, const or_options_t *options)
 {
@@ -1499,6 +1898,11 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options)
     log_err(LD_BUG, "Error reinitializing TLS context");
     tor_assert_unreached();
   }
+  if (generate_ed_link_cert(options, now, 1)) {
+    log_err(LD_OR, "Unable to update Ed25519->TLS link certificate for "
+            "new TLS context.");
+    tor_assert_unreached();
+  }
 
   /* We also make sure to rotate the TLS connections themselves if they've
    * been up for too long -- but that's done via is_bad_for_new_circs in
@@ -1506,6 +1910,10 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options)
   return MAX_SSL_KEY_LIFETIME_INTERNAL;
 }
 
+/**
+ * Periodic callback: once an hour, grab some more entropy from the
+ * kernel and feed it to our CSPRNG.
+ **/
 static int
 add_entropy_callback(time_t now, const or_options_t *options)
 {
@@ -1522,6 +1930,10 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
 static int
 launch_reachability_tests_callback(time_t now, const or_options_t *options)
 {
@@ -1533,6 +1945,10 @@ launch_reachability_tests_callback(time_t now, const or_options_t *options)
   return REACHABILITY_TEST_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
 static int
 downrate_stability_callback(time_t now, const or_options_t *options)
 {
@@ -1544,6 +1960,10 @@ downrate_stability_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next);
 }
 
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
 static int
 save_stability_callback(time_t now, const or_options_t *options)
 {
@@ -1556,6 +1976,10 @@ save_stability_callback(time_t now, const or_options_t *options)
   return SAVE_STABILITY_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
 static int
 check_authority_cert_callback(time_t now, const or_options_t *options)
 {
@@ -1568,18 +1992,55 @@ check_authority_cert_callback(time_t now, const or_options_t *options)
   return CHECK_V3_CERTIFICATE_INTERVAL;
 }
 
+/**
+ * Scheduled callback: Run directory-authority voting functionality.
+ *
+ * The schedule is a bit complicated here, so dirvote_act() manages the
+ * schedule itself.
+ **/
+static int
+dirvote_callback(time_t now, const or_options_t *options)
+{
+  if (!authdir_mode_v3(options)) {
+    tor_assert_nonfatal_unreached();
+    return 3600;
+  }
+
+  time_t next = dirvote_act(options, now);
+  if (BUG(next == TIME_MAX)) {
+    /* This shouldn't be returned unless we called dirvote_act() without
+     * being an authority.  If it happens, maybe our configuration will
+     * fix itself in an hour or so? */
+    return 3600;
+  }
+  return safe_timer_diff(now, next);
+}
+
+/** Reschedule the directory-authority voting event.  Run this whenever the
+ * schedule has changed. */
+void
+reschedule_dirvote(const or_options_t *options)
+{
+  if (periodic_events_initialized && authdir_mode_v3(options)) {
+    periodic_event_reschedule(dirvote_event);
+  }
+}
+
+/**
+ * Periodic callback: If our consensus is too old, recalculate whether
+ * we can actually use it.
+ */
 static int
 check_expired_networkstatus_callback(time_t now, const or_options_t *options)
 {
   (void)options;
-  /* 1f. Check whether our networkstatus has expired.
-   */
+  /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
   /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
    * networkstatus_get_reasonably_live_consensus(), but that value is way
    * way too high.  Arma: is the bridge issue there resolved yet? -NM */
 #define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < now+NS_EXPIRY_SLOP &&
+  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }
@@ -1587,6 +2048,37 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   return CHECK_EXPIRED_NS_INTERVAL;
 }
 
+/**
+ * Scheduled callback: Save the state file to disk if appropriate.
+ */
+static int
+save_state_callback(time_t now, const or_options_t *options)
+{
+  (void) options;
+  (void) or_state_save(now); // only saves if appropriate
+  const time_t next_write = get_or_state()->next_write;
+  if (next_write == TIME_MAX) {
+    return 86400;
+  }
+  return safe_timer_diff(now, next_write);
+}
+
+/** Reschedule the event for saving the state file.
+ *
+ * Run this when the state becomes dirty. */
+void
+reschedule_or_state_save(void)
+{
+  if (save_state_event == NULL) {
+    /* This can happen early on during startup. */
+    return;
+  }
+  periodic_event_reschedule(save_state_event);
+}
+
+/**
+ * Periodic callback: Write statistics to disk if appropriate.
+ */
 static int
 write_stats_file_callback(time_t now, const or_options_t *options)
 {
@@ -1634,11 +2126,36 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
+#define CHANNEL_CHECK_INTERVAL (60*60)
 static int
-record_bridge_stats_callback(time_t now, const or_options_t *options)
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+static int
+reset_padding_counts_callback(time_t now, const or_options_t *options)
 {
-  static int should_init_bridge_stats = 1;
+  if (options->PaddingStatistics) {
+    rep_hist_prep_published_padding_counts(now);
+  }
 
+  rep_hist_reset_padding_counts();
+  return REPHIST_CELL_PADDING_COUNTS_INTERVAL;
+}
+
+static int should_init_bridge_stats = 1;
+
+/**
+ * Periodic callback: Write bridge statistics to disk if appropriate.
+ */
+static int
+record_bridge_stats_callback(time_t now, const or_options_t *options)
+{
   /* 1h. Check whether we should write bridge statistics to disk.
    */
   if (should_record_bridge_info(options)) {
@@ -1661,19 +2178,26 @@ record_bridge_stats_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: Clean in-memory caches every once in a while
+ */
 static int
 clean_caches_callback(time_t now, const or_options_t *options)
 {
   /* Remove old information from rephist and the rend cache. */
   rep_history_clean(now - options->RephistTrackTime);
-  rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
   rend_cache_clean(now, REND_CACHE_TYPE_SERVICE);
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  hs_cache_clean_as_client(now);
+  hs_cache_clean_as_dir(now);
   microdesc_cache_rebuild(NULL, 0);
 #define CLEAN_CACHES_INTERVAL (30*60)
   return CLEAN_CACHES_INTERVAL;
 }
 
+/**
+ * Periodic callback: Clean the cache of failed hidden service lookups
+ * frequently.
+ */
 static int
 rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 {
@@ -1682,23 +2206,25 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
    * clean it as soon as we can since we want to make sure the client waits
    * as little as possible for reachability reasons. */
   rend_cache_failure_clean(now);
+  hs_cache_client_intro_state_clean(now);
   return 30;
 }
 
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
 static int
 retry_dns_callback(time_t now, const or_options_t *options)
 {
   (void)now;
 #define RETRY_DNS_INTERVAL (10*60)
-  /* If we're a server and initializing dns failed, retry periodically. */
   if (server_mode(options) && has_dns_init_failed())
     dns_init();
   return RETRY_DNS_INTERVAL;
 }
 
-  /* 2. Periodically, we consider force-uploading our descriptor
-   * (if we've passed our internal checks). */
-
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
 static int
 check_descriptor_callback(time_t now, const or_options_t *options)
 {
@@ -1707,9 +2233,11 @@ check_descriptor_callback(time_t now, const or_options_t *options)
  * address has changed. */
 #define CHECK_DESCRIPTOR_INTERVAL (60)
 
+  (void)options;
+
   /* 2b. Once per minute, regenerate and upload the descriptor if the old
    * one is inaccurate. */
-  if (!options->DisableNetwork) {
+  if (!net_is_disabled()) {
     check_descriptor_bandwidth_changed(now);
     check_descriptor_ipaddress_changed(now);
     mark_my_descriptor_dirty_if_too_old(now);
@@ -1725,6 +2253,11 @@ check_descriptor_callback(time_t now, const or_options_t *options)
   return CHECK_DESCRIPTOR_INTERVAL;
 }
 
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
 static int
 check_for_reachability_bw_callback(time_t now, const or_options_t *options)
 {
@@ -1736,9 +2269,9 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options)
    * 20 minutes of our uptime. */
   if (server_mode(options) &&
       (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
-      !we_are_hibernating()) {
-    if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-      consider_testing_reachability(1, dirport_reachability_count==0);
+      !net_is_disabled()) {
+    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+      router_do_reachability_checks(1, dirport_reachability_count==0);
       if (++dirport_reachability_count > 5)
         dirport_reachability_count = 0;
       return 1;
@@ -1761,13 +2294,13 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options)
   return CHECK_DESCRIPTOR_INTERVAL;
 }
 
+/**
+ * Periodic event: once a minute, (or every second if TestingTorNetwork, or
+ * during client bootstrap), check whether we want to download any
+ * networkstatus documents. */
 static int
 fetch_networkstatus_callback(time_t now, const or_options_t *options)
 {
-  /* 2c. Every minute (or every second if TestingTorNetwork, or during
-   * client bootstrap), check whether we want to download any networkstatus
-   * documents. */
-
   /* How often do we check whether we should download network status
    * documents? */
   const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
@@ -1789,12 +2322,13 @@ fetch_networkstatus_callback(time_t now, const or_options_t *options)
   return networkstatus_dl_check_interval;
 }
 
+/**
+ * Periodic callback: Every 60 seconds, we relaunch listeners if any died. */
 static int
 retry_listeners_callback(time_t now, const or_options_t *options)
 {
   (void)now;
   (void)options;
-  /* 3d. And every 60 seconds, we relaunch listeners if any died. */
   if (!net_is_disabled()) {
     retry_all_listeners(NULL, NULL, 0);
     return 60;
@@ -1802,6 +2336,9 @@ retry_listeners_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
 static int
 expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
 {
@@ -1811,6 +2348,60 @@ expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
   return 11;
 }
 
+/**
+ * Callback: Send warnings if Tor doesn't find its ports reachable.
+ */
+static int
+reachability_warnings_callback(time_t now, const or_options_t *options)
+{
+  (void) now;
+
+  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
+  }
+
+  if (server_mode(options) &&
+      !net_is_disabled() &&
+      have_completed_a_circuit()) {
+    /* every 20 minutes, check and complain if necessary */
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me && !check_whether_orport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
+               "its ORPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->or_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
+                                  address, me->or_port);
+      tor_free(address);
+    }
+
+    if (me && !check_whether_dirport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,
+               "Your server (%s:%d) has not managed to confirm that its "
+               "DirPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->dir_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
+                                  address, me->dir_port);
+      tor_free(address);
+    }
+  }
+
+  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
+}
+
+static int dns_honesty_first_time = 1;
+
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
 static int
 check_dns_honesty_callback(time_t now, const or_options_t *options)
 {
@@ -1822,10 +2413,9 @@ check_dns_honesty_callback(time_t now, const or_options_t *options)
       router_my_exit_policy_is_reject_star())
     return PERIODIC_EVENT_NO_UPDATE;
 
-  static int first_time = 1;
-  if (first_time) {
+  if (dns_honesty_first_time) {
     /* Don't launch right when we start */
-    first_time = 0;
+    dns_honesty_first_time = 0;
     return crypto_rand_int_range(60, 180);
   }
 
@@ -1833,6 +2423,10 @@ check_dns_honesty_callback(time_t now, const or_options_t *options)
   return 12*3600 + crypto_rand_int(12*3600);
 }
 
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
 static int
 write_bridge_ns_callback(time_t now, const or_options_t *options)
 {
@@ -1845,54 +2439,168 @@ write_bridge_ns_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+static int heartbeat_callback_first_time = 1;
+
+/**
+ * Periodic callback: write the heartbeat message in the logs.
+ *
+ * If writing the heartbeat message to the logs fails for some reason, retry
+ * again after <b>MIN_HEARTBEAT_PERIOD</b> seconds.
+ */
 static int
-check_fw_helper_app_callback(time_t now, const or_options_t *options)
+heartbeat_callback(time_t now, const or_options_t *options)
 {
-  if (net_is_disabled() ||
-      ! server_mode(options) ||
-      ! options->PortForwarding) {
+  /* Check if heartbeat is disabled */
+  if (!options->HeartbeatPeriod) {
     return PERIODIC_EVENT_NO_UPDATE;
   }
-  /* 11. check the port forwarding app */
 
-#define PORT_FORWARDING_CHECK_INTERVAL 5
-  smartlist_t *ports_to_forward = get_list_of_ports_to_forward();
-  if (ports_to_forward) {
-    tor_check_port_forwarding(options->PortForwardingHelper,
-                              ports_to_forward,
-                              now);
+  /* Skip the first one. */
+  if (heartbeat_callback_first_time) {
+    heartbeat_callback_first_time = 0;
+    return options->HeartbeatPeriod;
+  }
 
-    SMARTLIST_FOREACH(ports_to_forward, char *, cp, tor_free(cp));
-    smartlist_free(ports_to_forward);
+  /* Write the heartbeat message */
+  if (log_heartbeat(now) == 0) {
+    return options->HeartbeatPeriod;
+  } else {
+    /* If we couldn't write the heartbeat log message, try again in the minimum
+     * interval of time. */
+    return MIN_HEARTBEAT_PERIOD;
   }
-  return PORT_FORWARDING_CHECK_INTERVAL;
 }
 
-/** Callback to write heartbeat message in the logs. */
+#define CDM_CLEAN_CALLBACK_INTERVAL 600
 static int
-heartbeat_callback(time_t now, const or_options_t *options)
+clean_consdiffmgr_callback(time_t now, const or_options_t *options)
 {
-  static int first = 1;
-
-  /* Check if heartbeat is disabled */
-  if (!options->HeartbeatPeriod) {
-    return PERIODIC_EVENT_NO_UPDATE;
+  (void)now;
+  if (dir_server_mode(options)) {
+    consdiffmgr_cleanup();
   }
+  return CDM_CLEAN_CALLBACK_INTERVAL;
+}
 
-  /* Write the heartbeat message */
-  if (first) {
-    first = 0; /* Skip the first one. */
-  } else {
-    log_heartbeat(now);
+/*
+ * Periodic callback: Run scheduled events for HS service. This is called
+ * every second.
+ */
+static int
+hs_service_callback(time_t now, const or_options_t *options)
+{
+  (void) options;
+
+  /* We need to at least be able to build circuits and that we actually have
+   * a working network. */
+  if (!have_completed_a_circuit() || net_is_disabled() ||
+      networkstatus_get_live_consensus(now) == NULL) {
+    goto end;
   }
 
-  return options->HeartbeatPeriod;
+  hs_service_run_scheduled_events(now);
+
+ end:
+  /* Every 1 second. */
+  return 1;
 }
 
 /** Timer: used to invoke second_elapsed_callback() once per second. */
 static periodic_timer_t *second_timer = NULL;
-/** Number of libevent errors in the last second: we die if we get too many. */
-static int n_libevent_errors = 0;
+
+/**
+ * Enable or disable the per-second timer as appropriate, creating it if
+ * necessary.
+ */
+void
+reschedule_per_second_timer(void)
+{
+  struct timeval one_second;
+  one_second.tv_sec = 1;
+  one_second.tv_usec = 0;
+
+  if (! second_timer) {
+    second_timer = periodic_timer_new(tor_libevent_get_base(),
+                                      &one_second,
+                                      second_elapsed_callback,
+                                      NULL);
+    tor_assert(second_timer);
+  }
+
+  const bool run_per_second_events =
+    control_any_per_second_event_enabled() || ! net_is_completely_disabled();
+
+  if (run_per_second_events) {
+    periodic_timer_launch(second_timer, &one_second);
+  } else {
+    periodic_timer_disable(second_timer);
+  }
+}
+
+/** Last time that update_current_time was called. */
+static time_t current_second = 0;
+/** Last time that update_current_time updated current_second. */
+static monotime_coarse_t current_second_last_changed;
+
+/**
+ * Set the current time to "now", which should be the value returned by
+ * time().  Check for clock jumps and track the total number of seconds we
+ * have been running.
+ */
+void
+update_current_time(time_t now)
+{
+  if (PREDICT_LIKELY(now == current_second)) {
+    /* We call this function a lot.  Most frequently, the current second
+     * will not have changed, so we just return. */
+    return;
+  }
+
+  const time_t seconds_elapsed = current_second ? (now - current_second) : 0;
+
+  /* Check the wall clock against the monotonic clock, so we can
+   * better tell idleness from clock jumps and/or other shenanigans. */
+  monotime_coarse_t last_updated;
+  memcpy(&last_updated, &current_second_last_changed, sizeof(last_updated));
+  monotime_coarse_get(&current_second_last_changed);
+
+  /** How much clock jumping do we tolerate? */
+#define NUM_JUMPED_SECONDS_BEFORE_WARN 100
+
+  /** How much idleness do we tolerate? */
+#define NUM_IDLE_SECONDS_BEFORE_WARN 3600
+
+  if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) {
+    // moving back in time is always a bad sign.
+    circuit_note_clock_jumped(seconds_elapsed, false);
+  } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
+    /* Compare the monotonic clock to the result of time(). */
+    const int32_t monotime_msec_passed =
+      monotime_coarse_diff_msec32(&last_updated,
+                                  &current_second_last_changed);
+    const int monotime_sec_passed = monotime_msec_passed / 1000;
+    const int discrepancy = monotime_sec_passed - (int)seconds_elapsed;
+    /* If the monotonic clock deviates from time(NULL), we have a couple of
+     * possibilities.  On some systems, this means we have been suspended or
+     * sleeping.  Everywhere, it can mean that the wall-clock time has
+     * been changed -- for example, with settimeofday().
+     *
+     * On the other hand, if the monotonic time matches with the wall-clock
+     * time, we've probably just been idle for a while, with no events firing.
+     * we tolerate much more of that.
+     */
+    const bool clock_jumped = abs(discrepancy) > 2;
+
+    if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) {
+      circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped);
+    }
+  } else if (seconds_elapsed > 0) {
+    stats_n_seconds_working += seconds_elapsed;
+  }
+
+  update_approx_time(now);
+  current_second = now;
+}
 
 /** Libevent callback: invoked once every second. */
 static void
@@ -1901,83 +2609,22 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg)
   /* XXXX This could be sensibly refactored into multiple callbacks, and we
    * could use Libevent's timers for this rather than checking the current
    * time against a bunch of timeouts every second. */
-  static time_t current_second = 0;
   time_t now;
-  size_t bytes_written;
-  size_t bytes_read;
-  int seconds_elapsed;
-  const or_options_t *options = get_options();
   (void)timer;
   (void)arg;
 
-  n_libevent_errors = 0;
-
-  /* log_notice(LD_GENERAL, "Tick."); */
   now = time(NULL);
-  update_approx_time(now);
 
-  /* the second has rolled over. check more stuff. */
-  seconds_elapsed = current_second ? (int)(now - current_second) : 0;
-  bytes_read = (size_t)(stats_n_bytes_read - stats_prev_n_read);
-  bytes_written = (size_t)(stats_n_bytes_written - stats_prev_n_written);
-  stats_prev_n_read = stats_n_bytes_read;
-  stats_prev_n_written = stats_n_bytes_written;
-
-  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
-  control_event_stream_bandwidth_used();
-  control_event_conn_bandwidth_used();
-  control_event_circ_bandwidth_used();
-  control_event_circuit_cell_stats();
-
-  if (server_mode(options) &&
-      !net_is_disabled() &&
-      seconds_elapsed > 0 &&
-      have_completed_a_circuit() &&
-      stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT !=
-      (stats_n_seconds_working+seconds_elapsed) /
-        TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-    /* every 20 minutes, check and complain if necessary */
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me && !check_whether_orport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
-               "its ORPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->or_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
-                                  address, me->or_port);
-      tor_free(address);
-    }
-
-    if (me && !check_whether_dirport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,
-               "Your server (%s:%d) has not managed to confirm that its "
-               "DirPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->dir_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
-                                  address, me->dir_port);
-      tor_free(address);
-    }
-  }
+  /* We don't need to do this once-per-second any more: time-updating is
+   * only in this callback _because it is a callback_. It should be fine
+   * to disable this callback, and the time will still get updated.
+   */
+  update_current_time(now);
 
-/** If more than this many seconds have elapsed, probably the clock
- * jumped: doesn't count. */
-#define NUM_JUMPED_SECONDS_BEFORE_WARN 100
-  if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN ||
-      seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
-    circuit_note_clock_jumped(seconds_elapsed);
-  } else if (seconds_elapsed > 0)
-    stats_n_seconds_working += seconds_elapsed;
+  /* Maybe some controller events are ready to fire */
+  control_per_second_events();
 
   run_scheduled_events(now);
-
-  current_second = now; /* remember which second it is, for next time */
 }
 
 #ifdef HAVE_SYSTEMD_209
@@ -1991,71 +2638,7 @@ systemd_watchdog_callback(periodic_timer_t *timer, void *arg)
   (void)arg;
   sd_notify(0, "WATCHDOG=1");
 }
-#endif
-
-/** Timer: used to invoke refill_callback(). */
-static periodic_timer_t *refill_timer = NULL;
-
-/** Libevent callback: invoked periodically to refill token buckets
- * and count r/w bytes. */
-static void
-refill_callback(periodic_timer_t *timer, void *arg)
-{
-  static struct timeval current_millisecond;
-  struct timeval now;
-
-  size_t bytes_written;
-  size_t bytes_read;
-  int milliseconds_elapsed = 0;
-  int seconds_rolled_over = 0;
-
-  const or_options_t *options = get_options();
-
-  (void)timer;
-  (void)arg;
-
-  tor_gettimeofday(&now);
-
-  /* If this is our first time, no time has passed. */
-  if (current_millisecond.tv_sec) {
-    long mdiff = tv_mdiff(&current_millisecond, &now);
-    if (mdiff > INT_MAX)
-      mdiff = INT_MAX;
-    milliseconds_elapsed = (int)mdiff;
-    seconds_rolled_over = (int)(now.tv_sec - current_millisecond.tv_sec);
-  }
-
-  bytes_written = stats_prev_global_write_bucket - global_write_bucket;
-  bytes_read = stats_prev_global_read_bucket - global_read_bucket;
-
-  stats_n_bytes_read += bytes_read;
-  stats_n_bytes_written += bytes_written;
-  if (accounting_is_enabled(options) && milliseconds_elapsed >= 0)
-    accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over);
-
-  if (milliseconds_elapsed > 0)
-    connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec);
-
-  stats_prev_global_read_bucket = global_read_bucket;
-  stats_prev_global_write_bucket = global_write_bucket;
-
-  current_millisecond = now; /* remember what time it is, for next time */
-}
-
-#ifndef _WIN32
-/** Called when a possibly ignorable libevent error occurs; ensures that we
- * don't get into an infinite loop by ignoring too many errors from
- * libevent. */
-static int
-got_libevent_error(void)
-{
-  if (++n_libevent_errors > 8) {
-    log_err(LD_NET, "Too many libevent errors in one second; dying");
-    return -1;
-  }
-  return 0;
-}
-#endif
+#endif /* defined(HAVE_SYSTEMD_209) */
 
 #define UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST (6*60*60)
 
@@ -2078,9 +2661,9 @@ ip_address_changed(int at_interface)
     }
   } else {
     if (server) {
-      if (stats_n_seconds_working > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST)
+      if (get_uptime() > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST)
         reset_bandwidth_test();
-      stats_n_seconds_working = 0;
+      reset_uptime();
       router_reset_reachability();
     }
   }
@@ -2130,10 +2713,18 @@ do_hup(void)
   /* first, reload config variables, in case they've changed */
   if (options->ReloadTorrcOnSIGHUP) {
     /* no need to provide argc/v, they've been cached in init_from_config */
-    if (options_init_from_torrc(0, NULL) < 0) {
+    int init_rv = options_init_from_torrc(0, NULL);
+    if (init_rv < 0) {
       log_err(LD_CONFIG,"Reading config failed--see warnings above. "
               "For usage, try -h.");
       return -1;
+    } else if (BUG(init_rv > 0)) {
+      // LCOV_EXCL_START
+      /* This should be impossible: the only "return 1" cases in
+       * options_init_from_torrc are ones caused by command-line arguments;
+       * but they can't change while Tor is running. */
+      return -1;
+      // LCOV_EXCL_STOP
     }
     options = get_options(); /* they have changed now */
     /* Logs are only truncated the first time they are opened, but were
@@ -2152,7 +2743,7 @@ do_hup(void)
       tor_free(msg);
     }
   }
-  if (authdir_mode_handles_descs(options, -1)) {
+  if (authdir_mode(options)) {
     /* reload the approved-routers file */
     if (dirserv_load_fingerprint_file() < 0) {
       /* warnings are logged from dirserv_load_fingerprint_file() directly */
@@ -2169,7 +2760,7 @@ do_hup(void)
   /* retry appropriate downloads */
   router_reset_status_download_failures();
   router_reset_descriptor_download_failures();
-  if (!options->DisableNetwork)
+  if (!net_is_disabled())
     update_networkstatus_downloads(time(NULL));
 
   /* We'll retry routerstatus downloads in about 10 seconds; no need to
@@ -2179,8 +2770,9 @@ do_hup(void)
     /* Maybe we've been given a new ed25519 key or certificate?
      */
     time_t now = approx_time();
-    if (load_ed_keys(options, now) < 0 ||
-         generate_ed_link_cert(options, now)) {
+    int new_signing_key = load_ed_keys(options, now);
+    if (new_signing_key < 0 ||
+        generate_ed_link_cert(options, now, new_signing_key > 0)) {
       log_warn(LD_OR, "Problem reloading Ed25519 keys; still using old keys.");
     }
 
@@ -2192,6 +2784,20 @@ do_hup(void)
   return 0;
 }
 
+/** Initialize some mainloop_event_t objects that we require. */
+STATIC void
+initialize_mainloop_events(void)
+{
+  if (!schedule_active_linked_connections_event) {
+    schedule_active_linked_connections_event =
+      mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL);
+  }
+  if (!postloop_cleanup_ev) {
+    postloop_cleanup_ev =
+      mainloop_event_postloop_new(postloop_cleanup_cb, NULL);
+  }
+}
+
 /** Tor main loop. */
 int
 do_main_loop(void)
@@ -2205,6 +2811,8 @@ do_main_loop(void)
     initialize_periodic_events();
   }
 
+  initialize_mainloop_events();
+
   /* initialize dns resolve map, spawn workers if needed */
   if (dns_init() < 0) {
     if (get_options()->ServerDNSAllowBrokenConfig)
@@ -2216,7 +2824,9 @@ do_main_loop(void)
     }
   }
 
-  handle_signals(1);
+  handle_signals();
+  monotime_init();
+  timers_initialize();
 
   /* load the private keys, if we're supposed to have them, and set up the
    * TLS context. */
@@ -2229,8 +2839,6 @@ do_main_loop(void)
 
   /* Set up our buckets */
   connection_bucket_init();
-  stats_prev_global_read_bucket = global_read_bucket;
-  stats_prev_global_write_bucket = global_write_bucket;
 
   /* initialize the bootstrap status events to know we're starting up */
   control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
@@ -2280,30 +2888,21 @@ do_main_loop(void)
   now = time(NULL);
   directory_info_has_arrived(now, 1, 0);
 
-  if (server_mode(get_options())) {
+  if (server_mode(get_options()) || dir_server_mode(get_options())) {
     /* launch cpuworkers. Need to do this *after* we've read the onion key. */
     cpu_init();
   }
+  consdiffmgr_enable_background_compression();
 
   /* Setup shared random protocol subsystem. */
-  if (authdir_mode_publishes_statuses(get_options())) {
+  if (authdir_mode_v3(get_options())) {
     if (sr_init(1) < 0) {
       return -1;
     }
   }
 
   /* set up once-a-second callback. */
-  if (! second_timer) {
-    struct timeval one_second;
-    one_second.tv_sec = 1;
-    one_second.tv_usec = 0;
-
-    second_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &one_second,
-                                      second_elapsed_callback,
-                                      NULL);
-    tor_assert(second_timer);
-  }
+  reschedule_per_second_timer();
 
 #ifdef HAVE_SYSTEMD_209
   uint64_t watchdog_delay;
@@ -2325,21 +2924,7 @@ do_main_loop(void)
       tor_assert(systemd_watchdog_timer);
     }
   }
-#endif
-
-  if (!refill_timer) {
-    struct timeval refill_interval;
-    int msecs = get_options()->TokenBucketRefillInterval;
-
-    refill_interval.tv_sec =  msecs/1000;
-    refill_interval.tv_usec = (msecs%1000)*1000;
-
-    refill_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &refill_interval,
-                                      refill_callback,
-                                      NULL);
-    tor_assert(refill_timer);
-  }
+#endif /* defined(HAVE_SYSTEMD_209) */
 
 #ifdef HAVE_SYSTEMD
   {
@@ -2353,11 +2938,44 @@ do_main_loop(void)
       log_info(LD_GENERAL, "Systemd NOTIFY_SOCKET not present.");
     }
   }
+#endif /* defined(HAVE_SYSTEMD) */
+
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+
+#ifdef ENABLE_RESTART_DEBUGGING
+  {
+    static int first_time = 1;
+
+    if (first_time && getenv("TOR_DEBUG_RESTART")) {
+      first_time = 0;
+      const char *sec_str = getenv("TOR_DEBUG_RESTART_AFTER_SECONDS");
+      long sec;
+      int sec_ok=0;
+      if (sec_str &&
+          (sec = tor_parse_long(sec_str, 10, 0, INT_MAX, &sec_ok, NULL)) &&
+          sec_ok) {
+        /* Okay, we parsed the seconds. */
+      } else {
+        sec = 5;
+      }
+      struct timeval restart_after = { (time_t) sec, 0 };
+      tor_shutdown_event_loop_for_restart_event =
+        tor_evtimer_new(tor_libevent_get_base(),
+                        tor_shutdown_event_loop_for_restart_cb, NULL);
+      event_add(tor_shutdown_event_loop_for_restart_event, &restart_after);
+    }
+  }
 #endif
 
   return run_main_loop_until_done();
 }
 
+#ifndef _WIN32
+/** Rate-limiter for EINVAL-type libevent warnings. */
+static ratelim_t libevent_error_ratelim = RATELIM_INIT(10);
+#endif
+
 /**
  * Run the main loop a single time. Return 0 for "exit"; -1 for "exit with
  * error", and 1 for "run this again."
@@ -2370,23 +2988,50 @@ run_main_loop_once(void)
   if (nt_service_is_stopping())
     return 0;
 
+  if (main_loop_should_exit)
+    return 0;
+
 #ifndef _WIN32
   /* Make it easier to tell whether libevent failure is our fault or not. */
   errno = 0;
 #endif
-  /* All active linked conns should get their read events activated. */
-  SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn,
-                    event_active(conn->read_event, EV_READ, 1));
-  called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0;
 
+  if (get_options()->MainloopStats) {
+    /* We always enforce that EVLOOP_ONCE is passed to event_base_loop() if we
+     * are collecting main loop statistics. */
+    called_loop_once = 1;
+  } else {
+    called_loop_once = 0;
+  }
+
+  /* Make sure we know (about) what time it is. */
   update_approx_time(time(NULL));
 
-  /* poll until we have an event, or the second ends, or until we have
-   * some active linked connections to trigger events for. */
-  loop_result = event_base_loop(tor_libevent_get_base(),
-                                called_loop_once ? EVLOOP_ONCE : 0);
+  /* Here it is: the main loop.  Here we tell Libevent to poll until we have
+   * an event, or the second ends, or until we have some active linked
+   * connections to trigger events for.  Libevent will wait till one
+   * of these happens, then run all the appropriate callbacks. */
+  loop_result = tor_libevent_run_event_loop(tor_libevent_get_base(),
+                                            called_loop_once);
+
+  if (get_options()->MainloopStats) {
+    /* Update our main loop counters. */
+    if (loop_result == 0) {
+      // The call was successful.
+      increment_main_loop_success_count();
+    } else if (loop_result == -1) {
+      // The call was erroneous.
+      increment_main_loop_error_count();
+    } else if (loop_result == 1) {
+      // The call didn't have any active or pending events
+      // to handle.
+      increment_main_loop_idle_count();
+    }
+  }
 
-  /* let catch() handle things like ^c, and otherwise don't worry about it */
+  /* Oh, the loop failed.  That might be an error that we need to
+   * catch, but more likely, it's just an interrupted poll() call or something,
+   * and we should try again. */
   if (loop_result < 0) {
     int e = tor_socket_errno(-1);
     /* let the program survive things like ^z */
@@ -2396,10 +3041,13 @@ run_main_loop_once(void)
       return -1;
 #ifndef _WIN32
     } else if (e == EINVAL) {
-      log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?");
-      if (got_libevent_error())
+      log_fn_ratelim(&libevent_error_ratelim, LOG_WARN, LD_NET,
+                     "EINVAL from libevent: should you upgrade libevent?");
+      if (libevent_error_ratelim.n_calls_since_last_time > 8) {
+        log_err(LD_NET, "Too many libevent errors, too fast: dying");
         return -1;
-#endif
+      }
+#endif /* !defined(_WIN32) */
     } else {
       tor_assert_nonfatal_once(! ERRNO_IS_EINPROGRESS(e));
       log_debug(LD_NET,"libevent call interrupted.");
@@ -2409,10 +3057,8 @@ run_main_loop_once(void)
     }
   }
 
-  /* This will be pretty fast if nothing new is pending. Note that this gets
-   * called once per libevent loop, which will make it happen once per group
-   * of events that fire, or once per second. */
-  connection_ap_attach_pending(0);
+  if (main_loop_should_exit)
+    return 0;
 
   return 1;
 }
@@ -2429,7 +3075,11 @@ run_main_loop_until_done(void)
   do {
     loop_result = run_main_loop_once();
   } while (loop_result == 1);
-  return loop_result;
+
+  if (main_loop_should_exit)
+    return main_loop_exit_value;
+  else
+    return loop_result;
 }
 
 /** Libevent callback: invoked when we get a signal.
@@ -2442,6 +3092,7 @@ signal_callback(evutil_socket_t fd, short events, void *arg)
   (void)fd;
   (void)events;
 
+  update_current_time(time(NULL));
   process_signal(sig);
 }
 
@@ -2453,14 +3104,13 @@ process_signal(int sig)
     {
     case SIGTERM:
       log_notice(LD_GENERAL,"Catching signal TERM, exiting cleanly.");
-      tor_cleanup();
-      exit(0);
+      tor_shutdown_event_loop_and_exit(0);
       break;
     case SIGINT:
       if (!server_mode(get_options())) { /* do it now */
         log_notice(LD_GENERAL,"Interrupt: exiting cleanly.");
-        tor_cleanup();
-        exit(0);
+        tor_shutdown_event_loop_and_exit(0);
+        return;
       }
 #ifdef HAVE_SYSTEMD
       sd_notify(0, "STOPPING=1");
@@ -2489,8 +3139,8 @@ process_signal(int sig)
 #endif
       if (do_hup() < 0) {
         log_warn(LD_CONFIG,"Restart failed (config error?). Exiting.");
-        tor_cleanup();
-        exit(1);
+        tor_shutdown_event_loop_and_exit(1);
+        return;
       }
 #ifdef HAVE_SYSTEMD
       sd_notify(0, "READY=1");
@@ -2505,10 +3155,20 @@ process_signal(int sig)
     case SIGNEWNYM: {
       time_t now = time(NULL);
       if (time_of_last_signewnym + MAX_SIGNEWNYM_RATE > now) {
-        signewnym_is_pending = 1;
+        const time_t delay_sec =
+          time_of_last_signewnym + MAX_SIGNEWNYM_RATE - now;
+        if (! signewnym_is_pending) {
+          signewnym_is_pending = 1;
+          if (!handle_deferred_signewnym_ev) {
+            handle_deferred_signewnym_ev =
+              mainloop_event_postloop_new(handle_deferred_signewnym_cb, NULL);
+          }
+          const struct timeval delay_tv = { delay_sec, 0 };
+          mainloop_event_schedule(handle_deferred_signewnym_ev, &delay_tv);
+        }
         log_notice(LD_CONTROL,
-            "Rate limiting NEWNYM request: delaying by %d second(s)",
-            (int)(MAX_SIGNEWNYM_RATE+time_of_last_signewnym-now));
+                   "Rate limiting NEWNYM request: delaying by %d second(s)",
+                   (int)(delay_sec));
       } else {
         signewnym_impl(now);
       }
@@ -2532,6 +3192,13 @@ get_uptime,(void))
   return stats_n_seconds_working;
 }
 
+/** Reset Tor's uptime. */
+MOCK_IMPL(void,
+reset_uptime,(void))
+{
+  stats_n_seconds_working = 0;
+}
+
 /**
  * Write current memory usage information to the log.
  */
@@ -2575,13 +3242,13 @@ dumpstats(int severity)
           i,
           (int)connection_get_inbuf_len(conn),
           (int)buf_allocation(conn->inbuf),
-          (int)(now - conn->timestamp_lastread));
+          (int)(now - conn->timestamp_last_read_allowed));
       tor_log(severity,LD_GENERAL,
           "Conn %d: %d bytes waiting on outbuf "
           "(len %d, last written %d secs ago)",i,
           (int)connection_get_outbuf_len(conn),
           (int)buf_allocation(conn->outbuf),
-          (int)(now - conn->timestamp_lastwritten));
+          (int)(now - conn->timestamp_last_write_allowed));
       if (conn->type == CONN_TYPE_OR) {
         or_connection_t *or_conn = TO_OR_CONN(conn);
         if (or_conn->tls) {
@@ -2652,7 +3319,6 @@ dumpstats(int severity)
 
   rep_hist_dump_stats(now,severity);
   rend_service_dump_stats(severity);
-  dump_pk_ops(severity);
   dump_distinct_digest_count(severity);
 }
 
@@ -2673,9 +3339,15 @@ exit_function(void)
 #else
 #define UNIX_ONLY 1
 #endif
+
 static struct {
+  /** A numeric code for this signal. Must match the signal value if
+   * try_to_register is true. */
   int signal_value;
+  /** True if we should try to register this signal with libevent and catch
+   * corresponding posix signals. False otherwise. */
   int try_to_register;
+  /** Pointer to hold the event object constructed for this signal. */
   struct event *signal_event;
 } signal_handlers[] = {
 #ifdef SIGINT
@@ -2709,50 +3381,40 @@ static struct {
   { -1, -1, NULL }
 };
 
-/** Set up the signal handlers for either parent or child process */
+/** Set up the signal handler events for this process, and register them
+ * with libevent if appropriate. */
 void
-handle_signals(int is_parent)
+handle_signals(void)
 {
   int i;
-  if (is_parent) {
-    for (i = 0; signal_handlers[i].signal_value >= 0; ++i) {
-      if (signal_handlers[i].try_to_register) {
-        signal_handlers[i].signal_event =
-          tor_evsignal_new(tor_libevent_get_base(),
-                           signal_handlers[i].signal_value,
-                           signal_callback,
-                           &signal_handlers[i].signal_value);
-        if (event_add(signal_handlers[i].signal_event, NULL))
-          log_warn(LD_BUG, "Error from libevent when adding "
-                   "event for signal %d",
-                   signal_handlers[i].signal_value);
-      } else {
-        signal_handlers[i].signal_event =
-          tor_event_new(tor_libevent_get_base(), -1,
-                        EV_SIGNAL, signal_callback,
-                        &signal_handlers[i].signal_value);
-      }
+  const int enabled = !get_options()->DisableSignalHandlers;
+
+  for (i = 0; signal_handlers[i].signal_value >= 0; ++i) {
+    /* Signal handlers are only registered with libevent if they need to catch
+     * real POSIX signals.  We construct these signal handler events in either
+     * case, though, so that controllers can activate them with the SIGNAL
+     * command.
+     */
+    if (enabled && signal_handlers[i].try_to_register) {
+      signal_handlers[i].signal_event =
+        tor_evsignal_new(tor_libevent_get_base(),
+                         signal_handlers[i].signal_value,
+                         signal_callback,
+                         &signal_handlers[i].signal_value);
+      if (event_add(signal_handlers[i].signal_event, NULL))
+        log_warn(LD_BUG, "Error from libevent when adding "
+                 "event for signal %d",
+                 signal_handlers[i].signal_value);
+    } else {
+      signal_handlers[i].signal_event =
+        tor_event_new(tor_libevent_get_base(), -1,
+                      EV_SIGNAL, signal_callback,
+                      &signal_handlers[i].signal_value);
     }
-  } else {
-#ifndef _WIN32
-    struct sigaction action;
-    action.sa_flags = 0;
-    sigemptyset(&action.sa_mask);
-    action.sa_handler = SIG_IGN;
-    sigaction(SIGINT,  &action, NULL);
-    sigaction(SIGTERM, &action, NULL);
-    sigaction(SIGPIPE, &action, NULL);
-    sigaction(SIGUSR1, &action, NULL);
-    sigaction(SIGUSR2, &action, NULL);
-    sigaction(SIGHUP,  &action, NULL);
-#ifdef SIGXFSZ
-    sigaction(SIGXFSZ, &action, NULL);
-#endif
-#endif
   }
 }
 
-/* Make sure the signal handler for signal_num will be called. */
+/* Cause the signal handler for signal_num to be called in the event loop. */
 void
 activate_signal(int signal_num)
 {
@@ -2765,7 +3427,8 @@ activate_signal(int signal_num)
   }
 }
 
-/** Main entry point for the Tor command-line client.
+/** Main entry point for the Tor command-line client.  Return 0 on "success",
+ * negative on "failure", and positive on "success and exit".
  */
 int
 tor_init(int argc, char *argv[])
@@ -2791,6 +3454,8 @@ tor_init(int argc, char *argv[])
   rend_cache_init();
   addressmap_init(); /* Init the client dns cache. Do it always, since it's
                       * cheap. */
+  /* Initialize the HS subsystem. */
+  hs_init();
 
   {
   /* We search for the "quiet" option first, since it decides whether we
@@ -2835,11 +3500,16 @@ tor_init(int argc, char *argv[])
     const char *version = get_version();
 
     log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, "
-               "OpenSSL %s and Zlib %s.", version,
+               "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version,
                get_uname(),
                tor_libevent_get_version_str(),
                crypto_openssl_get_version_str(),
-               tor_zlib_get_version_str());
+               tor_compress_supports_method(ZLIB_METHOD) ?
+                 tor_compress_version_str(ZLIB_METHOD) : "N/A",
+               tor_compress_supports_method(LZMA_METHOD) ?
+                 tor_compress_version_str(LZMA_METHOD) : "N/A",
+               tor_compress_supports_method(ZSTD_METHOD) ?
+                 tor_compress_version_str(ZSTD_METHOD) : "N/A");
 
     log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! "
                "Learn how to be safe at "
@@ -2848,22 +3518,40 @@ tor_init(int argc, char *argv[])
     if (strstr(version, "alpha") || strstr(version, "beta"))
       log_notice(LD_GENERAL, "This version is not a stable Tor release. "
                  "Expect more bugs than usual.");
+
+    tor_compress_log_init_warnings();
   }
 
+#ifdef HAVE_RUST
+  rust_log_welcome_string();
+#endif /* defined(HAVE_RUST) */
+
   if (network_init()<0) {
     log_err(LD_BUG,"Error initializing network; exiting.");
     return -1;
   }
   atexit(exit_function);
 
-  if (options_init_from_torrc(argc,argv) < 0) {
+  int init_rv = options_init_from_torrc(argc,argv);
+  if (init_rv < 0) {
     log_err(LD_CONFIG,"Reading config failed--see warnings above.");
     return -1;
+  } else if (init_rv > 0) {
+    // We succeeded, and should exit anyway -- probably the user just said
+    // "--version" or something like that.
+    return 1;
   }
 
   /* The options are now initialised */
   const or_options_t *options = get_options();
 
+  /* Initialize channelpadding parameters to defaults until we get
+   * a consensus */
+  channelpadding_new_consensus_params(NULL);
+
+  /* Initialize predicted ports list after loading options */
+  predicted_ports_init();
+
 #ifndef _WIN32
   if (geteuid()==0)
     log_warn(LD_GENERAL,"You are running Tor as root. You don't need to, "
@@ -2881,7 +3569,7 @@ tor_init(int argc, char *argv[])
     log_warn(LD_NET, "Problem initializing libevent RNG.");
   }
 
-  /* Scan/clean unparseable descroptors; after reading config */
+  /* Scan/clean unparseable descriptors; after reading config */
   routerparse_init();
 
   return 0;
@@ -2903,7 +3591,7 @@ try_locking(const or_options_t *options, int err_if_locked)
   if (lockfile)
     return 0;
   else {
-    char *fname = options_get_datadir_fname2_suffix(options, "lock",NULL,NULL);
+    char *fname = options_get_datadir_fname(options, "lock");
     int already_locked = 0;
     tor_lockfile_t *lf = tor_lockfile_lock(fname, 0, &already_locked);
     tor_free(fname);
@@ -2921,7 +3609,7 @@ try_locking(const or_options_t *options, int err_if_locked)
         r = try_locking(options, 0);
         if (r<0) {
           log_err(LD_GENERAL, "No, it's still there.  Exiting.");
-          exit(0);
+          return -1;
         }
         return r;
       }
@@ -2969,7 +3657,6 @@ tor_free_all(int postfork)
   networkstatus_free_all();
   addressmap_free_all();
   dirserv_free_all();
-  rend_service_free_all();
   rend_cache_free_all();
   rend_service_authorization_free_all();
   rep_hist_free_all();
@@ -2990,7 +3677,13 @@ tor_free_all(int postfork)
   control_free_all();
   sandbox_free_getaddrinfo_cache();
   protover_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
   dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+
   if (!postfork) {
     config_free_all();
     or_state_free_all();
@@ -3011,19 +3704,62 @@ tor_free_all(int postfork)
   smartlist_free(active_linked_connection_lst);
   periodic_timer_free(second_timer);
   teardown_periodic_events();
-  periodic_timer_free(refill_timer);
+  tor_event_free(shutdown_did_not_work_event);
+  tor_event_free(initialize_periodic_events_event);
+  mainloop_event_free(directory_all_unreachable_cb_event);
+  mainloop_event_free(schedule_active_linked_connections_event);
+  mainloop_event_free(postloop_cleanup_ev);
+  mainloop_event_free(handle_deferred_signewnym_ev);
+
+#ifdef HAVE_SYSTEMD_209
+  periodic_timer_free(systemd_watchdog_timer);
+#endif
+
+  memset(&global_bucket, 0, sizeof(global_bucket));
+  memset(&global_relayed_bucket, 0, sizeof(global_relayed_bucket));
+  stats_n_bytes_read = stats_n_bytes_written = 0;
+  time_of_process_start = 0;
+  time_of_last_signewnym = 0;
+  signewnym_is_pending = 0;
+  newnym_epoch = 0;
+  called_loop_once = 0;
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+  can_complete_circuits = 0;
+  quiet_level = 0;
+  should_init_bridge_stats = 1;
+  dns_honesty_first_time = 1;
+  heartbeat_callback_first_time = 1;
+  current_second = 0;
+  memset(&current_second_last_changed, 0,
+         sizeof(current_second_last_changed));
 
   if (!postfork) {
     release_lockfile();
   }
+  tor_libevent_free_all();
   /* Stuff in util.c and address.c*/
   if (!postfork) {
     escaped(NULL);
     esc_router_info(NULL);
+    clean_up_backtrace_handler();
     logs_free_all(); /* free log strings. do this last so logs keep working. */
   }
 }
 
+/**
+ * Remove the specified file, and log a warning if the operation fails for
+ * any reason other than the file not existing. Ignores NULL filenames.
+ */
+void
+tor_remove_file(const char *filename)
+{
+  if (filename && tor_unlink(filename) != 0 && errno != ENOENT) {
+    log_warn(LD_FS, "Couldn't unlink %s: %s",
+               filename, strerror(errno));
+  }
+}
+
 /** Do whatever cleanup is necessary before shutting Tor down. */
 void
 tor_cleanup(void)
@@ -3033,18 +3769,20 @@ tor_cleanup(void)
     time_t now = time(NULL);
     /* Remove our pid file. We don't care if there was an error when we
      * unlink, nothing we could do about it anyways. */
-    if (options->PidFile) {
-      if (unlink(options->PidFile) != 0) {
-        log_warn(LD_FS, "Couldn't unlink pid file %s: %s",
-                 options->PidFile, strerror(errno));
-      }
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
     }
-    if (options->ControlPortWriteToFile) {
-      if (unlink(options->ControlPortWriteToFile) != 0) {
-        log_warn(LD_FS, "Couldn't unlink control port file %s: %s",
-                 options->ControlPortWriteToFile,
-                 strerror(errno));
-      }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
     }
     if (accounting_is_enabled(options))
       accounting_record_bandwidth_usage(now, get_or_state());
@@ -3057,6 +3795,9 @@ tor_cleanup(void)
       rep_hist_record_mtbf_data(now, 0);
     keypin_close_journal();
   }
+
+  timers_shutdown();
+
 #ifdef USE_DMALLOC
   dmalloc_log_stats();
 #endif
@@ -3174,7 +3915,7 @@ sandbox_init_filter(void)
   int i;
 
   sandbox_cfg_allow_openat_filename(&cfg,
-      get_datadir_fname("cached-status"));
+      get_cachedir_fname("cached-status"));
 
 #define OPEN(name)                              \
   sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name))
@@ -3195,21 +3936,38 @@ sandbox_init_filter(void)
     OPEN_DATADIR2(name, name2 suffix);                  \
   } while (0)
 
+#define OPEN_KEY_DIRECTORY() \
+  sandbox_cfg_allow_open_filename(&cfg, tor_strdup(options->KeyDirectory))
+#define OPEN_CACHEDIR(name)                      \
+  sandbox_cfg_allow_open_filename(&cfg, get_cachedir_fname(name))
+#define OPEN_CACHEDIR_SUFFIX(name, suffix) do {  \
+    OPEN_CACHEDIR(name);                         \
+    OPEN_CACHEDIR(name suffix);                  \
+  } while (0)
+#define OPEN_KEYDIR(name)                      \
+  sandbox_cfg_allow_open_filename(&cfg, get_keydir_fname(name))
+#define OPEN_KEYDIR_SUFFIX(name, suffix) do {    \
+    OPEN_KEYDIR(name);                           \
+    OPEN_KEYDIR(name suffix);                    \
+  } while (0)
+
   OPEN(options->DataDirectory);
-  OPEN_DATADIR("keys");
-  OPEN_DATADIR_SUFFIX("cached-certs", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("unverified-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("unverified-microdesc-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-microdesc-consensus", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-microdescs", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-microdescs.new", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-descriptors", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-descriptors.new", ".tmp");
-  OPEN_DATADIR("cached-descriptors.tmp.tmp");
-  OPEN_DATADIR_SUFFIX("cached-extrainfo", ".tmp");
-  OPEN_DATADIR_SUFFIX("cached-extrainfo.new", ".tmp");
-  OPEN_DATADIR("cached-extrainfo.tmp.tmp");
+  OPEN_KEY_DIRECTORY();
+
+  OPEN_CACHEDIR_SUFFIX("cached-certs", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("unverified-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("unverified-microdesc-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-microdesc-consensus", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-microdescs", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-microdescs.new", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-descriptors", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-descriptors.new", ".tmp");
+  OPEN_CACHEDIR("cached-descriptors.tmp.tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-extrainfo", ".tmp");
+  OPEN_CACHEDIR_SUFFIX("cached-extrainfo.new", ".tmp");
+  OPEN_CACHEDIR("cached-extrainfo.tmp.tmp");
+
   OPEN_DATADIR_SUFFIX("state", ".tmp");
   OPEN_DATADIR_SUFFIX("sr-state", ".tmp");
   OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp");
@@ -3224,7 +3982,7 @@ sandbox_init_filter(void)
   if (options->BridgeAuthoritativeDir)
     OPEN_DATADIR_SUFFIX("networkstatus-bridges", ".tmp");
 
-  if (authdir_mode_handles_descs(options, -1))
+  if (authdir_mode(options))
     OPEN_DATADIR("approved-routers");
 
   if (options->ServerDNSResolvConfFile)
@@ -3239,6 +3997,10 @@ sandbox_init_filter(void)
     }
   }
 
+  SMARTLIST_FOREACH(options->FilesOpenedByIncludes, char *, f, {
+    OPEN(f);
+  });
+
 #define RENAME_SUFFIX(name, suffix)        \
   sandbox_cfg_allow_rename(&cfg,           \
       get_datadir_fname(name suffix),      \
@@ -3249,20 +4011,31 @@ sandbox_init_filter(void)
                            get_datadir_fname2(prefix, name suffix),     \
                            get_datadir_fname2(prefix, name))
 
-  RENAME_SUFFIX("cached-certs", ".tmp");
-  RENAME_SUFFIX("cached-consensus", ".tmp");
-  RENAME_SUFFIX("unverified-consensus", ".tmp");
-  RENAME_SUFFIX("unverified-microdesc-consensus", ".tmp");
-  RENAME_SUFFIX("cached-microdesc-consensus", ".tmp");
-  RENAME_SUFFIX("cached-microdescs", ".tmp");
-  RENAME_SUFFIX("cached-microdescs", ".new");
-  RENAME_SUFFIX("cached-microdescs.new", ".tmp");
-  RENAME_SUFFIX("cached-descriptors", ".tmp");
-  RENAME_SUFFIX("cached-descriptors", ".new");
-  RENAME_SUFFIX("cached-descriptors.new", ".tmp");
-  RENAME_SUFFIX("cached-extrainfo", ".tmp");
-  RENAME_SUFFIX("cached-extrainfo", ".new");
-  RENAME_SUFFIX("cached-extrainfo.new", ".tmp");
+#define RENAME_CACHEDIR_SUFFIX(name, suffix)        \
+  sandbox_cfg_allow_rename(&cfg,           \
+      get_cachedir_fname(name suffix),      \
+      get_cachedir_fname(name))
+
+#define RENAME_KEYDIR_SUFFIX(name, suffix)    \
+  sandbox_cfg_allow_rename(&cfg,           \
+      get_keydir_fname(name suffix),      \
+      get_keydir_fname(name))
+
+  RENAME_CACHEDIR_SUFFIX("cached-certs", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("unverified-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("unverified-microdesc-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-microdesc-consensus", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-microdescs", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-microdescs", ".new");
+  RENAME_CACHEDIR_SUFFIX("cached-microdescs.new", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-descriptors", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-descriptors", ".new");
+  RENAME_CACHEDIR_SUFFIX("cached-descriptors.new", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-extrainfo", ".tmp");
+  RENAME_CACHEDIR_SUFFIX("cached-extrainfo", ".new");
+  RENAME_CACHEDIR_SUFFIX("cached-extrainfo.new", ".tmp");
+
   RENAME_SUFFIX("state", ".tmp");
   RENAME_SUFFIX("sr-state", ".tmp");
   RENAME_SUFFIX("unparseable-desc", ".tmp");
@@ -3274,14 +4047,21 @@ sandbox_init_filter(void)
 #define STAT_DATADIR(name)                      \
   sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname(name))
 
+#define STAT_CACHEDIR(name)                                             \
+  sandbox_cfg_allow_stat_filename(&cfg, get_cachedir_fname(name))
+
 #define STAT_DATADIR2(name, name2)                                      \
   sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname2((name), (name2)))
 
+#define STAT_KEY_DIRECTORY() \
+  sandbox_cfg_allow_stat_filename(&cfg, tor_strdup(options->KeyDirectory))
+
   STAT_DATADIR(NULL);
   STAT_DATADIR("lock");
   STAT_DATADIR("state");
   STAT_DATADIR("router-stability");
-  STAT_DATADIR("cached-extrainfo.new");
+
+  STAT_CACHEDIR("cached-extrainfo.new");
 
   {
     smartlist_t *files = smartlist_new();
@@ -3296,7 +4076,7 @@ sandbox_init_filter(void)
   {
     smartlist_t *files = smartlist_new();
     smartlist_t *dirs = smartlist_new();
-    rend_services_add_filenames_to_lists(files, dirs);
+    hs_service_lists_fnames_for_sandbox(files, dirs);
     SMARTLIST_FOREACH(files, char *, file_name, {
       char *tmp_name = NULL;
       tor_asprintf(&tmp_name, "%s.tmp", file_name);
@@ -3346,22 +4126,20 @@ sandbox_init_filter(void)
   // orport
   if (server_mode(get_options())) {
 
-    OPEN_DATADIR2_SUFFIX("keys", "secret_id_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key_ntor", ".tmp");
-    OPEN_DATADIR2("keys", "secret_id_key.old");
-    OPEN_DATADIR2("keys", "secret_onion_key.old");
-    OPEN_DATADIR2("keys", "secret_onion_key_ntor.old");
-
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_secret_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_secret_key_encrypted",
-                         ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_public_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_secret_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_secret_key_encrypted",
-                         ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_public_key", ".tmp");
-    OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_cert", ".tmp");
+    OPEN_KEYDIR_SUFFIX("secret_id_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("secret_onion_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("secret_onion_key_ntor", ".tmp");
+    OPEN_KEYDIR("secret_id_key.old");
+    OPEN_KEYDIR("secret_onion_key.old");
+    OPEN_KEYDIR("secret_onion_key_ntor.old");
+
+    OPEN_KEYDIR_SUFFIX("ed25519_master_id_secret_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_master_id_secret_key_encrypted", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_master_id_public_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_secret_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_secret_key_encrypted", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_public_key", ".tmp");
+    OPEN_KEYDIR_SUFFIX("ed25519_signing_cert", ".tmp");
 
     OPEN_DATADIR2_SUFFIX("stats", "bridge-stats", ".tmp");
     OPEN_DATADIR2_SUFFIX("stats", "dirreq-stats", ".tmp");
@@ -3380,11 +4158,13 @@ sandbox_init_filter(void)
     OPEN("/etc/resolv.conf");
 
     RENAME_SUFFIX("fingerprint", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_id_key", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_id_key.old", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_onion_key", ".tmp");
-    RENAME_SUFFIX2("keys", "secret_onion_key.old", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_onion_key_ntor", ".tmp");
+
+    RENAME_KEYDIR_SUFFIX("secret_id_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_id_key.old", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_onion_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("secret_onion_key.old", ".tmp");
+
     RENAME_SUFFIX2("stats", "bridge-stats", ".tmp");
     RENAME_SUFFIX2("stats", "dirreq-stats", ".tmp");
     RENAME_SUFFIX2("stats", "entry-stats", ".tmp");
@@ -3395,23 +4175,25 @@ sandbox_init_filter(void)
     RENAME_SUFFIX("hashed-fingerprint", ".tmp");
     RENAME_SUFFIX("router-stability", ".tmp");
 
-    RENAME_SUFFIX2("keys", "ed25519_master_id_secret_key", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_master_id_secret_key_encrypted", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_master_id_public_key", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_signing_secret_key", ".tmp");
-    RENAME_SUFFIX2("keys", "ed25519_signing_cert", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_master_id_secret_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_master_id_secret_key_encrypted", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_master_id_public_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_signing_secret_key", ".tmp");
+    RENAME_KEYDIR_SUFFIX("ed25519_signing_cert", ".tmp");
 
     sandbox_cfg_allow_rename(&cfg,
-             get_datadir_fname2("keys", "secret_onion_key"),
-             get_datadir_fname2("keys", "secret_onion_key.old"));
+             get_keydir_fname("secret_onion_key"),
+             get_keydir_fname("secret_onion_key.old"));
     sandbox_cfg_allow_rename(&cfg,
-             get_datadir_fname2("keys", "secret_onion_key_ntor"),
-             get_datadir_fname2("keys", "secret_onion_key_ntor.old"));
+             get_keydir_fname("secret_onion_key_ntor"),
+             get_keydir_fname("secret_onion_key_ntor.old"));
 
-    STAT_DATADIR("keys");
+    STAT_KEY_DIRECTORY();
     OPEN_DATADIR("stats");
     STAT_DATADIR("stats");
     STAT_DATADIR2("stats", "dirreq-stats");
+
+    consdiffmgr_register_with_sandbox(&cfg);
   }
 
   init_addrinfo();
@@ -3419,14 +4201,16 @@ sandbox_init_filter(void)
   return cfg;
 }
 
-/** Main entry point for the Tor process.  Called from main(). */
-/* This function is distinct from main() only so we can link main.c into
- * the unittest binary without conflicting with the unittests' main. */
+/* Main entry point for the Tor process.  Called from tor_main(), and by
+ * anybody embedding Tor. */
 int
-tor_main(int argc, char *argv[])
+tor_run_main(const tor_main_configuration_t *tor_cfg)
 {
   int result = 0;
 
+  int argc = tor_cfg->argc;
+  char **argv = tor_cfg->argv;
+
 #ifdef _WIN32
 #ifndef HeapEnableTerminationOnCorruption
 #define HeapEnableTerminationOnCorruption 1
@@ -3454,36 +4238,45 @@ tor_main(int argc, char *argv[])
     }
   }
 #endif /* !defined(_WIN64) */
-#endif
+#endif /* defined(_WIN32) */
 
   configure_backtrace_handler(get_version());
+  init_protocol_warning_severity_level();
 
   update_approx_time(time(NULL));
   tor_threads_init();
+  tor_compress_init();
   init_logging(0);
+  monotime_init();
 #ifdef USE_DMALLOC
   {
     /* Instruct OpenSSL to use our internal wrappers for malloc,
        realloc and free. */
-    int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
-    tor_assert(r);
+    int r = crypto_use_tor_alloc_functions();
+    tor_assert(r == 0);
   }
-#endif
+#endif /* defined(USE_DMALLOC) */
 #ifdef NT_SERVICE
   {
      int done = 0;
      result = nt_service_parse_options(argc, argv, &done);
      if (done) return result;
   }
-#endif
-  if (tor_init(argc, argv)<0)
-    return -1;
+#endif /* defined(NT_SERVICE) */
+  {
+    int init_rv = tor_init(argc, argv);
+    if (init_rv) {
+      tor_free_all(0);
+      return (init_rv < 0) ? -1 : 0;
+    }
+  }
 
   if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
     sandbox_cfg_t* cfg = sandbox_init_filter();
 
     if (sandbox_init(cfg)) {
       log_err(LD_BUG,"Failed to create syscall sandbox filter");
+      tor_free_all(0);
       return -1;
     }
 
@@ -3494,8 +4287,6 @@ tor_main(int argc, char *argv[])
 #endif
   }
 
-  monotime_init();
-
   switch (get_options()->command) {
   case CMD_RUN_TOR:
 #ifdef NT_SERVICE
@@ -3504,7 +4295,11 @@ tor_main(int argc, char *argv[])
     result = do_main_loop();
     break;
   case CMD_KEYGEN:
-    result = load_ed_keys(get_options(), time(NULL));
+    result = load_ed_keys(get_options(), time(NULL)) < 0;
+    break;
+  case CMD_KEY_EXPIRATION:
+    init_keys();
+    result = log_cert_expiration();
     break;
   case CMD_LIST_FINGERPRINT:
     result = do_list_fingerprint();
diff --git a/src/or/main.h b/src/or/main.h
index 07b22598b1..9dbbc6e5ee 100644
--- a/src/or/main.h
+++ b/src/or/main.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,6 +28,7 @@ int connection_is_on_closeable_list(connection_t *conn);
 MOCK_DECL(smartlist_t *, get_connection_array, (void));
 MOCK_DECL(uint64_t,get_bytes_read,(void));
 MOCK_DECL(uint64_t,get_bytes_written,(void));
+void stats_increment_bytes_read_and_written(uint64_t r, uint64_t w);
 
 /** Bitmask for events that we can turn on and off with
  * connection_watch_events. */
@@ -45,7 +46,8 @@ int connection_is_writing(connection_t *conn);
 MOCK_DECL(void,connection_stop_writing,(connection_t *conn));
 MOCK_DECL(void,connection_start_writing,(connection_t *conn));
 
-void tell_event_loop_to_finish(void);
+void tor_shutdown_event_loop_and_exit(int exitcode);
+int tor_event_loop_shutdown_is_pending(void);
 
 void connection_stop_reading_from_linked_conn(connection_t *conn);
 
@@ -59,40 +61,61 @@ void dns_servers_relaunch_checks(void);
 void reset_all_main_loop_timers(void);
 void reschedule_descriptor_update_check(void);
 void reschedule_directory_downloads(void);
+void reschedule_or_state_save(void);
+void reschedule_dirvote(const or_options_t *options);
+void mainloop_schedule_postloop_cleanup(void);
+void rescan_periodic_events(const or_options_t *options);
+
+void update_current_time(time_t now);
 
 MOCK_DECL(long,get_uptime,(void));
+MOCK_DECL(void,reset_uptime,(void));
 
 unsigned get_signewnym_epoch(void);
 
-void handle_signals(int is_parent);
+void handle_signals(void);
 void activate_signal(int signal_num);
 
 int try_locking(const or_options_t *options, int err_if_locked);
 int have_lockfile(void);
 void release_lockfile(void);
 
+void tor_remove_file(const char *filename);
+
 void tor_cleanup(void);
 void tor_free_all(int postfork);
 
-int tor_main(int argc, char *argv[]);
-
 int do_main_loop(void);
 int tor_init(int argc, char **argv);
 
+void reset_main_loop_counters(void);
+uint64_t get_main_loop_success_count(void);
+uint64_t get_main_loop_error_count(void);
+uint64_t get_main_loop_idle_count(void);
+
+void periodic_events_on_new_options(const or_options_t *options);
+void reschedule_per_second_timer(void);
+
 extern time_t time_of_process_start;
-extern long stats_n_seconds_working;
 extern int quiet_level;
-extern int global_read_bucket;
-extern int global_write_bucket;
-extern int global_relayed_read_bucket;
-extern int global_relayed_write_bucket;
+extern token_bucket_rw_t global_bucket;
+extern token_bucket_rw_t global_relayed_bucket;
 
 #ifdef MAIN_PRIVATE
 STATIC void init_connection_lists(void);
+STATIC void initialize_mainloop_events(void);
 STATIC void close_closeable_connections(void);
 STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
-#endif
+STATIC int get_my_roles(const or_options_t *options);
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *connection_array;
 
+/* We need the periodic_event_item_t definition. */
+#include "periodic.h"
+extern periodic_event_item_t periodic_events[];
 #endif
+#endif /* defined(MAIN_PRIVATE) */
+
+#endif /* !defined(TOR_MAIN_H) */
 
diff --git a/src/or/microdesc.c b/src/or/microdesc.c
index a81dc54628..b4a934e095 100644
--- a/src/or/microdesc.c
+++ b/src/or/microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,6 +74,102 @@ HT_GENERATE2(microdesc_map, microdesc_t, node,
              microdesc_hash_, microdesc_eq_, 0.6,
              tor_reallocarray_, tor_free_)
 
+/************************* md fetch fail cache *****************************/
+
+/* If we end up with too many outdated dirservers, something probably went
+ * wrong so clean up the list. */
+#define TOO_MANY_OUTDATED_DIRSERVERS 30
+
+/** List of dirservers with outdated microdesc information. The smartlist is
+ *  filled with the hex digests of outdated dirservers. */
+static smartlist_t *outdated_dirserver_list = NULL;
+
+/** Note that we failed to fetch a microdescriptor from the relay with
+ *  <b>relay_digest</b> (of size DIGEST_LEN). */
+void
+microdesc_note_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  /* Don't register outdated dirservers if we don't have a live consensus,
+   * since we might be trying to fetch microdescriptors that are not even
+   * currently active. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    return;
+  }
+
+  if (!outdated_dirserver_list) {
+    outdated_dirserver_list = smartlist_new();
+  }
+
+  tor_assert(outdated_dirserver_list);
+
+  /* If the list grows too big, clean it up */
+  if (BUG(smartlist_len(outdated_dirserver_list) >
+          TOO_MANY_OUTDATED_DIRSERVERS)) {
+    microdesc_reset_outdated_dirservers_list();
+  }
+
+  /* Turn the binary relay digest to a hex since smartlists have better support
+   * for strings than digests. */
+  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Make sure we don't add a dirauth as an outdated dirserver */
+  if (router_get_trusteddirserver_by_digest(relay_digest)) {
+    log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
+    return;
+  }
+
+  /* Don't double-add outdated dirservers */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return;
+  }
+
+  /* Add it to the list of outdated dirservers */
+  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
+
+  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
+}
+
+/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
+ *  outdated dirserver */
+int
+microdesc_relay_is_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  if (!outdated_dirserver_list) {
+    return 0;
+  }
+
+  /* Convert identity digest to hex digest */
+  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Last time we tried to fetch microdescs, was this directory mirror missing
+   * any mds we asked for? */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Reset the list of outdated dirservers. */
+void
+microdesc_reset_outdated_dirservers_list(void)
+{
+  if (!outdated_dirserver_list) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+  smartlist_clear(outdated_dirserver_list);
+}
+
+/****************************************************************************/
+
 /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  * On success, return the total number of bytes written, and set
  * *<b>annotation_len_out</b> to the number of bytes written as
@@ -142,8 +238,8 @@ get_microdesc_cache_noload(void)
   if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
     microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
     HT_INIT(microdesc_map, &cache->map);
-    cache->cache_fname = get_datadir_fname("cached-microdescs");
-    cache->journal_fname = get_datadir_fname("cached-microdescs.new");
+    cache->cache_fname = get_cachedir_fname("cached-microdescs");
+    cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
     the_microdesc_cache = cache;
   }
   return the_microdesc_cache;
@@ -789,6 +885,11 @@ microdesc_free_all(void)
     tor_free(the_microdesc_cache->journal_fname);
     tor_free(the_microdesc_cache);
   }
+
+  if (outdated_dirserver_list) {
+    SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+    smartlist_free(outdated_dirserver_list);
+  }
 }
 
 /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
@@ -804,18 +905,6 @@ microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
   return md;
 }
 
-/** Return the mean size of decriptors added to <b>cache</b> since it was last
- * cleared.  Used to estimate the size of large downloads. */
-size_t
-microdesc_average_size(microdesc_cache_t *cache)
-{
-  if (!cache)
-    cache = get_microdesc_cache();
-  if (!cache->n_seen)
-    return 512;
-  return (size_t)(cache->total_len_seen / cache->n_seen);
-}
-
 /** Return a smartlist of all the sha256 digest of the microdescriptors that
  * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  * to internals of <b>ns</b>; you should not free the members of the resulting
@@ -831,8 +920,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
     if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
       continue;
     if (downloadable_only &&
-        !download_status_is_ready(&rs->dl_status, now,
-                  get_options()->TestingMicrodescMaxDownloadTries))
+        !download_status_is_ready(&rs->dl_status, now))
       continue;
     if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
       continue;
@@ -888,7 +976,7 @@ update_microdesc_downloads(time_t now)
   smartlist_free(missing);
 }
 
-/** For every microdescriptor listed in the current microdecriptor consensus,
+/** For every microdescriptor listed in the current microdescriptor consensus,
  * update its last_listed field to be at least as recent as the publication
  * time of the current microdescriptor consensus.
  */
@@ -917,20 +1005,9 @@ update_microdescs_from_networkstatus(time_t now)
 int
 we_use_microdescriptors_for_circuits(const or_options_t *options)
 {
-  int ret = options->UseMicrodescriptors;
-  if (ret == -1) {
-    /* UseMicrodescriptors is "auto"; we need to decide: */
-    /* If we are configured to use bridges and none of our bridges
-     * know what a microdescriptor is, the answer is no. */
-    if (options->UseBridges && !any_bridge_supports_microdescriptors())
-      return 0;
-    /* Otherwise, we decide that we'll use microdescriptors iff we are
-     * not a server, and we're not autofetching everything. */
-    /* XXXX++ what does not being a server have to do with it? also there's
-     * a partitioning issue here where bridges differ from clients. */
-    ret = !server_mode(options) && !options->FetchUselessDescriptors;
-  }
-  return ret;
+  if (options->UseMicrodescriptors == 0)
+    return 0; /* the user explicitly picked no */
+  return 1; /* yes and auto both mean yes */
 }
 
 /** Return true iff we should try to download microdescriptors at all. */
diff --git a/src/or/microdesc.h b/src/or/microdesc.h
index 40c83139e9..83a90bd8ff 100644
--- a/src/or/microdesc.h
+++ b/src/or/microdesc.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,16 +32,16 @@ void microdesc_cache_clear(microdesc_cache_t *cache);
 microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
                                                  const char *d);
 
-size_t microdesc_average_size(microdesc_cache_t *cache);
-
 smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
                                               microdesc_cache_t *cache,
                                               int downloadable_only,
                                               digest256map_t *skip);
 
 void microdesc_free_(microdesc_t *md, const char *fname, int line);
-#define microdesc_free(md) \
-  microdesc_free_((md), __FILE__, __LINE__)
+#define microdesc_free(md) do {                 \
+    microdesc_free_((md), __FILE__, __LINE__);  \
+    (md) = NULL;                                \
+  } while (0)
 void microdesc_free_all(void);
 
 void update_microdesc_downloads(time_t now);
@@ -52,5 +52,9 @@ int we_fetch_microdescriptors(const or_options_t *options);
 int we_fetch_router_descriptors(const or_options_t *options);
 int we_use_microdescriptors_for_circuits(const or_options_t *options);
 
-#endif
+void microdesc_note_outdated_dirserver(const char *relay_digest);
+int microdesc_relay_is_outdated_dirserver(const char *relay_digest);
+void microdesc_reset_outdated_dirservers_list(void);
+
+#endif /* !defined(TOR_MICRODESC_H) */
 
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index d8e2c00273..998eaf74e6 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -1,17 +1,44 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file networkstatus.c
- * \brief Functions and structures for handling network status documents as a
- * client or cache.
+ * \brief Functions and structures for handling networkstatus documents as a
+ * client or as a directory cache.
+ *
+ * A consensus networkstatus object is created by the directory
+ * authorities.  It authenticates a set of network parameters--most
+ * importantly, the list of all the relays in the network.  This list
+ * of relays is represented as an array of routerstatus_t objects.
+ *
+ * There are currently two flavors of consensus.  With the older "NS"
+ * flavor, each relay is associated with a digest of its router
+ * descriptor. Tor instances that use this consensus keep the list of
+ * router descriptors as routerinfo_t objects stored and managed in
+ * routerlist.c.  With the newer "microdesc" flavor, each relay is
+ * associated with a digest of the microdescriptor that the authorities
+ * made for it.  These are stored and managed in microdesc.c. Information
+ * about the router is divided between the the networkstatus and the
+ * microdescriptor according to the general rule that microdescriptors
+ * should hold information that changes much less frequently than the
+ * information in the networkstatus.
+ *
+ * Modern clients use microdescriptor networkstatuses.  Directory caches
+ * need to keep both kinds of networkstatus document, so they can serve them.
+ *
+ * This module manages fetching, holding, storing, updating, and
+ * validating networkstatus objects.  The download-and-validate process
+ * is slightly complicated by the fact that the keys you need to
+ * validate a consensus are stored in the authority certificates, which
+ * you might not have yet when you download the consensus.
  */
 
 #define NETWORKSTATUS_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "channel.h"
 #include "circuitmux.h"
 #include "circuitmux_ewma.h"
@@ -19,12 +46,15 @@
 #include "config.h"
 #include "connection.h"
 #include "connection_or.h"
+#include "consdiffmgr.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "dos.h"
 #include "entrynodes.h"
+#include "hibernate.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
@@ -34,23 +64,23 @@
 #include "router.h"
 #include "routerlist.h"
 #include "routerparse.h"
-#include "shared_random.h"
+#include "scheduler.h"
 #include "transports.h"
 #include "torcert.h"
+#include "channelpadding.h"
+#include "voting_schedule.h"
 
-/** Map from lowercase nickname to identity digest of named server, if any. */
-static strmap_t *named_server_map = NULL;
-/** Map from lowercase nickname to (void*)1 for all names that are listed
- * as unnamed for some server in the consensus. */
-static strmap_t *unnamed_server_map = NULL;
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
+#include "dirauth/shared_random.h"
 
 /** Most recently received and validated v3 "ns"-flavored consensus network
  * status. */
-static networkstatus_t *current_ns_consensus = NULL;
+STATIC networkstatus_t *current_ns_consensus = NULL;
 
-/** Most recently received and validated v3 "microdec"-flavored consensus
+/** Most recently received and validated v3 "microdesc"-flavored consensus
  * network status. */
-static networkstatus_t *current_md_consensus = NULL;
+STATIC networkstatus_t *current_md_consensus = NULL;
 
 /** A v3 consensus networkstatus that we've received, but which we don't
  * have enough certificates to be happy about. */
@@ -82,9 +112,9 @@ static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS];
 static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] =
   {
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-      DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-      DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_FAILURE, 0, 0 },
   };
 
 #define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2
@@ -101,10 +131,10 @@ static download_status_t
               consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] =
   {
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
-      DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
     /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
     { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-      DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 },
+      DL_SCHED_INCREMENT_ATTEMPT, 0, 0 },
   };
 
 /** True iff we have logged a warning about this OR's version being older than
@@ -114,7 +144,6 @@ static int have_warned_about_old_version = 0;
  * listed by the authorities. */
 static int have_warned_about_new_version = 0;
 
-static void routerstatus_list_update_named_server_map(void);
 static void update_consensus_bootstrap_multiple_downloads(
                                                   time_t now,
                                                   const or_options_t *options);
@@ -152,60 +181,74 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static char *
+networkstatus_read_cached_consensus_impl(int flav,
+                                         const char *flavorname,
+                                         int unverified_consensus)
+{
+  char buf[128];
+  const char *prefix;
+  if (unverified_consensus) {
+    prefix = "unverified";
+  } else {
+    prefix = "cached";
+  }
+  if (flav == FLAV_NS) {
+    tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
+  }
+
+  char *filename = get_cachedir_fname(buf);
+  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  return result;
+}
+
+/** Return a new string containing the current cached consensus of flavor
+ * <b>flavorname</b>. */
+char *
+networkstatus_read_cached_consensus(const char *flavorname)
+ {
+  int flav = networkstatus_parse_flavor_name(flavorname);
+  if (flav < 0)
+    return NULL;
+  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+}
+
 /** Read every cached v3 consensus networkstatus from the disk. */
 int
 router_reload_consensus_networkstatus(void)
 {
-  char *filename;
-  char *s;
   const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
   int flav;
 
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
-    char buf[128];
     const char *flavor = networkstatus_get_flavor_name(flav);
-    if (flav == FLAV_NS) {
-      filename = get_datadir_fname("cached-consensus");
-    } else {
-      tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
-      filename = get_datadir_fname(buf);
-    }
-    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
     if (s) {
       if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
-                 flavor, filename);
+        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
+                 flavor);
       }
       tor_free(s);
     }
-    tor_free(filename);
-
-    if (flav == FLAV_NS) {
-      filename = get_datadir_fname("unverified-consensus");
-    } else {
-      tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
-      filename = get_datadir_fname(buf);
-    }
 
-    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
     if (s) {
       if (networkstatus_set_current_consensus(s, flavor,
-                                     flags|NSSET_WAS_WAITING_FOR_CERTS,
+                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
                                      NULL)) {
-      log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
-               flavor, filename);
-    }
+        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
+                 "from cache", flavor);
+      }
       tor_free(s);
     }
-    tor_free(filename);
-  }
-
-  if (!networkstatus_get_latest_consensus()) {
-    if (!named_server_map)
-      named_server_map = strmap_new();
-    if (!unnamed_server_map)
-      unnamed_server_map = strmap_new();
   }
 
   update_certificate_downloads(time(NULL));
@@ -217,8 +260,8 @@ router_reload_consensus_networkstatus(void)
 }
 
 /** Free all storage held by the vote_routerstatus object <b>rs</b>. */
-STATIC void
-vote_routerstatus_free(vote_routerstatus_t *rs)
+void
+vote_routerstatus_free_(vote_routerstatus_t *rs)
 {
   vote_microdesc_hash_t *h, *next;
   if (!rs)
@@ -236,7 +279,7 @@ vote_routerstatus_free(vote_routerstatus_t *rs)
 
 /** Free all storage held by the routerstatus object <b>rs</b>. */
 void
-routerstatus_free(routerstatus_t *rs)
+routerstatus_free_(routerstatus_t *rs)
 {
   if (!rs)
     return;
@@ -246,7 +289,7 @@ routerstatus_free(routerstatus_t *rs)
 
 /** Free all storage held in <b>sig</b> */
 void
-document_signature_free(document_signature_t *sig)
+document_signature_free_(document_signature_t *sig)
 {
   tor_free(sig->signature);
   tor_free(sig);
@@ -264,7 +307,7 @@ document_signature_dup(const document_signature_t *sig)
 
 /** Free all storage held in <b>ns</b>. */
 void
-networkstatus_vote_free(networkstatus_t *ns)
+networkstatus_vote_free_(networkstatus_t *ns)
 {
   if (!ns)
     return;
@@ -327,9 +370,7 @@ networkstatus_vote_free(networkstatus_t *ns)
   digestmap_free(ns->desc_digest_map, NULL);
 
   if (ns->sr_info.commits) {
-    SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c,
-                      sr_commit_free(c));
-    smartlist_free(ns->sr_info.commits);
+    dirvote_clear_commits(ns);
   }
   tor_free(ns->sr_info.previous_srv);
   tor_free(ns->sr_info.current_srv);
@@ -353,6 +394,20 @@ networkstatus_get_voter_by_id(networkstatus_t *vote,
   return NULL;
 }
 
+/** Return the signature made by <b>voter</b> using the algorithm
+ * <b>alg</b>, or NULL if none is found. */
+document_signature_t *
+networkstatus_get_voter_sig_by_alg(const networkstatus_voter_info_t *voter,
+                                   digest_algorithm_t alg)
+{
+  if (!voter->sigs)
+    return NULL;
+  SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
+                    if (sig->alg == alg)
+                    return sig);
+  return NULL;
+}
+
 /** Check whether the signature <b>sig</b> is correctly signed with the
  * signing key in <b>cert</b>.  Return -1 if <b>cert</b> doesn't match the
  * signing key; otherwise set the good_signature or bad_signature flag on
@@ -745,41 +800,6 @@ router_get_consensus_status_by_id(const char *digest)
   return router_get_mutable_consensus_status_by_id(digest);
 }
 
-/** Given a nickname (possibly verbose, possibly a hexadecimal digest), return
- * the corresponding routerstatus_t, or NULL if none exists.  Warn the
- * user if <b>warn_if_unnamed</b> is set, and they have specified a router by
- * nickname, but the Named flag isn't set for that router. */
-const routerstatus_t *
-router_get_consensus_status_by_nickname(const char *nickname,
-                                        int warn_if_unnamed)
-{
-  const node_t *node = node_get_by_nickname(nickname, warn_if_unnamed);
-  if (node)
-    return node->rs;
-  else
-    return NULL;
-}
-
-/** Return the identity digest that's mapped to officially by
- * <b>nickname</b>. */
-const char *
-networkstatus_get_router_digest_by_nickname(const char *nickname)
-{
-  if (!named_server_map)
-    return NULL;
-  return strmap_get_lc(named_server_map, nickname);
-}
-
-/** Return true iff <b>nickname</b> is disallowed from being the nickname
- * of any server. */
-int
-networkstatus_nickname_is_unnamed(const char *nickname)
-{
-  if (!unnamed_server_map)
-    return 0;
-  return strmap_get_lc(unnamed_server_map, nickname) != NULL;
-}
-
 /** How frequently do directory authorities re-download fresh networkstatus
  * documents? */
 #define AUTHORITY_NS_CACHE_INTERVAL (10*60)
@@ -789,8 +809,11 @@ networkstatus_nickname_is_unnamed(const char *nickname)
 #define NONAUTHORITY_NS_CACHE_INTERVAL (60*60)
 
 /** Return true iff, given the options listed in <b>options</b>, <b>flavor</b>
- *  is the flavor of a consensus networkstatus that we would like to fetch. */
-static int
+ *  is the flavor of a consensus networkstatus that we would like to fetch.
+ *
+ * For certificate fetches, use we_want_to_fetch_unknown_auth_certs, and
+ * for serving fetched documents, use directory_caches_dir_info. */
+int
 we_want_to_fetch_flavor(const or_options_t *options, int flavor)
 {
   if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
@@ -812,6 +835,29 @@ we_want_to_fetch_flavor(const or_options_t *options, int flavor)
   return flavor == usable_consensus_flavor();
 }
 
+/** Return true iff, given the options listed in <b>options</b>, we would like
+ * to fetch and store unknown authority certificates.
+ *
+ * For consensus and descriptor fetches, use we_want_to_fetch_flavor, and
+ * for serving fetched certificates, use directory_caches_unknown_auth_certs.
+ */
+int
+we_want_to_fetch_unknown_auth_certs(const or_options_t *options)
+{
+  if (authdir_mode_v3(options) ||
+      directory_caches_unknown_auth_certs((options))) {
+    /* We want to serve all certs to others, regardless if we would use
+     * them ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* Unknown certificates are definitely useless. */
+    return 1;
+  }
+  /* Otherwise, don't fetch unknown certificates. */
+  return 0;
+}
+
 /** How long will we hang onto a possibly live consensus for which we're
  * fetching certs before we check whether there is a better one? */
 #define DELAY_WHILE_FETCHING_CERTS (20*60)
@@ -914,20 +960,20 @@ update_consensus_networkstatus_downloads(time_t now)
       update_consensus_bootstrap_multiple_downloads(now, options);
     } else {
       /* Check if we failed downloading a consensus too recently */
-      int max_dl_tries = options->TestingConsensusMaxDownloadTries;
 
       /* Let's make sure we remembered to update consensus_dl_status */
       tor_assert(consensus_dl_status[i].schedule == DL_SCHED_CONSENSUS);
 
-      if (!download_status_is_ready(&consensus_dl_status[i],
-                                    now,
-                                    max_dl_tries)) {
+      if (!download_status_is_ready(&consensus_dl_status[i], now)) {
         continue;
       }
 
-      /* Check if we're waiting for certificates to download */
-      if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i]))
+      /** Check if we're waiting for certificates to download. If we are,
+       * launch download for missing directory authority certificates. */
+      if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) {
+        update_certificate_downloads(now);
         continue;
+      }
 
       /* Try the requested attempt */
       log_info(LD_DIR, "Launching %s standard networkstatus consensus "
@@ -948,17 +994,9 @@ update_consensus_networkstatus_downloads(time_t now)
 static void
 update_consensus_bootstrap_attempt_downloads(
                                       time_t now,
-                                      const or_options_t *options,
                                       download_status_t *dls,
                                       download_want_authority_t want_authority)
 {
-  int use_fallbacks = networkstatus_consensus_can_use_extra_fallbacks(options);
-  int max_dl_tries = options->ClientBootstrapConsensusMaxDownloadTries;
-  if (!use_fallbacks) {
-    max_dl_tries =
-              options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries;
-  }
-
   const char *resource = networkstatus_get_flavor_name(
                                                   usable_consensus_flavor());
 
@@ -967,7 +1005,7 @@ update_consensus_bootstrap_attempt_downloads(
 
   /* Allow for multiple connections in the same second, if the schedule value
    * is 0. */
-  while (download_status_is_ready(dls, now, max_dl_tries)) {
+  while (download_status_is_ready(dls, now)) {
     log_info(LD_DIR, "Launching %s bootstrap %s networkstatus consensus "
              "download.", resource, (want_authority == DL_WANT_AUTHORITY
                                      ? "authority"
@@ -1018,7 +1056,7 @@ update_consensus_bootstrap_multiple_downloads(time_t now,
 
     if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_f)) {
       /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */
-      update_consensus_bootstrap_attempt_downloads(now, options, dls_f,
+      update_consensus_bootstrap_attempt_downloads(now, dls_f,
                                                    DL_WANT_ANY_DIRSERVER);
     }
   }
@@ -1028,7 +1066,7 @@ update_consensus_bootstrap_multiple_downloads(time_t now,
     &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY];
 
   if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_a)) {
-    update_consensus_bootstrap_attempt_downloads(now, options, dls_a,
+    update_consensus_bootstrap_attempt_downloads(now, dls_a,
                                                  DL_WANT_AUTHORITY);
   }
 }
@@ -1181,8 +1219,18 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
     return 1;
   }
 
+  if (we_are_hibernating()) {
+    if (msg_out) {
+      *msg_out = "We are hibernating or shutting down.";
+    }
+    log_info(LD_DIR, "Delaying dir fetches (Hibernating or shutting down)");
+    return 1;
+  }
+
   if (options->UseBridges) {
-    if (!any_bridge_descriptors_known()) {
+    /* If we know that none of our bridges can possibly work, avoid fetching
+     * directory documents. But if some of them might work, try again. */
+    if (num_bridges_usable(1) == 0) {
       if (msg_out) {
         *msg_out = "No running bridges";
       }
@@ -1202,16 +1250,20 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
   return 0;
 }
 
-/** Launch requests for networkstatus documents and authority certificates as
- * appropriate. */
+/** Launch requests for networkstatus documents as appropriate. This is called
+ * when we retry all the connections on a SIGHUP and periodically by a Periodic
+ * event which checks whether we want to download any networkstatus documents.
+ */
 void
 update_networkstatus_downloads(time_t now)
 {
   const or_options_t *options = get_options();
   if (should_delay_dir_fetches(options, NULL))
     return;
+  /** Launch a consensus download request, we will wait for the consensus to
+   * download and when it completes we will launch a certificate download
+   * request. */
   update_consensus_networkstatus_downloads(now);
-  update_certificate_downloads(now);
 }
 
 /** Launch requests as appropriate for missing directory authority
@@ -1318,14 +1370,47 @@ networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_live_consensus,(time_t now))
 {
-  if (networkstatus_get_latest_consensus() &&
-      networkstatus_get_latest_consensus()->valid_after <= now &&
-      now <= networkstatus_get_latest_consensus()->valid_until)
-    return networkstatus_get_latest_consensus();
+  networkstatus_t *ns = networkstatus_get_latest_consensus();
+  if (ns && networkstatus_is_live(ns, now))
+    return ns;
   else
     return NULL;
 }
 
+/** Given a consensus in <b>ns</b>, return true iff currently live and
+ *  unexpired. */
+int
+networkstatus_is_live(const networkstatus_t *ns, time_t now)
+{
+  return (ns->valid_after <= now && now <= ns->valid_until);
+}
+
+/** Determine if <b>consensus</b> is valid or expired recently enough that
+ * we can still use it.
+ *
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ */
+int
+networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                        time_t now)
+{
+  if (BUG(!consensus))
+    return 0;
+
+  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+                                                      now);
+}
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_until
+ * time rather than an entire consensus. */
+int
+networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                             time_t now)
+{
+#define REASONABLY_LIVE_TIME (24*60*60)
+  return (now <= valid_until + REASONABLY_LIVE_TIME);
+}
+
 /* XXXX remove this in favor of get_live_consensus. But actually,
  * leave something like it for bridge users, who need to not totally
  * lose if they spend a while fetching a new consensus. */
@@ -1334,12 +1419,11 @@ networkstatus_get_live_consensus,(time_t now))
 networkstatus_t *
 networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
       consensus->valid_after <= now &&
-      now <= consensus->valid_until+REASONABLY_LIVE_TIME)
+      networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
     return NULL;
@@ -1439,6 +1523,32 @@ networkstatus_consensus_is_already_downloading(const char *resource)
   return answer;
 }
 
+/* Does the current, reasonably live consensus have IPv6 addresses?
+ * Returns 1 if there is a reasonably live consensus and its consensus method
+ * includes IPv6 addresses in the consensus.
+ * Otherwise, if there is no consensus, or the method does not include IPv6
+ * addresses, returns 0. */
+int
+networkstatus_consensus_has_ipv6(const or_options_t* options)
+{
+  const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus(
+                                                    approx_time(),
+                                                    usable_consensus_flavor());
+
+  /* If we have no consensus, we have no IPv6 in it */
+  if (!cons) {
+    return 0;
+  }
+
+  /* Different flavours of consensus gained IPv6 at different times */
+  if (we_use_microdescriptors_for_circuits(options)) {
+    return
+       cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+  } else {
+    return 1;
+  }
+}
+
 /** Given two router status entries for the same router identity, return 1 if
  * if the contents have changed between them. Otherwise, return 0. */
 static int
@@ -1503,13 +1613,22 @@ notify_control_networkstatus_changed(const networkstatus_t *old_c,
   smartlist_free(changed);
 }
 
-/* Called when the consensus has changed from old_c to new_c. */
+/* Called before the consensus changes from old_c to new_c. */
 static void
-notify_networkstatus_changed(const networkstatus_t *old_c,
-                             const networkstatus_t *new_c)
+notify_before_networkstatus_changes(const networkstatus_t *old_c,
+                                    const networkstatus_t *new_c)
 {
   notify_control_networkstatus_changed(old_c, new_c);
   dos_consensus_has_changed(new_c);
+  relay_consensus_has_changed(new_c);
+}
+
+/* Called after a new consensus has been put in the global state. It is safe
+ * to use the consensus getters in this function. */
+static void
+notify_after_networkstatus_changes(void)
+{
+  scheduler_notify_networkstatus_changed();
 }
 
 /** Copy all the ancillary information (like router download status and so on)
@@ -1569,23 +1688,7 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
   }
   return current_md_consensus ? 0 : -1;
 }
-#endif //TOR_UNIT_TESTS
-
-/**
- * Return true if any option is set in <b>options</b> to make us behave
- * as a client.
- *
- * XXXX If we need this elsewhere at any point, we should make it nonstatic
- * XXXX and move it into another file.
- */
-static int
-any_client_port_set(const or_options_t *options)
-{
-  return (options->SocksPort_set ||
-          options->TransPort_set ||
-          options->NATDPort_set ||
-          options->DNSPort_set);
-}
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /**
  * Helper for handle_missing_protocol_warning: handles either the
@@ -1610,7 +1713,7 @@ handle_missing_protocol_warning_impl(const networkstatus_t *c,
   }
   tor_free(protocol_warning);
   if (should_exit)
-    exit(1);
+    exit(1); // XXXX bad exit: should return from main.
 }
 
 /** Called when we have received a networkstatus <b>c</b>. If there are
@@ -1622,7 +1725,7 @@ handle_missing_protocol_warning(const networkstatus_t *c,
                                 const or_options_t *options)
 {
   const int is_server = server_mode(options);
-  const int is_client = any_client_port_set(options) || !is_server;
+  const int is_client = options_any_client_port_set(options) || !is_server;
 
   if (is_server)
     handle_missing_protocol_warning_impl(c, 0);
@@ -1630,6 +1733,57 @@ handle_missing_protocol_warning(const networkstatus_t *c,
     handle_missing_protocol_warning_impl(c, 1);
 }
 
+/**
+ * Check whether we received a consensus that appears to be coming
+ * from the future.  Because we implicitly trust the directory
+ * authorities' idea of the current time, we produce a warning if we
+ * get an early consensus.
+ *
+ * If we got a consensus that is time stamped far in the past, that
+ * could simply have come from a stale cache.  Possible ways to get a
+ * consensus from the future can include:
+ *
+ * - enough directory authorities have wrong clocks
+ * - directory authorities collude to produce misleading time stamps
+ * - our own clock is wrong (this is by far the most likely)
+ *
+ * We neglect highly improbable scenarios that involve actual time
+ * travel.
+ */
+STATIC void
+warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                     time_t now)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  char dbuf[64];
+  long delta = now - c->valid_after;
+  char *flavormsg = NULL;
+
+/** If a consensus appears more than this many seconds before it could
+ * possibly be a sufficiently-signed consensus, declare that our clock
+ * is skewed. */
+#define EARLY_CONSENSUS_NOTICE_SKEW 60
+
+  /* We assume that if a majority of dirauths have accurate clocks,
+   * the earliest that a dirauth with a skewed clock could possibly
+   * publish a sufficiently-signed consensus is (valid_after -
+   * dist_seconds).  Before that time, the skewed dirauth would be
+   * unable to obtain enough authority signatures for the consensus to
+   * be valid. */
+  if (now >= c->valid_after - c->dist_seconds - EARLY_CONSENSUS_NOTICE_SKEW)
+    return;
+
+  format_iso_time(tbuf, c->valid_after);
+  format_time_interval(dbuf, sizeof(dbuf), delta);
+  log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
+           "consensus network status document (%s UTC).  Tor needs an "
+           "accurate clock to work correctly. Please check your time and "
+           "date settings!", dbuf, tbuf);
+  tor_asprintf(&flavormsg, "%s flavor consensus", flavor);
+  clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS");
+  tor_free(flavormsg);
+}
+
 /** Try to replace the current cached v3 networkstatus with the one in
  * <b>consensus</b>.  If we don't have enough certificates to validate it,
  * store it in consensus_waiting_for_certs and launch a certificate fetch.
@@ -1659,7 +1813,7 @@ networkstatus_set_current_consensus(const char *consensus,
 {
   networkstatus_t *c=NULL;
   int r, result = -1;
-  time_t now = time(NULL);
+  time_t now = approx_time();
   const or_options_t *options = get_options();
   char *unverified_fname = NULL, *consensus_fname = NULL;
   int flav = networkstatus_parse_flavor_name(flavor);
@@ -1672,7 +1826,6 @@ networkstatus_set_current_consensus(const char *consensus,
   consensus_waiting_for_certs_t *waiting = NULL;
   time_t current_valid_after = 0;
   int free_consensus = 1; /* Free 'c' at the end of the function */
-  int old_ewma_enabled;
   int checked_protocols_already = 0;
 
   if (flav < 0) {
@@ -1691,7 +1844,7 @@ networkstatus_set_current_consensus(const char *consensus,
 
   if (from_cache && !was_waiting_for_certs) {
     /* We previously stored this; check _now_ to make sure that version-kills
-     * really work.  This happens even before we check signatures: we did so
+     * really work. This happens even before we check signatures: we did so
      * before when we stored this to disk. This does mean an attacker who can
      * write to the datadir can make us not start: such an attacker could
      * already harm us by replacing our guards, which would be worse. */
@@ -1711,9 +1864,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (flav != usable_consensus_flavor() &&
-      !directory_caches_dir_info(options)) {
-    /* This consensus is totally boring to us: we won't use it, and we won't
-     * serve it.  Drop it. */
+      !we_want_to_fetch_flavor(options, flav)) {
+    /* This consensus is totally boring to us: we won't use it, we didn't want
+     * it, and we won't serve it.  Drop it. */
     goto done;
   }
 
@@ -1724,31 +1877,23 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (!strcmp(flavor, "ns")) {
-    consensus_fname = get_datadir_fname("cached-consensus");
-    unverified_fname = get_datadir_fname("unverified-consensus");
+    consensus_fname = get_cachedir_fname("cached-consensus");
+    unverified_fname = get_cachedir_fname("unverified-consensus");
     if (current_ns_consensus) {
       current_digests = &current_ns_consensus->digests;
       current_valid_after = current_ns_consensus->valid_after;
     }
   } else if (!strcmp(flavor, "microdesc")) {
-    consensus_fname = get_datadir_fname("cached-microdesc-consensus");
-    unverified_fname = get_datadir_fname("unverified-microdesc-consensus");
+    consensus_fname = get_cachedir_fname("cached-microdesc-consensus");
+    unverified_fname = get_cachedir_fname("unverified-microdesc-consensus");
     if (current_md_consensus) {
       current_digests = &current_md_consensus->digests;
       current_valid_after = current_md_consensus->valid_after;
     }
   } else {
-    cached_dir_t *cur;
-    char buf[128];
-    tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
-    consensus_fname = get_datadir_fname(buf);
-    tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
-    unverified_fname = get_datadir_fname(buf);
-    cur = dirserv_get_consensus(flavor);
-    if (cur) {
-      current_digests = &cur->digests;
-      current_valid_after = cur->published;
-    }
+    tor_assert_nonfatal_unreached();
+    result = -2;
+    goto done;
   }
 
   if (current_digests &&
@@ -1821,6 +1966,15 @@ networkstatus_set_current_consensus(const char *consensus,
     }
   }
 
+  /* Signatures from the consensus are verified */
+  if (from_cache && was_waiting_for_certs) {
+    /* We check if the consensus is loaded from disk cache and that it
+     * it is an unverified consensus. If it is unverified, rename it to
+     * cached-*-consensus since it has been verified. */
+    log_info(LD_DIR, "Unverified consensus signatures verified.");
+    tor_rename(unverified_fname, consensus_fname);
+  }
+
   if (!from_cache && flav == usable_consensus_flavor())
     control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED");
 
@@ -1834,8 +1988,11 @@ networkstatus_set_current_consensus(const char *consensus,
 
   const int is_usable_flavor = flav == usable_consensus_flavor();
 
+  /* Before we switch to the new consensus, notify that we are about to change
+   * it using the old consensus and the new one. */
   if (is_usable_flavor) {
-    notify_networkstatus_changed(networkstatus_get_latest_consensus(), c);
+    notify_before_networkstatus_changes(networkstatus_get_latest_consensus(),
+                                        c);
   }
   if (flav == FLAV_NS) {
     if (current_ns_consensus) {
@@ -1878,25 +2035,24 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (is_usable_flavor) {
+    /* Notify that we just changed the consensus so the current global value
+     * can be looked at. */
+    notify_after_networkstatus_changes();
+
+    /* The "current" consensus has just been set and it is a usable flavor so
+     * the first thing we need to do is recalculate the voting schedule static
+     * object so we can use the timings in there needed by some subsystems
+     * such as hidden service and shared random. */
+    voting_schedule_recalculate_timing(options, now);
+    reschedule_dirvote(options);
+
     nodelist_set_consensus(c);
 
     /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
-    dirvote_recalculate_timing(options, now);
-    routerstatus_list_update_named_server_map();
-
-    /* Update ewma and adjust policy if needed; first cache the old value */
-    old_ewma_enabled = cell_ewma_enabled();
     /* Change the cell EWMA settings */
-    cell_ewma_set_scale_factor(options, c);
-    /* If we just enabled ewma, set the cmux policy on all active channels */
-    if (cell_ewma_enabled() && !old_ewma_enabled) {
-      channel_set_cmux_policy_everywhere(&ewma_policy);
-    } else if (!cell_ewma_enabled() && old_ewma_enabled) {
-      /* Turn it off everywhere */
-      channel_set_cmux_policy_everywhere(NULL);
-    }
+    cmux_ewma_set_options(options, c);
 
     /* XXXX this call might be unnecessary here: can changing the
      * current consensus really alter our view of any OR's rate limits? */
@@ -1904,6 +2060,7 @@ networkstatus_set_current_consensus(const char *consensus,
 
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
+    channelpadding_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -1914,34 +2071,25 @@ networkstatus_set_current_consensus(const char *consensus,
       download_status_failed(&consensus_dl_status[flav], 0);
   }
 
-  if (directory_caches_dir_info(options)) {
+  if (we_want_to_fetch_flavor(options, flav)) {
     dirserv_set_cached_consensus_networkstatus(consensus,
                                                flavor,
                                                &c->digests,
+                                               c->digest_sha3_as_signed,
                                                c->valid_after);
+    if (dir_server_mode(get_options())) {
+      consdiffmgr_add_consensus(consensus, c);
+    }
   }
 
   if (!from_cache) {
     write_str_to_file(consensus_fname, consensus, 0);
   }
 
-/** If a consensus appears more than this many seconds before its declared
- * valid-after time, declare that our clock is skewed. */
-#define EARLY_CONSENSUS_NOTICE_SKEW 60
+  warn_early_consensus(c, flavor, now);
 
-  if (now < c->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) {
-    char tbuf[ISO_TIME_LEN+1];
-    char dbuf[64];
-    long delta = now - c->valid_after;
-    format_iso_time(tbuf, c->valid_after);
-    format_time_interval(dbuf, sizeof(dbuf), delta);
-    log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
-             "consensus network status document (%s UTC).  Tor needs an "
-             "accurate clock to work correctly. Please check your time and "
-             "date settings!", dbuf, tbuf);
-    control_event_general_status(LOG_WARN,
-                    "CLOCK_SKEW MIN_SKEW=%ld SOURCE=CONSENSUS", delta);
-  }
+  /* We got a new consesus. Reset our md fetch fail cache */
+  microdesc_reset_outdated_dirservers_list();
 
   router_dir_info_changed();
 
@@ -1966,6 +2114,7 @@ networkstatus_note_certs_arrived(const char *source_dir)
 {
   int i;
   for (i=0; i<N_CONSENSUS_FLAVORS; ++i) {
+    const char *flavor_name = networkstatus_get_flavor_name(i);
     consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
     if (!waiting->consensus)
       continue;
@@ -1973,7 +2122,7 @@ networkstatus_note_certs_arrived(const char *source_dir)
       char *waiting_body = waiting->body;
       if (!networkstatus_set_current_consensus(
                                  waiting_body,
-                                 networkstatus_get_flavor_name(i),
+                                 flavor_name,
                                  NSSET_WAS_WAITING_FOR_CERTS,
                                  source_dir)) {
         tor_free(waiting_body);
@@ -2048,31 +2197,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version)
   }
 }
 
-/** Update our view of the list of named servers from the most recently
- * retrieved networkstatus consensus. */
-static void
-routerstatus_list_update_named_server_map(void)
-{
-  networkstatus_t *ns = networkstatus_get_latest_consensus();
-  if (!ns)
-    return;
-
-  strmap_free(named_server_map, tor_free_);
-  named_server_map = strmap_new();
-  strmap_free(unnamed_server_map, NULL);
-  unnamed_server_map = strmap_new();
-  smartlist_t *rslist = ns->routerstatus_list;
-  SMARTLIST_FOREACH_BEGIN(rslist, const routerstatus_t *, rs) {
-      if (rs->is_named) {
-        strmap_set_lc(named_server_map, rs->nickname,
-                      tor_memdup(rs->identity_digest, DIGEST_LEN));
-      }
-      if (rs->is_unnamed) {
-        strmap_set_lc(unnamed_server_map, rs->nickname, (void*)1);
-      }
-  } SMARTLIST_FOREACH_END(rs);
-}
-
 /** Given a list <b>routers</b> of routerinfo_t *, update each status field
  * according to our current consensus networkstatus.  May re-order
  * <b>routers</b>. */
@@ -2151,7 +2275,9 @@ signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
 char *
 networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 {
-  return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT, NULL);
+  return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT,
+                                   ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
+                                   NULL);
 }
 
 /** Alloc and return a string describing routerstatuses for the most
@@ -2164,13 +2290,13 @@ networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 char *
 networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
 {
-  time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
+  const time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
   char *answer;
   routerlist_t *rl = router_get_routerlist();
   smartlist_t *statuses;
-  uint8_t purpose = router_purpose_from_string(purpose_string);
+  const uint8_t purpose = router_purpose_from_string(purpose_string);
   routerstatus_t rs;
-  int bridge_auth = authdir_mode_bridge(get_options());
+  const int bridge_auth = authdir_mode_bridge(get_options());
 
   if (purpose == ROUTER_PURPOSE_UNKNOWN) {
     log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
@@ -2187,6 +2313,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
       continue;
     if (ri->purpose != purpose)
       continue;
+    /* TODO: modifying the running flag in a getinfo is a bad idea */
     if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
       dirserv_set_router_is_running(ri, now);
     /* then generate and write out status lines for each of them */
@@ -2205,25 +2332,34 @@ void
 networkstatus_dump_bridge_status_to_file(time_t now)
 {
   char *status = networkstatus_getinfo_by_purpose("bridge", now);
-  const or_options_t *options = get_options();
   char *fname = NULL;
   char *thresholds = NULL;
   char *published_thresholds_and_status = NULL;
   char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
 
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
   format_iso_time(published, now);
   dirserv_compute_bridge_flag_thresholds();
   thresholds = dirserv_get_flag_thresholds_line();
   tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s",
-               published, thresholds, status);
-  tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges",
-               options->DataDirectory);
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
   write_str_to_file(fname,published_thresholds_and_status,0);
   tor_free(thresholds);
   tor_free(published_thresholds_and_status);
   tor_free(fname);
   tor_free(status);
+  tor_free(fingerprint_line);
 }
 
 /* DOCDOC get_net_param_from_list */
@@ -2270,9 +2406,9 @@ get_net_param_from_list(smartlist_t *net_params, const char *param_name,
  * Make sure the value parsed from the consensus is at least
  * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value
  * if necessary. */
-int32_t
-networkstatus_get_param(const networkstatus_t *ns, const char *param_name,
-                        int32_t default_val, int32_t min_val, int32_t max_val)
+MOCK_IMPL(int32_t,
+networkstatus_get_param, (const networkstatus_t *ns, const char *param_name,
+                        int32_t default_val, int32_t min_val, int32_t max_val))
 {
   if (!ns) /* if they pass in null, go find it ourselves */
     ns = networkstatus_get_latest_consensus();
@@ -2285,6 +2421,25 @@ networkstatus_get_param(const networkstatus_t *ns, const char *param_name,
 }
 
 /**
+ * As networkstatus_get_param(), but check torrc_value before checking the
+ * consensus. If torrc_value is in-range, then return it instead of the
+ * value from the consensus.
+ */
+int32_t
+networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                    int32_t torrc_value,
+                                    const char *param_name,
+                                    int32_t default_val,
+                                    int32_t min_val, int32_t max_val)
+{
+  if (torrc_value >= min_val && torrc_value <= max_val)
+    return torrc_value;
+  else
+    return networkstatus_get_param(
+                         ns, param_name, default_val, min_val, max_val);
+}
+
+/**
  * Retrieve the consensus parameter that governs the
  * fixed-point precision of our network balancing 'bandwidth-weights'
  * (which are themselves integer consensus values). We divide them
@@ -2360,12 +2515,11 @@ networkstatus_parse_flavor_name(const char *flavname)
  * running, or otherwise not a descriptor that we would make any
  * use of even if we had it. Else return 1. */
 int
-client_would_use_router(const routerstatus_t *rs, time_t now,
-                        const or_options_t *options)
+client_would_use_router(const routerstatus_t *rs, time_t now)
 {
-  if (!rs->is_flagged_running && !options->FetchUselessDescriptors) {
+  if (!rs->is_flagged_running) {
     /* If we had this router descriptor, we wouldn't even bother using it.
-     * But, if we want to have a complete list, fetch it anyway. */
+     * (Fetching and storing depends on by we_want_to_fetch_flavor().) */
     return 0;
   }
   if (rs->published_on + OLD_ROUTER_DESC_MAX_AGE < now) {
@@ -2421,7 +2575,8 @@ getinfo_helper_networkstatus(control_connection_t *conn,
     }
     status = router_get_consensus_status_by_id(d);
   } else if (!strcmpstart(question, "ns/name/")) {
-    status = router_get_consensus_status_by_nickname(question+8, 0);
+    const node_t *n = node_get_by_nickname(question+8, 0);
+    status = n ? n->rs : NULL;
   } else if (!strcmpstart(question, "ns/purpose/")) {
     *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL));
     return *answer ? 0 : -1;
@@ -2511,6 +2666,25 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
   return 0;
 }
 
+/** Release all storage held in <b>s</b>. */
+void
+ns_detached_signatures_free_(ns_detached_signatures_t *s)
+{
+  if (!s)
+    return;
+  if (s->signatures) {
+    STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) {
+      SMARTLIST_FOREACH(sigs, document_signature_t *, sig,
+                        document_signature_free(sig));
+      smartlist_free(sigs);
+    } STRMAP_FOREACH_END;
+    strmap_free(s->signatures, NULL);
+    strmap_free(s->digests, tor_free_);
+  }
+
+  tor_free(s);
+}
+
 /** Free all storage held locally in this module. */
 void
 networkstatus_free_all(void)
@@ -2528,8 +2702,5 @@ networkstatus_free_all(void)
     }
     tor_free(waiting->body);
   }
-
-  strmap_free(named_server_map, tor_free_);
-  strmap_free(unnamed_server_map, NULL);
 }
 
diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h
index 71f36b69ed..94f85c3c29 100644
--- a/src/or/networkstatus.h
+++ b/src/or/networkstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,12 +16,24 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
+char *networkstatus_read_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
-void routerstatus_free(routerstatus_t *rs);
-void networkstatus_vote_free(networkstatus_t *ns);
+void routerstatus_free_(routerstatus_t *rs);
+#define routerstatus_free(rs) \
+  FREE_AND_NULL(routerstatus_t, routerstatus_free_, (rs))
+void networkstatus_vote_free_(networkstatus_t *ns);
+#define networkstatus_vote_free(ns) \
+  FREE_AND_NULL(networkstatus_t, networkstatus_vote_free_, (ns))
+void ns_detached_signatures_free_(ns_detached_signatures_t *s);
+#define ns_detached_signatures_free(s) \
+  FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
 networkstatus_voter_info_t *networkstatus_get_voter_by_id(
                                        networkstatus_t *vote,
                                        const char *identity);
+document_signature_t *networkstatus_get_voter_sig_by_alg(
+                                    const networkstatus_voter_info_t *voter,
+                                    digest_algorithm_t alg);
+
 int networkstatus_check_consensus_signature(networkstatus_t *consensus,
                                             int warn);
 int networkstatus_check_document_signature(const networkstatus_t *consensus,
@@ -61,11 +73,8 @@ const routerstatus_t *router_get_consensus_status_by_descriptor_digest(
 MOCK_DECL(routerstatus_t *,
           router_get_mutable_consensus_status_by_descriptor_digest,
           (networkstatus_t *consensus, const char *digest));
-const routerstatus_t *router_get_consensus_status_by_nickname(
-                                   const char *nickname,
-                                   int warn_if_unnamed);
-const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
-int networkstatus_nickname_is_unnamed(const char *nickname);
+int we_want_to_fetch_flavor(const or_options_t *options, int flavor);
+int we_want_to_fetch_unknown_auth_certs(const or_options_t *options);
 void networkstatus_consensus_download_failed(int status_code,
                                              const char *flavname);
 void update_consensus_networkstatus_fetch_time(time_t now);
@@ -73,12 +82,16 @@ int should_delay_dir_fetches(const or_options_t *options,const char **msg_out);
 void update_networkstatus_downloads(time_t now);
 void update_certificate_downloads(time_t now);
 int consensus_is_waiting_for_certs(void);
-int client_would_use_router(const routerstatus_t *rs, time_t now,
-                            const or_options_t *options);
+int client_would_use_router(const routerstatus_t *rs, time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void));
 MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor,
           (consensus_flavor_t f));
 MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
+int networkstatus_is_live(const networkstatus_t *ns, time_t now);
+int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                            time_t now);
+int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                                 time_t now);
 networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now,
                                                              int flavor);
 MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now));
@@ -87,6 +100,7 @@ int networkstatus_consensus_can_use_multiple_directories(
 MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,(
                                                 const or_options_t *options));
 int networkstatus_consensus_is_already_downloading(const char *resource);
+int networkstatus_consensus_has_ipv6(const or_options_t* options);
 
 #define NSSET_FROM_CACHE 1
 #define NSSET_WAS_WAITING_FOR_CERTS 2
@@ -107,10 +121,14 @@ void signed_descs_update_status_from_consensus_networkstatus(
 char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
 char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
 void networkstatus_dump_bridge_status_to_file(time_t now);
-int32_t networkstatus_get_param(const networkstatus_t *ns,
-                                const char *param_name,
-                                int32_t default_val, int32_t min_val,
-                                int32_t max_val);
+MOCK_DECL(int32_t, networkstatus_get_param,
+          (const networkstatus_t *ns, const char *param_name,
+           int32_t default_val, int32_t min_val, int32_t max_val));
+int32_t networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                            int32_t torrc_value,
+                                            const char *param_name,
+                                            int32_t default_val,
+                                            int32_t min_val, int32_t max_val);
 int getinfo_helper_networkstatus(control_connection_t *conn,
                                  const char *question, char **answer,
                                  const char **errmsg);
@@ -118,18 +136,27 @@ int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight,
                                     int32_t default_val);
 const char *networkstatus_get_flavor_name(consensus_flavor_t flav);
 int networkstatus_parse_flavor_name(const char *flavname);
-void document_signature_free(document_signature_t *sig);
+void document_signature_free_(document_signature_t *sig);
+#define document_signature_free(sig) \
+  FREE_AND_NULL(document_signature_t, document_signature_free_, (sig))
 document_signature_t *document_signature_dup(const document_signature_t *sig);
 void networkstatus_free_all(void);
 int networkstatus_get_weight_scale_param(networkstatus_t *ns);
 
+void vote_routerstatus_free_(vote_routerstatus_t *rs);
+#define vote_routerstatus_free(rs) \
+  FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
+
 #ifdef NETWORKSTATUS_PRIVATE
-STATIC void vote_routerstatus_free(vote_routerstatus_t *rs);
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
                                                 const char *flavor);
-#endif // TOR_UNIT_TESTS
-#endif
+STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor,
+                                 time_t now);
+extern networkstatus_t *current_ns_consensus;
+extern networkstatus_t *current_md_consensus;
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(NETWORKSTATUS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NETWORKSTATUS_H) */
 
diff --git a/src/or/nodelist.c b/src/or/nodelist.c
index 26f990b08c..032e8d669f 100644
--- a/src/or/nodelist.c
+++ b/src/or/nodelist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,43 +10,90 @@
  * \brief Structures and functions for tracking what we know about the routers
  *   on the Tor network, and correlating information from networkstatus,
  *   routerinfo, and microdescs.
+ *
+ * The key structure here is node_t: that's the canonical way to refer
+ * to a Tor relay that we might want to build a circuit through.  Every
+ * node_t has either a routerinfo_t, or a routerstatus_t from the current
+ * networkstatus consensus.  If it has a routerstatus_t, it will also
+ * need to have a microdesc_t before you can use it for circuits.
+ *
+ * The nodelist_t is a global singleton that maps identities to node_t
+ * objects.  Access them with the node_get_*() functions.  The nodelist_t
+ * is maintained by calls throughout the codebase
+ *
+ * Generally, other code should not have to reach inside a node_t to
+ * see what information it has.  Instead, you should call one of the
+ * many accessor functions that works on a generic node_t.  If there
+ * isn't one that does what you need, it's better to make such a function,
+ * and then use it.
+ *
+ * For historical reasons, some of the functions that select a node_t
+ * from the list of all usable node_t objects are in the routerlist.c
+ * module, since they originally selected a routerinfo_t. (TODO: They
+ * should move!)
+ *
+ * (TODO: Perhaps someday we should abstract the remaining ways of
+ * talking about a relay to also be node_t instances. Those would be
+ * routerstatus_t as used for directory requests, and dir_server_t as
+ * used for authorities and fallback directories.)
  */
 
+#define NODELIST_PRIVATE
+
 #include "or.h"
 #include "address.h"
 #include "address_set.h"
+#include "backtrace.h"
+#include "bridges.h"
 #include "config.h"
 #include "control.h"
 #include "dirserv.h"
+#include "entrynodes.h"
 #include "geoip.h"
+#include "hs_common.h"
+#include "hs_client.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "protover.h"
 #include "rendservice.h"
 #include "router.h"
 #include "routerlist.h"
+#include "routerparse.h"
 #include "routerset.h"
+#include "torcert.h"
+#include "util_format.h"
 
 #include <string.h>
 
+#include "dirauth/mode.h"
+
 static void nodelist_drop_node(node_t *node, int remove_from_ht);
-static void node_free(node_t *node);
+#define node_free(val) \
+  FREE_AND_NULL(node_t, node_free_, (val))
+static void node_free_(node_t *node);
 
 /** count_usable_descriptors counts descriptors with these flag(s)
  */
 typedef enum {
-  /* All descriptors regardless of flags */
-  USABLE_DESCRIPTOR_ALL = 0,
-  /* Only descriptors with the Exit flag */
-  USABLE_DESCRIPTOR_EXIT_ONLY = 1
+  /* All descriptors regardless of flags or exit policies */
+  USABLE_DESCRIPTOR_ALL         = 0U,
+  /* Only count descriptors with an exit policy that allows at least one port
+   */
+  USABLE_DESCRIPTOR_EXIT_POLICY = 1U << 0,
+  /* Only count descriptors for relays that have the exit flag in the
+   * consensus */
+  USABLE_DESCRIPTOR_EXIT_FLAG   = 1U << 1,
+  /* Only count descriptors for relays that have the policy and the flag */
+  USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG = (USABLE_DESCRIPTOR_EXIT_POLICY |
+                                            USABLE_DESCRIPTOR_EXIT_FLAG)
 } usable_descriptor_t;
 static void count_usable_descriptors(int *num_present,
                                      int *num_usable,
                                      smartlist_t *descs_out,
                                      const networkstatus_t *consensus,
-                                     const or_options_t *options,
                                      time_t now,
                                      routerset_t *in_set,
                                      usable_descriptor_t exit_only);
@@ -64,9 +111,22 @@ typedef struct nodelist_t {
   smartlist_t *nodes;
   /* Hash table to map from node ID digest to node. */
   HT_HEAD(nodelist_map, node_t) nodes_by_id;
+  /* Hash table to map from node Ed25519 ID to node.
+   *
+   * Whenever a node's routerinfo or microdescriptor is about to change,
+   * you should remove it from this map with node_remove_from_ed25519_map().
+   * Whenever a node's routerinfo or microdescriptor has just chaned,
+   * you should add it to this map with node_add_to_ed25519_map().
+   */
+  HT_HEAD(nodelist_ed_map, node_t) nodes_by_ed_id;
 
   /* Set of addresses that belong to nodes we believe in. */
   address_set_t *node_addrs;
+
+  /* The valid-after time of the last live consensus that initialized the
+   * nodelist.  We use this to detect outdated nodelists that need to be
+   * rebuilt using a newer consensus. */
+  time_t live_consensus_valid_after;
 } nodelist_t;
 
 static inline unsigned int
@@ -85,6 +145,23 @@ HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq)
 HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq,
              0.6, tor_reallocarray_, tor_free_)
 
+static inline unsigned int
+node_ed_id_hash(const node_t *node)
+{
+  return (unsigned) siphash24g(node->ed25519_id.pubkey, ED25519_PUBKEY_LEN);
+}
+
+static inline unsigned int
+node_ed_id_eq(const node_t *node1, const node_t *node2)
+{
+  return ed25519_pubkey_eq(&node1->ed25519_id, &node2->ed25519_id);
+}
+
+HT_PROTOTYPE(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq)
+HT_GENERATE2(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
+             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_)
+
 /** The global nodelist. */
 static nodelist_t *the_nodelist=NULL;
 
@@ -95,13 +172,14 @@ init_nodelist(void)
   if (PREDICT_UNLIKELY(the_nodelist == NULL)) {
     the_nodelist = tor_malloc_zero(sizeof(nodelist_t));
     HT_INIT(nodelist_map, &the_nodelist->nodes_by_id);
+    HT_INIT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
     the_nodelist->nodes = smartlist_new();
   }
 }
 
 /** As node_get_by_id, but returns a non-const pointer */
-node_t *
-node_get_mutable_by_id(const char *identity_digest)
+MOCK_IMPL(node_t *,
+node_get_mutable_by_id,(const char *identity_digest))
 {
   node_t search, *node;
   if (PREDICT_UNLIKELY(the_nodelist == NULL))
@@ -112,6 +190,21 @@ node_get_mutable_by_id(const char *identity_digest)
   return node;
 }
 
+/** As node_get_by_ed25519_id, but returns a non-const pointer */
+node_t *
+node_get_mutable_by_ed25519_id(const ed25519_public_key_t *ed_id)
+{
+  node_t search, *node;
+  if (PREDICT_UNLIKELY(the_nodelist == NULL))
+    return NULL;
+  if (BUG(ed_id == NULL) || BUG(ed25519_public_key_is_zero(ed_id)))
+    return NULL;
+
+  memcpy(&search.ed25519_id, ed_id, sizeof(search.ed25519_id));
+  node = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, &search);
+  return node;
+}
+
 /** Return the node_t whose identity is <b>identity_digest</b>, or NULL
  * if no such node exists. */
 MOCK_IMPL(const node_t *,
@@ -120,6 +213,14 @@ node_get_by_id,(const char *identity_digest))
   return node_get_mutable_by_id(identity_digest);
 }
 
+/** Return the node_t whose ed25519 identity is <b>ed_id</b>, or NULL
+ * if no such node exists. */
+MOCK_IMPL(const node_t *,
+node_get_by_ed25519_id,(const ed25519_public_key_t *ed_id))
+{
+  return node_get_mutable_by_ed25519_id(ed_id);
+}
+
 /** Internal: return the node_t whose identity_digest is
  * <b>identity_digest</b>.  If none exists, create a new one, add it to the
  * nodelist, and return it.
@@ -146,6 +247,180 @@ node_get_or_create(const char *identity_digest)
   return node;
 }
 
+/** Remove <b>node</b> from the ed25519 map (if it present), and
+ * set its ed25519_id field to zero. */
+static int
+node_remove_from_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  int rv = 0;
+  node_t *search =
+    HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (BUG(search != node)) {
+    goto clear_and_return;
+  }
+
+  search = HT_REMOVE(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  tor_assert(search == node);
+  rv = 1;
+
+ clear_and_return:
+  memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+  return rv;
+}
+
+/** Helper function to log details of duplicated ed2559_ids */
+static void
+node_log_dup_ed_id(const node_t *old, const node_t *node, const char *ed_id)
+{
+  char *s;
+  char *olddesc = tor_strdup(node_describe(old));
+
+  tor_asprintf(&s, "Reused ed25519_id %s: old %s new %s", ed_id,
+               olddesc, node_describe(node));
+  log_backtrace(LOG_NOTICE, LD_DIR, s);
+  tor_free(olddesc);
+  tor_free(s);
+}
+
+/** If <b>node</b> has an ed25519 id, and it is not already in the ed25519 id
+ * map, set its ed25519_id field, and add it to the ed25519 map.
+ */
+static int
+node_add_to_ed25519_map(node_t *node)
+{
+  tor_assert(the_nodelist);
+  tor_assert(node);
+
+  if (! ed25519_public_key_is_zero(&node->ed25519_id)) {
+    return 0;
+  }
+
+  const ed25519_public_key_t *key = node_get_ed25519_id(node);
+  if (!key) {
+    return 0;
+  }
+
+  node_t *old;
+  memcpy(&node->ed25519_id, key, sizeof(node->ed25519_id));
+  old = HT_FIND(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  if (old) {
+    char ed_id[BASE32_BUFSIZE(sizeof(key->pubkey))];
+
+    base32_encode(ed_id, sizeof(ed_id), (const char *)key->pubkey,
+                  sizeof(key->pubkey));
+    if (BUG(old == node)) {
+      /* Actual bug: all callers of this function call
+       * node_remove_from_ed25519_map first. */
+      log_err(LD_BUG,
+              "Unexpectedly found deleted node with ed25519_id %s", ed_id);
+    } else {
+      /* Distinct nodes sharing a ed25519 id, possibly due to relay
+       * misconfiguration.  The key pinning might not catch this,
+       * possibly due to downloading a missing descriptor during
+       * consensus voting. */
+      node_log_dup_ed_id(old, node, ed_id);
+      memset(&node->ed25519_id, 0, sizeof(node->ed25519_id));
+    }
+    return 0;
+  }
+
+  HT_INSERT(nodelist_ed_map, &the_nodelist->nodes_by_ed_id, node);
+  return 1;
+}
+
+/* For a given <b>node</b> for the consensus <b>ns</b>, set the hsdir index
+ * for the node, both current and next if possible. This can only fails if the
+ * node_t ed25519 identity key can't be found which would be a bug. */
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns)
+{
+  time_t now = approx_time();
+  const ed25519_public_key_t *node_identity_pk;
+  uint8_t *fetch_srv = NULL, *store_first_srv = NULL, *store_second_srv = NULL;
+  uint64_t next_time_period_num, current_time_period_num;
+  uint64_t fetch_tp, store_first_tp, store_second_tp;
+
+  tor_assert(node);
+  tor_assert(ns);
+
+  if (!networkstatus_is_live(ns, now)) {
+    static struct ratelim_t live_consensus_ratelim = RATELIM_INIT(30 * 60);
+    log_fn_ratelim(&live_consensus_ratelim, LOG_INFO, LD_GENERAL,
+                   "Not setting hsdir index with a non-live consensus.");
+    goto done;
+  }
+
+  node_identity_pk = node_get_ed25519_id(node);
+  if (node_identity_pk == NULL) {
+    log_debug(LD_GENERAL, "ed25519 identity public key not found when "
+                          "trying to build the hsdir indexes for node %s",
+              node_describe(node));
+    goto done;
+  }
+
+  /* Get the current and next time period number. */
+  current_time_period_num = hs_get_time_period_num(0);
+  next_time_period_num = hs_get_next_time_period_num(0);
+
+  /* We always use the current time period for fetching descs */
+  fetch_tp = current_time_period_num;
+
+  /* Now extract the needed SRVs and time periods for building hsdir indices */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    fetch_srv = hs_get_current_srv(fetch_tp, ns);
+
+    store_first_tp = hs_get_previous_time_period_num(0);
+    store_second_tp = current_time_period_num;
+  } else {
+    fetch_srv = hs_get_previous_srv(fetch_tp, ns);
+
+    store_first_tp = current_time_period_num;
+    store_second_tp = next_time_period_num;
+  }
+
+  /* We always use the old SRV for storing the first descriptor and the latest
+   * SRV for storing the second descriptor */
+  store_first_srv = hs_get_previous_srv(store_first_tp, ns);
+  store_second_srv = hs_get_current_srv(store_second_tp, ns);
+
+  /* Build the fetch index. */
+  hs_build_hsdir_index(node_identity_pk, fetch_srv, fetch_tp,
+                       node->hsdir_index.fetch);
+
+  /* If we are in the time segment between SRV#N and TP#N, the fetch index is
+     the same as the first store index */
+  if (!hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_first, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_first));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_first_srv, store_first_tp,
+                         node->hsdir_index.store_first);
+  }
+
+  /* If we are in the time segment between TP#N and SRV#N+1, the fetch index is
+     the same as the second store index */
+  if (hs_in_period_between_tp_and_srv(ns, now)) {
+    memcpy(node->hsdir_index.store_second, node->hsdir_index.fetch,
+           sizeof(node->hsdir_index.store_second));
+  } else {
+    hs_build_hsdir_index(node_identity_pk, store_second_srv, store_second_tp,
+                         node->hsdir_index.store_second);
+  }
+
+ done:
+  tor_free(fetch_srv);
+  tor_free(store_first_srv);
+  tor_free(store_second_srv);
+  return;
+}
+
 /** Called when a node's address changes. */
 static void
 node_addrs_changed(node_t *node)
@@ -214,6 +489,8 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
   id_digest = ri->cache_info.identity_digest;
   node = node_get_or_create(id_digest);
 
+  node_remove_from_ed25519_map(node);
+
   if (node->ri) {
     if (!routers_have_same_or_addrs(node->ri, ri)) {
       node_addrs_changed(node);
@@ -227,6 +504,8 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
   }
   node->ri = ri;
 
+  node_add_to_ed25519_map(node);
+
   if (node->country == -1)
     node_set_country(node);
 
@@ -236,6 +515,14 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out)
     dirserv_set_node_flags_from_authoritative_status(node, status);
   }
 
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (node->rs && node->rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node,
+                         networkstatus_get_latest_consensus());
+  }
+
   node_add_to_address_set(node);
 
   return node;
@@ -265,10 +552,20 @@ nodelist_add_microdesc(microdesc_t *md)
   node = node_get_mutable_by_id(rs->identity_digest);
   if (node == NULL)
     return NULL;
+
+  node_remove_from_ed25519_map(node);
   if (node->md)
     node->md->held_by_nodes--;
+
   node->md = md;
   md->held_by_nodes++;
+  /* Setting the HSDir index requires the ed25519 identity key which can
+   * only be found either in the ri or md. This is why this is called here.
+   * Only nodes supporting HSDir=2 protocol version needs this index. */
+  if (rs->pv.supports_v3_hsdir) {
+    node_set_hsdir_index(node, ns);
+  }
+  node_add_to_ed25519_map(node);
   node_add_to_address_set(node);
 
   return node;
@@ -315,15 +612,20 @@ nodelist_set_consensus(networkstatus_t *ns)
     if (ns->flavor == FLAV_MICRODESC) {
       if (node->md == NULL ||
           tor_memneq(node->md->digest,rs->descriptor_digest,DIGEST256_LEN)) {
+        node_remove_from_ed25519_map(node);
         if (node->md)
           node->md->held_by_nodes--;
         node->md = microdesc_cache_lookup_by_digest256(NULL,
                                                        rs->descriptor_digest);
         if (node->md)
           node->md->held_by_nodes++;
+        node_add_to_ed25519_map(node);
       }
     }
 
+    if (rs->pv.supports_v3_hsdir) {
+      node_set_hsdir_index(node, ns);
+    }
     node_set_country(node);
 
     /* If we're not an authdir, believe others. */
@@ -369,6 +671,12 @@ nodelist_set_consensus(networkstatus_t *ns)
       }
     } SMARTLIST_FOREACH_END(node);
   }
+
+  /* If the consensus is live, note down the consensus valid-after that formed
+   * the nodelist. */
+  if (networkstatus_is_live(ns, approx_time())) {
+    the_nodelist->live_consensus_valid_after = ns->valid_after;
+  }
 }
 
 /** Helper: return true iff a node has a usable amount of information*/
@@ -387,6 +695,9 @@ nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md)
   if (node && node->md == md) {
     node->md = NULL;
     md->held_by_nodes--;
+    if (! node_get_ed25519_id(node)) {
+      node_remove_from_ed25519_map(node);
+    }
   }
 }
 
@@ -415,6 +726,7 @@ nodelist_drop_node(node_t *node, int remove_from_ht)
     tmp = HT_REMOVE(nodelist_map, &the_nodelist->nodes_by_id, node);
     tor_assert(tmp == node);
   }
+  node_remove_from_ed25519_map(node);
 
   idx = node->nodelist_idx;
   tor_assert(idx >= 0);
@@ -449,7 +761,7 @@ nodelist_find_nodes_with_microdesc(const microdesc_t *md)
 
 /** Release storage held by <b>node</b>  */
 static void
-node_free(node_t *node)
+node_free_(node_t *node)
 {
   if (!node)
     return;
@@ -497,6 +809,7 @@ nodelist_free_all(void)
     return;
 
   HT_CLEAR(nodelist_map, &the_nodelist->nodes_by_id);
+  HT_CLEAR(nodelist_ed_map, &the_nodelist->nodes_by_ed_id);
   SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
     node->nodelist_idx = -1;
     node_free(node);
@@ -564,12 +877,49 @@ nodelist_assert_ok(void)
     tor_assert(node_sl_idx == node->nodelist_idx);
   } SMARTLIST_FOREACH_END(node);
 
+  /* Every node listed with an ed25519 identity should be listed by that
+   * identity.
+   */
+  SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
+    if (!ed25519_public_key_is_zero(&node->ed25519_id)) {
+      tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  node_t **idx;
+  HT_FOREACH(idx, nodelist_ed_map, &the_nodelist->nodes_by_ed_id) {
+    node_t *node = *idx;
+    tor_assert(node == node_get_by_ed25519_id(&node->ed25519_id));
+  }
+
   tor_assert((long)smartlist_len(the_nodelist->nodes) ==
              (long)HT_SIZE(&the_nodelist->nodes_by_id));
 
+  tor_assert((long)smartlist_len(the_nodelist->nodes) >=
+             (long)HT_SIZE(&the_nodelist->nodes_by_ed_id));
+
   digestmap_free(dm, NULL);
 }
 
+/** Ensure that the nodelist has been created with the most recent consensus.
+ *  If that's not the case, make it so.  */
+void
+nodelist_ensure_freshness(networkstatus_t *ns)
+{
+  tor_assert(ns);
+
+  /* We don't even have a nodelist: this is a NOP. */
+  if (!the_nodelist) {
+    return;
+  }
+
+  if (the_nodelist->live_consensus_valid_after != ns->valid_after) {
+    log_info(LD_GENERAL, "Nodelist was not fresh: rebuilding. (%d / %d)",
+             (int) the_nodelist->live_consensus_valid_after,
+             (int) ns->valid_after);
+    nodelist_set_consensus(ns);
+  }
+}
 /** Return a list of a node_t * for every node we know about.  The caller
  * MUST NOT modify the list. (You can set and clear flags in the nodes if
  * you must, but you must not add or remove nodes.) */
@@ -583,28 +933,23 @@ nodelist_get_list,(void))
 /** Given a hex-encoded nickname of the format DIGEST, $DIGEST, $DIGEST=name,
  * or $DIGEST~name, return the node with the matching identity digest and
  * nickname (if any).  Return NULL if no such node exists, or if <b>hex_id</b>
- * is not well-formed. */
+ * is not well-formed. DOCDOC flags */
 const node_t *
-node_get_by_hex_id(const char *hex_id)
+node_get_by_hex_id(const char *hex_id, unsigned flags)
 {
   char digest_buf[DIGEST_LEN];
   char nn_buf[MAX_NICKNAME_LEN+1];
   char nn_char='\0';
 
+  (void) flags; // XXXX
+
   if (hex_digest_nickname_decode(hex_id, digest_buf, &nn_char, nn_buf)==0) {
     const node_t *node = node_get_by_id(digest_buf);
     if (!node)
       return NULL;
-    if (nn_char) {
-      const char *real_name = node_get_nickname(node);
-      if (!real_name || strcasecmp(real_name, nn_buf))
-        return NULL;
-      if (nn_char == '=') {
-        const char *named_id =
-          networkstatus_get_router_digest_by_nickname(nn_buf);
-        if (!named_id || tor_memneq(named_id, digest_buf, DIGEST_LEN))
-          return NULL;
-      }
+    if (nn_char == '=') {
+      /* "=" indicates a Named relay, but there aren't any of those now. */
+      return NULL;
     }
     return node;
   }
@@ -613,42 +958,27 @@ node_get_by_hex_id(const char *hex_id)
 }
 
 /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return
- * the corresponding node_t, or NULL if none exists.  Warn the user if
- * <b>warn_if_unnamed</b> is set, and they have specified a router by
- * nickname, but the Named flag isn't set for that router. */
+ * the corresponding node_t, or NULL if none exists. Warn the user if they
+ * have specified a router by nickname, unless the NNF_NO_WARN_UNNAMED bit is
+ * set in <b>flags</b>. */
 MOCK_IMPL(const node_t *,
-node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
+node_get_by_nickname,(const char *nickname, unsigned flags))
 {
+  const int warn_if_unnamed = !(flags & NNF_NO_WARN_UNNAMED);
+
   if (!the_nodelist)
     return NULL;
 
   /* Handle these cases: DIGEST, $DIGEST, $DIGEST=name, $DIGEST~name. */
   {
     const node_t *node;
-    if ((node = node_get_by_hex_id(nickname)) != NULL)
+    if ((node = node_get_by_hex_id(nickname, flags)) != NULL)
       return node;
   }
 
   if (!strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME))
     return NULL;
 
-  /* Okay, so if we get here, the nickname is just a nickname.  Is there
-   * a binding for it in the consensus? */
-  {
-    const char *named_id =
-      networkstatus_get_router_digest_by_nickname(nickname);
-    if (named_id)
-      return node_get_by_id(named_id);
-  }
-
-  /* Is it marked as owned-by-someone-else? */
-  if (networkstatus_nickname_is_unnamed(nickname)) {
-    log_info(LD_GENERAL, "The name %s is listed as Unnamed: there is some "
-             "router that holds it, but not one listed in the current "
-             "consensus.", escaped(nickname));
-    return NULL;
-  }
-
   /* Okay, so the name is not canonical for anybody. */
   {
     smartlist_t *matches = smartlist_new();
@@ -679,11 +1009,9 @@ node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
       if (! node->name_lookup_warned) {
         base16_encode(fp, sizeof(fp), node->identity, DIGEST_LEN);
         log_warn(LD_CONFIG,
-                 "You specified a server \"%s\" by name, but the directory "
-                 "authorities do not have any key registered for this "
-                 "nickname -- so it could be used by any server, not just "
-                 "the one you meant. "
-                 "To make sure you get the same server in the future, refer "
+                 "You specified a relay \"%s\" by name, but nicknames can be "
+                 "used by any relay, not just the one you meant. "
+                 "To make sure you get the same relay in the future, refer "
                  "to it by key, as \"$%s\".", nickname, fp);
         node->name_lookup_warned = 1;
       }
@@ -697,6 +1025,150 @@ node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
   }
 }
 
+/** Return the Ed25519 identity key for the provided node, or NULL if it
+ * doesn't have one. */
+const ed25519_public_key_t *
+node_get_ed25519_id(const node_t *node)
+{
+  const ed25519_public_key_t *ri_pk = NULL;
+  const ed25519_public_key_t *md_pk = NULL;
+
+  if (node->ri) {
+    if (node->ri->cache_info.signing_key_cert) {
+      ri_pk = &node->ri->cache_info.signing_key_cert->signing_key;
+      /* Checking whether routerinfo ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(ri_pk)))
+        ri_pk = NULL;
+    }
+  }
+
+  if (node->md) {
+    if (node->md->ed25519_identity_pkey) {
+      md_pk = node->md->ed25519_identity_pkey;
+      /* Checking whether microdesc ed25519 is all zero.
+       * Our descriptor parser should make sure this never happens. */
+      if (BUG(ed25519_public_key_is_zero(md_pk)))
+        md_pk = NULL;
+    }
+  }
+
+  if (ri_pk && md_pk) {
+    if (ed25519_pubkey_eq(ri_pk, md_pk)) {
+      return ri_pk;
+    } else {
+      /* This can happen if the relay gets flagged NoEdConsensus which will be
+       * triggered on all relays of the network. Thus a protocol warning. */
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Inconsistent ed25519 identities in the nodelist");
+      return NULL;
+    }
+  } else if (ri_pk) {
+    return ri_pk;
+  } else {
+    return md_pk;
+  }
+}
+
+/** Return true iff this node's Ed25519 identity matches <b>id</b>.
+ * (An absent Ed25519 identity matches NULL or zero.) */
+int
+node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
+{
+  const ed25519_public_key_t *node_id = node_get_ed25519_id(node);
+  if (node_id == NULL || ed25519_public_key_is_zero(node_id)) {
+    return id == NULL || ed25519_public_key_is_zero(id);
+  } else {
+    return id && ed25519_pubkey_eq(node_id, id);
+  }
+}
+
+/** Dummy object that should be unreturnable.  Used to ensure that
+ * node_get_protover_summary_flags() always returns non-NULL. */
+static const protover_summary_flags_t zero_protover_flags = {
+  0,0,0,0,0,0,0
+};
+
+/** Return the protover_summary_flags for a given node. */
+static const protover_summary_flags_t *
+node_get_protover_summary_flags(const node_t *node)
+{
+  if (node->rs) {
+    return &node->rs->pv;
+  } else if (node->ri) {
+    return &node->ri->pv;
+  } else {
+    /* This should be impossible: every node should have a routerstatus or a
+     * router descriptor or both. But just in case we've messed up somehow,
+     * return a nice empty set of flags to indicate "this node supports
+     * nothing." */
+    tor_assert_nonfatal_unreached_once();
+    return &zero_protover_flags;
+  }
+}
+
+/** Return true iff <b>node</b> supports authenticating itself
+ * by ed25519 ID during the link handshake.  If <b>compatible_with_us</b>,
+ * it needs to be using a link authentication method that we understand.
+ * If not, any plausible link authentication method will do. */
+int
+node_supports_ed25519_link_authentication(const node_t *node,
+                                          int compatible_with_us)
+{
+  if (! node_get_ed25519_id(node))
+    return 0;
+
+  const protover_summary_flags_t *pv = node_get_protover_summary_flags(node);
+
+  if (compatible_with_us)
+    return pv->supports_ed25519_link_handshake_compat;
+  else
+    return pv->supports_ed25519_link_handshake_any;
+}
+
+/** Return true iff <b>node</b> supports the hidden service directory version
+ * 3 protocol (proposal 224). */
+int
+node_supports_v3_hsdir(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_v3_hsdir;
+}
+
+/** Return true iff <b>node</b> supports ed25519 authentication as an hidden
+ * service introduction point.*/
+int
+node_supports_ed25519_hs_intro(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
+}
+
+/** Return true iff <b>node</b> supports to be a rendezvous point for hidden
+ * service version 3 (HSRend=2). */
+int
+node_supports_v3_rendezvous_point(const node_t *node)
+{
+  tor_assert(node);
+
+  /* We can't use a v3 rendezvous point without the curve25519 onion pk. */
+  if (!node_get_curve25519_onion_key(node)) {
+    return 0;
+  }
+
+  return node_get_protover_summary_flags(node)->supports_v3_rendezvous_point;
+}
+
+/** Return the RSA ID key's SHA1 digest for the provided node. */
+const uint8_t *
+node_get_rsa_id_digest(const node_t *node)
+{
+  tor_assert(node);
+  return (const uint8_t*)node->identity;
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -710,21 +1182,6 @@ node_get_nickname(const node_t *node)
     return NULL;
 }
 
-/** Return true iff the nickname of <b>node</b> is canonical, based on the
- * latest consensus. */
-int
-node_is_named(const node_t *node)
-{
-  const char *named_id;
-  const char *nickname = node_get_nickname(node);
-  if (!nickname)
-    return 0;
-  named_id = networkstatus_get_router_digest_by_nickname(nickname);
-  if (!named_id)
-    return 0;
-  return tor_memeq(named_id, node->identity, DIGEST_LEN);
-}
-
 /** Return true iff <b>node</b> appears to be a directory authority or
  * directory cache */
 int
@@ -745,15 +1202,44 @@ node_is_dir(const node_t *node)
   }
 }
 
-/** Return true iff <b>node</b> has either kind of usable descriptor -- that
- * is, a routerdescriptor or a microdescriptor. */
+/** Return true iff <b>node</b> has either kind of descriptor -- that
+ * is, a routerdescriptor or a microdescriptor.
+ *
+ * You should probably use node_has_preferred_descriptor() instead.
+ **/
 int
-node_has_descriptor(const node_t *node)
+node_has_any_descriptor(const node_t *node)
 {
   return (node->ri ||
           (node->rs && node->md));
 }
 
+/** Return true iff <b>node</b> has the kind of descriptor we would prefer to
+ * use for it, given our configuration and how we intend to use the node.
+ *
+ * If <b>for_direct_connect</b> is true, we intend to connect to the node
+ * directly, as the first hop of a circuit; otherwise, we intend to connect to
+ * it indirectly, or use it as if we were connecting to it indirectly. */
+int
+node_has_preferred_descriptor(const node_t *node,
+                              int for_direct_connect)
+{
+  const int is_bridge = node_is_a_configured_bridge(node);
+  const int we_use_mds = we_use_microdescriptors_for_circuits(get_options());
+
+  if ((is_bridge && for_direct_connect) || !we_use_mds) {
+    /* We need an ri in this case. */
+    if (!node->ri)
+      return 0;
+  } else {
+    /* Otherwise we need an rs and an md. */
+    if (node->rs == NULL || node->md == NULL)
+      return 0;
+  }
+
+  return 1;
+}
+
 /** Return the router_purpose of <b>node</b>. */
 int
 node_get_purpose(const node_t *node)
@@ -772,13 +1258,12 @@ node_get_verbose_nickname(const node_t *node,
                           char *verbose_name_out)
 {
   const char *nickname = node_get_nickname(node);
-  int is_named = node_is_named(node);
   verbose_name_out[0] = '$';
   base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, node->identity,
                 DIGEST_LEN);
   if (!nickname)
     return;
-  verbose_name_out[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
+  verbose_name_out[1+HEX_DIGEST_LEN] = '~';
   strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
 }
 
@@ -992,16 +1477,6 @@ node_get_platform(const node_t *node)
     return NULL;
 }
 
-/** Return <b>node</b>'s time of publication, or 0 if we don't have one. */
-time_t
-node_get_published_on(const node_t *node)
-{
-  if (node->ri)
-    return node->ri->cache_info.published_on;
-  else
-    return 0;
-}
-
 /** Return true iff <b>node</b> is one representing this router. */
 int
 node_is_me(const node_t *node)
@@ -1082,9 +1557,11 @@ node_ipv6_or_preferred(const node_t *node)
   /* XX/teor - node->ipv6_preferred is set from
    * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded.
    */
+  node_get_prim_orport(node, &ipv4_addr);
   if (!fascist_firewall_use_ipv6(options)) {
     return 0;
-  } else if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) {
+  } else if (node->ipv6_preferred ||
+             !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) {
     return node_has_ipv6_orport(node);
   }
   return 0;
@@ -1095,14 +1572,12 @@ node_ipv6_or_preferred(const node_t *node)
     if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \
       tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \
       (ap_out)->port = (r)->port_field; \
-      return 0; \
     } \
   STMT_END
 
-/** Copy the primary (IPv4) OR port (IP address and TCP port) for
- * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and
- * port was copied, else return non-zero.*/
-int
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
 node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out)
 {
   node_assert_ok(node);
@@ -1119,8 +1594,6 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out)
   RETURN_IPV4_AP(node->ri, or_port, ap_out);
   RETURN_IPV4_AP(node->rs, or_port, ap_out);
   /* Microdescriptors only have an IPv6 address */
-
-  return -1;
 }
 
 /** Copy the preferred OR port (IP address and TCP port) for
@@ -1145,10 +1618,13 @@ node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out)
 {
   node_assert_ok(node);
   tor_assert(ap_out);
+  memset(ap_out, 0, sizeof(*ap_out));
 
-  /* Prefer routerstatus over microdesc for consistency with the
-   * fascist_firewall_* functions. Also check if the address or port are valid,
-   * and try another alternative if they are not. */
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
 
   if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
                                          node->ri->ipv6_orport, 0)) {
@@ -1190,29 +1666,35 @@ node_ipv6_dir_preferred(const node_t *node)
    * so we can't use it to determine DirPort IPv6 preference.
    * This means that bridge clients will use IPv4 DirPorts by default.
    */
+  node_get_prim_dirport(node, &ipv4_addr);
   if (!fascist_firewall_use_ipv6(options)) {
     return 0;
-  } else if (node_get_prim_dirport(node, &ipv4_addr)
+  } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0)
       || fascist_firewall_prefer_ipv6_dirport(get_options())) {
     return node_has_ipv6_dirport(node);
   }
   return 0;
 }
 
-/** Copy the primary (IPv4) Dir port (IP address and TCP port) for
- * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and
- * port was copied, else return non-zero.*/
-int
+/** Copy the primary (IPv4) Dir port (IP address and TCP port) for <b>node</b>
+ * into *<b>ap_out</b>. */
+void
 node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out)
 {
   node_assert_ok(node);
   tor_assert(ap_out);
 
+  /* Clear the address, as a safety precaution if calling functions ignore the
+   * return value */
+  tor_addr_make_null(&ap_out->addr, AF_INET);
+  ap_out->port = 0;
+
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
   RETURN_IPV4_AP(node->ri, dir_port, ap_out);
   RETURN_IPV4_AP(node->rs, dir_port, ap_out);
   /* Microdescriptors only have an IPv6 address */
-
-  return -1;
 }
 
 #undef RETURN_IPV4_AP
@@ -1240,8 +1722,11 @@ node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out)
   node_assert_ok(node);
   tor_assert(ap_out);
 
-  /* Check if the address or port are valid, and try another alternative if
-   * they are not. Note that microdescriptors have no dir_port. */
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
 
   /* Assume IPv4 and IPv6 dirports are the same */
   if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
@@ -1283,15 +1768,21 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md)
 int
 node_has_curve25519_onion_key(const node_t *node)
 {
-  if (!node)
-    return 0;
+  return node_get_curve25519_onion_key(node) != NULL;
+}
 
-  if (node->ri)
-    return routerinfo_has_curve25519_onion_key(node->ri);
-  else if (node->md)
-    return microdesc_has_curve25519_onion_key(node->md);
+/** Return the curve25519 key of <b>node</b>, or NULL if none. */
+const curve25519_public_key_t *
+node_get_curve25519_onion_key(const node_t *node)
+{
+  if (!node)
+    return NULL;
+  if (routerinfo_has_curve25519_onion_key(node->ri))
+    return node->ri->onion_curve25519_pkey;
+  else if (microdesc_has_curve25519_onion_key(node->md))
+    return node->md->onion_curve25519_pkey;
   else
-    return 0;
+    return NULL;
 }
 
 /** Refresh the country code of <b>ri</b>.  This function MUST be called on
@@ -1321,7 +1812,7 @@ nodelist_refresh_countries(void)
 
 /** Return true iff router1 and router2 have similar enough network addresses
  * that we should treat them as being in the same family */
-static inline int
+int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
@@ -1340,8 +1831,7 @@ node_nickname_matches(const node_t *node, const char *nickname)
     return 1;
   return hex_digest_nickname_matches(nickname,
                                      node->identity,
-                                     n,
-                                     node_is_named(node));
+                                     n);
 }
 
 /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
@@ -1443,7 +1933,7 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
     SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
       const node_t *node2;
       const smartlist_t *family2;
-      if (!(node2 = node_get_by_nickname(name, 0)))
+      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
         continue;
       if (!(family2 = node_get_declared_family(node2)))
         continue;
@@ -1595,8 +2085,8 @@ static char dir_info_status[512] = "";
  * no exits in the consensus."
  * To obtain the final weighted bandwidth, we multiply the
  * weighted bandwidth fraction for each position (guard, middle, exit). */
-int
-router_have_minimum_dir_info(void)
+MOCK_IMPL(int,
+router_have_minimum_dir_info,(void))
 {
   static int logged_delay=0;
   const char *delay_fetches_msg = NULL;
@@ -1628,8 +2118,8 @@ router_have_minimum_dir_info(void)
  * this can cause router_have_consensus_path() to be set to
  * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies.
  */
-consensus_path_type_t
-router_have_consensus_path(void)
+MOCK_IMPL(consensus_path_type_t,
+router_have_consensus_path, (void))
 {
   return have_consensus_path;
 }
@@ -1643,6 +2133,8 @@ router_dir_info_changed(void)
 {
   need_to_update_have_min_dir_info = 1;
   rend_hsdir_routers_changed();
+  hs_service_dir_info_changed();
+  hs_client_dir_info_changed();
 }
 
 /** Return a string describing what we're missing before we have enough
@@ -1659,8 +2151,11 @@ get_dir_info_status_string(void)
  * *<b>num_present</b>).
  *
  * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>.
- * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes
- * with the Exit flag.
+ * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_POLICY, only consider nodes
+ * present if they have an exit policy that accepts at least one port.
+ * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_FLAG, only consider nodes
+ * usable if they have the exit flag in the consensus.
+ *
  * If *<b>descs_out</b> is present, add a node_t for each usable descriptor
  * to it.
  */
@@ -1668,7 +2163,7 @@ static void
 count_usable_descriptors(int *num_present, int *num_usable,
                          smartlist_t *descs_out,
                          const networkstatus_t *consensus,
-                         const or_options_t *options, time_t now,
+                         time_t now,
                          routerset_t *in_set,
                          usable_descriptor_t exit_only)
 {
@@ -1681,11 +2176,11 @@ count_usable_descriptors(int *num_present, int *num_usable,
        if (!node)
          continue; /* This would be a bug: every entry in the consensus is
                     * supposed to have a node. */
-       if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit)
+       if ((exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) && ! rs->is_exit)
          continue;
        if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1))
          continue;
-       if (client_would_use_router(rs, now, options)) {
+       if (client_would_use_router(rs, now)) {
          const char * const digest = rs->descriptor_digest;
          int present;
          ++*num_usable; /* the consensus says we want it. */
@@ -1694,7 +2189,14 @@ count_usable_descriptors(int *num_present, int *num_usable,
          else
            present = NULL != router_get_by_descriptor_digest(digest);
          if (present) {
-           /* we have the descriptor listed in the consensus. */
+           /* Do the policy check last, because it requires a descriptor,
+            * and is potentially expensive */
+           if ((exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) &&
+               node_exit_policy_rejects_all(node)) {
+               continue;
+           }
+           /* we have the descriptor listed in the consensus, and it
+            * satisfies our exit constraints (if any) */
            ++*num_present;
          }
          if (descs_out)
@@ -1703,10 +2205,17 @@ count_usable_descriptors(int *num_present, int *num_usable,
      }
   SMARTLIST_FOREACH_END(rs);
 
-  log_debug(LD_DIR, "%d usable, %d present (%s%s).",
+  log_debug(LD_DIR, "%d usable, %d present (%s%s%s%s%s).",
             *num_usable, *num_present,
             md ? "microdesc" : "desc",
-            exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s");
+            (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ?
+              " exit"     : "s",
+            (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) ?
+              " policies" : "" ,
+            (exit_only == USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ?
+              " and" : "" ,
+            (exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) ?
+              " flags"    : "" );
 }
 
 /** Return an estimate of which fraction of usable paths through the Tor
@@ -1718,9 +2227,9 @@ count_usable_descriptors(int *num_present, int *num_usable,
  * If **<b>status_out</b> is present, allocate a new string and print the
  * available percentages of guard, middle, and exit nodes to it, noting
  * whether there are exits in the consensus.
- * If there are no guards in the consensus,
- * we treat the exit fraction as 100%.
- */
+ * If there are no exits in the consensus, we treat the exit fraction as 100%,
+ * but set router_have_consensus_path() so that we can only build internal
+ * paths. */
 static double
 compute_frac_paths_available(const networkstatus_t *consensus,
                              const or_options_t *options, time_t now,
@@ -1739,11 +2248,22 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, options, now, NULL,
+                           mid, consensus, now, NULL,
                            USABLE_DESCRIPTOR_ALL);
+  log_debug(LD_NET,
+            "%s: %d present, %d usable",
+            "mid",
+            np,
+            nu);
+
   if (options->EntryNodes) {
-    count_usable_descriptors(&np, &nu, guards, consensus, options, now,
+    count_usable_descriptors(&np, &nu, guards, consensus, now,
                              options->EntryNodes, USABLE_DESCRIPTOR_ALL);
+    log_debug(LD_NET,
+              "%s: %d present, %d usable",
+              "guard",
+              np,
+              nu);
   } else {
     SMARTLIST_FOREACH(mid, const node_t *, node, {
       if (authdir) {
@@ -1754,42 +2274,45 @@ compute_frac_paths_available(const networkstatus_t *consensus,
           smartlist_add(guards, (node_t*)node);
       }
     });
+    log_debug(LD_NET,
+              "%s: %d possible",
+              "guard",
+              smartlist_len(guards));
   }
 
-  /* All nodes with exit flag
-   * If we're in a network with TestingDirAuthVoteExit set,
-   * this can cause false positives on have_consensus_path,
-   * incorrectly setting it to CONSENSUS_PATH_EXIT. This is
-   * an unavoidable feature of forcing authorities to declare
-   * certain nodes as exits.
-   */
-  count_usable_descriptors(&np, &nu, exits, consensus, options, now,
-                           NULL, USABLE_DESCRIPTOR_EXIT_ONLY);
+  /* All nodes with exit policy and flag */
+  count_usable_descriptors(&np, &nu, exits, consensus, now,
+                           NULL, USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
             "exits",
             np,
             nu);
 
-  /* We need at least 1 exit present in the consensus to consider
+  /* We need at least 1 exit (flag and policy) in the consensus to consider
    * building exit paths */
   /* Update our understanding of whether the consensus has exits */
   consensus_path_type_t old_have_consensus_path = have_consensus_path;
-  have_consensus_path = ((nu > 0) ?
+  have_consensus_path = ((np > 0) ?
                          CONSENSUS_PATH_EXIT :
                          CONSENSUS_PATH_INTERNAL);
 
-  if (have_consensus_path == CONSENSUS_PATH_INTERNAL
-      && old_have_consensus_path != have_consensus_path) {
-    log_notice(LD_NET,
-               "The current consensus has no exit nodes. "
-               "Tor can only build internal paths, "
-               "such as paths to hidden services.");
-
-    /* However, exit nodes can reachability self-test using this consensus,
-     * join the network, and appear in a later consensus. This will allow
-     * the network to build exit paths, such as paths for world wide web
-     * browsing (as distinct from hidden service web browsing). */
+  if (old_have_consensus_path != have_consensus_path) {
+    if (have_consensus_path == CONSENSUS_PATH_INTERNAL) {
+      log_notice(LD_NET,
+                 "The current consensus has no exit nodes. "
+                 "Tor can only build internal paths, "
+                 "such as paths to onion services.");
+
+      /* However, exit nodes can reachability self-test using this consensus,
+       * join the network, and appear in a later consensus. This will allow
+       * the network to build exit paths, such as paths for world wide web
+       * browsing (as distinct from hidden service web browsing). */
+    } else if (old_have_consensus_path == CONSENSUS_PATH_INTERNAL) {
+      log_notice(LD_NET,
+                 "The current consensus contains exit nodes. "
+                 "Tor can build exit and internal paths.");
+    }
   }
 
   f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD);
@@ -1811,42 +2334,28 @@ compute_frac_paths_available(const networkstatus_t *consensus,
     smartlist_t *myexits= smartlist_new();
     smartlist_t *myexits_unflagged = smartlist_new();
 
-    /* All nodes with exit flag in ExitNodes option */
-    count_usable_descriptors(&np, &nu, myexits, consensus, options, now,
-                             options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY);
+    /* All nodes with exit policy and flag in ExitNodes option */
+    count_usable_descriptors(&np, &nu, myexits, consensus, now,
+                             options->ExitNodes,
+                             USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG);
     log_debug(LD_NET,
               "%s: %d present, %d usable",
               "myexits",
               np,
               nu);
 
-    /* Now compute the nodes in the ExitNodes option where which we don't know
-     * what their exit policy is, or we know it permits something. */
+    /* Now compute the nodes in the ExitNodes option where we know their exit
+     * policy permits something. */
     count_usable_descriptors(&np, &nu, myexits_unflagged,
-                             consensus, options, now,
-                             options->ExitNodes, USABLE_DESCRIPTOR_ALL);
+                             consensus, now,
+                             options->ExitNodes,
+                             USABLE_DESCRIPTOR_EXIT_POLICY);
     log_debug(LD_NET,
               "%s: %d present, %d usable",
               "myexits_unflagged (initial)",
               np,
               nu);
 
-    SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) {
-      if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) {
-        SMARTLIST_DEL_CURRENT(myexits_unflagged, node);
-        /* this node is not actually an exit */
-        np--;
-        /* this node is unusable as an exit */
-        nu--;
-      }
-    } SMARTLIST_FOREACH_END(node);
-
-    log_debug(LD_NET,
-              "%s: %d present, %d usable",
-              "myexits_unflagged (final)",
-              np,
-              nu);
-
     f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT);
     f_myexit_unflagged=
               frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT);
@@ -1887,14 +2396,14 @@ compute_frac_paths_available(const networkstatus_t *consensus,
     tor_asprintf(status_out,
                  "%d%% of guards bw, "
                  "%d%% of midpoint bw, and "
-                 "%d%% of exit bw%s = "
+                 "%d%% of %s = "
                  "%d%% of path bw",
                  (int)(f_guard*100),
                  (int)(f_mid*100),
                  (int)(f_exit*100),
                  (router_have_consensus_path() == CONSENSUS_PATH_EXIT ?
-                  "" :
-                  " (no exits in consensus)"),
+                  "exit bw" :
+                  "end bw (no exits in consensus)"),
                  (int)(f_path*100));
 
   return f_path;
@@ -1952,6 +2461,7 @@ update_router_have_minimum_dir_info(void)
 {
   time_t now = time(NULL);
   int res;
+  int num_present=0, num_usable=0;
   const or_options_t *options = get_options();
   const networkstatus_t *consensus =
     networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor());
@@ -1973,7 +2483,6 @@ update_router_have_minimum_dir_info(void)
   /* Check fraction of available paths */
   {
     char *status = NULL;
-    int num_present=0, num_usable=0;
     double paths = compute_frac_paths_available(consensus, options, now,
                                                 &num_present, &num_usable,
                                                 &status);
@@ -1994,6 +2503,18 @@ update_router_have_minimum_dir_info(void)
     res = 1;
   }
 
+  { /* Check entry guard dirinfo status */
+    char *guard_error = entry_guards_get_err_str_if_dir_info_missing(using_md,
+                                                             num_present,
+                                                             num_usable);
+    if (guard_error) {
+      strlcpy(dir_info_status, guard_error, sizeof(dir_info_status));
+      tor_free(guard_error);
+      res = 0;
+      goto done;
+    }
+  }
+
  done:
 
   /* If paths have just become available in this update. */
diff --git a/src/or/nodelist.h b/src/or/nodelist.h
index 098f1d1555..dbe9ad18ff 100644
--- a/src/or/nodelist.h
+++ b/src/or/nodelist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,12 +16,20 @@
     tor_assert((n)->ri || (n)->rs);                             \
   } STMT_END
 
-node_t *node_get_mutable_by_id(const char *identity_digest);
+MOCK_DECL(node_t *, node_get_mutable_by_id,(const char *identity_digest));
 MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest));
-const node_t *node_get_by_hex_id(const char *identity_digest);
+node_t *node_get_mutable_by_ed25519_id(const ed25519_public_key_t *ed_id);
+MOCK_DECL(const node_t *, node_get_by_ed25519_id,
+          (const ed25519_public_key_t *ed_id));
+
+#define NNF_NO_WARN_UNNAMED (1u<<0)
+
+const node_t *node_get_by_hex_id(const char *identity_digest,
+                                 unsigned flags);
 node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out);
 node_t *nodelist_add_microdesc(microdesc_t *md);
 void nodelist_set_consensus(networkstatus_t *ns);
+void nodelist_ensure_freshness(networkstatus_t *ns);
 int nodelist_probably_contains_address(const tor_addr_t *addr);
 
 void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
@@ -33,14 +41,15 @@ void nodelist_free_all(void);
 void nodelist_assert_ok(void);
 
 MOCK_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unnamed));
+          (const char *nickname, unsigned flags));
 void node_get_verbose_nickname(const node_t *node,
                                char *verbose_name_out);
 void node_get_verbose_nickname_by_id(const char *id_digest,
                                 char *verbose_name_out);
-int node_is_named(const node_t *node);
 int node_is_dir(const node_t *node);
-int node_has_descriptor(const node_t *node);
+int node_has_any_descriptor(const node_t *node);
+int node_has_preferred_descriptor(const node_t *node,
+                                  int for_direct_connect);
 int node_get_purpose(const node_t *node);
 #define node_is_bridge(node) \
   (node_get_purpose((node)) == ROUTER_PURPOSE_BRIDGE)
@@ -54,8 +63,16 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-time_t node_get_published_on(const node_t *node);
 const smartlist_t *node_get_declared_family(const node_t *node);
+const ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+int node_ed25519_id_matches(const node_t *node,
+                            const ed25519_public_key_t *id);
+int node_supports_ed25519_link_authentication(const node_t *node,
+                                              int compatible_with_us);
+int node_supports_v3_hsdir(const node_t *node);
+int node_supports_ed25519_hs_intro(const node_t *node);
+int node_supports_v3_rendezvous_point(const node_t *node);
+const uint8_t *node_get_rsa_id_digest(const node_t *node);
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -63,14 +80,16 @@ int node_has_ipv6_dirport(const node_t *node);
 /* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */
 #define node_ipv6_preferred(node) node_ipv6_or_preferred(node)
 int node_ipv6_or_preferred(const node_t *node);
-int node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out);
 int node_ipv6_dir_preferred(const node_t *node);
-int node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out);
+void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out);
 void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out);
 int node_has_curve25519_onion_key(const node_t *node);
+const curve25519_public_key_t *node_get_curve25519_onion_key(
+                                  const node_t *node);
 
 MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
 
@@ -90,6 +109,8 @@ int node_is_unreliable(const node_t *router, int need_uptime,
 int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
                                         int need_uptime);
 void router_set_status(const char *digest, int up);
+int addrs_in_same_network_family(const tor_addr_t *a1,
+                                 const tor_addr_t *a2);
 
 /** router_have_minimum_dir_info tests to see if we have enough
  * descriptor information to create circuits.
@@ -98,7 +119,7 @@ void router_set_status(const char *digest, int up);
  * no exits in the consensus, we wait for enough info to create internal
  * paths, and should avoid creating exit paths, as they will simply fail.
  * We make sure we create all available circuit types at the same time. */
-int router_have_minimum_dir_info(void);
+MOCK_DECL(int, router_have_minimum_dir_info,(void));
 
 /** Set to CONSENSUS_PATH_EXIT if there is at least one exit node
  * in the consensus. We update this flag in compute_frac_paths_available if
@@ -119,13 +140,25 @@ typedef enum {
    * create exit and internal paths, circuits, streams, ... */
   CONSENSUS_PATH_EXIT = 1
 } consensus_path_type_t;
-consensus_path_type_t router_have_consensus_path(void);
+
+MOCK_DECL(consensus_path_type_t, router_have_consensus_path, (void));
 
 void router_dir_info_changed(void);
 const char *get_dir_info_status_string(void);
 int count_loading_descriptors_progress(void);
 
+#ifdef NODELIST_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(NODELIST_PRIVATE) */
+
 MOCK_DECL(int, get_estimated_address_per_node, (void));
 
-#endif
+#endif /* !defined(TOR_NODELIST_H) */
 
diff --git a/src/or/ntmain.c b/src/or/ntmain.c
index 0e6f296d24..e9a299807a 100644
--- a/src/or/ntmain.c
+++ b/src/or/ntmain.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,8 +24,6 @@
 #include "main.h"
 #include "ntmain.h"
 
-#include <event2/event.h>
-
 #include <windows.h>
 #define GENSRV_SERVICENAME  "tor"
 #define GENSRV_DISPLAYNAME  "Tor Win32 Service"
@@ -195,7 +193,7 @@ nt_service_loadlibrary(void)
   return;
  err:
   printf("Unable to load library support for NT services: exiting.\n");
-  exit(1);
+  exit(1); // exit ok: ntmain can't read libraries
 }
 
 /** If we're compiled to run as an NT service, and the service wants to
@@ -245,7 +243,8 @@ nt_service_control(DWORD request)
           log_notice(LD_GENERAL,
                      "Got stop/shutdown request; shutting down cleanly.");
           service_status.dwCurrentState = SERVICE_STOP_PENDING;
-          event_base_loopexit(tor_libevent_get_base(), &exit_now);
+          tor_libevent_exit_loop_after_delay(tor_libevent_get_base(),
+                                             &exit_now);
           return;
   }
   service_fns.SetServiceStatus_fn(hStatus, &service_status);
@@ -318,7 +317,7 @@ nt_service_main(void)
     printf("Service error %d : %s\n", (int) result, errmsg);
     tor_free(errmsg);
     if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
-      if (tor_init(backup_argc, backup_argv) < 0)
+      if (tor_init(backup_argc, backup_argv))
         return;
       switch (get_options()->command) {
       case CMD_RUN_TOR:
@@ -329,9 +328,10 @@ nt_service_main(void)
       case CMD_VERIFY_CONFIG:
       case CMD_DUMP_CONFIG:
       case CMD_KEYGEN:
+      case CMD_KEY_EXPIRATION:
         log_err(LD_CONFIG, "Unsupported command (--list-fingerprint, "
-               "--hash-password, --keygen, --dump-config, or --verify-config) "
-                "in NT service.");
+               "--hash-password, --keygen, --dump-config, --verify-config, "
+               "or --key-expiration) in NT service.");
         break;
       case CMD_RUN_UNITTESTS:
       default:
@@ -501,7 +501,7 @@ nt_service_command_line(int *using_default_torrc)
   tor_exe_ascii[sizeof(tor_exe_ascii)-1] = '\0';
 #else
   strlcpy(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii));
-#endif
+#endif /* defined(UNICODE) */
 
   /* Allocate a string for the NT service command line and */
   /* Format the service command */
@@ -777,5 +777,5 @@ nt_service_parse_options(int argc, char **argv, int *should_exit)
   return 0;
 }
 
-#endif
+#endif /* defined(_WIN32) */
 
diff --git a/src/or/ntmain.h b/src/or/ntmain.h
index 31bf38c62c..81b7159855 100644
--- a/src/or/ntmain.h
+++ b/src/or/ntmain.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,7 @@ int nt_service_is_stopping(void);
 void nt_service_set_state(DWORD state);
 #else
 #define nt_service_is_stopping() 0
-#endif
+#endif /* defined(NT_SERVICE) */
 
-#endif
+#endif /* !defined(TOR_NTMAIN_H) */
 
diff --git a/src/or/onion.c b/src/or/onion.c
index 4b803a785c..829be12bae 100644
--- a/src/or/onion.c
+++ b/src/or/onion.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -67,6 +67,7 @@
 #include "circuitlist.h"
 #include "config.h"
 #include "cpuworker.h"
+#include "crypto_util.h"
 #include "networkstatus.h"
 #include "onion.h"
 #include "onion_fast.h"
@@ -76,6 +77,9 @@
 #include "rephist.h"
 #include "router.h"
 
+// trunnel
+#include "ed25519_cert.h"
+
 /** Type for a linked list of circuits that are waiting for a free CPU worker
  * to process a waiting onion handshake. */
 typedef struct onion_queue_t {
@@ -420,7 +424,7 @@ server_onion_keys_new(void)
 
 /** Release all storage held in <b>keys</b>. */
 void
-server_onion_keys_free(server_onion_keys_t *keys)
+server_onion_keys_free_(server_onion_keys_t *keys)
 {
   if (! keys)
     return;
@@ -518,6 +522,11 @@ onion_skin_create(int type,
   return r;
 }
 
+/* This is the maximum value for keys_out_len passed to
+ * onion_skin_server_handshake, plus 16. We can make it bigger if needed:
+ * It just defines how many bytes to stack-allocate. */
+#define MAX_KEYS_TMP_LEN 128
+
 /** Perform the second (server-side) step of a circuit-creation handshake of
  * type <b>type</b>, responding to the client request in <b>onion_skin</b>
  * using the keys in <b>keys</b>.  On success, write our response into
@@ -560,20 +569,21 @@ onion_skin_server_handshake(int type,
       return -1;
     {
       size_t keys_tmp_len = keys_out_len + DIGEST_LEN;
-      uint8_t *keys_tmp = tor_malloc(keys_out_len + DIGEST_LEN);
+      tor_assert(keys_tmp_len <= MAX_KEYS_TMP_LEN);
+      uint8_t keys_tmp[MAX_KEYS_TMP_LEN];
 
       if (onion_skin_ntor_server_handshake(
                                    onion_skin, keys->curve25519_key_map,
                                    keys->junk_keypair,
                                    keys->my_identity,
                                    reply_out, keys_tmp, keys_tmp_len)<0) {
-        tor_free(keys_tmp);
+        /* no need to memwipe here, since the output will never be used */
         return -1;
       }
+
       memcpy(keys_out, keys_tmp, keys_out_len);
       memcpy(rend_nonce_out, keys_tmp+keys_out_len, DIGEST_LEN);
-      memwipe(keys_tmp, 0, keys_tmp_len);
-      tor_free(keys_tmp);
+      memwipe(keys_tmp, 0, sizeof(keys_tmp));
       r = NTOR_REPLY_LEN;
     }
     break;
@@ -873,13 +883,114 @@ check_extend_cell(const extend_cell_t *cell)
   return check_create_cell(&cell->create_cell, 1);
 }
 
-/** Protocol constants for specifier types in EXTEND2
- * @{
- */
-#define SPECTYPE_IPV4 0
-#define SPECTYPE_IPV6 1
-#define SPECTYPE_LEGACY_ID 2
-/** @} */
+static int
+extend_cell_from_extend1_cell_body(extend_cell_t *cell_out,
+                                   const extend1_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  memset(cell_out, 0, sizeof(*cell_out));
+  tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
+  tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
+
+  cell_out->cell_type = RELAY_COMMAND_EXTEND;
+  tor_addr_from_ipv4h(&cell_out->orport_ipv4.addr, cell->ipv4addr);
+  cell_out->orport_ipv4.port = cell->port;
+  if (tor_memeq(cell->onionskin, NTOR_CREATE_MAGIC, 16)) {
+    cell_out->create_cell.cell_type = CELL_CREATE2;
+    cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_NTOR;
+    cell_out->create_cell.handshake_len = NTOR_ONIONSKIN_LEN;
+    memcpy(cell_out->create_cell.onionskin, cell->onionskin + 16,
+           NTOR_ONIONSKIN_LEN);
+  } else {
+    cell_out->create_cell.cell_type = CELL_CREATE;
+    cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_TAP;
+    cell_out->create_cell.handshake_len = TAP_ONIONSKIN_CHALLENGE_LEN;
+    memcpy(cell_out->create_cell.onionskin, cell->onionskin,
+           TAP_ONIONSKIN_CHALLENGE_LEN);
+  }
+  memcpy(cell_out->node_id, cell->identity, DIGEST_LEN);
+  return 0;
+}
+
+static int
+create_cell_from_create2_cell_body(create_cell_t *cell_out,
+                                   const create2_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  memset(cell_out, 0, sizeof(create_cell_t));
+  if (BUG(cell->handshake_len > sizeof(cell_out->onionskin))) {
+    /* This should be impossible because there just isn't enough room in the
+     * input cell to make the handshake_len this large and provide a
+     * handshake_data to match. */
+    return -1;
+  }
+
+  cell_out->cell_type = CELL_CREATE2;
+  cell_out->handshake_type = cell->handshake_type;
+  cell_out->handshake_len = cell->handshake_len;
+  memcpy(cell_out->onionskin,
+       create2_cell_body_getconstarray_handshake_data(cell),
+       cell->handshake_len);
+  return 0;
+}
+
+static int
+extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
+                                   const extend2_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  int found_ipv4 = 0, found_ipv6 = 0, found_rsa_id = 0, found_ed_id = 0;
+  memset(cell_out, 0, sizeof(*cell_out));
+  tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
+  tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
+  cell_out->cell_type = RELAY_COMMAND_EXTEND2;
+
+  unsigned i;
+  for (i = 0; i < cell->n_spec; ++i) {
+    const link_specifier_t *ls = extend2_cell_body_getconst_ls(cell, i);
+    switch (ls->ls_type) {
+      case LS_IPV4:
+        if (found_ipv4)
+          continue;
+        found_ipv4 = 1;
+        tor_addr_from_ipv4h(&cell_out->orport_ipv4.addr, ls->un_ipv4_addr);
+        cell_out->orport_ipv4.port = ls->un_ipv4_port;
+        break;
+      case LS_IPV6:
+        if (found_ipv6)
+          continue;
+        found_ipv6 = 1;
+        tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
+                                 (const char *)ls->un_ipv6_addr);
+        cell_out->orport_ipv6.port = ls->un_ipv6_port;
+        break;
+      case LS_LEGACY_ID:
+        if (found_rsa_id)
+          return -1;
+        found_rsa_id = 1;
+        memcpy(cell_out->node_id, ls->un_legacy_id, 20);
+        break;
+      case LS_ED25519_ID:
+        if (found_ed_id)
+          return -1;
+        found_ed_id = 1;
+        memcpy(cell_out->ed_pubkey.pubkey, ls->un_ed25519_id, 32);
+        break;
+      default:
+        /* Ignore this, whatever it is. */
+        break;
+    }
+  }
+
+  if (!found_rsa_id || !found_ipv4) /* These are mandatory */
+    return -1;
+
+  return create_cell_from_create2_cell_body(&cell_out->create_cell,
+                                            cell->create2);
+}
 
 /** Parse an EXTEND or EXTEND2 cell (according to <b>command</b>) from the
  * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
@@ -888,101 +999,44 @@ int
 extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
                   const uint8_t *payload, size_t payload_length)
 {
-  const uint8_t *eop;
 
-  memset(cell_out, 0, sizeof(*cell_out));
+  tor_assert(cell_out);
+  tor_assert(payload);
+
   if (payload_length > RELAY_PAYLOAD_SIZE)
     return -1;
-  eop = payload + payload_length;
 
   switch (command) {
   case RELAY_COMMAND_EXTEND:
     {
-      if (payload_length != 6 + TAP_ONIONSKIN_CHALLENGE_LEN + DIGEST_LEN)
+      extend1_cell_body_t *cell = NULL;
+      if (extend1_cell_body_parse(&cell, payload, payload_length)<0 ||
+          cell == NULL) {
+        if (cell)
+          extend1_cell_body_free(cell);
         return -1;
-
-      cell_out->cell_type = RELAY_COMMAND_EXTEND;
-      tor_addr_from_ipv4n(&cell_out->orport_ipv4.addr, get_uint32(payload));
-      cell_out->orport_ipv4.port = ntohs(get_uint16(payload+4));
-      tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
-      if (tor_memeq(payload + 6, NTOR_CREATE_MAGIC, 16)) {
-        cell_out->create_cell.cell_type = CELL_CREATE2;
-        cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_NTOR;
-        cell_out->create_cell.handshake_len = NTOR_ONIONSKIN_LEN;
-        memcpy(cell_out->create_cell.onionskin, payload + 22,
-               NTOR_ONIONSKIN_LEN);
-      } else {
-        cell_out->create_cell.cell_type = CELL_CREATE;
-        cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_TAP;
-        cell_out->create_cell.handshake_len = TAP_ONIONSKIN_CHALLENGE_LEN;
-        memcpy(cell_out->create_cell.onionskin, payload + 6,
-               TAP_ONIONSKIN_CHALLENGE_LEN);
       }
-      memcpy(cell_out->node_id, payload + 6 + TAP_ONIONSKIN_CHALLENGE_LEN,
-             DIGEST_LEN);
-      break;
+      int r = extend_cell_from_extend1_cell_body(cell_out, cell);
+      extend1_cell_body_free(cell);
+      if (r < 0)
+        return r;
     }
+    break;
   case RELAY_COMMAND_EXTEND2:
     {
-      uint8_t n_specs, spectype, speclen;
-      int i;
-      int found_ipv4 = 0, found_ipv6 = 0, found_id = 0;
-      tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
-      tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
-
-      if (payload_length == 0)
+      extend2_cell_body_t *cell = NULL;
+      if (extend2_cell_body_parse(&cell, payload, payload_length) < 0 ||
+          cell == NULL) {
+        if (cell)
+          extend2_cell_body_free(cell);
         return -1;
-
-      cell_out->cell_type = RELAY_COMMAND_EXTEND2;
-      n_specs = *payload++;
-      /* Parse the specifiers. We'll only take the first IPv4 and first IPv6
-       * address, and the node ID, and ignore everything else */
-      for (i = 0; i < n_specs; ++i) {
-        if (eop - payload < 2)
-          return -1;
-        spectype = payload[0];
-        speclen = payload[1];
-        payload += 2;
-        if (eop - payload < speclen)
-          return -1;
-        switch (spectype) {
-        case SPECTYPE_IPV4:
-          if (speclen != 6)
-            return -1;
-          if (!found_ipv4) {
-            tor_addr_from_ipv4n(&cell_out->orport_ipv4.addr,
-                                get_uint32(payload));
-            cell_out->orport_ipv4.port = ntohs(get_uint16(payload+4));
-            found_ipv4 = 1;
-          }
-          break;
-        case SPECTYPE_IPV6:
-          if (speclen != 18)
-            return -1;
-          if (!found_ipv6) {
-            tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
-                                     (const char*)payload);
-            cell_out->orport_ipv6.port = ntohs(get_uint16(payload+16));
-            found_ipv6 = 1;
-          }
-          break;
-        case SPECTYPE_LEGACY_ID:
-          if (speclen != 20)
-            return -1;
-          if (found_id)
-            return -1;
-          memcpy(cell_out->node_id, payload, 20);
-          found_id = 1;
-          break;
-        }
-        payload += speclen;
       }
-      if (!found_id || !found_ipv4)
-        return -1;
-      if (parse_create2_payload(&cell_out->create_cell,payload,eop-payload)<0)
-        return -1;
-      break;
+      int r = extend_cell_from_extend2_cell_body(cell_out, cell);
+      extend2_cell_body_free(cell);
+      if (r < 0)
+        return r;
     }
+    break;
   default:
     return -1;
   }
@@ -994,6 +1048,7 @@ extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
 static int
 check_extended_cell(const extended_cell_t *cell)
 {
+  tor_assert(cell);
   if (cell->created_cell.cell_type == CELL_CREATED) {
     if (cell->cell_type != RELAY_COMMAND_EXTENDED)
       return -1;
@@ -1015,6 +1070,9 @@ extended_cell_parse(extended_cell_t *cell_out,
                     const uint8_t command, const uint8_t *payload,
                     size_t payload_len)
 {
+  tor_assert(cell_out);
+  tor_assert(payload);
+
   memset(cell_out, 0, sizeof(*cell_out));
   if (payload_len > RELAY_PAYLOAD_SIZE)
     return -1;
@@ -1131,6 +1189,21 @@ created_cell_format(cell_t *cell_out, const created_cell_t *cell_in)
   return 0;
 }
 
+/** Return true iff we are configured (by torrc or by the networkstatus
+ * parameters) to use Ed25519 identities in our Extend2 cells. */
+static int
+should_include_ed25519_id_extend_cells(const networkstatus_t *ns,
+                                       const or_options_t *options)
+{
+  if (options->ExtendByEd25519ID != -1)
+    return options->ExtendByEd25519ID; /* The user has an opinion. */
+
+  return (int) networkstatus_get_param(ns, "ExtendByEd25519ID",
+                                       0 /* default */,
+                                       0 /* min */,
+                                       1 /*max*/);
+}
+
 /** Format the EXTEND{,2} cell in <b>cell_in</b>, storing its relay payload in
  * <b>payload_out</b>, the number of bytes used in *<b>len_out</b>, and the
  * relay command in *<b>command_out</b>. The <b>payload_out</b> must have
@@ -1139,12 +1212,11 @@ int
 extend_cell_format(uint8_t *command_out, uint16_t *len_out,
                    uint8_t *payload_out, const extend_cell_t *cell_in)
 {
-  uint8_t *p, *eop;
+  uint8_t *p;
   if (check_extend_cell(cell_in) < 0)
     return -1;
 
   p = payload_out;
-  eop = payload_out + RELAY_PAYLOAD_SIZE;
 
   memset(p, 0, RELAY_PAYLOAD_SIZE);
 
@@ -1154,7 +1226,7 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
       *command_out = RELAY_COMMAND_EXTEND;
       *len_out = 6 + TAP_ONIONSKIN_CHALLENGE_LEN + DIGEST_LEN;
       set_uint32(p, tor_addr_to_ipv4n(&cell_in->orport_ipv4.addr));
-      set_uint16(p+4, ntohs(cell_in->orport_ipv4.port));
+      set_uint16(p+4, htons(cell_in->orport_ipv4.port));
       if (cell_in->create_cell.handshake_type == ONION_HANDSHAKE_TYPE_NTOR) {
         memcpy(p+6, NTOR_CREATE_MAGIC, 16);
         memcpy(p+22, cell_in->create_cell.onionskin, NTOR_ONIONSKIN_LEN);
@@ -1167,33 +1239,56 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
     break;
   case RELAY_COMMAND_EXTEND2:
     {
-      uint8_t n = 2;
+      uint8_t n_specifiers = 2;
       *command_out = RELAY_COMMAND_EXTEND2;
-
-      *p++ = n; /* 2 identifiers */
-      *p++ = SPECTYPE_IPV4; /* First is IPV4. */
-      *p++ = 6; /* It's 6 bytes long. */
-      set_uint32(p, tor_addr_to_ipv4n(&cell_in->orport_ipv4.addr));
-      set_uint16(p+4, htons(cell_in->orport_ipv4.port));
-      p += 6;
-      *p++ = SPECTYPE_LEGACY_ID; /* Next is an identity digest. */
-      *p++ = 20; /* It's 20 bytes long */
-      memcpy(p, cell_in->node_id, DIGEST_LEN);
-      p += 20;
-
-      /* Now we can send the handshake */
-      set_uint16(p, htons(cell_in->create_cell.handshake_type));
-      set_uint16(p+2, htons(cell_in->create_cell.handshake_len));
-      p += 4;
-
-      if (cell_in->create_cell.handshake_len > eop - p)
-        return -1;
-
-      memcpy(p, cell_in->create_cell.onionskin,
+      extend2_cell_body_t *cell = extend2_cell_body_new();
+      link_specifier_t *ls;
+      {
+        /* IPv4 specifier first. */
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_IPV4;
+        ls->ls_len = 6;
+        ls->un_ipv4_addr = tor_addr_to_ipv4h(&cell_in->orport_ipv4.addr);
+        ls->un_ipv4_port = cell_in->orport_ipv4.port;
+      }
+      {
+        /* Then RSA id */
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_LEGACY_ID;
+        ls->ls_len = DIGEST_LEN;
+        memcpy(ls->un_legacy_id, cell_in->node_id, DIGEST_LEN);
+      }
+      if (should_include_ed25519_id_extend_cells(NULL, get_options()) &&
+          !ed25519_public_key_is_zero(&cell_in->ed_pubkey)) {
+        /* Then, maybe, the ed25519 id! */
+        ++n_specifiers;
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_ED25519_ID;
+        ls->ls_len = 32;
+        memcpy(ls->un_ed25519_id, cell_in->ed_pubkey.pubkey, 32);
+      }
+      cell->n_spec = n_specifiers;
+
+      /* Now, the handshake */
+      cell->create2 = create2_cell_body_new();
+      cell->create2->handshake_type = cell_in->create_cell.handshake_type;
+      cell->create2->handshake_len = cell_in->create_cell.handshake_len;
+      create2_cell_body_setlen_handshake_data(cell->create2,
+                                         cell_in->create_cell.handshake_len);
+      memcpy(create2_cell_body_getarray_handshake_data(cell->create2),
+             cell_in->create_cell.onionskin,
              cell_in->create_cell.handshake_len);
 
-      p += cell_in->create_cell.handshake_len;
-      *len_out = p - payload_out;
+      ssize_t len_encoded = extend2_cell_body_encode(
+                             payload_out, RELAY_PAYLOAD_SIZE,
+                             cell);
+      extend2_cell_body_free(cell);
+      if (len_encoded < 0 || len_encoded > UINT16_MAX)
+        return -1;
+      *len_out = (uint16_t) len_encoded;
     }
     break;
   default:
diff --git a/src/or/onion.h b/src/or/onion.h
index 0275fa00d2..3b738debeb 100644
--- a/src/or/onion.h
+++ b/src/or/onion.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,7 +31,9 @@ typedef struct server_onion_keys_t {
 #define MAX_ONIONSKIN_REPLY_LEN 255
 
 server_onion_keys_t *server_onion_keys_new(void);
-void server_onion_keys_free(server_onion_keys_t *keys);
+void server_onion_keys_free_(server_onion_keys_t *keys);
+#define server_onion_keys_free(keys) \
+  FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys))
 
 void onion_handshake_state_release(onion_handshake_state_t *state);
 
@@ -85,6 +87,8 @@ typedef struct extend_cell_t {
   tor_addr_port_t orport_ipv6;
   /** Identity fingerprint of the node we're conecting to.*/
   uint8_t node_id[DIGEST_LEN];
+  /** Ed25519 public identity key. Zero if not set. */
+  ed25519_public_key_t ed_pubkey;
   /** The "create cell" embedded in this extend cell. Note that unlike the
    * create cells we generate ourself, this once can have a handshake type we
    * don't recognize. */
@@ -117,5 +121,5 @@ int extend_cell_format(uint8_t *command_out, uint16_t *len_out,
 int extended_cell_format(uint8_t *command_out, uint16_t *len_out,
                          uint8_t *payload_out, const extended_cell_t *cell_in);
 
-#endif
+#endif /* !defined(TOR_ONION_H) */
 
diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c
index 8dcbfe22d8..9f9b2199d4 100644
--- a/src/or/onion_fast.c
+++ b/src/or/onion_fast.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,16 +23,18 @@
  * [*]Actually, it's possible that TAP _was_ a little better than TLS with
  * RSA1024 certificates and EDH1024 for forward secrecy, if you
  * hypothesize an adversary who can compute discrete logarithms on a
- * small number of targetted DH1024 fields, but who can't break all that
+ * small number of targeted DH1024 fields, but who can't break all that
  * many RSA1024 keys.
  **/
 
 #include "or.h"
 #include "onion_fast.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 
 /** Release all state held in <b>victim</b>. */
 void
-fast_handshake_state_free(fast_handshake_state_t *victim)
+fast_handshake_state_free_(fast_handshake_state_t *victim)
 {
   if (! victim)
     return;
diff --git a/src/or/onion_fast.h b/src/or/onion_fast.h
index b9626002c3..c56712e2c2 100644
--- a/src/or/onion_fast.h
+++ b/src/or/onion_fast.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,9 @@ typedef struct fast_handshake_state_t {
   uint8_t state[DIGEST_LEN];
 } fast_handshake_state_t;
 
-void fast_handshake_state_free(fast_handshake_state_t *victim);
+void fast_handshake_state_free_(fast_handshake_state_t *victim);
+#define fast_handshake_state_free(st) \
+  FREE_AND_NULL(fast_handshake_state_t, fast_handshake_state_free_, (st))
 
 int fast_onionskin_create(fast_handshake_state_t **handshake_state_out,
                           uint8_t *handshake_out);
@@ -35,5 +37,5 @@ int fast_client_handshake(const fast_handshake_state_t *handshake_state,
                           size_t key_out_len,
                           const char **msg_out);
 
-#endif
+#endif /* !defined(TOR_ONION_FAST_H) */
 
diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c
index ded97ee73d..02d43cb722 100644
--- a/src/or/onion_ntor.c
+++ b/src/or/onion_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,13 +22,15 @@
 
 #define ONION_NTOR_PRIVATE
 #include "crypto.h"
+#include "crypto_digest.h"
+#include "crypto_util.h"
 #include "onion_ntor.h"
 #include "torlog.h"
 #include "util.h"
 
 /** Free storage held in an ntor handshake state. */
 void
-ntor_handshake_state_free(ntor_handshake_state_t *state)
+ntor_handshake_state_free_(ntor_handshake_state_t *state)
 {
   if (!state)
     return;
diff --git a/src/or/onion_ntor.h b/src/or/onion_ntor.h
index f637b437fd..f7c962b7d0 100644
--- a/src/or/onion_ntor.h
+++ b/src/or/onion_ntor.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ONION_NTOR_H
@@ -17,7 +17,9 @@ typedef struct ntor_handshake_state_t ntor_handshake_state_t;
 /** Length of an ntor reply, as sent from server to client. */
 #define NTOR_REPLY_LEN 64
 
-void ntor_handshake_state_free(ntor_handshake_state_t *state);
+void ntor_handshake_state_free_(ntor_handshake_state_t *state);
+#define ntor_handshake_state_free(state) \
+  FREE_AND_NULL(ntor_handshake_state_t, ntor_handshake_state_free_, (state))
 
 int onion_skin_ntor_create(const uint8_t *router_id,
                            const curve25519_public_key_t *router_key,
@@ -55,7 +57,7 @@ struct ntor_handshake_state_t {
   curve25519_public_key_t pubkey_X;
   /** @} */
 };
-#endif
+#endif /* defined(ONION_NTOR_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_ONION_NTOR_H) */
 
diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c
index 2769300945..44737034f4 100644
--- a/src/or/onion_tap.c
+++ b/src/or/onion_tap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,8 @@
 
 #include "or.h"
 #include "config.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "onion_tap.h"
 #include "rephist.h"
 
@@ -72,10 +74,8 @@ onion_skin_TAP_create(crypto_pk_t *dest_router_key,
   if (crypto_dh_get_public(dh, challenge, dhbytes))
     goto err;
 
-  note_crypto_pk_op(ENC_ONIONSKIN);
-
   /* set meeting point, meeting cookie, etc here. Leave zero for now. */
-  if (crypto_pk_public_hybrid_encrypt(dest_router_key, onion_skin_out,
+  if (crypto_pk_obsolete_public_hybrid_encrypt(dest_router_key, onion_skin_out,
                                       TAP_ONIONSKIN_CHALLENGE_LEN,
                                       challenge, DH_KEY_LEN,
                                       PK_PKCS1_OAEP_PADDING, 1)<0)
@@ -124,8 +124,7 @@ onion_skin_TAP_server_handshake(
     k = i==0?private_key:prev_private_key;
     if (!k)
       break;
-    note_crypto_pk_op(DEC_ONIONSKIN);
-    len = crypto_pk_private_hybrid_decrypt(k, challenge,
+    len = crypto_pk_obsolete_private_hybrid_decrypt(k, challenge,
                                            TAP_ONIONSKIN_CHALLENGE_LEN,
                                            onion_skin,
                                            TAP_ONIONSKIN_CHALLENGE_LEN,
@@ -159,7 +158,7 @@ onion_skin_TAP_server_handshake(
      * big. That should be impossible. */
     log_info(LD_GENERAL, "crypto_dh_get_public failed.");
     goto err;
-    /* LCOV_EXCP_STOP */
+    /* LCOV_EXCL_STOP */
   }
 
   key_material_len = DIGEST_LEN+key_out_len;
diff --git a/src/or/onion_tap.h b/src/or/onion_tap.h
index a2880f6e98..713c1d7391 100644
--- a/src/or/onion_tap.h
+++ b/src/or/onion_tap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,5 +34,5 @@ int onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state,
                                 size_t key_out_len,
                                 const char **msg_out);
 
-#endif
+#endif /* !defined(TOR_ONION_TAP_H) */
 
diff --git a/src/or/or.h b/src/or/or.h
index 9f53c80644..0de07ed7ab 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -64,22 +64,25 @@
 #include <process.h>
 #include <direct.h>
 #include <windows.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include "crypto.h"
 #include "crypto_format.h"
 #include "tortls.h"
 #include "torlog.h"
 #include "container.h"
-#include "torgzip.h"
+#include "compress.h"
 #include "address.h"
 #include "compat_libevent.h"
 #include "ht.h"
+#include "confline.h"
 #include "replaycache.h"
 #include "crypto_curve25519.h"
 #include "crypto_ed25519.h"
 #include "tor_queue.h"
+#include "token_bucket.h"
 #include "util_format.h"
+#include "hs_circuitmap.h"
 
 /* These signals are defined to help handle_control_signal work.
  */
@@ -114,6 +117,9 @@
 #define NON_ANONYMOUS_MODE_ENABLED 1
 #endif
 
+/** Helper macro: Given a pointer to to.base_, of type from*, return &to. */
+#define DOWNCAST(to, ptr) ((to*)SUBTYPE_P(ptr, to, base_))
+
 /** Length of longest allowable configured nickname. */
 #define MAX_NICKNAME_LEN 19
 /** Length of a router identity encoded as a hexadecimal digest, plus
@@ -143,8 +149,27 @@
 /** Maximum size of a single extrainfo document, as above. */
 #define MAX_EXTRAINFO_UPLOAD_SIZE 50000
 
-/** How often do we rotate onion keys? */
-#define MIN_ONION_KEY_LIFETIME (7*24*60*60)
+/** Minimum lifetime for an onion key in days. */
+#define MIN_ONION_KEY_LIFETIME_DAYS (1)
+
+/** Maximum lifetime for an onion key in days. */
+#define MAX_ONION_KEY_LIFETIME_DAYS (90)
+
+/** Default lifetime for an onion key in days. */
+#define DEFAULT_ONION_KEY_LIFETIME_DAYS (28)
+
+/** Minimum grace period for acceptance of an onion key in days.
+ * The maximum value is defined in proposal #274 as being the current network
+ * consensus parameter for "onion-key-rotation-days". */
+#define MIN_ONION_KEY_GRACE_PERIOD_DAYS (1)
+
+/** Default grace period for acceptance of an onion key in days. */
+#define DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS (7)
+
+/** How often we should check the network consensus if it is time to rotate or
+ * expire onion keys. */
+#define ONION_KEY_CONSENSUS_CHECK_INTERVAL (60*60)
+
 /** How often do we rotate TLS contexts? */
 #define MAX_SSL_KEY_LIFETIME_INTERNAL (2*60*60)
 
@@ -202,8 +227,10 @@ typedef enum {
 #define CONN_TYPE_EXT_OR 16
 /** Type for sockets listening for Extended ORPort connections. */
 #define CONN_TYPE_EXT_OR_LISTENER 17
+/** Type for sockets listening for HTTP CONNECT tunnel connections. */
+#define CONN_TYPE_AP_HTTP_CONNECT_LISTENER 18
 
-#define CONN_TYPE_MAX_ 17
+#define CONN_TYPE_MAX_ 19
 /* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in
  * connection_t. */
 
@@ -324,7 +351,9 @@ typedef enum {
 /** State for a transparent natd connection: waiting for original
  * destination. */
 #define AP_CONN_STATE_NATD_WAIT 12
-#define AP_CONN_STATE_MAX_ 12
+/** State for an HTTP tunnel: waiting for an HTTP CONNECT command. */
+#define AP_CONN_STATE_HTTP_CONNECT_WAIT 13
+#define AP_CONN_STATE_MAX_ 13
 
 /** True iff the AP_CONN_STATE_* value <b>s</b> means that the corresponding
  * edge connection is not attached to any circuit. */
@@ -397,14 +426,23 @@ typedef enum {
 #define DIR_PURPOSE_FETCH_RENDDESC_V2 18
 /** A connection to a directory server: download a microdescriptor. */
 #define DIR_PURPOSE_FETCH_MICRODESC 19
-#define DIR_PURPOSE_MAX_ 19
-
-/** True iff <b>p</b> is a purpose corresponding to uploading data to a
- * directory server. */
+/** A connection to a hidden service directory: upload a v3 descriptor. */
+#define DIR_PURPOSE_UPLOAD_HSDESC 20
+/** A connection to a hidden service directory: fetch a v3 descriptor. */
+#define DIR_PURPOSE_FETCH_HSDESC 21
+/** A connection to a directory server: set after a hidden service descriptor
+ * is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
+#define DIR_PURPOSE_MAX_ 22
+
+/** True iff <b>p</b> is a purpose corresponding to uploading
+ * data to a directory server. */
 #define DIR_PURPOSE_IS_UPLOAD(p)                \
   ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
    (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
-   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES)
+   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES ||        \
+   (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 ||       \
+   (p)==DIR_PURPOSE_UPLOAD_HSDESC)
 
 #define EXIT_PURPOSE_MIN_ 1
 /** This exit stream wants to do an ordinary connect. */
@@ -423,8 +461,12 @@ typedef enum {
 /** Circuit state: I'd like to deliver a create, but my n_chan is still
  * connecting. */
 #define CIRCUIT_STATE_CHAN_WAIT 2
+/** Circuit state: the circuit is open but we don't want to actually use it
+ * until we find out if a better guard will be available.
+ */
+#define CIRCUIT_STATE_GUARD_WAIT 3
 /** Circuit state: onionskin(s) processed, ready to send/receive cells. */
-#define CIRCUIT_STATE_OPEN 3
+#define CIRCUIT_STATE_OPEN 4
 
 #define CIRCUIT_PURPOSE_MIN_ 1
 
@@ -465,6 +507,7 @@ typedef enum {
  */
 /** Client-side circuit purpose: Normal circuit, with cpath. */
 #define CIRCUIT_PURPOSE_C_GENERAL 5
+#define CIRCUIT_PURPOSE_C_HS_MIN_ 6
 /** Client-side circuit purpose: at the client, connecting to intro point. */
 #define CIRCUIT_PURPOSE_C_INTRODUCING 6
 /** Client-side circuit purpose: at the client, sent INTRODUCE1 to intro point,
@@ -482,28 +525,46 @@ typedef enum {
 #define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11
 /** Client-side circuit purpose: at the client, rendezvous established. */
 #define CIRCUIT_PURPOSE_C_REND_JOINED 12
+/** This circuit is used for getting hsdirs */
+#define CIRCUIT_PURPOSE_C_HSDIR_GET 13
+#define CIRCUIT_PURPOSE_C_HS_MAX_ 13
 /** This circuit is used for build time measurement only */
-#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13
-#define CIRCUIT_PURPOSE_C_MAX_ 13
+#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14
+#define CIRCUIT_PURPOSE_C_MAX_ 14
+
+#define CIRCUIT_PURPOSE_S_HS_MIN_ 15
 /** Hidden-service-side circuit purpose: at the service, waiting for
  * introductions. */
-#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14
+#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15
 /** Hidden-service-side circuit purpose: at the service, successfully
  * established intro. */
-#define CIRCUIT_PURPOSE_S_INTRO 15
+#define CIRCUIT_PURPOSE_S_INTRO 16
 /** Hidden-service-side circuit purpose: at the service, connecting to rend
  * point. */
-#define CIRCUIT_PURPOSE_S_CONNECT_REND 16
+#define CIRCUIT_PURPOSE_S_CONNECT_REND 17
 /** Hidden-service-side circuit purpose: at the service, rendezvous
  * established. */
-#define CIRCUIT_PURPOSE_S_REND_JOINED 17
+#define CIRCUIT_PURPOSE_S_REND_JOINED 18
+/** This circuit is used for uploading hsdirs */
+#define CIRCUIT_PURPOSE_S_HSDIR_POST 19
+#define CIRCUIT_PURPOSE_S_HS_MAX_ 19
+
 /** A testing circuit; not meant to be used for actual traffic. */
-#define CIRCUIT_PURPOSE_TESTING 18
+#define CIRCUIT_PURPOSE_TESTING 20
 /** A controller made this circuit and Tor should not use it. */
-#define CIRCUIT_PURPOSE_CONTROLLER 19
+#define CIRCUIT_PURPOSE_CONTROLLER 21
 /** This circuit is used for path bias probing only */
-#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 20
-#define CIRCUIT_PURPOSE_MAX_ 20
+#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22
+
+/** This circuit is used for vanguards/restricted paths.
+ *
+ *  This type of circuit is *only* created preemptively and never
+ *  on-demand. When an HS operation needs to take place (e.g. connect to an
+ *  intro point), these circuits are then cannibalized and repurposed to the
+ *  actual needed HS purpose. */
+#define CIRCUIT_PURPOSE_HS_VANGUARDS 23
+
+#define CIRCUIT_PURPOSE_MAX_ 23
 /** A catch-all for unrecognized purposes. Currently we don't expect
  * to make or see any circuits with this purpose. */
 #define CIRCUIT_PURPOSE_UNKNOWN 255
@@ -526,6 +587,14 @@ typedef enum {
 /** True iff the circuit_t c is actually an or_circuit_t */
 #define CIRCUIT_IS_ORCIRC(c) (((circuit_t *)(c))->magic == OR_CIRCUIT_MAGIC)
 
+/** True iff this circuit purpose should count towards the global
+ * pending rate limit (set by MaxClientCircuitsPending). We count all
+ * general purpose circuits, as well as the first step of client onion
+ * service connections (HSDir gets). */
+#define CIRCUIT_PURPOSE_COUNTS_TOWARDS_MAXPENDING(p) \
+    ((p) == CIRCUIT_PURPOSE_C_GENERAL || \
+     (p) == CIRCUIT_PURPOSE_C_HSDIR_GET)
+
 /** How many circuits do we want simultaneously in-progress to handle
  * a given stream? */
 #define MIN_CIRCUITS_HANDLING_STREAM 2
@@ -611,6 +680,10 @@ typedef enum {
 /** The target address is in a private network (like 127.0.0.1 or 10.0.0.1);
  * you don't want to do that over a randomly chosen exit */
 #define END_STREAM_REASON_PRIVATE_ADDR 262
+/** This is an HTTP tunnel connection and the client used or misused HTTP in a
+ * way we can't handle.
+ */
+#define END_STREAM_REASON_HTTPPROTOCOL 263
 
 /** Bitwise-and this value with endreason to mask out all flags. */
 #define END_STREAM_REASON_MASK 511
@@ -655,8 +728,8 @@ typedef enum {
 /** Catch-all "other" reason for closing origin circuits. */
 #define END_CIRC_AT_ORIGIN              -1
 
-/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt for
- * documentation of these. */
+/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt
+ * section 5.4 for documentation of these. */
 #define END_CIRC_REASON_MIN_            0
 #define END_CIRC_REASON_NONE            0
 #define END_CIRC_REASON_TORPROTOCOL     1
@@ -702,15 +775,15 @@ typedef enum {
 #define REND_NUMBER_OF_CONSECUTIVE_REPLICAS 3
 
 /** Length of v2 descriptor ID (32 base32 chars = 160 bits). */
-#define REND_DESC_ID_V2_LEN_BASE32 32
+#define REND_DESC_ID_V2_LEN_BASE32 BASE32_DIGEST_LEN
 
 /** Length of the base32-encoded secret ID part of versioned hidden service
  * descriptors. */
-#define REND_SECRET_ID_PART_LEN_BASE32 32
+#define REND_SECRET_ID_PART_LEN_BASE32 BASE32_DIGEST_LEN
 
 /** Length of the base32-encoded hash of an introduction point's
  * identity key. */
-#define REND_INTRO_POINT_ID_LEN_BASE32 32
+#define REND_INTRO_POINT_ID_LEN_BASE32 BASE32_DIGEST_LEN
 
 /** Length of the descriptor cookie that is used for client authorization
  * to hidden services. */
@@ -767,6 +840,24 @@ typedef struct rend_service_authorization_t {
  * establishment. Not all fields contain data depending on where this struct
  * is used. */
 typedef struct rend_data_t {
+  /* Hidden service protocol version of this base object. */
+  uint32_t version;
+
+  /** List of HSDir fingerprints on which this request has been sent to. This
+   * contains binary identity digest of the directory of size DIGEST_LEN. */
+  smartlist_t *hsdirs_fp;
+
+  /** Rendezvous cookie used by both, client and service. */
+  char rend_cookie[REND_COOKIE_LEN];
+
+  /** Number of streams associated with this rendezvous circuit. */
+  int nr_streams;
+} rend_data_t;
+
+typedef struct rend_data_v2_t {
+  /* Rendezvous base data. */
+  rend_data_t base_;
+
   /** Onion address (without the .onion part) that a client requests. */
   char onion_address[REND_SERVICE_ID_LEN_BASE32+1];
 
@@ -788,17 +879,34 @@ typedef struct rend_data_t {
 
   /** Hash of the hidden service's PK used by a service. */
   char rend_pk_digest[DIGEST_LEN];
+} rend_data_v2_t;
 
-  /** Rendezvous cookie used by both, client and service. */
-  char rend_cookie[REND_COOKIE_LEN];
+/* From a base rend_data_t object <b>d</d>, return the v2 object. */
+static inline
+rend_data_v2_t *TO_REND_DATA_V2(const rend_data_t *d)
+{
+  tor_assert(d);
+  tor_assert(d->version == 2);
+  return DOWNCAST(rend_data_v2_t, d);
+}
 
-  /** List of HSDir fingerprints on which this request has been sent to.
-   * This contains binary identity digest of the directory. */
-  smartlist_t *hsdirs_fp;
+/* Stub because we can't include hs_ident.h. */
+struct hs_ident_edge_conn_t;
+struct hs_ident_dir_conn_t;
+struct hs_ident_circuit_t;
 
-  /** Number of streams associated with this rendezvous circuit. */
-  int nr_streams;
-} rend_data_t;
+/* Hidden service directory index used in a node_t which is set once we set
+ * the consensus. */
+typedef struct hsdir_index_t {
+  /* HSDir index to use when fetching a descriptor. */
+  uint8_t fetch[DIGEST256_LEN];
+
+  /* HSDir index used by services to store their first and second
+   * descriptor. The first descriptor is chronologically older than the second
+   * one and uses older TP and SRV values. */
+  uint8_t store_first[DIGEST256_LEN];
+  uint8_t store_second[DIGEST256_LEN];
+} hsdir_index_t;
 
 /** Time interval for tracking replays of DH public keys received in
  * INTRODUCE2 cells.  Used only to avoid launching multiple
@@ -821,6 +929,7 @@ typedef enum {
 /** Initial value on both sides of a stream transmission window when the
  * stream is initialized.  Measured in cells. */
 #define STREAMWINDOW_START 500
+#define STREAMWINDOW_START_MAX 500
 /** Amount to increment a stream window when we get a stream SENDME. */
 #define STREAMWINDOW_INCREMENT 50
 
@@ -850,6 +959,7 @@ typedef enum {
 #define CELL_RELAY_EARLY 9
 #define CELL_CREATE2 10
 #define CELL_CREATED2 11
+#define CELL_PADDING_NEGOTIATE 12
 
 #define CELL_VPADDING 128
 #define CELL_CERTS 129
@@ -1096,8 +1206,8 @@ typedef struct packed_cell_t {
   /** Next cell queued on this circuit. */
   TOR_SIMPLEQ_ENTRY(packed_cell_t) next;
   char body[CELL_MAX_NETWORK_SIZE]; /**< Cell as packed for network. */
-  uint32_t inserted_time; /**< Time (in milliseconds since epoch, with high
-                           * bits truncated) when this cell was inserted. */
+  uint32_t inserted_timestamp; /**< Time (in timestamp units) when this cell
+                                * was inserted */
 } packed_cell_t;
 
 /** A queue of cells on a circuit, waiting to be added to the
@@ -1112,7 +1222,8 @@ typedef struct cell_queue_t {
 typedef struct destroy_cell_t {
   TOR_SIMPLEQ_ENTRY(destroy_cell_t) next;
   circid_t circid;
-  uint32_t inserted_time; /** Timestamp when this was queued. */
+  uint32_t inserted_timestamp; /**< Time (in timestamp units) when this cell
+                                * was inserted */
   uint8_t reason;
 } destroy_cell_t;
 
@@ -1132,11 +1243,8 @@ typedef struct {
   uint16_t length; /**< How long is the payload body? */
 } relay_header_t;
 
-typedef struct buf_t buf_t;
 typedef struct socks_request_t socks_request_t;
 
-#define buf_t buf_t
-
 typedef struct entry_port_cfg_t {
   /* Client port types (socks, dns, trans, natd) only: */
   uint8_t isolation_flags; /**< Zero or more isolation flags */
@@ -1196,6 +1304,8 @@ typedef struct server_port_cfg_t {
 #define CONTROL_CONNECTION_MAGIC 0x8abc765du
 #define LISTENER_CONNECTION_MAGIC 0x1a1ac741u
 
+struct buf_t;
+
 /** Description of a connection to another host or process, and associated
  * data.
  *
@@ -1267,14 +1377,15 @@ typedef struct connection_t {
 
   struct event *read_event; /**< Libevent event structure. */
   struct event *write_event; /**< Libevent event structure. */
-  buf_t *inbuf; /**< Buffer holding data read over this connection. */
-  buf_t *outbuf; /**< Buffer holding data to write over this connection. */
+  struct buf_t *inbuf; /**< Buffer holding data read over this connection. */
+  struct buf_t *outbuf; /**< Buffer holding data to write over this
+                         * connection. */
   size_t outbuf_flushlen; /**< How much data should we try to flush from the
                            * outbuf? */
-  time_t timestamp_lastread; /**< When was the last time libevent said we could
-                              * read? */
-  time_t timestamp_lastwritten; /**< When was the last time libevent said we
-                                 * could write? */
+  time_t timestamp_last_read_allowed; /**< When was the last time libevent said
+                                       * we could read? */
+  time_t timestamp_last_write_allowed; /**< When was the last time libevent
+                                        * said we could write? */
 
   time_t timestamp_created; /**< When was this connection_t created? */
 
@@ -1351,13 +1462,30 @@ typedef struct listener_connection_t {
 #define OR_CERT_TYPE_RSA_ED_CROSSCERT 7
 /**@}*/
 
-/** The one currently supported type of AUTHENTICATE cell.  It contains
+/** The first supported type of AUTHENTICATE cell.  It contains
  * a bunch of structures signed with an RSA1024 key.  The signed
  * structures include a HMAC using negotiated TLS secrets, and a digest
  * of all cells sent or received before the AUTHENTICATE cell (including
  * the random server-generated AUTH_CHALLENGE cell).
  */
 #define AUTHTYPE_RSA_SHA256_TLSSECRET 1
+/** As AUTHTYPE_RSA_SHA256_TLSSECRET, but instead of using the
+ * negotiated TLS secrets, uses exported keying material from the TLS
+ * session as described in RFC 5705.
+ *
+ * Not used by today's tors, since everything that supports this
+ * also supports ED25519_SHA256_5705, which is better.
+ **/
+#define AUTHTYPE_RSA_SHA256_RFC5705 2
+/** As AUTHTYPE_RSA_SHA256_RFC5705, but uses an Ed25519 identity key to
+ * authenticate.  */
+#define AUTHTYPE_ED25519_SHA256_RFC5705 3
+/*
+ * NOTE: authchallenge_type_is_better() relies on these AUTHTYPE codes
+ * being sorted in order of preference.  If we someday add one with
+ * a higher numerical value that we don't like as much, we should revise
+ * authchallenge_type_is_better().
+ */
 
 /** The length of the part of the AUTHENTICATE cell body that the client and
  * server can generate independently (when using RSA_SHA256_TLSSECRET). It
@@ -1368,6 +1496,34 @@ typedef struct listener_connection_t {
  * signs. */
 #define V3_AUTH_BODY_LEN (V3_AUTH_FIXED_PART_LEN + 8 + 16)
 
+/** Structure to hold all the certificates we've received on an OR connection
+ */
+typedef struct or_handshake_certs_t {
+  /** True iff we originated this connection. */
+  int started_here;
+  /** The cert for the 'auth' RSA key that's supposed to sign the AUTHENTICATE
+   * cell. Signed with the RSA identity key. */
+  tor_x509_cert_t *auth_cert;
+  /** The cert for the 'link' RSA key that was used to negotiate the TLS
+   *  connection.  Signed with the RSA identity key. */
+  tor_x509_cert_t *link_cert;
+  /** A self-signed identity certificate: the RSA identity key signed
+   * with itself.  */
+  tor_x509_cert_t *id_cert;
+  /** The Ed25519 signing key, signed with the Ed25519 identity key. */
+  struct tor_cert_st *ed_id_sign;
+  /** A digest of the X509 link certificate for the TLS connection, signed
+   * with the Ed25519 siging key. */
+  struct tor_cert_st *ed_sign_link;
+  /** The Ed25519 authentication key (that's supposed to sign an AUTHENTICATE
+   * cell) , signed with the Ed25519 siging key. */
+  struct tor_cert_st *ed_sign_auth;
+  /** The Ed25519 identity key, crosssigned with the RSA identity key. */
+  uint8_t *ed_rsa_crosscert;
+  /** The length of <b>ed_rsa_crosscert</b> in bytes */
+  size_t ed_rsa_crosscert_len;
+} or_handshake_certs_t;
+
 /** Stores flags and information related to the portion of a v2/v3 Tor OR
  * connection handshake that happens after the TLS handshake is finished.
  */
@@ -1388,10 +1544,18 @@ typedef struct or_handshake_state_t {
 
   /* True iff we've received valid authentication to some identity. */
   unsigned int authenticated : 1;
+  unsigned int authenticated_rsa : 1;
+  unsigned int authenticated_ed25519 : 1;
 
   /* True iff we have sent a netinfo cell */
   unsigned int sent_netinfo : 1;
 
+  /** The signing->ed25519 link certificate corresponding to the x509
+   * certificate we used on the TLS connection (if this is a server-side
+   * connection). We make a copy of this here to prevent a race condition
+   * caused by TLS context rotation. */
+  struct tor_cert_st *own_link_cert;
+
   /** True iff we should feed outgoing cells into digest_sent and
    * digest_received respectively.
    *
@@ -1405,9 +1569,12 @@ typedef struct or_handshake_state_t {
   unsigned int digest_received_data : 1;
   /**@}*/
 
-  /** Identity digest that we have received and authenticated for our peer
+  /** Identity RSA digest that we have received and authenticated for our peer
    * on this connection. */
-  uint8_t authenticated_peer_id[DIGEST_LEN];
+  uint8_t authenticated_rsa_peer_id[DIGEST_LEN];
+  /** Identity Ed25519 public key that we have received and authenticated for
+   * our peer on this connection. */
+  ed25519_public_key_t authenticated_ed25519_peer_id;
 
   /** Digests of the cells that we have sent or received as part of a V3
    * handshake.  Used for making and checking AUTHENTICATE cells.
@@ -1420,14 +1587,8 @@ typedef struct or_handshake_state_t {
 
   /** Certificates that a connection initiator sent us in a CERTS cell; we're
    * holding on to them until we get an AUTHENTICATE cell.
-   *
-   * @{
    */
-  /** The cert for the key that's supposed to sign the AUTHENTICATE cell */
-  tor_x509_cert_t *auth_cert;
-  /** A self-signed identity certificate */
-  tor_x509_cert_t *id_cert;
-  /**@}*/
+  or_handshake_certs_t *certs;
 } or_handshake_state_t;
 
 /** Length of Extended ORPort connection identifier. */
@@ -1489,10 +1650,6 @@ typedef struct or_connection_t {
    * NETINFO cell listed the address we're connected to as recognized. */
   unsigned int is_canonical:1;
 
-  /** True iff we have decided that the other end of this connection
-   * is a client.  Connections with this flag set should never be used
-   * to satisfy an EXTEND request.  */
-  unsigned int is_connection_with_client:1;
   /** True iff this is an outgoing connection. */
   unsigned int is_outgoing:1;
   unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
@@ -1516,22 +1673,8 @@ typedef struct or_connection_t {
 
   time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/
 
-  /* bandwidth* and *_bucket only used by ORs in OPEN state: */
-  int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */
-  int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */
-  int read_bucket; /**< When this hits 0, stop receiving. Every second we
-                    * add 'bandwidthrate' to this, capping it at
-                    * bandwidthburst. (OPEN ORs only) */
-  int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */
-
-  struct or_connection_t *next_with_same_id; /**< Next connection with same
-                                              * identity digest as this one. */
-  /** Last emptied read token bucket in msec since midnight; only used if
-   * TB_EMPTY events are enabled. */
-  uint32_t read_emptied_time;
-  /** Last emptied write token bucket in msec since midnight; only used if
-   * TB_EMPTY events are enabled. */
-  uint32_t write_emptied_time;
+  token_bucket_rw_t bucket; /**< Used for rate limiting when the connection is
+                          * in state CONN_OPEN. */
 
   /*
    * Count the number of bytes flushed out on this orconn, and the number of
@@ -1561,6 +1704,11 @@ typedef struct edge_connection_t {
    * an exit)? */
   rend_data_t *rend_data;
 
+  /* Hidden service connection identifier for edge connections. Used by the HS
+   * client-side code to identify client SOCKS connections and by the
+   * service-side code to match HS circuits with their streams. */
+  struct hs_ident_edge_conn_t *hs_ident;
+
   uint32_t address_ttl; /**< TTL for address-to-addr mapping on exit
                          * connection.  Exit connections only. */
   uint32_t begincell_flags; /** Flags sent or received in the BEGIN cell
@@ -1597,12 +1745,35 @@ typedef struct edge_connection_t {
   uint64_t dirreq_id;
 } edge_connection_t;
 
+/**
+ * Struct to track a connection that we closed that the other end
+ * still thinks is open. Exists in origin_circuit_t.half_streams until
+ * we get an end cell or a resolved cell for this stream id.
+ */
+typedef struct half_edge_t {
+  /** stream_id for the half-closed connection */
+  streamid_t stream_id;
+
+  /** How many sendme's can the other end still send, based on how
+   * much data we had sent at the time of close */
+  int sendmes_pending;
+
+  /** How much more data can the other end still send, based on
+   * our deliver window */
+  int data_pending;
+
+  /** Is there a connected cell pending? */
+  int connected_pending : 1;
+} half_edge_t;
+
 /** Subtype of edge_connection_t for an "entry connection" -- that is, a SOCKS
  * connection, a DNS request, a TransPort connection or a NATD connection */
 typedef struct entry_connection_t {
   edge_connection_t edge_;
 
   /** Nickname of planned exit node -- used with .exit support. */
+  /* XXX prop220: we need to make chosen_exit_name able to encode Ed IDs too.
+   * That's logically part of the UI parts for prop220 though. */
   char *chosen_exit_name;
 
   socks_request_t *socks_request; /**< SOCKS structure describing request (AP
@@ -1628,11 +1799,11 @@ typedef struct entry_connection_t {
   /** For AP connections only: buffer for data that we have sent
    * optimistically, which we might need to re-send if we have to
    * retry this connection. */
-  buf_t *pending_optimistic_data;
+  struct buf_t *pending_optimistic_data;
   /* For AP connections only: buffer for data that we previously sent
   * optimistically which we are currently re-sending as we retry this
   * connection. */
-  buf_t *sending_optimistic_data;
+  struct buf_t *sending_optimistic_data;
 
   /** If this is a DNSPort connection, this field holds the pending DNS
    * request that we're going to try to answer.  */
@@ -1677,19 +1848,8 @@ typedef struct entry_connection_t {
    * the exit has sent a CONNECTED cell) and we have chosen to use it.
    */
   unsigned int may_use_optimistic_data : 1;
-
-  /** Are we a socks SocksSocket listener? */
-  unsigned int is_socks_socket:1;
 } entry_connection_t;
 
-typedef enum {
-    DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP,
-    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
-    DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS,
-    DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */
-} dir_spool_source_t;
-#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
-
 /** Subtype of connection_t for an "directory connection" -- that is, an HTTP
  * connection to retrieve or serve directory material. */
 typedef struct dir_connection_t {
@@ -1698,33 +1858,34 @@ typedef struct dir_connection_t {
  /** Which 'resource' did we ask the directory for? This is typically the part
   * of the URL string that defines, relative to the directory conn purpose,
   * what thing we want.  For example, in router descriptor downloads by
-  * descriptor digest, it contains "d/", then one ore more +-separated
+  * descriptor digest, it contains "d/", then one or more +-separated
   * fingerprints.
   **/
   char *requested_resource;
   unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
 
-  /* Used only for server sides of some dir connections, to implement
-   * "spooling" of directory material to the outbuf.  Otherwise, we'd have
-   * to append everything to the outbuf in one enormous chunk. */
-  /** What exactly are we spooling right now? */
-  dir_spool_source_bitfield_t  dir_spool_src : 3;
-
   /** If we're fetching descriptors, what router purpose shall we assign
    * to them? */
   uint8_t router_purpose;
-  /** List of fingerprints for networkstatuses or descriptors to be spooled. */
-  smartlist_t *fingerprint_stack;
-  /** A cached_dir_t object that we're currently spooling out */
-  struct cached_dir_t *cached_dir;
-  /** The current offset into cached_dir. */
-  off_t cached_dir_offset;
-  /** The zlib object doing on-the-fly compression for spooled data. */
-  tor_zlib_state_t *zlib_state;
+
+  /** List of spooled_resource_t for objects that we're spooling. We use
+   * it from back to front. */
+  smartlist_t *spool;
+  /** The compression object doing on-the-fly compression for spooled data. */
+  tor_compress_state_t *compress_state;
 
   /** What rendezvous service are we querying for? */
   rend_data_t *rend_data;
 
+  /* Hidden service connection identifier for dir connections: Used by HS
+     client-side code to fetch HS descriptors, and by the service-side code to
+     upload descriptors. */
+  struct hs_ident_dir_conn_t *hs_ident;
+
+  /** If this is a one-hop connection, tracks the state of the directory guard
+   * for this connection (if any). */
+  struct circuit_guard_state_t *guard_state;
+
   char identity_digest[DIGEST_LEN]; /**< Hash of the public RSA key for
                                      * the directory server's signing key. */
 
@@ -1732,6 +1893,14 @@ typedef struct dir_connection_t {
    * that's going away and being used on channels instead.  The dirserver still
    * needs this for the incoming side, so it's moved here. */
   uint64_t dirreq_id;
+
+#ifdef MEASUREMENTS_21206
+  /** Number of RELAY_DATA cells received. */
+  uint32_t data_cells_received;
+
+  /** Number of RELAY_DATA cells sent. */
+  uint32_t data_cells_sent;
+#endif /* defined(MEASUREMENTS_21206) */
 } dir_connection_t;
 
 /** Subtype of connection_t for an connection to a controller. */
@@ -1768,8 +1937,6 @@ typedef struct control_connection_t {
 
 /** Cast a connection_t subtype pointer to a connection_t **/
 #define TO_CONN(c) (&(((c)->base_)))
-/** Helper macro: Given a pointer to to.base_, of type from*, return &to. */
-#define DOWNCAST(to, ptr) ((to*)SUBTYPE_P(ptr, to, base_))
 
 /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
 #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
@@ -1872,11 +2039,13 @@ typedef struct addr_policy_t {
  * compressed form. */
 typedef struct cached_dir_t {
   char *dir; /**< Contents of this object, NUL-terminated. */
-  char *dir_z; /**< Compressed contents of this object. */
+  char *dir_compressed; /**< Compressed contents of this object. */
   size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */
-  size_t dir_z_len; /**< Length of <b>dir_z</b>. */
+  size_t dir_compressed_len; /**< Length of <b>dir_compressed</b>. */
   time_t published; /**< When was this object published. */
   common_digests_t digests; /**< Digests of this object (networkstatus only) */
+  /** Sha3 digest (also ns only) */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
   int refcnt; /**< Reference count for this cached_dir_t. */
 } cached_dir_t;
 
@@ -1931,15 +2100,6 @@ typedef enum {
 #define download_schedule_increment_bitfield_t \
                                         ENUM_BF(download_schedule_increment_t)
 
-/** Enumeration: do we want to use the random exponential backoff
- * mechanism? */
-typedef enum {
-  DL_SCHED_DETERMINISTIC = 0,
-  DL_SCHED_RANDOM_EXPONENTIAL = 1,
-} download_schedule_backoff_t;
-#define download_schedule_backoff_bitfield_t \
-                                        ENUM_BF(download_schedule_backoff_t)
-
 /** Information about our plans for retrying downloads for a downloadable
  * directory object.
  * Each type of downloadable directory object has a corresponding retry
@@ -1986,9 +2146,6 @@ typedef struct download_status_t {
   download_schedule_increment_bitfield_t increment_on : 1; /**< does this
                                         * schedule increment on each attempt,
                                         * or after each failure? */
-  download_schedule_backoff_bitfield_t backoff : 1; /**< do we use the
-                                        * deterministic schedule, or random
-                                        * exponential backoffs? */
   uint8_t last_backoff_position; /**< number of attempts/failures, depending
                                   * on increment_on, when we last recalculated
                                   * the delay.  Only updated if backoff
@@ -2056,6 +2213,43 @@ typedef struct signed_descriptor_t {
 /** A signed integer representing a country code. */
 typedef int16_t country_t;
 
+/** Flags used to summarize the declared protocol versions of a relay,
+ * so we don't need to parse them again and again. */
+typedef struct protover_summary_flags_t {
+  /** True iff we have a proto line for this router, or a versions line
+   * from which we could infer the protocols. */
+  unsigned int protocols_known:1;
+
+  /** True iff this router has a version or protocol list that allows it to
+   * accept EXTEND2 cells. This requires Relay=2. */
+  unsigned int supports_extend2_cells:1;
+
+  /** True iff this router has a protocol list that allows it to negotiate
+   * ed25519 identity keys on a link handshake with us. This
+   * requires LinkAuth=3. */
+  unsigned int supports_ed25519_link_handshake_compat:1;
+
+  /** True iff this router has a protocol list that allows it to negotiate
+   * ed25519 identity keys on a link handshake, at all. This requires some
+   * LinkAuth=X for X >= 3. */
+  unsigned int supports_ed25519_link_handshake_any:1;
+
+  /** True iff this router has a protocol list that allows it to be an
+   * introduction point supporting ed25519 authentication key which is part of
+   * the v3 protocol detailed in proposal 224. This requires HSIntro=4. */
+  unsigned int supports_ed25519_hs_intro : 1;
+
+  /** True iff this router has a protocol list that allows it to be an hidden
+   * service directory supporting version 3 as seen in proposal 224. This
+   * requires HSDir=2. */
+  unsigned int supports_v3_hsdir : 1;
+
+  /** True iff this router has a protocol list that allows it to be an hidden
+   * service rendezvous point supporting version 3 as seen in proposal 224.
+   * This requires HSRend=2. */
+  unsigned int supports_v3_rendezvous_point: 1;
+} protover_summary_flags_t;
+
 /** Information about another onion router in the network. */
 typedef struct {
   signed_descriptor_t cache_info;
@@ -2124,23 +2318,26 @@ typedef struct {
    * this routerinfo. Used only during voting. */
   unsigned int omit_from_vote:1;
 
+  /** Flags to summarize the protocol versions for this routerinfo_t. */
+  protover_summary_flags_t pv;
+
 /** Tor can use this router for general positions in circuits; we got it
  * from a directory server as usual, or we're an authority and a server
  * uploaded it. */
 #define ROUTER_PURPOSE_GENERAL 0
 /** Tor should avoid using this router for circuit-building: we got it
- * from a crontroller.  If the controller wants to use it, it'll have to
+ * from a controller.  If the controller wants to use it, it'll have to
  * ask for it by identity. */
 #define ROUTER_PURPOSE_CONTROLLER 1
 /** Tor should use this router only for bridge positions in circuits: we got
  * it via a directory request from the bridge itself, or a bridge
- * authority. x*/
+ * authority. */
 #define ROUTER_PURPOSE_BRIDGE 2
 /** Tor should not use this router; it was marked in cached-descriptors with
  * a purpose we didn't recognize. */
 #define ROUTER_PURPOSE_UNKNOWN 255
 
-  /* In what way did we find out about this router?  One of ROUTER_PURPOSE_*.
+  /** In what way did we find out about this router?  One of ROUTER_PURPOSE_*.
    * Routers of different purposes are kept segregated and used for different
    * things; see notes on ROUTER_PURPOSE_* macros above.
    */
@@ -2176,10 +2373,10 @@ typedef struct routerstatus_t {
    * If it's a descriptor, we only use the first DIGEST_LEN bytes. */
   char descriptor_digest[DIGEST256_LEN];
   uint32_t addr; /**< IPv4 address for this router, in host order. */
-  uint16_t or_port; /**< OR port for this router. */
+  uint16_t or_port; /**< IPv4 OR port for this router. */
   uint16_t dir_port; /**< Directory port for this router. */
   tor_addr_t ipv6_addr; /**< IPv6 address for this router. */
-  uint16_t ipv6_orport; /**<IPV6 OR port for this router. */
+  uint16_t ipv6_orport; /**< IPv6 OR port for this router. */
   unsigned int is_authority:1; /**< True iff this router is an authority. */
   unsigned int is_exit:1; /**< True iff this router is a good exit. */
   unsigned int is_stable:1; /**< True iff this router stays up a long time. */
@@ -2202,19 +2399,15 @@ typedef struct routerstatus_t {
   unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
                              * or it claims to accept tunnelled dir requests.
                              */
-  /** True iff we have a proto line for this router, or a versions line
-   * from which we could infer the protocols. */
-  unsigned int protocols_known:1;
-
-  /** True iff this router has a version or protocol list that allows it to
-   * accept EXTEND2 cells */
-  unsigned int supports_extend2_cells:1;
 
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
   unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
                                     * the Unmeasured flag set. */
 
+  /** Flags to summarize the protocol versions for this routerstatus_t. */
+  protover_summary_flags_t pv;
+
   uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in
                        * the vote/consensus, in kilobytes/sec. */
 
@@ -2335,6 +2528,8 @@ typedef struct node_t {
 
   /** Used to look up the node_t by its identity digest. */
   HT_ENTRY(node_t) ht_ent;
+  /** Used to look up the node_t by its ed25519 identity digest. */
+  HT_ENTRY(node_t) ed_ht_ent;
   /** Position of the node within the list of nodes */
   int nodelist_idx;
 
@@ -2342,6 +2537,13 @@ typedef struct node_t {
    * identity may exist at a time. */
   char identity[DIGEST_LEN];
 
+  /** The ed25519 identity of this node_t. This field is nonzero iff we
+   * currently have an ed25519 identity for this node in either md or ri,
+   * _and_ this node has been inserted to the ed25519-to-node map in the
+   * nodelist.
+   */
+  ed25519_public_key_t ed25519_id;
+
   microdesc_t *md;
   routerinfo_t *ri;
   routerstatus_t *rs;
@@ -2372,9 +2574,6 @@ typedef struct node_t {
   /** Local info: we treat this node as if it rejects everything */
   unsigned int rejects_all:1;
 
-  /** Local info: this node is in our list of guards */
-  unsigned int using_as_guard:1;
-
   /* Local info: derived. */
 
   /** True if the IPv6 OR port is preferred over the IPv4 OR port.
@@ -2392,6 +2591,10 @@ typedef struct node_t {
   time_t last_reachable;        /* IPv4. */
   time_t last_reachable6;       /* IPv6. */
 
+  /* Hidden service directory index data. This is used by a service or client
+   * in order to know what's the hs directory index for this node at the time
+   * the consensus is set. */
+  struct hsdir_index_t hsdir_index;
 } node_t;
 
 /** Linked list of microdesc hash lines for a single router in a directory
@@ -2566,6 +2769,9 @@ typedef struct networkstatus_t {
 
   /** Digests of this document, as signed. */
   common_digests_t digests;
+  /** A SHA3-256 digest of the document, not including signatures: used for
+   * consensus diffs */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
 
   /** List of router statuses, sorted by identity digest.  For a vote,
    * the elements are vote_routerstatus_t; for a consensus, the elements
@@ -2654,7 +2860,10 @@ typedef struct {
 typedef struct extend_info_t {
   char nickname[MAX_HEX_NICKNAME_LEN+1]; /**< This router's nickname for
                                           * display. */
-  char identity_digest[DIGEST_LEN]; /**< Hash of this router's identity key. */
+  /** Hash of this router's RSA identity key. */
+  char identity_digest[DIGEST_LEN];
+  /** Ed25519 identity for this router, if any. */
+  ed25519_public_key_t ed_identity;
   uint16_t port; /**< OR port. */
   tor_addr_t addr; /**< IP address. */
   crypto_pk_t *onion_key; /**< Current onionskin key. */
@@ -2720,11 +2929,7 @@ typedef struct {
   } u;
 } onion_handshake_state_t;
 
-/** Holds accounting information for a single step in the layered encryption
- * performed by a circuit.  Used only at the client edge of a circuit. */
-typedef struct crypt_path_t {
-  uint32_t magic;
-
+typedef struct relay_crypto_t {
   /* crypto environments */
   /** Encryption key and counter for cells heading towards the OR at this
    * step. */
@@ -2738,6 +2943,17 @@ typedef struct crypt_path_t {
   /** Digest state for cells heading away from the OR at this step. */
   crypto_digest_t *b_digest;
 
+} relay_crypto_t;
+
+/** Holds accounting information for a single step in the layered encryption
+ * performed by a circuit.  Used only at the client edge of a circuit. */
+typedef struct crypt_path_t {
+  uint32_t magic;
+
+  /** Cryptographic state used for encrypting and authenticating relay
+   * cells to and from this hop. */
+  relay_crypto_t crypto;
+
   /** Current state of the handshake as performed with the OR at this
    * step. */
   onion_handshake_state_t handshake_state;
@@ -2958,7 +3174,7 @@ typedef struct circuit_t {
 
   /** When the circuit was first used, or 0 if the circuit is clean.
    *
-   * XXXX Note that some code will artifically adjust this value backward
+   * XXXX Note that some code will artificially adjust this value backward
    * in time in order to indicate that a circuit shouldn't be used for new
    * streams, but that it can stay alive as long as it has streams on it.
    * That's a kludge we should fix.
@@ -2995,19 +3211,17 @@ typedef struct circuit_t {
   /** Index in smartlist of all circuits (global_circuitlist). */
   int global_circuitlist_idx;
 
-  /** Next circuit in the doubly-linked ring of circuits waiting to add
-   * cells to n_conn.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *next_active_on_n_chan;
-  /** Previous circuit in the doubly-linked ring of circuits waiting to add
-   * cells to n_conn.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *prev_active_on_n_chan;
-
   /** Various statistics about cells being added to or removed from this
    * circuit's queues; used only if CELL_STATS events are enabled and
    * cleared after being sent to control port. */
   smartlist_t *testing_cell_stats;
+
+  /** If set, points to an HS token that this circuit might be carrying.
+   *  Used by the HS circuitmap.  */
+  hs_token_t *hs_token;
+  /** Hashtable node: used to look up the circuit by its HS token using the HS
+      circuitmap. */
+  HT_ENTRY(circuit_t) hs_circuitmap_node;
 } circuit_t;
 
 /** Largest number of relay_early cells that we can send on a given
@@ -3080,16 +3294,40 @@ typedef struct origin_circuit_t {
    * associated with this circuit. */
   edge_connection_t *p_streams;
 
-  /** Bytes read from any attached stream since last call to
+  /** Smartlist of half-closed streams (half_edge_t*) that still
+   * have pending activity */
+  smartlist_t *half_streams;
+
+  /** Bytes read on this circuit since last call to
    * control_event_circ_bandwidth_used().  Only used if we're configured
    * to emit CIRC_BW events. */
   uint32_t n_read_circ_bw;
 
-  /** Bytes written to any attached stream since last call to
+  /** Bytes written to on this circuit since last call to
    * control_event_circ_bandwidth_used().  Only used if we're configured
    * to emit CIRC_BW events. */
   uint32_t n_written_circ_bw;
 
+  /** Total known-valid relay cell bytes since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_delivered_read_circ_bw;
+
+  /** Total written relay cell bytes since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_delivered_written_circ_bw;
+
+  /** Total overhead data in all known-valid relay data cells since last
+   * call to control_event_circ_bandwidth_used().  Only used if we're
+   * configured to emit CIRC_BW events. */
+  uint32_t n_overhead_read_circ_bw;
+
+  /** Total written overhead data in all relay data cells since last call to
+   * control_event_circ_bandwidth_used().  Only used if we're configured
+   * to emit CIRC_BW events. */
+  uint32_t n_overhead_written_circ_bw;
+
   /** Build state for this circuit. It includes the intended path
    * length, the chosen exit router, rendezvous information, etc.
    */
@@ -3104,6 +3342,19 @@ typedef struct origin_circuit_t {
   /** Holds all rendezvous data on either client or service side. */
   rend_data_t *rend_data;
 
+  /** Holds hidden service identifier on either client or service side. This
+   * is for both introduction and rendezvous circuit. */
+  struct hs_ident_circuit_t *hs_ident;
+
+  /** Holds the data that the entry guard system uses to track the
+   * status of the guard this circuit is using, and thereby to determine
+   * whether this circuit can be used. */
+  struct circuit_guard_state_t *guard_state;
+
+  /** Index into global_origin_circuit_list for this circuit. -1 if not
+   * present. */
+  int global_origin_circuit_list_idx;
+
   /** How many more relay_early cells can we send on this circuit, according
    * to the specification? */
   unsigned int remaining_relay_early_cells : 4;
@@ -3196,7 +3447,7 @@ typedef struct origin_circuit_t {
   uint32_t global_identifier;
 
   /** True if we have associated one stream to this circuit, thereby setting
-   * the isolation paramaters for this circuit.  Note that this doesn't
+   * the isolation parameters for this circuit.  Note that this doesn't
    * necessarily mean that we've <em>attached</em> any streams to the circuit:
    * we may only have marked up this circuit during the launch process.
    */
@@ -3247,6 +3498,13 @@ typedef struct origin_circuit_t {
    * adjust_exit_policy_from_exitpolicy_failure.
    */
   smartlist_t *prepend_policy;
+
+  /** How long do we wait before closing this circuit if it remains
+   * completely idle after it was built, in seconds? This value
+   * is randomized on a per-circuit basis from CircuitsAvailableTimoeut
+   * to 2*CircuitsAvailableTimoeut. */
+  int circuit_idle_timeout;
+
 } origin_circuit_t;
 
 struct onion_queue_t;
@@ -3256,14 +3514,6 @@ struct onion_queue_t;
 typedef struct or_circuit_t {
   circuit_t base_;
 
-  /** Next circuit in the doubly-linked ring of circuits waiting to add
-   * cells to p_chan.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *next_active_on_p_chan;
-  /** Previous circuit in the doubly-linked ring of circuits waiting to add
-   * cells to p_chan.  NULL if we have no cells pending, or if we're not
-   * linked to an OR connection. */
-  struct circuit_t *prev_active_on_p_chan;
   /** Pointer to an entry on the onion queue, if this circuit is waiting for a
    * chance to give an onionskin to a cpuworker. Used only in onion.c */
   struct onion_queue_t *onionqueue_entry;
@@ -3288,28 +3538,15 @@ typedef struct or_circuit_t {
   /** Linked list of Exit streams associated with this circuit that are
    * still being resolved. */
   edge_connection_t *resolving_streams;
-  /** The cipher used by intermediate hops for cells heading toward the
-   * OP. */
-  crypto_cipher_t *p_crypto;
-  /** The cipher used by intermediate hops for cells heading away from
-   * the OP. */
-  crypto_cipher_t *n_crypto;
-
-  /** The integrity-checking digest used by intermediate hops, for
-   * cells packaged here and heading towards the OP.
-   */
-  crypto_digest_t *p_digest;
-  /** The integrity-checking digest used by intermediate hops, for
-   * cells packaged at the OP and arriving here.
-   */
-  crypto_digest_t *n_digest;
+
+  /** Cryptographic state used for encrypting and authenticating relay
+   * cells to and from this hop. */
+  relay_crypto_t crypto;
 
   /** Points to spliced circuit if purpose is REND_ESTABLISHED, and circuit
    * is not marked for close. */
   struct or_circuit_t *rend_splice;
 
-  struct or_circuit_rendinfo_s *rendinfo;
-
   /** Stores KH for the handshake. */
   char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */
 
@@ -3320,9 +3557,6 @@ typedef struct or_circuit_t {
   /* We have already received an INTRODUCE1 cell on this circuit. */
   unsigned int already_received_introduce1 : 1;
 
-  /** True iff this circuit was made with a CREATE_FAST cell. */
-  unsigned int is_first_hop : 1;
-
   /** If set, this circuit carries HS traffic. Consider it in any HS
    *  statistics. */
   unsigned int circuit_carries_hs_traffic_stats : 1;
@@ -3335,33 +3569,13 @@ typedef struct or_circuit_t {
    * exit-ward queues of this circuit; reset every time when writing
    * buffer stats to disk. */
   uint64_t total_cell_waiting_time;
-
-  /** Maximum cell queue size for a middle relay; this is stored per circuit
-   * so append_cell_to_circuit_queue() can adjust it if it changes.  If set
-   * to zero, it is initialized to the default value.
-   */
-  uint32_t max_middle_cells;
 } or_circuit_t;
 
-typedef struct or_circuit_rendinfo_s {
-
 #if REND_COOKIE_LEN != DIGEST_LEN
 #error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN"
 #endif
 #define REND_TOKEN_LEN DIGEST_LEN
 
-  /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a
-   * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes
-   * otherwise.
-   */
-  char rend_token[REND_TOKEN_LEN];
-
-  /** True if this is a rendezvous point circuit; false if this is an
-   * introduction point. */
-  unsigned is_rend_circ;
-
-} or_circuit_rendinfo_t;
-
 /** Convert a circuit subtype to a circuit_t. */
 #define TO_CIRCUIT(x)  (&((x)->base_))
 
@@ -3404,15 +3618,6 @@ static inline const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(
   return DOWNCAST(origin_circuit_t, x);
 }
 
-/** Bitfield type: things that we're willing to use invalid routers for. */
-typedef enum invalid_router_usage_t {
-  ALLOW_INVALID_ENTRY       =1,
-  ALLOW_INVALID_EXIT        =2,
-  ALLOW_INVALID_MIDDLE      =4,
-  ALLOW_INVALID_RENDEZVOUS  =8,
-  ALLOW_INVALID_INTRODUCTION=16,
-} invalid_router_usage_t;
-
 /* limits for TCP send and recv buffer size used for constrained sockets */
 #define MIN_CONSTRAINED_TCP_BUFFER 2048
 #define MAX_CONSTRAINED_TCP_BUFFER 262144  /* 256k */
@@ -3474,33 +3679,18 @@ typedef struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 } port_cfg_t;
 
-/** Ordinary configuration line. */
-#define CONFIG_LINE_NORMAL 0
-/** Appends to previous configuration for the same option, even if we
- * would ordinary replace it. */
-#define CONFIG_LINE_APPEND 1
-/* Removes all previous configuration for an option. */
-#define CONFIG_LINE_CLEAR 2
-
-/** A linked list of lines in a config file. */
-typedef struct config_line_t {
-  char *key;
-  char *value;
-  struct config_line_t *next;
-  /** What special treatment (if any) does this line require? */
-  unsigned int command:2;
-  /** If true, subsequent assignments to this linelist should replace
-   * it, not extend it.  Set only on the first item in a linelist in an
-   * or_options_t. */
-  unsigned int fragile:1;
-} config_line_t;
-
 typedef struct routerset_t routerset_t;
 
 /** A magic value for the (Socks|OR|...)Port options below, telling Tor
  * to pick its own port. */
 #define CFG_AUTO_PORT 0xc4005e
 
+/** Enumeration of outbound address configuration types:
+ * Exit-only, OR-only, or both */
+typedef enum {OUTBOUND_ADDR_EXIT, OUTBOUND_ADDR_OR,
+              OUTBOUND_ADDR_EXIT_AND_OR,
+              OUTBOUND_ADDR_MAX} outbound_addr_t;
+
 /** Configuration options for a Tor process. */
 typedef struct {
   uint32_t magic_;
@@ -3509,7 +3699,8 @@ typedef struct {
   enum {
     CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD,
     CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG,
-    CMD_KEYGEN
+    CMD_KEYGEN,
+    CMD_KEY_EXPIRATION,
   } command;
   char *command_arg; /**< Argument for command-line option. */
 
@@ -3522,10 +3713,24 @@ typedef struct {
   int TruncateLogFile; /**< Boolean: Should we truncate the log file
                             before we start writing? */
   char *SyslogIdentityTag; /**< Identity tag to add for syslog logging. */
+  char *AndroidIdentityTag; /**< Identity tag to add for Android logging. */
 
   char *DebugLogFile; /**< Where to send verbose log messages. */
-  char *DataDirectory; /**< OR only: where to store long-term data. */
+  char *DataDirectory_option; /**< Where to store long-term data, as
+                               * configured by the user. */
+  char *DataDirectory; /**< Where to store long-term data, as modified. */
   int DataDirectoryGroupReadable; /**< Boolean: Is the DataDirectory g+r? */
+
+  char *KeyDirectory_option; /**< Where to store keys, as
+                               * configured by the user. */
+  char *KeyDirectory; /**< Where to store keys data, as modified. */
+  int KeyDirectoryGroupReadable; /**< Boolean: Is the KeyDirectory g+r? */
+
+  char *CacheDirectory_option; /**< Where to store cached data, as
+                               * configured by the user. */
+  char *CacheDirectory; /**< Where to store cached data, as modified. */
+  int CacheDirectoryGroupReadable; /**< Boolean: Is the CacheDirectory g+r? */
+
   char *Nickname; /**< OR only: nickname of this onion router. */
   char *Address; /**< OR only: configured address for this onion router. */
   char *PidFile; /**< Where to store PID of Tor process. */
@@ -3553,10 +3758,6 @@ typedef struct {
   int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our
                        * process for all current and future memory. */
 
-  /** List of "entry", "middle", "exit", "introduction", "rendezvous". */
-  smartlist_t *AllowInvalidNodes;
-  /** Bitmask; derived from AllowInvalidNodes. */
-  invalid_router_usage_t AllowInvalid_;
   config_line_t *ExitPolicy; /**< Lists of exit policy components. */
   int ExitPolicyRejectPrivate; /**< Should we not exit to reserved private
                                 * addresses, and our own published addresses?
@@ -3565,29 +3766,19 @@ typedef struct {
                                         * interface addresses?
                                         * Includes OutboundBindAddresses and
                                         * configured ports. */
+  int ReducedExitPolicy; /**<Should we use the Reduced Exit Policy? */
   config_line_t *SocksPolicy; /**< Lists of socks policy components */
   config_line_t *DirPolicy; /**< Lists of dir policy components */
-  /** Addresses to bind for listening for SOCKS connections. */
-  config_line_t *SocksListenAddress;
-  /** Addresses to bind for listening for transparent pf/netfilter
-   * connections. */
-  config_line_t *TransListenAddress;
-  /** Addresses to bind for listening for transparent natd connections */
-  config_line_t *NATDListenAddress;
-  /** Addresses to bind for listening for SOCKS connections. */
-  config_line_t *DNSListenAddress;
-  /** Addresses to bind for listening for OR connections. */
-  config_line_t *ORListenAddress;
-  /** Addresses to bind for listening for directory connections. */
-  config_line_t *DirListenAddress;
-  /** Addresses to bind for listening for control connections. */
-  config_line_t *ControlListenAddress;
   /** Local address to bind outbound sockets */
   config_line_t *OutboundBindAddress;
-  /** IPv4 address derived from OutboundBindAddress. */
-  tor_addr_t OutboundBindAddressIPv4_;
-  /** IPv6 address derived from OutboundBindAddress. */
-  tor_addr_t OutboundBindAddressIPv6_;
+  /** Local address to bind outbound relay sockets */
+  config_line_t *OutboundBindAddressOR;
+  /** Local address to bind outbound exit sockets */
+  config_line_t *OutboundBindAddressExit;
+  /** Addresses derived from the various OutboundBindAddress lines.
+   * [][0] is IPv4, [][1] is IPv6
+   */
+  tor_addr_t OutboundBindAddresses[OUTBOUND_ADDR_MAX][2];
   /** Directory server only: which versions of
    * Tor should we tell users to run? */
   config_line_t *RecommendedVersions;
@@ -3599,7 +3790,6 @@ typedef struct {
   /** Whether routers accept EXTEND cells to routers with private IPs. */
   int ExtendAllowPrivateAddresses;
   char *User; /**< Name of user to run Tor as. */
-  char *Group; /**< Name of group to run Tor as. */
   config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */
   /** Ports to listen on for extended OR connections. */
   config_line_t *ExtORPort_lines;
@@ -3607,8 +3797,8 @@ typedef struct {
   config_line_t *SocksPort_lines;
   /** Ports to listen on for transparent pf/netfilter connections. */
   config_line_t *TransPort_lines;
-  const char *TransProxyType; /**< What kind of transparent proxy
-                               * implementation are we using? */
+  char *TransProxyType; /**< What kind of transparent proxy
+                         * implementation are we using? */
   /** Parsed value of TransProxyType. */
   enum {
     TPT_DEFAULT,
@@ -3618,13 +3808,15 @@ typedef struct {
   } TransProxyType_parsed;
   config_line_t *NATDPort_lines; /**< Ports to listen on for transparent natd
                             * connections. */
+  /** Ports to listen on for HTTP Tunnel connections. */
+  config_line_t *HTTPTunnelPort_lines;
   config_line_t *ControlPort_lines; /**< Ports to listen on for control
                                * connections. */
   config_line_t *ControlSocket; /**< List of Unix Domain Sockets to listen on
                                  * for control connections. */
 
   int ControlSocketsGroupWritable; /**< Boolean: Are control sockets g+rw? */
-  int SocksSocketsGroupWritable; /**< Boolean: Are SOCKS sockets g+rw? */
+  int UnixSocksGroupWritable; /**< Boolean: Are SOCKS Unix sockets g+rw? */
   /** Ports to listen on for directory connections. */
   config_line_t *DirPort_lines;
   config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */
@@ -3644,7 +3836,8 @@ typedef struct {
    * configured in one of the _lines options above.
    * For client ports, also true if there is a unix socket configured.
    * If you are checking for client ports, you may want to use:
-   *   SocksPort_set || TransPort_set || NATDPort_set || DNSPort_set
+   *   SocksPort_set || TransPort_set || NATDPort_set || DNSPort_set ||
+   *   HTTPTunnelPort_set
    * rather than SocksPort_set.
    *
    * @{
@@ -3657,6 +3850,7 @@ typedef struct {
   unsigned int DirPort_set : 1;
   unsigned int DNSPort_set : 1;
   unsigned int ExtORPort_set : 1;
+  unsigned int HTTPTunnelPort_set : 1;
   /**@}*/
 
   int AssumeReachable; /**< Whether to publish our descriptor regardless. */
@@ -3669,6 +3863,10 @@ typedef struct {
   int BridgeAuthoritativeDir; /**< Boolean: is this an authoritative directory
                                * that aggregates bridge descriptors? */
 
+  /** If set on a bridge relay, it will include this value on a new
+   * "bridge-distribution-request" line in its bridge descriptor. */
+  char *BridgeDistribution;
+
   /** If set on a bridge authority, it will answer requests on its dirport
    * for bridge statuses -- but only if the requests use this password. */
   char *BridgePassword;
@@ -3702,6 +3900,15 @@ typedef struct {
   int AvoidDiskWrites; /**< Boolean: should we never cache things to disk?
                         * Not used yet. */
   int ClientOnly; /**< Boolean: should we never evolve into a server role? */
+
+  int ReducedConnectionPadding; /**< Boolean: Should we try to keep connections
+                                  open shorter and pad them less against
+                                  connection-level traffic analysis? */
+  /** Autobool: if auto, then connection padding will be negotiated by client
+   * and server. If 0, it will be fully disabled. If 1, the client will still
+   * pad to the server regardless of server support. */
+  int ConnectionPadding;
+
   /** To what authority types do we publish our descriptor? Choices are
    * "v1", "v2", "v3", "bridge", or "". */
   smartlist_t *PublishServerDescriptor;
@@ -3727,14 +3934,13 @@ typedef struct {
   /** A routerset that should be used when picking RPs for HS circuits. */
   routerset_t *Tor2webRendezvousPoints;
 
-  /** Close hidden service client circuits immediately when they reach
-   * the normal circuit-build timeout, even if they have already sent
-   * an INTRODUCE1 cell on its way to the service. */
-  int CloseHSClientCircuitsImmediatelyOnTimeout;
+  /** A routerset that should be used when picking middle nodes for HS
+   *  circuits. */
+  routerset_t *HSLayer2Nodes;
 
-  /** Close hidden-service-side rendezvous circuits immediately when
-   * they reach the normal circuit-build timeout. */
-  int CloseHSServiceRendCircuitsImmediatelyOnTimeout;
+  /** A routerset that should be used when picking third-hop nodes for HS
+   *  circuits. */
+  routerset_t *HSLayer3Nodes;
 
   /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct)
    * circuits between the onion service server, and the introduction and
@@ -3816,8 +4022,8 @@ typedef struct {
   int CircuitBuildTimeout; /**< Cull non-open circuits that were born at
                             * least this many seconds ago. Used until
                             * adaptive algorithm learns a new value. */
-  int CircuitIdleTimeout; /**< Cull open clean circuits that were born
-                           * at least this many seconds ago. */
+  int CircuitsAvailableTimeout; /**< Try to have an open circuit for at
+                                     least this long after last activity */
   int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits
                              * and try a new circuit if the stream has been
                              * waiting for this many seconds. If zero, use
@@ -3827,16 +4033,12 @@ typedef struct {
                          * a new one? */
   int MaxCircuitDirtiness; /**< Never use circs that were first used more than
                                 this interval ago. */
-  int PredictedPortsRelevanceTime; /** How long after we've requested a
-                                    * connection for a given port, do we want
-                                    * to continue to pick exits that support
-                                    * that port?  */
   uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing
                            * to use in a second? */
   uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing
                             * to use in a second? */
   uint64_t MaxAdvertisedBandwidth; /**< How much bandwidth are we willing to
-                                    * tell people we have? */
+                                    * tell other nodes we have? */
   uint64_t RelayBandwidthRate; /**< How much bandwidth, on average, are we
                                  * willing to use for all relayed conns? */
   uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we
@@ -3844,8 +4046,6 @@ typedef struct {
   uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */
   uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */
   int NumCPUs; /**< How many CPUs should we try to use? */
-//int RunTesting; /**< If true, create testing circuits to measure how well the
-//                 * other ORs are running. */
   config_line_t *RendConfigLines; /**< List of configuration lines
                                           * for rendezvous services. */
   config_line_t *HidServAuth; /**< List of configuration lines for client-side
@@ -3854,6 +4054,8 @@ typedef struct {
 
   int HeartbeatPeriod; /**< Log heartbeat messages after this many seconds
                         * have passed. */
+  int MainloopStats; /**< Log main loop statistics as part of the
+                      * heartbeat messages. */
 
   char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */
   tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */
@@ -3896,7 +4098,8 @@ typedef struct {
   /** If set, use these bridge authorities and not the default one. */
   config_line_t *AlternateBridgeAuthority;
 
-  char *MyFamily; /**< Declared family for this OR. */
+  config_line_t *MyFamily_lines; /**< Declared family for this OR. */
+  config_line_t *MyFamily; /**< Declared family for this OR, normalized */
   config_line_t *NodeFamilies; /**< List of config lines for
                                 * node families */
   smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */
@@ -3922,9 +4125,6 @@ typedef struct {
                             * and vote for all other exits as good. */
   int AuthDirMaxServersPerAddr; /**< Do not permit more than this
                                  * number of servers per IP address. */
-  int AuthDirMaxServersPerAuthAddr; /**< Do not permit more than this
-                                     * number of servers per IP address shared
-                                     * with an authority. */
   int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6?  */
   int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */
 
@@ -3973,6 +4173,8 @@ typedef struct {
   /** Process specifier for a controller that ‘owns’ this Tor
    * instance.  Tor will terminate if its owning controller does. */
   char *OwningControllerProcess;
+  /** FD specifier for a controller that owns this Tor instance. */
+  int OwningControllerFD;
 
   int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how
                            * long do we wait before exiting? */
@@ -3986,8 +4188,6 @@ typedef struct {
   int Sandbox; /**< Boolean: should sandboxing be enabled? */
   int SafeSocks; /**< Boolean: should we outright refuse application
                   * connections that use socks4 or socks5-with-local-dns? */
-#define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \
-                           LOG_WARN : LOG_INFO)
   int ProtocolWarnings; /**< Boolean: when other parties screw up the Tor
                          * protocol, is it a warn or an info in our logs? */
   int TestSocks; /**< Boolean: when we get a socks connection, do we loudly
@@ -4004,13 +4204,11 @@ typedef struct {
   int UseEntryGuards_option;
   /** Internal variable to remember whether we're actually acting on
    * UseEntryGuards_option -- when we're a non-anonymous Tor2web client or
-   * Single Onion Service, it is alwasy false, otherwise we use the value of
+   * Single Onion Service, it is always false, otherwise we use the value of
    * UseEntryGuards_option. */
   int UseEntryGuards;
 
   int NumEntryGuards; /**< How many entry guards do we try to establish? */
-  int UseEntryGuardsAsDirGuards; /** Boolean: Do we try to get directory info
-                                  * from a smallish number of fixed nodes? */
 
   /** If 1, we use any guardfraction information we see in the
    * consensus.  If 0, we don't.  If -1, let the consensus parameter
@@ -4019,9 +4217,9 @@ typedef struct {
 
   int NumDirectoryGuards; /**< How many dir guards do we try to establish?
                            * If 0, use value from NumEntryGuards. */
+  int NumPrimaryGuards; /**< How many primary guards do we want? */
+
   int RephistTrackTime; /**< How many seconds do we keep rephist info? */
-  int FastFirstHopPK; /**< If Tor believes it is safe, should we save a third
-                       * of our PK time by sending CREATE_FAST cells? */
   /** Should we always fetch our dir info on the mirror schedule (which
    * means directly from the authorities) no matter our other config? */
   int FetchDirInfoEarly;
@@ -4064,10 +4262,6 @@ typedef struct {
                                         * testing our DNS server. */
   int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the
                                * same network zone in the same circuit. */
-  int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically
-                       * forward the DirPort and ORPort on the NAT device */
-  char *PortForwardingHelper; /** < Filename or full path of the port
-                                  forwarding helper executable */
   int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames
                                 * with weird characters. */
   /** If true, we try resolving hostnames with weird characters. */
@@ -4077,27 +4271,6 @@ typedef struct {
    * if we are a cache).  For authorities, this is always true. */
   int DownloadExtraInfo;
 
-  /** If true, and we are acting as a relay, allow exit circuits even when
-   * we are the first hop of a circuit. */
-  int AllowSingleHopExits;
-  /** If true, don't allow relays with AllowSingleHopExits=1 to be used in
-   * circuits that we build. */
-  int ExcludeSingleHopRelays;
-  /** If true, and the controller tells us to use a one-hop circuit, and the
-   * exit allows it, we use it. */
-  int AllowSingleHopCircuits;
-
-  /** If true, we convert "www.google.com.foo.exit" addresses on the
-   * socks/trans/natd ports into "www.google.com" addresses that
-   * exit from the node "foo". Disabled by default since attacking
-   * websites and exit relays can use it to manipulate your path
-   * selection. */
-  int AllowDotExit;
-
-  /** If true, we will warn if a user gives us only an IP address
-   * instead of a hostname. */
-  int WarnUnsafeSocks;
-
   /** If true, we're configured to collect statistics on clients
    * requesting network statuses from us as directory. */
   int DirReqStatistics_option;
@@ -4114,11 +4287,18 @@ typedef struct {
   /** If true, the user wants us to collect cell statistics. */
   int CellStatistics;
 
+  /** If true, the user wants us to collect padding statistics. */
+  int PaddingStatistics;
+
   /** If true, the user wants us to collect statistics as entry node. */
   int EntryStatistics;
 
   /** If true, the user wants us to collect statistics as hidden service
    * directory, introduction point, or rendezvous point. */
+  int HiddenServiceStatistics_option;
+  /** Internal variable to remember whether we're actually acting on
+   * HiddenServiceStatistics_option -- yes if it's set and we're a server,
+   * else no. */
   int HiddenServiceStatistics;
 
   /** If true, include statistics file contents in extra-info documents. */
@@ -4207,19 +4387,19 @@ typedef struct {
 
   /** Schedule for when servers should download things in general.  Only
    * altered on testing networks. */
-  smartlist_t *TestingServerDownloadSchedule;
+  int TestingServerDownloadInitialDelay;
 
   /** Schedule for when clients should download things in general.  Only
    * altered on testing networks. */
-  smartlist_t *TestingClientDownloadSchedule;
+  int TestingClientDownloadInitialDelay;
 
   /** Schedule for when servers should download consensuses.  Only altered
    * on testing networks. */
-  smartlist_t *TestingServerConsensusDownloadSchedule;
+  int TestingServerConsensusDownloadInitialDelay;
 
   /** Schedule for when clients should download consensuses.  Only altered
    * on testing networks. */
-  smartlist_t *TestingClientConsensusDownloadSchedule;
+  int TestingClientConsensusDownloadInitialDelay;
 
   /** Schedule for when clients should download consensuses from authorities
    * if they are bootstrapping (that is, they don't have a usable, reasonably
@@ -4229,7 +4409,7 @@ typedef struct {
    * This schedule is incremented by (potentially concurrent) connection
    * attempts, unlike other schedules, which are incremented by connection
    * failures.  Only altered on testing networks. */
-  smartlist_t *ClientBootstrapConsensusAuthorityDownloadSchedule;
+  int ClientBootstrapConsensusAuthorityDownloadInitialDelay;
 
   /** Schedule for when clients should download consensuses from fallback
    * directory mirrors if they are bootstrapping (that is, they don't have a
@@ -4239,7 +4419,7 @@ typedef struct {
    * This schedule is incremented by (potentially concurrent) connection
    * attempts, unlike other schedules, which are incremented by connection
    * failures.  Only altered on testing networks. */
-  smartlist_t *ClientBootstrapConsensusFallbackDownloadSchedule;
+  int ClientBootstrapConsensusFallbackDownloadInitialDelay;
 
   /** Schedule for when clients should download consensuses from authorities
    * if they are bootstrapping (that is, they don't have a usable, reasonably
@@ -4249,11 +4429,15 @@ typedef struct {
    * This schedule is incremented by (potentially concurrent) connection
    * attempts, unlike other schedules, which are incremented by connection
    * failures.  Only altered on testing networks. */
-  smartlist_t *ClientBootstrapConsensusAuthorityOnlyDownloadSchedule;
+  int ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
 
   /** Schedule for when clients should download bridge descriptors.  Only
    * altered on testing networks. */
-  smartlist_t *TestingBridgeDownloadSchedule;
+  int TestingBridgeDownloadInitialDelay;
+
+  /** Schedule for when clients should download bridge descriptors when they
+   * have no running bridges.  Only altered on testing networks. */
+  int TestingBridgeBootstrapDownloadInitialDelay;
 
   /** When directory clients have only a few descriptors to request, they
    * batch them until they have more, or until this amount of time has
@@ -4264,37 +4448,11 @@ typedef struct {
    * it?  Only altered on testing networks. */
   int TestingDirConnectionMaxStall;
 
-  /** How many times will we try to fetch a consensus before we give
-   * up?  Only altered on testing networks. */
-  int TestingConsensusMaxDownloadTries;
-
-  /** How many times will a client try to fetch a consensus while
-   * bootstrapping using a list of fallback directories, before it gives up?
-   * Only altered on testing networks. */
-  int ClientBootstrapConsensusMaxDownloadTries;
-
-  /** How many times will a client try to fetch a consensus while
-   * bootstrapping using only a list of authorities, before it gives up?
-   * Only altered on testing networks. */
-  int ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries;
-
   /** How many simultaneous in-progress connections will we make when trying
    * to fetch a consensus before we wait for one to complete, timeout, or
    * error out?  Only altered on testing networks. */
   int ClientBootstrapConsensusMaxInProgressTries;
 
-  /** How many times will we try to download a router's descriptor before
-   * giving up?  Only altered on testing networks. */
-  int TestingDescriptorMaxDownloadTries;
-
-  /** How many times will we try to download a microdescriptor before
-   * giving up?  Only altered on testing networks. */
-  int TestingMicrodescMaxDownloadTries;
-
-  /** How many times will we try to fetch a certificate before giving
-   * up?  Only altered on testing networks. */
-  int TestingCertMaxDownloadTries;
-
   /** If true, we take part in a testing network. Change the defaults of a
    * couple of other configuration options and allow to change the values
    * of certain configuration options. */
@@ -4317,8 +4475,7 @@ typedef struct {
   int TestingDirAuthVoteGuardIsStrict;
 
   /** Relays in a testing network which should be voted HSDir
-   * regardless of uptime and DirPort.
-   * Respects VoteOnHidServDirectoriesV2. */
+   * regardless of uptime and DirPort. */
   routerset_t *TestingDirAuthVoteHSDir;
   int TestingDirAuthVoteHSDirIsStrict;
 
@@ -4328,9 +4485,6 @@ typedef struct {
   /** Enable CELL_STATS events.  Only altered on testing networks. */
   int TestingEnableCellStatsEvent;
 
-  /** Enable TB_EMPTY events.  Only altered on testing networks. */
-  int TestingEnableTbEmptyEvent;
-
   /** If true, and we have GeoIP data, and we're a bridge, keep a per-country
    * count of how many client addresses have contacted us so that we can help
    * the bridge authority guess which countries have blocked access to us. */
@@ -4450,8 +4604,6 @@ typedef struct {
 
   int IPv6Exit; /**< Do we support exiting to IPv6 addresses? */
 
-  char *TLSECGroup; /**< One of "P256", "P224", or nil for auto */
-
   /** Fraction: */
   double PathsNeededToBuildCircuits;
 
@@ -4462,19 +4614,6 @@ typedef struct {
   /** How long (seconds) do we keep a guard before picking a new one? */
   int GuardLifetime;
 
-  /** Low-water mark for global scheduler - start sending when estimated
-   * queued size falls below this threshold.
-   */
-  uint64_t SchedulerLowWaterMark__;
-  /** High-water mark for global scheduler - stop sending when estimated
-   * queued size exceeds this threshold.
-   */
-  uint64_t SchedulerHighWaterMark__;
-  /** Flush size for global scheduler - flush this many cells at a time
-   * when sending.
-   */
-  int SchedulerMaxFlushCells__;
-
   /** Is this an exit node?  This is a tristate, where "1" means "yes, and use
    * the default exit policy if none is given" and "0" means "no; exit policy
    * is 'reject *'" and "auto" (-1) means "same as 1, but warn the user."
@@ -4482,7 +4621,7 @@ typedef struct {
    * XXXX Eventually, the default will be 0. */
   int ExitRelay;
 
-  /** For how long (seconds) do we declare our singning keys to be valid? */
+  /** For how long (seconds) do we declare our signing keys to be valid? */
   int SigningKeyLifetime;
   /** For how long (seconds) do we declare our link keys to be valid? */
   int TestingLinkCertLifetime;
@@ -4527,6 +4666,50 @@ typedef struct {
   /** If 1, we skip all OOS checks. */
   int DisableOOSCheck;
 
+  /** Autobool: Should we include Ed25519 identities in extend2 cells?
+   * If -1, we should do whatever the consensus parameter says. */
+  int ExtendByEd25519ID;
+
+  /** Bool (default: 1): When testing routerinfos as a directory authority,
+   * do we enforce Ed25519 identity match? */
+  /* NOTE: remove this option someday. */
+  int AuthDirTestEd25519LinkKeys;
+
+  /** Bool (default: 0): Tells if a %include was used on torrc */
+  int IncludeUsed;
+
+  /** The seconds after expiration which we as a relay should keep old
+   * consensuses around so that we can generate diffs from them.  If 0,
+   * use the default. */
+  int MaxConsensusAgeForDiffs;
+
+  /** Bool (default: 0). Tells Tor to never try to exec another program.
+   */
+  int NoExec;
+
+  /** Have the KIST scheduler run every X milliseconds. If less than zero, do
+   * not use the KIST scheduler but use the old vanilla scheduler instead. If
+   * zero, do what the consensus says and fall back to using KIST as if this is
+   * set to "10 msec" if the consensus doesn't say anything. */
+  int KISTSchedRunInterval;
+
+  /** A multiplier for the KIST per-socket limit calculation. */
+  double KISTSockBufSizeFactor;
+
+  /** The list of scheduler type string ordered by priority that is first one
+   * has to be tried first. Default: KIST,KISTLite,Vanilla */
+  smartlist_t *Schedulers;
+  /* An ordered list of scheduler_types mapped from Schedulers. */
+  smartlist_t *SchedulerTypes_;
+
+  /** List of files that were opened by %include in torrc and torrc-defaults */
+  smartlist_t *FilesOpenedByIncludes;
+
+  /** If true, Tor shouldn't install any posix signal handlers, since it is
+   * running embedded inside another process.
+   */
+  int DisableSignalHandlers;
+
   /** Autobool: Is the circuit creation DoS mitigation subsystem enabled? */
   int DoSCircuitCreationEnabled;
   /** Minimum concurrent connection needed from one single address before any
@@ -4557,6 +4740,8 @@ typedef struct {
   int DoSRefuseSingleHopClientRendezvous;
 } or_options_t;
 
+#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
+
 /** Persistent state for an onion router, as saved to disk. */
 typedef struct {
   uint32_t magic_;
@@ -4578,11 +4763,17 @@ typedef struct {
   uint64_t AccountingBytesAtSoftLimit;
   uint64_t AccountingExpectedUsage;
 
-  /** A list of Entry Guard-related configuration lines. */
+  /** A list of Entry Guard-related configuration lines. (pre-prop271) */
   config_line_t *EntryGuards;
 
+  /** A list of guard-related configuration lines. (post-prop271) */
+  config_line_t *Guard;
+
   config_line_t *TransportProxies;
 
+  /** Cached revision counters for active hidden services on this host */
+  config_line_t *HidServRevCounter;
+
   /** These fields hold information on the history of bandwidth usage for
    * servers.  The "Ends" fields hold the time when we last updated the
    * bandwidth usage. The "Interval" fields hold the granularity, in seconds,
@@ -4610,8 +4801,8 @@ typedef struct {
 
   /** Build time histogram */
   config_line_t * BuildtimeHistogram;
-  unsigned int TotalBuildTimes;
-  unsigned int CircuitBuildAbandonedCount;
+  int TotalBuildTimes;
+  int CircuitBuildAbandonedCount;
 
   /** What version of Tor wrote this state file? */
   char *TorVersion;
@@ -4624,15 +4815,6 @@ typedef struct {
   time_t LastRotatedOnionKey;
 } or_state_t;
 
-/** Change the next_write time of <b>state</b> to <b>when</b>, unless the
- * state is already scheduled to be written to disk earlier than <b>when</b>.
- */
-static inline void or_state_mark_dirty(or_state_t *state, time_t when)
-{
-  if (state->next_write > when)
-    state->next_write = when;
-}
-
 #define MAX_SOCKS_REPLY_LEN 1024
 #define MAX_SOCKS_ADDR_LEN 256
 #define SOCKS_NO_AUTH 0x00
@@ -4777,7 +4959,7 @@ typedef uint32_t build_time_t;
 double circuit_build_times_quantile_cutoff(void);
 
 /** How often in seconds should we build a test circuit */
-#define CBT_DEFAULT_TEST_FREQUENCY 60
+#define CBT_DEFAULT_TEST_FREQUENCY 10
 #define CBT_MIN_TEST_FREQUENCY 1
 #define CBT_MAX_TEST_FREQUENCY INT32_MAX
 
@@ -4972,7 +5154,7 @@ typedef struct measured_bw_line_t {
   long int bw_kb;
 } measured_bw_line_t;
 
-#endif
+#endif /* defined(DIRSERV_PRIVATE) */
 
 /********************************* dirvote.c ************************/
 
@@ -5180,7 +5362,7 @@ typedef struct rend_intro_point_t {
    */
   int accepted_introduce2_count;
 
-  /** (Service side only) Number of maximum INTRODUCE2 cells that this IP
+  /** (Service side only) Maximum number of INTRODUCE2 cells that this IP
    * will accept. This is a random value between
    * INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS and
    * INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS. */
@@ -5266,7 +5448,8 @@ typedef struct dir_server_t {
                            * address information from published? */
 
   routerstatus_t fake_status; /**< Used when we need to pass this trusted
-                               * dir_server_t to directory_initiate_command_*
+                               * dir_server_t to
+                               * directory_request_set_routerstatus.
                                * as a routerstatus_t.  Not updated by the
                                * router-status management code!
                                **/
@@ -5306,10 +5489,6 @@ typedef struct dir_server_t {
  */
 #define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
 
-/** This node is to be chosen as a directory guard, so don't choose any
- * node that's currently a guard. */
-#define PDS_FOR_GUARD (1<<5)
-
 /** Possible ways to weight routers when choosing one randomly.  See
  * routerlist_sl_choose_by_bandwidth() for more information.*/
 typedef enum bandwidth_weight_rule_t {
@@ -5323,7 +5502,6 @@ typedef enum {
   CRN_NEED_UPTIME = 1<<0,
   CRN_NEED_CAPACITY = 1<<1,
   CRN_NEED_GUARD = 1<<2,
-  CRN_ALLOW_INVALID = 1<<3,
   /* XXXX not used, apparently. */
   CRN_WEIGHT_AS_EXIT = 1<<5,
   CRN_NEED_DESC = 1<<6,
@@ -5331,15 +5509,16 @@ typedef enum {
   CRN_PREF_ADDR = 1<<7,
   /* On clients, only provide nodes that we can connect to directly, based on
    * our firewall rules */
-  CRN_DIRECT_CONN = 1<<8
+  CRN_DIRECT_CONN = 1<<8,
+  /* On clients, only provide nodes with HSRend >= 2 protocol version which
+   * is required for hidden service version >= 3. */
+  CRN_RENDEZVOUS_V3 = 1<<9,
 } router_crn_flags_t;
 
 /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */
 typedef enum was_router_added_t {
   /* Router was added successfully. */
   ROUTER_ADDED_SUCCESSFULLY = 1,
-  /* Router descriptor was added with warnings to submitter. */
-  ROUTER_ADDED_NOTIFY_GENERATOR = 0,
   /* Extrainfo document was rejected because no corresponding router
    * descriptor was found OR router descriptor was rejected because
    * it was incompatible with its extrainfo document. */
@@ -5388,5 +5567,5 @@ typedef struct tor_version_t {
   char git_tag[DIGEST_LEN];
 } tor_version_t;
 
-#endif
+#endif /* !defined(TOR_OR_H) */
 
diff --git a/src/or/parsecommon.c b/src/or/parsecommon.c
new file mode 100644
index 0000000000..cd1a0c2521
--- /dev/null
+++ b/src/or/parsecommon.c
@@ -0,0 +1,450 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.c
+ * \brief Common code to parse and validate various type of descriptors.
+ **/
+
+#include "parsecommon.h"
+#include "torlog.h"
+#include "util_format.h"
+
+#define MIN_ANNOTATION A_PURPOSE
+#define MAX_ANNOTATION A_UNKNOWN_
+
+#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
+#define ALLOC(sz) memarea_alloc(area,sz)
+#define STRDUP(str) memarea_strdup(area,str)
+#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
+
+#define RET_ERR(msg)                                               \
+  STMT_BEGIN                                                       \
+    if (tok) token_clear(tok);                                      \
+    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
+    tok->tp = ERR_;                                                \
+    tok->error = STRDUP(msg);                                      \
+    goto done_tokenizing;                                          \
+  STMT_END
+
+/** Free all resources allocated for <b>tok</b> */
+void
+token_clear(directory_token_t *tok)
+{
+  if (tok->key)
+    crypto_pk_free(tok->key);
+}
+
+/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
+ * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
+ * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
+ * entire string.
+ */
+int
+tokenize_string(memarea_t *area,
+                const char *start, const char *end, smartlist_t *out,
+                token_rule_t *table, int flags)
+{
+  const char **s;
+  directory_token_t *tok = NULL;
+  int counts[NIL_];
+  int i;
+  int first_nonannotation;
+  int prev_len = smartlist_len(out);
+  tor_assert(area);
+
+  s = &start;
+  if (!end) {
+    end = start+strlen(start);
+  } else {
+    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
+    if (memchr(start, '\0', end-start)) {
+      log_warn(LD_DIR, "parse error: internal NUL character.");
+      return -1;
+    }
+  }
+  for (i = 0; i < NIL_; ++i)
+    counts[i] = 0;
+
+  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
+
+  while (*s < end && (!tok || tok->tp != EOF_)) {
+    tok = get_next_token(area, s, end, table);
+    if (tok->tp == ERR_) {
+      log_warn(LD_DIR, "parse error: %s", tok->error);
+      token_clear(tok);
+      return -1;
+    }
+    ++counts[tok->tp];
+    smartlist_add(out, tok);
+    *s = eat_whitespace_eos(*s, end);
+  }
+
+  if (flags & TS_NOCHECK)
+    return 0;
+
+  if ((flags & TS_ANNOTATIONS_OK)) {
+    first_nonannotation = -1;
+    for (i = 0; i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
+        first_nonannotation = i;
+        break;
+      }
+    }
+    if (first_nonannotation < 0) {
+      log_warn(LD_DIR, "parse error: item contains only annotations");
+      return -1;
+    }
+    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
+        return -1;
+      }
+    }
+    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
+      if (first_nonannotation != prev_len) {
+        log_warn(LD_DIR, "parse error: Unexpected annotations.");
+        return -1;
+      }
+    }
+  } else {
+    for (i=0;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: no annotations allowed.");
+        return -1;
+      }
+    }
+    first_nonannotation = 0;
+  }
+  for (i = 0; table[i].t; ++i) {
+    if (counts[table[i].v] < table[i].min_cnt) {
+      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
+      return -1;
+    }
+    if (counts[table[i].v] > table[i].max_cnt) {
+      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
+      return -1;
+    }
+    if (table[i].pos & AT_START) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+    if (table[i].pos & AT_END) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/** Helper: parse space-separated arguments from the string <b>s</b> ending at
+ * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
+ * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
+ * all storage in <b>area</b>.  Return the number of arguments parsed, or
+ * return -1 if there was an insanely high number of arguments. */
+static inline int
+get_token_arguments(memarea_t *area, directory_token_t *tok,
+                    const char *s, const char *eol)
+{
+/** Largest number of arguments we'll accept to any token, ever. */
+#define MAX_ARGS 512
+  char *mem = memarea_strndup(area, s, eol-s);
+  char *cp = mem;
+  int j = 0;
+  char *args[MAX_ARGS];
+  memset(args, 0, sizeof(args));
+  while (*cp) {
+    if (j == MAX_ARGS)
+      return -1;
+    args[j++] = cp;
+    cp = (char*)find_whitespace(cp);
+    if (!cp || !*cp)
+      break; /* End of the line. */
+    *cp++ = '\0';
+    cp = (char*)eat_whitespace(cp);
+  }
+  tok->n_args = j;
+  tok->args = memarea_memdup(area, args, j*sizeof(char*));
+  return j;
+#undef MAX_ARGS
+}
+
+/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
+ * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
+ * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
+ * conform to the syntax we wanted.
+ **/
+static inline directory_token_t *
+token_check_object(memarea_t *area, const char *kwd,
+                   directory_token_t *tok, obj_syntax o_syn)
+{
+  char ebuf[128];
+  switch (o_syn) {
+    case NO_OBJ:
+      /* No object is allowed for this token. */
+      if (tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_OBJ:
+      /* There must be a (non-key) object. */
+      if (!tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
+    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
+      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
+                     kwd, crypto_pk_num_bits(tok->key));
+        RET_ERR(ebuf);
+      }
+      /* fall through */
+    case NEED_KEY: /* There must be some kind of key. */
+      if (!tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (o_syn != NEED_SKEY_1024) {
+        if (crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Private key given for %s, which wants a public key", kwd);
+          RET_ERR(ebuf);
+        }
+      } else { /* o_syn == NEED_SKEY_1024 */
+        if (!crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Public key given for %s, which wants a private key", kwd);
+          RET_ERR(ebuf);
+        }
+      }
+      break;
+    case OBJ_OK:
+      /* Anything goes with this token. */
+      break;
+  }
+
+ done_tokenizing:
+  return tok;
+}
+
+/** Helper function: read the next token from *s, advance *s to the end of the
+ * token, and return the parsed token.  Parse *<b>s</b> according to the list
+ * of tokens in <b>table</b>.
+ */
+directory_token_t *
+get_next_token(memarea_t *area,
+               const char **s, const char *eos, token_rule_t *table)
+{
+  /** Reject any object at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
+  /** Reject any line at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_LINE_LENGTH (128*1024)
+
+  const char *next, *eol, *obstart;
+  size_t obname_len;
+  int i;
+  directory_token_t *tok;
+  obj_syntax o_syn = NO_OBJ;
+  char ebuf[128];
+  const char *kwd = "";
+
+  tor_assert(area);
+  tok = ALLOC_ZERO(sizeof(directory_token_t));
+  tok->tp = ERR_;
+
+  /* Set *s to first token, eol to end-of-line, next to after first token */
+  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol)
+    eol = eos;
+  if (eol - *s > MAX_LINE_LENGTH) {
+    RET_ERR("Line far too long");
+  }
+
+  next = find_whitespace_eos(*s, eol);
+
+  if (!strcmp_len(*s, "opt", next-*s)) {
+    /* Skip past an "opt" at the start of the line. */
+    *s = eat_whitespace_eos_no_nl(next, eol);
+    next = find_whitespace_eos(*s, eol);
+  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
+    RET_ERR("Unexpected EOF");
+  }
+
+  /* Search the table for the appropriate entry.  (I tried a binary search
+   * instead, but it wasn't any faster.) */
+  for (i = 0; table[i].t ; ++i) {
+    if (!strcmp_len(*s, table[i].t, next-*s)) {
+      /* We've found the keyword. */
+      kwd = table[i].t;
+      tok->tp = table[i].v;
+      o_syn = table[i].os;
+      *s = eat_whitespace_eos_no_nl(next, eol);
+      /* We go ahead whether there are arguments or not, so that tok->args is
+       * always set if we want arguments. */
+      if (table[i].concat_args) {
+        /* The keyword takes the line as a single argument */
+        tok->args = ALLOC(sizeof(char*));
+        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
+        tok->n_args = 1;
+      } else {
+        /* This keyword takes multiple arguments. */
+        if (get_token_arguments(area, tok, *s, eol)<0) {
+          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
+          RET_ERR(ebuf);
+        }
+        *s = eol;
+      }
+      if (tok->n_args < table[i].min_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
+        RET_ERR(ebuf);
+      } else if (tok->n_args > table[i].max_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    }
+  }
+
+  if (tok->tp == ERR_) {
+    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
+    if (*s < eol && **s == '@')
+      tok->tp = A_UNKNOWN_;
+    else
+      tok->tp = K_OPT;
+    tok->args = ALLOC(sizeof(char*));
+    tok->args[0] = STRNDUP(*s, eol-*s);
+    tok->n_args = 1;
+    o_syn = OBJ_OK;
+  }
+
+  /* Check whether there's an object present */
+  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
+    goto check_object;
+
+  obstart = *s; /* Set obstart to start of object spec */
+  if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
+      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
+    RET_ERR("Malformed object: bad begin line");
+  }
+  tok->object_type = STRNDUP(*s+11, eol-*s-16);
+  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
+  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
+
+  /* Go to the end of the object */
+  next = tor_memstr(*s, eos-*s, "-----END ");
+  if (!next) {
+    RET_ERR("Malformed object: missing object end line");
+  }
+  tor_assert(eos >= next);
+  eol = memchr(next, '\n', eos-next);
+  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
+    eol = eos;
+  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
+  if ((size_t)(eol-next) != 9+obname_len+5 ||
+      strcmp_len(next+9, tok->object_type, obname_len) ||
+      strcmp_len(eol-5, "-----", 5)) {
+    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
+             tok->object_type);
+    ebuf[sizeof(ebuf)-1] = '\0';
+    RET_ERR(ebuf);
+  }
+  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
+    RET_ERR("Couldn't parse object: missing footer or object much too big.");
+
+  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse public key.");
+  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse private key.");
+  } else { /* If it's something else, try to base64-decode it */
+    int r;
+    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
+    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+  *s = eol;
+
+ check_object:
+  tok = token_check_object(area, kwd, tok, o_syn);
+
+ done_tokenizing:
+  return tok;
+
+#undef RET_ERR
+#undef ALLOC
+#undef ALLOC_ZERO
+#undef STRDUP
+#undef STRNDUP
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
+ * with an assert if no such keyword is found.
+ */
+directory_token_t *
+find_by_keyword_(smartlist_t *s, directory_keyword keyword,
+                 const char *keyword_as_string)
+{
+  directory_token_t *tok = find_opt_by_keyword(s, keyword);
+  if (PREDICT_UNLIKELY(!tok)) {
+    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
+         "been validated. Internal error.", keyword_as_string, (int)keyword);
+    tor_assert(tok);
+  }
+  return tok;
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
+ * NULL if no such keyword is found.
+ */
+directory_token_t *
+find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword)
+{
+  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
+  return NULL;
+}
+
+/** If there are any directory_token_t entries in <b>s</b> whose keyword is
+ * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
+ * in the same order in which they occur in <b>s</b>.  Otherwise return
+ * NULL. */
+smartlist_t *
+find_all_by_keyword(const smartlist_t *s, directory_keyword k)
+{
+  smartlist_t *out = NULL;
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+                    if (t->tp == k) {
+                    if (!out)
+                    out = smartlist_new();
+                    smartlist_add(out, t);
+                    });
+  return out;
+}
diff --git a/src/or/parsecommon.h b/src/or/parsecommon.h
new file mode 100644
index 0000000000..d33faf8ec7
--- /dev/null
+++ b/src/or/parsecommon.h
@@ -0,0 +1,322 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.h
+ * \brief Header file for parsecommon.c
+ **/
+
+#ifndef TOR_PARSECOMMON_H
+#define TOR_PARSECOMMON_H
+
+#include "container.h"
+#include "crypto.h"
+#include "memarea.h"
+
+/** Enumeration of possible token types.  The ones starting with K_ correspond
+* to directory 'keywords'. A_ is for an annotation, R or C is related to
+* hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
+* end-of-file marker, and NIL_ is used to encode not-a-token.
+*/
+typedef enum {
+  K_ACCEPT = 0,
+  K_ACCEPT6,
+  K_DIRECTORY_SIGNATURE,
+  K_RECOMMENDED_SOFTWARE,
+  K_REJECT,
+  K_REJECT6,
+  K_ROUTER,
+  K_SIGNED_DIRECTORY,
+  K_SIGNING_KEY,
+  K_ONION_KEY,
+  K_ONION_KEY_NTOR,
+  K_ROUTER_SIGNATURE,
+  K_PUBLISHED,
+  K_RUNNING_ROUTERS,
+  K_ROUTER_STATUS,
+  K_PLATFORM,
+  K_PROTO,
+  K_OPT,
+  K_BANDWIDTH,
+  K_CONTACT,
+  K_NETWORK_STATUS,
+  K_UPTIME,
+  K_DIR_SIGNING_KEY,
+  K_FAMILY,
+  K_FINGERPRINT,
+  K_HIBERNATING,
+  K_READ_HISTORY,
+  K_WRITE_HISTORY,
+  K_NETWORK_STATUS_VERSION,
+  K_DIR_SOURCE,
+  K_DIR_OPTIONS,
+  K_CLIENT_VERSIONS,
+  K_SERVER_VERSIONS,
+  K_RECOMMENDED_CLIENT_PROTOCOLS,
+  K_RECOMMENDED_RELAY_PROTOCOLS,
+  K_REQUIRED_CLIENT_PROTOCOLS,
+  K_REQUIRED_RELAY_PROTOCOLS,
+  K_OR_ADDRESS,
+  K_ID,
+  K_P,
+  K_P6,
+  K_R,
+  K_A,
+  K_S,
+  K_V,
+  K_W,
+  K_M,
+  K_EXTRA_INFO,
+  K_EXTRA_INFO_DIGEST,
+  K_CACHES_EXTRA_INFO,
+  K_HIDDEN_SERVICE_DIR,
+  K_ALLOW_SINGLE_HOP_EXITS,
+  K_IPV6_POLICY,
+  K_ROUTER_SIG_ED25519,
+  K_IDENTITY_ED25519,
+  K_MASTER_KEY_ED25519,
+  K_ONION_KEY_CROSSCERT,
+  K_NTOR_ONION_KEY_CROSSCERT,
+
+  K_DIRREQ_END,
+  K_DIRREQ_V2_IPS,
+  K_DIRREQ_V3_IPS,
+  K_DIRREQ_V2_REQS,
+  K_DIRREQ_V3_REQS,
+  K_DIRREQ_V2_SHARE,
+  K_DIRREQ_V3_SHARE,
+  K_DIRREQ_V2_RESP,
+  K_DIRREQ_V3_RESP,
+  K_DIRREQ_V2_DIR,
+  K_DIRREQ_V3_DIR,
+  K_DIRREQ_V2_TUN,
+  K_DIRREQ_V3_TUN,
+  K_ENTRY_END,
+  K_ENTRY_IPS,
+  K_CELL_END,
+  K_CELL_PROCESSED,
+  K_CELL_QUEUED,
+  K_CELL_TIME,
+  K_CELL_CIRCS,
+  K_EXIT_END,
+  K_EXIT_WRITTEN,
+  K_EXIT_READ,
+  K_EXIT_OPENED,
+
+  K_DIR_KEY_CERTIFICATE_VERSION,
+  K_DIR_IDENTITY_KEY,
+  K_DIR_KEY_PUBLISHED,
+  K_DIR_KEY_EXPIRES,
+  K_DIR_KEY_CERTIFICATION,
+  K_DIR_KEY_CROSSCERT,
+  K_DIR_ADDRESS,
+  K_DIR_TUNNELLED,
+
+  K_VOTE_STATUS,
+  K_VALID_AFTER,
+  K_FRESH_UNTIL,
+  K_VALID_UNTIL,
+  K_VOTING_DELAY,
+
+  K_KNOWN_FLAGS,
+  K_PARAMS,
+  K_BW_WEIGHTS,
+  K_VOTE_DIGEST,
+  K_CONSENSUS_DIGEST,
+  K_ADDITIONAL_DIGEST,
+  K_ADDITIONAL_SIGNATURE,
+  K_CONSENSUS_METHODS,
+  K_CONSENSUS_METHOD,
+  K_LEGACY_DIR_KEY,
+  K_DIRECTORY_FOOTER,
+  K_SIGNING_CERT_ED,
+  K_SR_FLAG,
+  K_COMMIT,
+  K_PREVIOUS_SRV,
+  K_CURRENT_SRV,
+  K_PACKAGE,
+
+  A_PURPOSE,
+  A_LAST_LISTED,
+  A_UNKNOWN_,
+
+  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+  R_VERSION,
+  R_PERMANENT_KEY,
+  R_SECRET_ID_PART,
+  R_PUBLICATION_TIME,
+  R_PROTOCOL_VERSIONS,
+  R_INTRODUCTION_POINTS,
+  R_SIGNATURE,
+
+  R_HS_DESCRIPTOR, /* From version 3, this MUST be generic to all future
+                      descriptor versions thus making it R_. */
+  R3_DESC_LIFETIME,
+  R3_DESC_SIGNING_CERT,
+  R3_REVISION_COUNTER,
+  R3_SUPERENCRYPTED,
+  R3_SIGNATURE,
+  R3_CREATE2_FORMATS,
+  R3_INTRO_AUTH_REQUIRED,
+  R3_SINGLE_ONION_SERVICE,
+  R3_INTRODUCTION_POINT,
+  R3_INTRO_ONION_KEY,
+  R3_INTRO_AUTH_KEY,
+  R3_INTRO_ENC_KEY,
+  R3_INTRO_ENC_KEY_CERT,
+  R3_INTRO_LEGACY_KEY,
+  R3_INTRO_LEGACY_KEY_CERT,
+  R3_DESC_AUTH_TYPE,
+  R3_DESC_AUTH_KEY,
+  R3_DESC_AUTH_CLIENT,
+  R3_ENCRYPTED,
+
+  R_IPO_IDENTIFIER,
+  R_IPO_IP_ADDRESS,
+  R_IPO_ONION_PORT,
+  R_IPO_ONION_KEY,
+  R_IPO_SERVICE_KEY,
+
+  C_CLIENT_NAME,
+  C_DESCRIPTOR_COOKIE,
+  C_CLIENT_KEY,
+
+  ERR_,
+  EOF_,
+  NIL_
+} directory_keyword;
+
+/** Structure to hold a single directory token.
+ *
+ * We parse a directory by breaking it into "tokens", each consisting
+ * of a keyword, a line full of arguments, and a binary object.  The
+ * arguments and object are both optional, depending on the keyword
+ * type.
+ *
+ * This structure is only allocated in memareas; do not allocate it on
+ * the heap, or token_clear() won't work.
+ */
+typedef struct directory_token_t {
+  directory_keyword tp;        /**< Type of the token. */
+  int n_args:30;               /**< Number of elements in args */
+  char **args;                 /**< Array of arguments from keyword line. */
+
+  char *object_type;           /**< -----BEGIN [object_type]-----*/
+  size_t object_size;          /**< Bytes in object_body */
+  char *object_body;           /**< Contents of object, base64-decoded. */
+
+  crypto_pk_t *key;        /**< For public keys only.  Heap-allocated. */
+
+  char *error;                 /**< For ERR_ tokens only. */
+} directory_token_t;
+
+/** We use a table of rules to decide how to parse each token type. */
+
+/** Rules for whether the keyword needs an object. */
+typedef enum {
+  NO_OBJ,        /**< No object, ever. */
+  NEED_OBJ,      /**< Object is required. */
+  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
+  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
+  NEED_KEY,      /**< Object is required, and must be a public key. */
+  OBJ_OK,        /**< Object is optional. */
+} obj_syntax;
+
+#define AT_START 1
+#define AT_END 2
+
+#define TS_ANNOTATIONS_OK 1
+#define TS_NOCHECK 2
+#define TS_NO_NEW_ANNOTATIONS 4
+
+/**
+ * @name macros for defining token rules
+ *
+ * Helper macros to define token tables.  's' is a string, 't' is a
+ * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
+ * object syntax.
+ */
+/**@{*/
+
+/** Appears to indicate the end of a table. */
+#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions: used for obsolete document types */
+#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions on multiplicity or location. */
+#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item that must appear exactly once */
+#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
+/** An item that must appear exactly once, at the start of the document */
+#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
+/** An item that must appear exactly once, at the end of the document */
+#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
+/** An item that must appear one or more times */
+#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
+/** An item that must appear no more than once */
+#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
+/** An annotation that must appear no more than once */
+#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
+
+/** Argument multiplicity: any number of arguments. */
+#define ARGS        0,INT_MAX,0
+/** Argument multiplicity: no arguments. */
+#define NO_ARGS     0,0,0
+/** Argument multiplicity: concatenate all arguments. */
+#define CONCAT_ARGS 1,1,1
+/** Argument multiplicity: at least <b>n</b> arguments. */
+#define GE(n)       n,INT_MAX,0
+/** Argument multiplicity: exactly <b>n</b> arguments. */
+#define EQ(n)       n,n,0
+/**@}*/
+
+/** Determines the parsing rules for a single token type. */
+typedef struct token_rule_t {
+  /** The string value of the keyword identifying the type of item. */
+  const char *t;
+  /** The corresponding directory_keyword enum. */
+  directory_keyword v;
+  /** Minimum number of arguments for this item */
+  int min_args;
+  /** Maximum number of arguments for this item */
+  int max_args;
+  /** If true, we concatenate all arguments for this item into a single
+   * string. */
+  int concat_args;
+  /** Requirements on object syntax for this item. */
+  obj_syntax os;
+  /** Lowest number of times this item may appear in a document. */
+  int min_cnt;
+  /** Highest number of times this item may appear in a document. */
+  int max_cnt;
+  /** One or more of AT_START/AT_END to limit where the item may appear in a
+   * document. */
+  int pos;
+  /** True iff this token is an annotation. */
+  int is_annotation;
+} token_rule_t;
+
+void token_clear(directory_token_t *tok);
+
+int tokenize_string(memarea_t *area,
+                    const char *start, const char *end,
+                    smartlist_t *out,
+                    token_rule_t *table,
+                    int flags);
+directory_token_t *get_next_token(memarea_t *area,
+                                  const char **s,
+                                  const char *eos,
+                                  token_rule_t *table);
+
+directory_token_t *find_by_keyword_(smartlist_t *s,
+                                    directory_keyword keyword,
+                                    const char *keyword_str);
+
+#define find_by_keyword(s, keyword) \
+  find_by_keyword_((s), (keyword), #keyword)
+
+directory_token_t *find_opt_by_keyword(const smartlist_t *s,
+                                       directory_keyword keyword);
+smartlist_t * find_all_by_keyword(const smartlist_t *s, directory_keyword k);
+
+#endif /* !defined(TOR_PARSECOMMON_H) */
+
diff --git a/src/or/periodic.c b/src/or/periodic.c
index d02d4a7bbb..9470376d06 100644
--- a/src/or/periodic.c
+++ b/src/or/periodic.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,10 +14,9 @@
 #include "or.h"
 #include "compat_libevent.h"
 #include "config.h"
+#include "main.h"
 #include "periodic.h"
 
-#include <event2/event.h>
-
 /** We disable any interval greater than this number of seconds, on the
  * grounds that it is probably an absolute time mistakenly passed in as a
  * relative time.
@@ -34,24 +33,34 @@ periodic_event_set_interval(periodic_event_item_t *event,
   struct timeval tv;
   tv.tv_sec = next_interval;
   tv.tv_usec = 0;
-  event_add(event->ev, &tv);
+  mainloop_event_schedule(event->ev, &tv);
 }
 
 /** Wraps dispatches for periodic events, <b>data</b> will be a pointer to the
  * event that needs to be called */
 static void
-periodic_event_dispatch(evutil_socket_t fd, short what, void *data)
+periodic_event_dispatch(mainloop_event_t *ev, void *data)
 {
-  (void)fd;
-  (void)what;
   periodic_event_item_t *event = data;
+  tor_assert(ev == event->ev);
+
+  if (BUG(!periodic_event_is_enabled(event))) {
+    return;
+  }
 
   time_t now = time(NULL);
+  update_current_time(now);
   const or_options_t *options = get_options();
 //  log_debug(LD_GENERAL, "Dispatching %s", event->name);
   int r = event->fn(now, options);
   int next_interval = 0;
 
+  if (!periodic_event_is_enabled(event)) {
+    /* The event got disabled from inside its callback; no need to
+     * reschedule. */
+    return;
+  }
+
   /* update the last run time if action was taken */
   if (r==0) {
     log_err(LD_BUG, "Invalid return value for periodic event from %s.",
@@ -74,14 +83,17 @@ periodic_event_dispatch(evutil_socket_t fd, short what, void *data)
 //  log_debug(LD_GENERAL, "Scheduling %s for %d seconds", event->name,
 //           next_interval);
   struct timeval tv = { next_interval , 0 };
-  event_add(event->ev, &tv);
+  mainloop_event_schedule(ev, &tv);
 }
 
 /** Schedules <b>event</b> to run as soon as possible from now. */
 void
 periodic_event_reschedule(periodic_event_item_t *event)
 {
-  periodic_event_set_interval(event, 1);
+  /* Don't reschedule a disabled event. */
+  if (periodic_event_is_enabled(event)) {
+    periodic_event_set_interval(event, 1);
+  }
 }
 
 /** Initializes the libevent backend for a periodic event. */
@@ -93,10 +105,8 @@ periodic_event_setup(periodic_event_item_t *event)
     tor_assert(0);
   }
 
-  event->ev = tor_event_new(tor_libevent_get_base(),
-                            -1, 0,
-                            periodic_event_dispatch,
-                            event);
+  event->ev = mainloop_event_new(periodic_event_dispatch,
+                                 event);
   tor_assert(event->ev);
 }
 
@@ -109,9 +119,15 @@ periodic_event_launch(periodic_event_item_t *event)
     log_err(LD_BUG, "periodic_event_launch without periodic_event_setup");
     tor_assert(0);
   }
+  /* Event already enabled? This is a bug */
+  if (periodic_event_is_enabled(event)) {
+    log_err(LD_BUG, "periodic_event_launch on an already enabled event");
+    tor_assert(0);
+  }
 
   // Initial dispatch
-  periodic_event_dispatch(-1, EV_TIMEOUT, event);
+  event->enabled = 1;
+  periodic_event_dispatch(event->ev, event);
 }
 
 /** Release all storage associated with <b>event</b> */
@@ -120,7 +136,38 @@ periodic_event_destroy(periodic_event_item_t *event)
 {
   if (!event)
     return;
-  tor_event_free(event->ev);
+  mainloop_event_free(event->ev);
   event->last_action_time = 0;
 }
 
+/** Enable the given event by setting its "enabled" flag and scheduling it to
+ * run immediately in the event loop. This can be called for an event that is
+ * already enabled. */
+void
+periodic_event_enable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  /* Safely and silently ignore if this event is already enabled. */
+  if (periodic_event_is_enabled(event)) {
+    return;
+  }
+
+  tor_assert(event->ev);
+  event->enabled = 1;
+  mainloop_event_activate(event->ev);
+}
+
+/** Disable the given event which means the event is destroyed and then the
+ * event's enabled flag is unset. This can be called for an event that is
+ * already disabled. */
+void
+periodic_event_disable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  /* Safely and silently ignore if this event is already disabled. */
+  if (!periodic_event_is_enabled(event)) {
+    return;
+  }
+  mainloop_event_cancel(event->ev);
+  event->enabled = 0;
+}
diff --git a/src/or/periodic.h b/src/or/periodic.h
index 021bb4ef5c..e8208b2475 100644
--- a/src/or/periodic.h
+++ b/src/or/periodic.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_PERIODIC_H
@@ -6,6 +6,39 @@
 
 #define PERIODIC_EVENT_NO_UPDATE (-1)
 
+/* Tor roles for which a periodic event item is for. An event can be for
+ * multiple roles, they can be combined. */
+#define PERIODIC_EVENT_ROLE_CLIENT      (1U << 0)
+#define PERIODIC_EVENT_ROLE_RELAY       (1U << 1)
+#define PERIODIC_EVENT_ROLE_BRIDGE      (1U << 2)
+#define PERIODIC_EVENT_ROLE_DIRAUTH     (1U << 3)
+#define PERIODIC_EVENT_ROLE_BRIDGEAUTH  (1U << 4)
+#define PERIODIC_EVENT_ROLE_HS_SERVICE  (1U << 5)
+#define PERIODIC_EVENT_ROLE_DIRSERVER   (1U << 6)
+
+/* Helper macro to make it a bit less annoying to defined groups of roles that
+ * are often used. */
+
+/* Router that is a Bridge or Relay. */
+#define PERIODIC_EVENT_ROLE_ROUTER \
+  (PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_RELAY)
+/* Authorities that is both bridge and directory. */
+#define PERIODIC_EVENT_ROLE_AUTHORITIES \
+  (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH)
+/* All roles. */
+#define PERIODIC_EVENT_ROLE_ALL \
+  (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \
+   PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER)
+
+/*
+ * Event flags which can change the behavior of an event.
+ */
+
+/* Indicate that the event needs the network meaning that if we are in
+ * DisableNetwork or hibernation mode, the event won't be enabled. This obey
+ * the net_is_disabled() check. */
+#define PERIODIC_EVENT_FLAG_NEED_NET  (1U << 0)
+
 /** Callback function for a periodic event to take action.  The return value
 * influences the next time the function will get called.  Return
 * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
@@ -14,24 +47,42 @@
 typedef int (*periodic_event_helper_t)(time_t now,
                                       const or_options_t *options);
 
-struct event;
+struct mainloop_event_t;
 
 /** A single item for the periodic-events-function table. */
 typedef struct periodic_event_item_t {
   periodic_event_helper_t fn; /**< The function to run the event */
   time_t last_action_time; /**< The last time the function did something */
-  struct event *ev; /**< Libevent callback we're using to implement this */
+  struct mainloop_event_t *ev; /**< Libevent callback we're using to implement
+                                * this */
   const char *name; /**< Name of the function -- for debug */
+
+  /* Bitmask of roles define above for which this event applies. */
+  uint32_t roles;
+  /* Bitmask of flags which can change the behavior of the event. */
+  uint32_t flags;
+  /* Indicate that this event has been enabled that is scheduled. */
+  unsigned int enabled : 1;
 } periodic_event_item_t;
 
 /** events will get their interval from first execution */
-#define PERIODIC_EVENT(fn) { fn##_callback, 0, NULL, #fn }
-#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL }
+#define PERIODIC_EVENT(fn, r, f) { fn##_callback, 0, NULL, #fn, r, f, 0 }
+#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL, 0, 0, 0 }
+
+/* Return true iff the given event was setup before thus is enabled to be
+ * scheduled. */
+static inline int
+periodic_event_is_enabled(const periodic_event_item_t *item)
+{
+  return item->enabled;
+}
 
 void periodic_event_launch(periodic_event_item_t *event);
 void periodic_event_setup(periodic_event_item_t *event);
 void periodic_event_destroy(periodic_event_item_t *event);
 void periodic_event_reschedule(periodic_event_item_t *event);
+void periodic_event_enable(periodic_event_item_t *event);
+void periodic_event_disable(periodic_event_item_t *event);
 
-#endif
+#endif /* !defined(TOR_PERIODIC_H) */
 
diff --git a/src/or/policies.c b/src/or/policies.c
index f58bf329ad..1210ca687d 100644
--- a/src/or/policies.c
+++ b/src/or/policies.c
@@ -1,18 +1,27 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file policies.c
  * \brief Code to parse and use address policies and exit policies.
+ *
+ * We have two key kinds of address policy: full and compressed.  A full
+ * policy is an array of accept/reject patterns, to be applied in order.
+ * A short policy is simply a list of ports.  This module handles both
+ * kinds, including generic functions to apply them to addresses, and
+ * also including code to manage the global policies that we apply to
+ * incoming and outgoing connections.
  **/
 
 #define POLICIES_PRIVATE
 
 #include "or.h"
+#include "bridges.h"
 #include "config.h"
 #include "dirserv.h"
+#include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
@@ -73,7 +82,8 @@ static int policies_parse_exit_policy_internal(
                                       const smartlist_t *configured_addresses,
                                       int reject_interface_addresses,
                                       int reject_configured_port_addresses,
-                                      int add_default_policy);
+                                      int add_default_policy,
+                                      int add_reduced_policy);
 
 /** Replace all "private" entries in *<b>policy</b> with their expanded
  * equivalents. */
@@ -290,8 +300,8 @@ parse_reachable_addresses(void)
     } else if (fascist_firewall_use_ipv6(options)
        && (policy_is_reject_star(reachable_or_addr_policy, AF_INET6, 0)
          || policy_is_reject_star(reachable_dir_addr_policy, AF_INET6, 0))) {
-          log_warn(LD_CONFIG, "You have configured tor to use IPv6 "
-                   "(ClientUseIPv6 1 or UseBridges 1), but "
+          log_warn(LD_CONFIG, "You have configured tor to use or prefer IPv6 "
+                   "(or UseBridges 1), but "
                    "ReachableAddresses, ReachableORAddresses, or "
                    "ReachableDirAddresses reject all IPv6 addresses. "
                    "Tor will not connect using IPv6.");
@@ -309,10 +319,8 @@ firewall_is_fascist_impl(void)
   const or_options_t *options = get_options();
   /* Assume every non-bridge relay has an IPv4 address.
    * Clients which use bridges may only know the IPv6 address of their
-   * bridge. */
-  return (options->ClientUseIPv4 == 0
-          || (!fascist_firewall_use_ipv6(options)
-              && options->UseBridges == 1));
+   * bridge, but they will connect regardless of the ClientUseIPv6 setting. */
+  return options->ClientUseIPv4 == 0;
 }
 
 /** Return true iff the firewall options, including ClientUseIPv4 0 and
@@ -419,6 +427,9 @@ fascist_firewall_allows_address(const tor_addr_t *addr,
 }
 
 /** Is this client configured to use IPv6?
+ * Returns true if the client might use IPv6 for some of its connections
+ * (including dual-stack and IPv6-only clients), and false if it will never
+ * use IPv6 for any connections.
  * Use node_ipv6_or/dir_preferred() when checking a specific node and OR/Dir
  * port: it supports bridge client per-node IPv6 preferences.
  */
@@ -426,9 +437,11 @@ int
 fascist_firewall_use_ipv6(const or_options_t *options)
 {
   /* Clients use IPv6 if it's set, or they use bridges, or they don't use
-   * IPv4 */
-  return (options->ClientUseIPv6 == 1 || options->UseBridges == 1
-          || options->ClientUseIPv4 == 0);
+   * IPv4, or they prefer it.
+   * ClientPreferIPv6DirPort is deprecated, but check it anyway. */
+  return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 ||
+          options->ClientPreferIPv6ORPort == 1 ||
+          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1);
 }
 
 /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and
@@ -812,9 +825,8 @@ fascist_firewall_choose_address(const tor_addr_port_t *a,
  * If pref_only, only choose preferred addresses. In either case, choose
  * a preferred address before an address that's not preferred.
  * If both addresses could be chosen (they are both preferred or both allowed)
- * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4.
- * If neither address is chosen, return 0, else return 1. */
-static int
+ * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. */
+static void
 fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr,
                                      uint16_t ipv4_orport,
                                      uint16_t ipv4_dirport,
@@ -832,6 +844,9 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr,
   tor_assert(ipv6_addr);
   tor_assert(ap);
 
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   tor_addr_port_t ipv4_ap;
   tor_addr_copy(&ipv4_ap.addr, ipv4_addr);
   ipv4_ap.port = (fw_connection == FIREWALL_OR_CONNECTION
@@ -852,17 +867,12 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr,
   if (result) {
     tor_addr_copy(&ap->addr, &result->addr);
     ap->port = result->port;
-    return 1;
-  } else {
-    tor_addr_make_null(&ap->addr, AF_UNSPEC);
-    ap->port = 0;
-    return 0;
   }
 }
 
 /** Like fascist_firewall_choose_address_base(), but takes a host-order IPv4
  * address as the first parameter. */
-static int
+static void
 fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
                                       uint16_t ipv4_orport,
                                       uint16_t ipv4_dirport,
@@ -876,11 +886,59 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
 {
   tor_addr_t ipv4_addr;
   tor_addr_from_ipv4h(&ipv4_addr, ipv4h_addr);
-  return fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport,
-                                              ipv4_dirport, ipv6_addr,
-                                              ipv6_orport, ipv6_dirport,
-                                              fw_connection, pref_only,
-                                              pref_ipv6, ap);
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
+  fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport,
+                                       ipv4_dirport, ipv6_addr,
+                                       ipv6_orport, ipv6_dirport,
+                                       fw_connection, pref_only,
+                                       pref_ipv6, ap);
+}
+
+/* Some microdescriptor consensus methods have no IPv6 addresses in rs: they
+ * are in the microdescriptors. For these consensus methods, we can't rely on
+ * the node's IPv6 address until its microdescriptor is available (when using
+ * microdescs).
+ * But for bridges, rewrite_node_address_for_bridge() updates node->ri with
+ * the configured address, so we can trust bridge addresses.
+ * (Bridges could gain an IPv6 address if their microdescriptor arrives, but
+ * this will never be their preferred address: that is in the config.)
+ * Returns true if the node needs a microdescriptor for its IPv6 address, and
+ * false if the addresses in the node are already up-to-date.
+ */
+static int
+node_awaiting_ipv6(const or_options_t* options, const node_t *node)
+{
+  tor_assert(node);
+
+  /* There's no point waiting for an IPv6 address if we'd never use it */
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  }
+
+  /* If the node has an IPv6 address, we're not waiting */
+  if (node_has_ipv6_addr(node)) {
+    return 0;
+  }
+
+  /* If the current consensus method and flavour has IPv6 addresses, we're not
+   * waiting */
+  if (networkstatus_consensus_has_ipv6(options)) {
+    return 0;
+  }
+
+  /* Bridge clients never use the address from a bridge's md, so there's no
+   * need to wait for it. */
+  if (node_is_a_configured_bridge(node)) {
+    return 0;
+  }
+
+  /* We are waiting if we_use_microdescriptors_for_circuits() and we have no
+   * md. */
+  return (!node->md && we_use_microdescriptors_for_circuits(options));
 }
 
 /** Like fascist_firewall_choose_address_base(), but takes <b>rs</b>.
@@ -888,58 +946,70 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
  * This should only happen when there's no valid consensus, and rs doesn't
  * correspond to a bridge client's bridge.
  */
-int
+void
 fascist_firewall_choose_address_rs(const routerstatus_t *rs,
                                    firewall_connection_t fw_connection,
                                    int pref_only, tor_addr_port_t* ap)
 {
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   if (!rs) {
-    return 0;
+    return;
   }
 
-  tor_assert(ap);
-
+  const or_options_t *options = get_options();
   const node_t *node = node_get_by_id(rs->identity_digest);
 
-  if (node) {
-    return fascist_firewall_choose_address_node(node, fw_connection, pref_only,
-                                                ap);
+  if (node && !node_awaiting_ipv6(options, node)) {
+    fascist_firewall_choose_address_node(node, fw_connection, pref_only, ap);
   } else {
     /* There's no node-specific IPv6 preference, so use the generic IPv6
      * preference instead. */
-    const or_options_t *options = get_options();
     int pref_ipv6 = (fw_connection == FIREWALL_OR_CONNECTION
                      ? fascist_firewall_prefer_ipv6_orport(options)
                      : fascist_firewall_prefer_ipv6_dirport(options));
 
     /* Assume IPv4 and IPv6 DirPorts are the same.
      * Assume the IPv6 OR and Dir addresses are the same. */
-    return fascist_firewall_choose_address_ipv4h(rs->addr,
-                                                 rs->or_port,
-                                                 rs->dir_port,
-                                                 &rs->ipv6_addr,
-                                                 rs->ipv6_orport,
-                                                 rs->dir_port,
-                                                 fw_connection,
-                                                 pref_only,
-                                                 pref_ipv6,
-                                                 ap);
+    fascist_firewall_choose_address_ipv4h(rs->addr, rs->or_port, rs->dir_port,
+                                          &rs->ipv6_addr, rs->ipv6_orport,
+                                          rs->dir_port, fw_connection,
+                                          pref_only, pref_ipv6, ap);
   }
 }
 
 /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and
  * looks up the node's IPv6 preference rather than taking an argument
  * for pref_ipv6. */
-int
+void
 fascist_firewall_choose_address_node(const node_t *node,
                                      firewall_connection_t fw_connection,
                                      int pref_only, tor_addr_port_t *ap)
 {
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   if (!node) {
-    return 0;
+    return;
   }
 
   node_assert_ok(node);
+  /* Calling fascist_firewall_choose_address_node() when the node is missing
+   * IPv6 information breaks IPv6-only clients.
+   * If the node is a hard-coded fallback directory or authority, call
+   * fascist_firewall_choose_address_rs() on the fake (hard-coded) routerstatus
+   * for the node.
+   * If it is not hard-coded, check that the node has a microdescriptor, full
+   * descriptor (routerinfo), or is one of our configured bridges before
+   * calling this function. */
+  if (BUG(node_awaiting_ipv6(get_options(), node))) {
+    return;
+  }
 
   const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION
                               ? node_ipv6_or_preferred(node)
@@ -956,27 +1026,27 @@ fascist_firewall_choose_address_node(const node_t *node,
   node_get_pref_ipv6_dirport(node, &ipv6_dir_ap);
 
   /* Assume the IPv6 OR and Dir addresses are the same. */
-  return fascist_firewall_choose_address_base(&ipv4_or_ap.addr,
-                                              ipv4_or_ap.port,
-                                              ipv4_dir_ap.port,
-                                              &ipv6_or_ap.addr,
-                                              ipv6_or_ap.port,
-                                              ipv6_dir_ap.port,
-                                              fw_connection,
-                                              pref_only,
-                                              pref_ipv6_node,
-                                              ap);
+  fascist_firewall_choose_address_base(&ipv4_or_ap.addr, ipv4_or_ap.port,
+                                       ipv4_dir_ap.port, &ipv6_or_ap.addr,
+                                       ipv6_or_ap.port, ipv6_dir_ap.port,
+                                       fw_connection, pref_only,
+                                       pref_ipv6_node, ap);
 }
 
 /** Like fascist_firewall_choose_address_rs(), but takes <b>ds</b>. */
-int
+void
 fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
                                            firewall_connection_t fw_connection,
                                            int pref_only,
                                            tor_addr_port_t *ap)
 {
+  tor_assert(ap);
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
   if (!ds) {
-    return 0;
+    return;
   }
 
   /* A dir_server_t always has a fake_status. As long as it has the same
@@ -984,8 +1054,8 @@ fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
    * (See #17867.)
    * This function relies on fascist_firewall_choose_address_rs looking up the
    * node if it can, because that will get the latest info for the relay. */
-  return fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection,
-                                            pref_only, ap);
+  fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection,
+                                     pref_only, ap);
 }
 
 /** Return 1 if <b>addr</b> is permitted to connect to our dir port,
@@ -1096,7 +1166,7 @@ validate_addr_policies(const or_options_t *options, char **msg)
              "to 1 to disable this warning, and for forward compatibility.",
              options->ExitPolicy == NULL ?
                  " with the default exit policy" : "");
-    if (options->ExitPolicy == NULL) {
+    if (options->ExitPolicy == NULL && options->ReducedExitPolicy == 0) {
       log_warn(LD_CONFIG,
                "In a future version of Tor, ExitRelay 0 may become the "
                "default when no ExitPolicy is given.");
@@ -1743,14 +1813,14 @@ policies_parse_exit_policy_reject_private(
     /* Reject public IPv4 addresses on any interface */
     public_addresses = get_interface_address6_list(LOG_INFO, AF_INET, 0);
     addr_policy_append_reject_addr_list_filter(dest, public_addresses, 1, 0);
-    free_interface_address6_list(public_addresses);
+    interface_address6_list_free(public_addresses);
 
     /* Don't look for IPv6 addresses if we're configured as IPv4-only */
     if (ipv6_exit) {
       /* Reject public IPv6 addresses on any interface */
       public_addresses = get_interface_address6_list(LOG_INFO, AF_INET6, 0);
       addr_policy_append_reject_addr_list_filter(dest, public_addresses, 0, 1);
-      free_interface_address6_list(public_addresses);
+      interface_address6_list_free(public_addresses);
     }
   }
 
@@ -1829,6 +1899,24 @@ policies_log_first_redundant_entry(const smartlist_t *policy)
   "reject *:563,reject *:1214,reject *:4661-4666,"                  \
   "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*"
 
+#define REDUCED_EXIT_POLICY                                                   \
+  "accept *:20-23,accept *:43,accept *:53,accept *:79-81,accept *:88,"        \
+  "accept *:110,accept *:143,accept *:194,accept *:220,accept *:389,"         \
+  "accept *:443,accept *:464,accept *:465,accept *:531,accept *:543-544,"     \
+  "accept *:554,accept *:563,accept *:587,accept *:636,accept *:706,"         \
+  "accept *:749,accept *:873,accept *:902-904,accept *:981,accept *:989-995," \
+  "accept *:1194,accept *:1220,accept *:1293,accept *:1500,accept *:1533,"    \
+  "accept *:1677,accept *:1723,accept *:1755,accept *:1863,"                  \
+  "accept *:2082-2083,accept *:2086-2087,accept *:2095-2096,"                 \
+  "accept *:2102-2104,accept *:3128,accept *:3389,accept *:3690,"             \
+  "accept *:4321,accept *:4643,accept *:5050,accept *:5190,"                  \
+  "accept *:5222-5223,accept *:5228,accept *:5900,accept *:6660-6669,"        \
+  "accept *:6679,accept *:6697,accept *:8000,accept *:8008,accept *:8074,"    \
+  "accept *:8080,accept *:8082,accept *:8087-8088,accept *:8232-8233,"        \
+  "accept *:8332-8333,accept *:8443,accept *:8888,accept *:9418,"             \
+  "accept *:9999,accept *:10000,accept *:11371,accept *:19294,"               \
+  "accept *:19638,accept *:50002,accept *:64738,reject *:*"
+
 /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>.
  *
  * If <b>ipv6_exit</b> is false, prepend "reject *6:*" to the policy.
@@ -1864,7 +1952,8 @@ policies_parse_exit_policy_internal(config_line_t *cfg,
                                     const smartlist_t *configured_addresses,
                                     int reject_interface_addresses,
                                     int reject_configured_port_addresses,
-                                    int add_default_policy)
+                                    int add_default_policy,
+                                    int add_reduced_policy)
 {
   if (!ipv6_exit) {
     append_exit_policy_string(dest, "reject *6:*");
@@ -1890,7 +1979,9 @@ policies_parse_exit_policy_internal(config_line_t *cfg,
    * effect, and are most likely an error. */
   policies_log_first_redundant_entry(*dest);
 
-  if (add_default_policy) {
+  if (add_reduced_policy) {
+    append_exit_policy_string(dest, REDUCED_EXIT_POLICY);
+  } else if (add_default_policy) {
     append_exit_policy_string(dest, DEFAULT_EXIT_POLICY);
   } else {
     append_exit_policy_string(dest, "reject *4:*");
@@ -1931,13 +2022,15 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest,
   int add_default = (options & EXIT_POLICY_ADD_DEFAULT) ? 1 : 0;
   int reject_local_interfaces = (options &
                                  EXIT_POLICY_REJECT_LOCAL_INTERFACES) ? 1 : 0;
+  int add_reduced = (options & EXIT_POLICY_ADD_REDUCED) ? 1 : 0;
 
   return policies_parse_exit_policy_internal(cfg,dest,ipv6_enabled,
                                              reject_private,
                                              configured_addresses,
                                              reject_local_interfaces,
                                              reject_local_interfaces,
-                                             add_default);
+                                             add_default,
+                                             add_reduced);
 }
 
 /** Helper function that adds a copy of addr to a smartlist as long as it is
@@ -1971,10 +2064,10 @@ policies_copy_ipv4h_to_smartlist(smartlist_t *addr_list, uint32_t ipv4h_addr)
   }
 }
 
-/** Helper function that adds copies of
- * or_options->OutboundBindAddressIPv[4|6]_ to a smartlist as tor_addr_t *, as
- * long as or_options is non-NULL, and the addresses are not
- * tor_addr_is_null(), by passing them to policies_add_addr_to_smartlist.
+/** Helper function that adds copies of or_options->OutboundBindAddresses
+ * to a smartlist as tor_addr_t *, as long as or_options is non-NULL, and
+ * the addresses are not tor_addr_is_null(), by passing them to
+ * policies_add_addr_to_smartlist.
  *
  * The caller is responsible for freeing all the tor_addr_t* in the smartlist.
  */
@@ -1983,10 +2076,14 @@ policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list,
                                               const or_options_t *or_options)
 {
   if (or_options) {
-    policies_copy_addr_to_smartlist(addr_list,
-                                    &or_options->OutboundBindAddressIPv4_);
-    policies_copy_addr_to_smartlist(addr_list,
-                                    &or_options->OutboundBindAddressIPv6_);
+    for (int i=0;i<OUTBOUND_ADDR_MAX;i++) {
+      for (int j=0;j<2;j++) {
+        if (!tor_addr_is_null(&or_options->OutboundBindAddresses[i][j])) {
+          policies_copy_addr_to_smartlist(addr_list,
+                          &or_options->OutboundBindAddresses[i][j]);
+        }
+      }
+    }
   }
 }
 
@@ -2003,10 +2100,10 @@ policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list,
  *  - if ipv6_local_address is non-NULL, and not the null tor_addr_t, add it
  *    to the list of configured addresses.
  * If <b>or_options->ExitPolicyRejectLocalInterfaces</b> is true:
- *  - if or_options->OutboundBindAddressIPv4_ is not the null tor_addr_t, add
- *    it to the list of configured addresses.
- *  - if or_options->OutboundBindAddressIPv6_ is not the null tor_addr_t, add
- *    it to the list of configured addresses.
+ *  - if or_options->OutboundBindAddresses[][0] (=IPv4) is not the null
+ *    tor_addr_t, add it to the list of configured addresses.
+ *  - if or_options->OutboundBindAddresses[][1] (=IPv6) is not the null
+ *    tor_addr_t, add it to the list of configured addresses.
  *
  * If <b>or_options->BridgeRelay</b> is false, append entries of default
  * Tor exit policy into <b>result</b> smartlist.
@@ -2043,7 +2140,10 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   }
 
   if (!or_options->BridgeRelay) {
-    parser_cfg |= EXIT_POLICY_ADD_DEFAULT;
+    if (or_options->ReducedExitPolicy)
+      parser_cfg |= EXIT_POLICY_ADD_REDUCED;
+    else
+      parser_cfg |= EXIT_POLICY_ADD_DEFAULT;
   }
 
   if (or_options->ExitPolicyRejectLocalInterfaces) {
@@ -2134,21 +2234,16 @@ exit_policy_is_general_exit_helper(smartlist_t *policy, int port)
 }
 
 /** Return true iff <b>ri</b> is "useful as an exit node", meaning
- * it allows exit to at least one /8 address space for at least
- * two of ports 80, 443, and 6667. */
+ * it allows exit to at least one /8 address space for each of ports 80
+ * and 443. */
 int
 exit_policy_is_general_exit(smartlist_t *policy)
 {
-  static const int ports[] = { 80, 443, 6667 };
-  int n_allowed = 0;
-  int i;
   if (!policy) /*XXXX disallow NULL policies? */
     return 0;
 
-  for (i = 0; i < 3; ++i) {
-    n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]);
-  }
-  return n_allowed >= 2;
+  return (exit_policy_is_general_exit_helper(policy, 80) &&
+          exit_policy_is_general_exit_helper(policy, 443));
 }
 
 /** Return false if <b>policy</b> might permit access to some addr:port;
@@ -2314,7 +2409,7 @@ policy_summary_item_split(policy_summary_item_t* old, uint16_t new_starts)
 #define REJECT_CUTOFF_SCALE_IPV6 (64)
 /* Ports are rejected in an IPv6 summary if they are rejected in more than one
  * IPv6 /16 address block.
- * This is rougly equivalent to the IPv4 cutoff, as only five IPv6 /12s (and
+ * This is roughly equivalent to the IPv4 cutoff, as only five IPv6 /12s (and
  * some scattered smaller blocks) have been allocated to the RIRs.
  * Network providers are typically allocated one or more IPv6 /32s.
  */
@@ -2528,9 +2623,9 @@ policy_summarize(smartlist_t *policy, sa_family_t family)
         tor_snprintf(buf, sizeof(buf), "%d-%d", start_prt, AT(i)->prt_max);
 
       if (AT(i)->accepted)
-        smartlist_add(accepts, tor_strdup(buf));
+        smartlist_add_strdup(accepts, buf);
       else
-        smartlist_add(rejects, tor_strdup(buf));
+        smartlist_add_strdup(rejects, buf);
 
       if (last)
         break;
@@ -2679,7 +2774,7 @@ parse_short_policy(const char *summary)
   }
 
   {
-    size_t size = STRUCT_OFFSET(short_policy_t, entries) +
+    size_t size = offsetof(short_policy_t, entries) +
       sizeof(short_policy_entry_t)*(n_entries);
     result = tor_malloc_zero(size);
 
@@ -2710,7 +2805,7 @@ write_short_policy(const short_policy_t *policy)
       smartlist_add_asprintf(sl, "%d-%d", e->min_port, e->max_port);
     }
     if (i < policy->n_entries-1)
-      smartlist_add(sl, tor_strdup(","));
+      smartlist_add_strdup(sl, ",");
   }
   answer = smartlist_join_strings(sl, "", 0, NULL);
   SMARTLIST_FOREACH(sl, char *, a, tor_free(a));
@@ -2720,7 +2815,7 @@ write_short_policy(const short_policy_t *policy)
 
 /** Release all storage held in <b>policy</b>. */
 void
-short_policy_free(short_policy_t *policy)
+short_policy_free_(short_policy_t *policy)
 {
   tor_free(policy);
 }
@@ -2904,11 +2999,12 @@ getinfo_helper_policies(control_connection_t *conn,
     smartlist_free(private_policy_strings);
   } else if (!strcmp(question, "exit-policy/reject-private/relay")) {
     const or_options_t *options = get_options();
-    const routerinfo_t *me = router_get_my_routerinfo();
+    int err = 0;
+    const routerinfo_t *me = router_get_my_routerinfo_with_err(&err);
 
     if (!me) {
-      *errmsg = "router_get_my_routerinfo returned NULL";
-      return -1;
+      *errmsg = routerinfo_err_to_string(err);
+      return routerinfo_err_is_transient(err) ? -1 : 0;
     }
 
     if (!options->ExitPolicyRejectPrivate &&
@@ -2943,11 +3039,17 @@ getinfo_helper_policies(control_connection_t *conn,
     SMARTLIST_FOREACH(configured_addresses, tor_addr_t *, a, tor_free(a));
     smartlist_free(configured_addresses);
   } else if (!strcmpstart(question, "exit-policy/")) {
-    const routerinfo_t *me = router_get_my_routerinfo();
-
     int include_ipv4 = 0;
     int include_ipv6 = 0;
 
+    int err = 0;
+    const routerinfo_t *me = router_get_my_routerinfo_with_err(&err);
+
+    if (!me) {
+      *errmsg = routerinfo_err_to_string(err);
+      return routerinfo_err_is_transient(err) ? -1 : 0;
+    }
+
     if (!strcmp(question, "exit-policy/ipv4")) {
       include_ipv4 = 1;
     } else if (!strcmp(question, "exit-policy/ipv6")) {
@@ -2958,19 +3060,16 @@ getinfo_helper_policies(control_connection_t *conn,
       return 0; /* No such key. */
     }
 
-    if (!me) {
-      *errmsg = "router_get_my_routerinfo returned NULL";
-      return -1;
-    }
-
-    *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6);
+    *answer = router_dump_exit_policy_to_string(me,include_ipv4,
+                                                include_ipv6);
   }
+
   return 0;
 }
 
 /** Release all storage held by <b>p</b>. */
 void
-addr_policy_list_free(smartlist_t *lst)
+addr_policy_list_free_(smartlist_t *lst)
 {
   if (!lst)
     return;
@@ -2980,7 +3079,7 @@ addr_policy_list_free(smartlist_t *lst)
 
 /** Release all storage held by <b>p</b>. */
 void
-addr_policy_free(addr_policy_t *p)
+addr_policy_free_(addr_policy_t *p)
 {
   if (!p)
     return;
diff --git a/src/or/policies.h b/src/or/policies.h
index f73f850c21..4879acdd8d 100644
--- a/src/or/policies.h
+++ b/src/or/policies.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,8 @@
 #define EXIT_POLICY_REJECT_PRIVATE           (1 << 1)
 #define EXIT_POLICY_ADD_DEFAULT              (1 << 2)
 #define EXIT_POLICY_REJECT_LOCAL_INTERFACES  (1 << 3)
-#define EXIT_POLICY_OPTION_MAX             EXIT_POLICY_REJECT_LOCAL_INTERFACES
+#define EXIT_POLICY_ADD_REDUCED              (1 << 4)
+#define EXIT_POLICY_OPTION_MAX             EXIT_POLICY_ADD_REDUCED
 /* All options set: used for unit testing */
 #define EXIT_POLICY_OPTION_ALL             ((EXIT_POLICY_OPTION_MAX << 1) - 1)
 
@@ -54,13 +55,13 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds,
                                        firewall_connection_t fw_connection,
                                        int pref_only);
 
-int fascist_firewall_choose_address_rs(const routerstatus_t *rs,
-                                       firewall_connection_t fw_connection,
-                                       int pref_only, tor_addr_port_t* ap);
-int fascist_firewall_choose_address_node(const node_t *node,
-                                         firewall_connection_t fw_connection,
-                                         int pref_only, tor_addr_port_t* ap);
-int fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
+void fascist_firewall_choose_address_rs(const routerstatus_t *rs,
+                                        firewall_connection_t fw_connection,
+                                        int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_node(const node_t *node,
+                                          firewall_connection_t fw_connection,
+                                          int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_dir_server(const dir_server_t *ds,
                                           firewall_connection_t fw_connection,
                                           int pref_only, tor_addr_port_t* ap);
 
@@ -114,15 +115,21 @@ int getinfo_helper_policies(control_connection_t *conn,
 int policy_write_item(char *buf, size_t buflen, const addr_policy_t *item,
                       int format_for_desc);
 
-void addr_policy_list_free(smartlist_t *p);
-void addr_policy_free(addr_policy_t *p);
+void addr_policy_list_free_(smartlist_t *p);
+#define addr_policy_list_free(lst) \
+  FREE_AND_NULL(smartlist_t, addr_policy_list_free_, (lst))
+void addr_policy_free_(addr_policy_t *p);
+#define addr_policy_free(p) \
+  FREE_AND_NULL(addr_policy_t, addr_policy_free_, (p))
 void policies_free_all(void);
 
 char *policy_summarize(smartlist_t *policy, sa_family_t family);
 
 short_policy_t *parse_short_policy(const char *summary);
 char *write_short_policy(const short_policy_t *policy);
-void short_policy_free(short_policy_t *policy);
+void short_policy_free_(short_policy_t *policy);
+#define short_policy_free(p) \
+  FREE_AND_NULL(short_policy_t, short_policy_free_, (p))
 int short_policy_is_reject_star(const short_policy_t *policy);
 addr_policy_result_t compare_tor_addr_to_short_policy(
                           const tor_addr_t *addr, uint16_t port,
@@ -141,7 +148,7 @@ STATIC const tor_addr_port_t * fascist_firewall_choose_address(
                                           firewall_connection_t fw_connection,
                                           int pref_only, int pref_ipv6);
 
-#endif
+#endif /* defined(POLICIES_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_POLICIES_H) */
 
diff --git a/src/or/proto_cell.c b/src/or/proto_cell.c
new file mode 100644
index 0000000000..75eb2a7e7f
--- /dev/null
+++ b/src/or/proto_cell.c
@@ -0,0 +1,84 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "buffers.h"
+#include "proto_cell.h"
+
+#include "connection_or.h"
+
+/** True iff the cell command <b>command</b> is one that implies a
+ * variable-length cell in Tor link protocol <b>linkproto</b>. */
+static inline int
+cell_command_is_var_length(uint8_t command, int linkproto)
+{
+  /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells
+   * work as implemented here. If it's 1, there are no variable-length cells.
+   * Tor does not support other versions right now, and so can't negotiate
+   * them.
+   */
+  switch (linkproto) {
+  case 1:
+    /* Link protocol version 1 has no variable-length cells. */
+    return 0;
+  case 2:
+    /* In link protocol version 2, VERSIONS is the only variable-length cell */
+    return command == CELL_VERSIONS;
+  case 0:
+  case 3:
+  default:
+    /* In link protocol version 3 and later, and in version "unknown",
+     * commands 128 and higher indicate variable-length. VERSIONS is
+     * grandfathered in. */
+    return command == CELL_VERSIONS || command >= 128;
+  }
+}
+
+/** Check <b>buf</b> for a variable-length cell according to the rules of link
+ * protocol version <b>linkproto</b>.  If one is found, pull it off the buffer
+ * and assign a newly allocated var_cell_t to *<b>out</b>, and return 1.
+ * Return 0 if whatever is on the start of buf_t is not a variable-length
+ * cell.  Return 1 and set *<b>out</b> to NULL if there seems to be the start
+ * of a variable-length cell on <b>buf</b>, but the whole thing isn't there
+ * yet. */
+int
+fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
+{
+  char hdr[VAR_CELL_MAX_HEADER_SIZE];
+  var_cell_t *result;
+  uint8_t command;
+  uint16_t length;
+  const int wide_circ_ids = linkproto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
+  const int circ_id_len = get_circ_id_size(wide_circ_ids);
+  const unsigned header_len = get_var_cell_header_size(wide_circ_ids);
+  *out = NULL;
+  if (buf_datalen(buf) < header_len)
+    return 0;
+  buf_peek(buf, hdr, header_len);
+
+  command = get_uint8(hdr + circ_id_len);
+  if (!(cell_command_is_var_length(command, linkproto)))
+    return 0;
+
+  length = ntohs(get_uint16(hdr + circ_id_len + 1));
+  if (buf_datalen(buf) < (size_t)(header_len+length))
+    return 1;
+
+  result = var_cell_new(length);
+  result->command = command;
+  if (wide_circ_ids)
+    result->circ_id = ntohl(get_uint32(hdr));
+  else
+    result->circ_id = ntohs(get_uint16(hdr));
+
+  buf_drain(buf, header_len);
+  buf_peek(buf, (char*) result->payload, length);
+  buf_drain(buf, length);
+
+  *out = result;
+  return 1;
+}
+
diff --git a/src/or/proto_cell.h b/src/or/proto_cell.h
new file mode 100644
index 0000000000..bbc14b9a02
--- /dev/null
+++ b/src/or/proto_cell.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CELL_H
+#define TOR_PROTO_CELL_H
+
+struct buf_t;
+struct var_cell_t;
+
+int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out,
+                            int linkproto);
+
+#endif /* !defined(TOR_PROTO_CELL_H) */
+
diff --git a/src/or/proto_control0.c b/src/or/proto_control0.c
new file mode 100644
index 0000000000..c17ba34948
--- /dev/null
+++ b/src/or/proto_control0.c
@@ -0,0 +1,26 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "buffers.h"
+#include "proto_control0.h"
+
+/** Return 1 iff buf looks more like it has an (obsolete) v0 controller
+ * command on it than any valid v1 controller command. */
+int
+peek_buf_has_control0_command(buf_t *buf)
+{
+  if (buf_datalen(buf) >= 4) {
+    char header[4];
+    uint16_t cmd;
+    buf_peek(buf, header, sizeof(header));
+    cmd = ntohs(get_uint16(header+2));
+    if (cmd <= 0x14)
+      return 1; /* This is definitely not a v1 control command. */
+  }
+  return 0;
+}
+
diff --git a/src/or/proto_control0.h b/src/or/proto_control0.h
new file mode 100644
index 0000000000..0cc8eacad0
--- /dev/null
+++ b/src/or/proto_control0.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_CONTROL0_H
+#define TOR_PROTO_CONTROL0_H
+
+struct buf_t;
+int peek_buf_has_control0_command(struct buf_t *buf);
+
+#endif /* !defined(TOR_PROTO_CONTROL0_H) */
+
diff --git a/src/or/proto_ext_or.c b/src/or/proto_ext_or.c
new file mode 100644
index 0000000000..057cf109ec
--- /dev/null
+++ b/src/or/proto_ext_or.c
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "buffers.h"
+#include "ext_orport.h"
+#include "proto_ext_or.h"
+
+/** The size of the header of an Extended ORPort message: 2 bytes for
+ *  COMMAND, 2 bytes for BODYLEN */
+#define EXT_OR_CMD_HEADER_SIZE 4
+
+/** Read <b>buf</b>, which should contain an Extended ORPort message
+ *  from a transport proxy. If well-formed, create and populate
+ *  <b>out</b> with the Extended ORport message. Return 0 if the
+ *  buffer was incomplete, 1 if it was well-formed and -1 if we
+ *  encountered an error while parsing it.  */
+int
+fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
+{
+  char hdr[EXT_OR_CMD_HEADER_SIZE];
+  uint16_t len;
+
+  if (buf_datalen(buf) < EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  buf_peek(buf, hdr, sizeof(hdr));
+  len = ntohs(get_uint16(hdr+2));
+  if (buf_datalen(buf) < (unsigned)len + EXT_OR_CMD_HEADER_SIZE)
+    return 0;
+  *out = ext_or_cmd_new(len);
+  (*out)->cmd = ntohs(get_uint16(hdr));
+  (*out)->len = len;
+  buf_drain(buf, EXT_OR_CMD_HEADER_SIZE);
+  buf_get_bytes(buf, (*out)->body, len);
+  return 1;
+}
+
diff --git a/src/or/proto_ext_or.h b/src/or/proto_ext_or.h
new file mode 100644
index 0000000000..cc504d18e3
--- /dev/null
+++ b/src/or/proto_ext_or.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_EXT_OR_H
+#define TOR_PROTO_EXT_OR_H
+
+struct buf_t;
+struct ext_or_cmt_t;
+
+int fetch_ext_or_command_from_buf(struct buf_t *buf,
+                                  struct ext_or_cmd_t **out);
+
+#endif /* !defined(TOR_PROTO_EXT_OR_H) */
+
diff --git a/src/or/proto_http.c b/src/or/proto_http.c
new file mode 100644
index 0000000000..3762429e1e
--- /dev/null
+++ b/src/or/proto_http.c
@@ -0,0 +1,171 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define PROTO_HTTP_PRIVATE
+#include "or.h"
+#include "buffers.h"
+#include "proto_http.h"
+
+/** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
+int
+peek_buf_has_http_command(const buf_t *buf)
+{
+  if (buf_peek_startswith(buf, "CONNECT ") ||
+      buf_peek_startswith(buf, "DELETE ") ||
+      buf_peek_startswith(buf, "GET ") ||
+      buf_peek_startswith(buf, "POST ") ||
+      buf_peek_startswith(buf, "PUT " ))
+    return 1;
+  return 0;
+}
+
+/** There is a (possibly incomplete) http statement on <b>buf</b>, of the
+ * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
+ * If a) the headers include a Content-Length field and all bytes in
+ * the body are present, or b) there's no Content-Length field and
+ * all headers are present, then:
+ *
+ *  - strdup headers into <b>*headers_out</b>, and NUL-terminate it.
+ *  - memdup body into <b>*body_out</b>, and NUL-terminate it.
+ *  - Then remove them from <b>buf</b>, and return 1.
+ *
+ *  - If headers or body is NULL, discard that part of the buf.
+ *  - If a headers or body doesn't fit in the arg, return -1.
+ *  (We ensure that the headers or body don't exceed max len,
+ *   _even if_ we're planning to discard them.)
+ *  - If force_complete is true, then succeed even if not all of the
+ *    content has arrived.
+ *
+ * Else, change nothing and return 0.
+ */
+int
+fetch_from_buf_http(buf_t *buf,
+                    char **headers_out, size_t max_headerlen,
+                    char **body_out, size_t *body_used, size_t max_bodylen,
+                    int force_complete)
+{
+  const char *headers;
+  size_t headerlen, bodylen, contentlen=0;
+  int crlf_offset;
+  int r;
+
+  if (buf_datalen(buf) == 0)
+    return 0;
+
+  crlf_offset = buf_find_string_offset(buf, "\r\n\r\n", 4);
+  if (crlf_offset > (int)max_headerlen ||
+      (crlf_offset < 0 && buf_datalen(buf) > max_headerlen)) {
+    log_debug(LD_HTTP,"headers too long.");
+    return -1;
+  } else if (crlf_offset < 0) {
+    log_debug(LD_HTTP,"headers not all here yet.");
+    return 0;
+  }
+  /* Okay, we have a full header.  Make sure it all appears in the first
+   * chunk. */
+  headerlen = crlf_offset + 4;
+  size_t headers_in_chunk = 0;
+  buf_pullup(buf, headerlen, &headers, &headers_in_chunk);
+
+  bodylen = buf_datalen(buf) - headerlen;
+  log_debug(LD_HTTP,"headerlen %d, bodylen %d.", (int)headerlen, (int)bodylen);
+
+  if (max_headerlen <= headerlen) {
+    log_warn(LD_HTTP,"headerlen %d larger than %d. Failing.",
+             (int)headerlen, (int)max_headerlen-1);
+    return -1;
+  }
+  if (max_bodylen <= bodylen) {
+    log_warn(LD_HTTP,"bodylen %d larger than %d. Failing.",
+             (int)bodylen, (int)max_bodylen-1);
+    return -1;
+  }
+
+  r = buf_http_find_content_length(headers, headerlen, &contentlen);
+  if (r == -1) {
+    log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe "
+             "someone is trying to crash us.");
+    return -1;
+  } else if (r == 1) {
+    /* if content-length is malformed, then our body length is 0. fine. */
+    log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
+    if (bodylen < contentlen) {
+      if (!force_complete) {
+        log_debug(LD_HTTP,"body not all here yet.");
+        return 0; /* not all there yet */
+      }
+    }
+    if (bodylen > contentlen) {
+      bodylen = contentlen;
+      log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
+    }
+  } else {
+    tor_assert(r == 0);
+    /* Leave bodylen alone */
+  }
+
+  /* all happy. copy into the appropriate places, and return 1 */
+  if (headers_out) {
+    *headers_out = tor_malloc(headerlen+1);
+    buf_get_bytes(buf, *headers_out, headerlen);
+    (*headers_out)[headerlen] = 0; /* NUL terminate it */
+  }
+  if (body_out) {
+    tor_assert(body_used);
+    *body_used = bodylen;
+    *body_out = tor_malloc(bodylen+1);
+    buf_get_bytes(buf, *body_out, bodylen);
+    (*body_out)[bodylen] = 0; /* NUL terminate it */
+  }
+  return 1;
+}
+
+/**
+ * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at
+ * <b>headers</b>, looking for a "Content-Length" header.  Try to set
+ * *<b>result_out</b> to the numeric value of that header if possible.
+ * Return -1 if the header was malformed, 0 if it was missing, and 1 if
+ * it was present and well-formed.
+ */
+STATIC int
+buf_http_find_content_length(const char *headers, size_t headerlen,
+                             size_t *result_out)
+{
+  const char *p, *newline;
+  char *len_str, *eos=NULL;
+  size_t remaining, result;
+  int ok;
+  *result_out = 0; /* The caller shouldn't look at this unless the
+                    * return value is 1, but let's prevent confusion */
+
+#define CONTENT_LENGTH "\r\nContent-Length: "
+  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
+  if (p == NULL)
+    return 0;
+
+  tor_assert(p >= headers && p < headers+headerlen);
+  remaining = (headers+headerlen)-p;
+  p += strlen(CONTENT_LENGTH);
+  remaining -= strlen(CONTENT_LENGTH);
+
+  newline = memchr(p, '\n', remaining);
+  if (newline == NULL)
+    return -1;
+
+  len_str = tor_memdup_nulterm(p, newline-p);
+  /* We limit the size to INT_MAX because other parts of the buffer.c
+   * code don't like buffers to be any bigger than that. */
+  result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos);
+  if (eos && !tor_strisspace(eos)) {
+    ok = 0;
+  } else {
+    *result_out = result;
+  }
+  tor_free(len_str);
+
+  return ok ? 1 : -1;
+}
+
diff --git a/src/or/proto_http.h b/src/or/proto_http.h
new file mode 100644
index 0000000000..805686070f
--- /dev/null
+++ b/src/or/proto_http.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_HTTP_H
+#define TOR_PROTO_HTTP_H
+
+struct buf_t;
+
+int fetch_from_buf_http(struct buf_t *buf,
+                        char **headers_out, size_t max_headerlen,
+                        char **body_out, size_t *body_used, size_t max_bodylen,
+                        int force_complete);
+int peek_buf_has_http_command(const struct buf_t *buf);
+
+#ifdef PROTO_HTTP_PRIVATE
+STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
+                                        size_t *result_out);
+#endif
+
+#endif /* !defined(TOR_PROTO_HTTP_H) */
+
diff --git a/src/or/proto_socks.c b/src/or/proto_socks.c
new file mode 100644
index 0000000000..57a7d1cd64
--- /dev/null
+++ b/src/or/proto_socks.c
@@ -0,0 +1,711 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "addressmap.h"
+#include "buffers.h"
+#include "control.h"
+#include "config.h"
+#include "crypto_util.h"
+#include "ext_orport.h"
+#include "proto_socks.h"
+#include "reasons.h"
+
+static void socks_request_set_socks5_error(socks_request_t *req,
+                              socks5_reply_status_t reason);
+
+static int parse_socks(const char *data, size_t datalen, socks_request_t *req,
+                       int log_sockstype, int safe_socks, ssize_t *drain_out,
+                       size_t *want_length_out);
+static int parse_socks_client(const uint8_t *data, size_t datalen,
+                              int state, char **reason,
+                              ssize_t *drain_out);
+/**
+ * Wait this many seconds before warning the user about using SOCKS unsafely
+ * again. */
+#define SOCKS_WARN_INTERVAL 5
+
+/** Warn that the user application has made an unsafe socks request using
+ * protocol <b>socks_protocol</b> on port <b>port</b>.  Don't warn more than
+ * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */
+static void
+log_unsafe_socks_warning(int socks_protocol, const char *address,
+                         uint16_t port, int safe_socks)
+{
+  static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
+
+  if (safe_socks) {
+    log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
+             "Your application (using socks%d to port %d) is giving "
+             "Tor only an IP address. Applications that do DNS resolves "
+             "themselves may leak information. Consider using Socks4A "
+             "(e.g. via privoxy or socat) instead. For more information, "
+             "please see https://wiki.torproject.org/TheOnionRouter/"
+             "TorFAQ#SOCKSAndDNS.%s",
+             socks_protocol,
+             (int)port,
+             safe_socks ? " Rejecting." : "");
+  }
+  control_event_client_status(LOG_WARN,
+                              "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d",
+                              socks_protocol, address, (int)port);
+}
+
+/** Do not attempt to parse socks messages longer than this.  This value is
+ * actually significantly higher than the longest possible socks message. */
+#define MAX_SOCKS_MESSAGE_LEN 512
+
+/** Return a new socks_request_t. */
+socks_request_t *
+socks_request_new(void)
+{
+  return tor_malloc_zero(sizeof(socks_request_t));
+}
+
+/** Free all storage held in the socks_request_t <b>req</b>. */
+void
+socks_request_free_(socks_request_t *req)
+{
+  if (!req)
+    return;
+  if (req->username) {
+    memwipe(req->username, 0x10, req->usernamelen);
+    tor_free(req->username);
+  }
+  if (req->password) {
+    memwipe(req->password, 0x04, req->passwordlen);
+    tor_free(req->password);
+  }
+  memwipe(req, 0xCC, sizeof(socks_request_t));
+  tor_free(req);
+}
+
+/** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one
+ * of the forms
+ *  - socks4: "socksheader username\\0"
+ *  - socks4a: "socksheader username\\0 destaddr\\0"
+ *  - socks5 phase one: "version #methods methods"
+ *  - socks5 phase two: "version command 0 addresstype..."
+ * If it's a complete and valid handshake, and destaddr fits in
+ *   MAX_SOCKS_ADDR_LEN bytes, then pull the handshake off the buf,
+ *   assign to <b>req</b>, and return 1.
+ *
+ * If it's invalid or too big, return -1.
+ *
+ * Else it's not all there yet, leave buf alone and return 0.
+ *
+ * If you want to specify the socks reply, write it into <b>req->reply</b>
+ *   and set <b>req->replylen</b>, else leave <b>req->replylen</b> alone.
+ *
+ * If <b>log_sockstype</b> is non-zero, then do a notice-level log of whether
+ * the connection is possibly leaking DNS requests locally or not.
+ *
+ * If <b>safe_socks</b> is true, then reject unsafe socks protocols.
+ *
+ * If returning 0 or -1, <b>req->address</b> and <b>req->port</b> are
+ * undefined.
+ */
+int
+fetch_from_buf_socks(buf_t *buf, socks_request_t *req,
+                     int log_sockstype, int safe_socks)
+{
+  int res;
+  ssize_t n_drain;
+  size_t want_length = 128;
+  const char *head = NULL;
+  size_t datalen = 0;
+
+  if (buf_datalen(buf) < 2) /* version and another byte */
+    return 0;
+
+  do {
+    n_drain = 0;
+    buf_pullup(buf, want_length, &head, &datalen);
+    tor_assert(head && datalen >= 2);
+    want_length = 0;
+
+    res = parse_socks(head, datalen, req, log_sockstype,
+                      safe_socks, &n_drain, &want_length);
+
+    if (n_drain < 0)
+      buf_clear(buf);
+    else if (n_drain > 0)
+      buf_drain(buf, n_drain);
+
+  } while (res == 0 && head && want_length < buf_datalen(buf) &&
+           buf_datalen(buf) >= 2);
+
+  return res;
+}
+
+/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and
+ * have Tor send it as error response to <b>req</b>.
+ */
+static void
+socks_request_set_socks5_error(socks_request_t *req,
+                  socks5_reply_status_t reason)
+{
+   req->replylen = 10;
+   memset(req->reply,0,10);
+
+   req->reply[0] = 0x05;   // VER field.
+   req->reply[1] = reason; // REP field.
+   req->reply[3] = 0x01;   // ATYP field.
+}
+
+static const char SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 501 Tor is not an HTTP Proxy\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>This is a SOCKS Proxy, Not An HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>This is a SOCKs proxy, not an HTTP proxy.</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use this Tor port as\n"
+  "an HTTP proxy.\n"
+  "</p><p>\n"
+  "This is not correct: This port is configured as a SOCKS proxy, not\n"
+  "an HTTP proxy. If you need an HTTP proxy tunnel, use the HTTPTunnelPort\n"
+  "configuration option in place of, or in addition to, SOCKSPort.\n"
+  "Please configure your client accordingly.\n"
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
+/** Implementation helper to implement fetch_from_*_socks.  Instead of looking
+ * at a buffer's contents, we look at the <b>datalen</b> bytes of data in
+ * <b>data</b>. Instead of removing data from the buffer, we set
+ * <b>drain_out</b> to the amount of data that should be removed (or -1 if the
+ * buffer should be cleared).  Instead of pulling more data into the first
+ * chunk of the buffer, we set *<b>want_length_out</b> to the number of bytes
+ * we'd like to see in the input buffer, if they're available. */
+static int
+parse_socks(const char *data, size_t datalen, socks_request_t *req,
+            int log_sockstype, int safe_socks, ssize_t *drain_out,
+            size_t *want_length_out)
+{
+  unsigned int len;
+  char tmpbuf[TOR_ADDR_BUF_LEN+1];
+  tor_addr_t destaddr;
+  uint32_t destip;
+  uint8_t socksver;
+  char *next, *startaddr;
+  unsigned char usernamelen, passlen;
+  struct in_addr in;
+
+  if (datalen < 2) {
+    /* We always need at least 2 bytes. */
+    *want_length_out = 2;
+    return 0;
+  }
+
+  if (req->socks_version == 5 && !req->got_auth) {
+    /* See if we have received authentication.  Strictly speaking, we should
+       also check whether we actually negotiated username/password
+       authentication.  But some broken clients will send us authentication
+       even if we negotiated SOCKS_NO_AUTH. */
+    if (*data == 1) { /* username/pass version 1 */
+      /* Format is: authversion [1 byte] == 1
+                    usernamelen [1 byte]
+                    username    [usernamelen bytes]
+                    passlen     [1 byte]
+                    password    [passlen bytes] */
+      usernamelen = (unsigned char)*(data + 1);
+      if (datalen < 2u + usernamelen + 1u) {
+        *want_length_out = 2u + usernamelen + 1u;
+        return 0;
+      }
+      passlen = (unsigned char)*(data + 2u + usernamelen);
+      if (datalen < 2u + usernamelen + 1u + passlen) {
+        *want_length_out = 2u + usernamelen + 1u + passlen;
+        return 0;
+      }
+      req->replylen = 2; /* 2 bytes of response */
+      req->reply[0] = 1; /* authversion == 1 */
+      req->reply[1] = 0; /* authentication successful */
+      log_debug(LD_APP,
+               "socks5: Accepted username/password without checking.");
+      if (usernamelen) {
+        req->username = tor_memdup(data+2u, usernamelen);
+        req->usernamelen = usernamelen;
+      }
+      if (passlen) {
+        req->password = tor_memdup(data+3u+usernamelen, passlen);
+        req->passwordlen = passlen;
+      }
+      *drain_out = 2u + usernamelen + 1u + passlen;
+      req->got_auth = 1;
+      *want_length_out = 7; /* Minimal socks5 command. */
+      return 0;
+    } else if (req->auth_type == SOCKS_USER_PASS) {
+      /* unknown version byte */
+      log_warn(LD_APP, "Socks5 username/password version %d not recognized; "
+               "rejecting.", (int)*data);
+      return -1;
+    }
+  }
+
+  socksver = *data;
+
+  switch (socksver) { /* which version of socks? */
+    case 5: /* socks5 */
+
+      if (req->socks_version != 5) { /* we need to negotiate a method */
+        unsigned char nummethods = (unsigned char)*(data+1);
+        int have_user_pass, have_no_auth;
+        int r=0;
+        tor_assert(!req->socks_version);
+        if (datalen < 2u+nummethods) {
+          *want_length_out = 2u+nummethods;
+          return 0;
+        }
+        if (!nummethods)
+          return -1;
+        req->replylen = 2; /* 2 bytes of response */
+        req->reply[0] = 5; /* socks5 reply */
+        have_user_pass = (memchr(data+2, SOCKS_USER_PASS, nummethods) !=NULL);
+        have_no_auth   = (memchr(data+2, SOCKS_NO_AUTH,   nummethods) !=NULL);
+        if (have_user_pass && !(have_no_auth && req->socks_prefer_no_auth)) {
+          req->auth_type = SOCKS_USER_PASS;
+          req->reply[1] = SOCKS_USER_PASS; /* tell client to use "user/pass"
+                                              auth method */
+          req->socks_version = 5; /* remember we've already negotiated auth */
+          log_debug(LD_APP,"socks5: accepted method 2 (username/password)");
+          r=0;
+        } else if (have_no_auth) {
+          req->reply[1] = SOCKS_NO_AUTH; /* tell client to use "none" auth
+                                            method */
+          req->socks_version = 5; /* remember we've already negotiated auth */
+          log_debug(LD_APP,"socks5: accepted method 0 (no authentication)");
+          r=0;
+        } else {
+          log_warn(LD_APP,
+                    "socks5: offered methods don't include 'no auth' or "
+                    "username/password. Rejecting.");
+          req->reply[1] = '\xFF'; /* reject all methods */
+          r=-1;
+        }
+        /* Remove packet from buf. Some SOCKS clients will have sent extra
+         * junk at this point; let's hope it's an authentication message. */
+        *drain_out = 2u + nummethods;
+
+        return r;
+      }
+      if (req->auth_type != SOCKS_NO_AUTH && !req->got_auth) {
+        log_warn(LD_APP,
+                 "socks5: negotiated authentication, but none provided");
+        return -1;
+      }
+      /* we know the method; read in the request */
+      log_debug(LD_APP,"socks5: checking request");
+      if (datalen < 7) {/* basic info plus >=1 for addr plus 2 for port */
+        *want_length_out = 7;
+        return 0; /* not yet */
+      }
+      req->command = (unsigned char) *(data+1);
+      if (req->command != SOCKS_COMMAND_CONNECT &&
+          req->command != SOCKS_COMMAND_RESOLVE &&
+          req->command != SOCKS_COMMAND_RESOLVE_PTR) {
+        /* not a connect or resolve or a resolve_ptr? we don't support it. */
+        socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED);
+
+        log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.",
+                 req->command);
+        return -1;
+      }
+      switch (*(data+3)) { /* address type */
+        case 1: /* IPv4 address */
+        case 4: /* IPv6 address */ {
+          const int is_v6 = *(data+3) == 4;
+          const unsigned addrlen = is_v6 ? 16 : 4;
+          log_debug(LD_APP,"socks5: ipv4 address type");
+          if (datalen < 6+addrlen) {/* ip/port there? */
+            *want_length_out = 6+addrlen;
+            return 0; /* not yet */
+          }
+
+          if (is_v6)
+            tor_addr_from_ipv6_bytes(&destaddr, data+4);
+          else
+            tor_addr_from_ipv4n(&destaddr, get_uint32(data+4));
+
+          tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1);
+
+          if (BUG(strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN)) {
+            /* LCOV_EXCL_START -- This branch is unreachable, given the
+             * size of tmpbuf and the actual value of MAX_SOCKS_ADDR_LEN */
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+            log_warn(LD_APP,
+                     "socks5 IP takes %d bytes, which doesn't fit in %d. "
+                     "Rejecting.",
+                     (int)strlen(tmpbuf)+1,(int)MAX_SOCKS_ADDR_LEN);
+            return -1;
+            /* LCOV_EXCL_STOP */
+          }
+          strlcpy(req->address,tmpbuf,sizeof(req->address));
+          req->port = ntohs(get_uint16(data+4+addrlen));
+          *drain_out = 6+addrlen;
+          if (req->command != SOCKS_COMMAND_RESOLVE_PTR &&
+              !addressmap_have_mapping(req->address,0)) {
+            log_unsafe_socks_warning(5, req->address, req->port, safe_socks);
+            if (safe_socks) {
+              socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED);
+              return -1;
+            }
+          }
+          return 1;
+        }
+        case 3: /* fqdn */
+          log_debug(LD_APP,"socks5: fqdn address type");
+          if (req->command == SOCKS_COMMAND_RESOLVE_PTR) {
+            socks_request_set_socks5_error(req,
+                                           SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+            log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
+                     "hostname type. Rejecting.");
+            return -1;
+          }
+          len = (unsigned char)*(data+4);
+          if (datalen < 7+len) { /* addr/port there? */
+            *want_length_out = 7+len;
+            return 0; /* not yet */
+          }
+          if (BUG(len+1 > MAX_SOCKS_ADDR_LEN)) {
+            /* LCOV_EXCL_START -- unreachable, since len is at most 255,
+             * and MAX_SOCKS_ADDR_LEN is 256. */
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+            log_warn(LD_APP,
+                     "socks5 hostname is %d bytes, which doesn't fit in "
+                     "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN);
+            return -1;
+            /* LCOV_EXCL_STOP */
+          }
+          memcpy(req->address,data+5,len);
+          req->address[len] = 0;
+          req->port = ntohs(get_uint16(data+5+len));
+          *drain_out = 5+len+2;
+
+          if (!string_is_valid_dest(req->address)) {
+            socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR);
+
+            log_warn(LD_PROTOCOL,
+                     "Your application (using socks5 to port %d) gave Tor "
+                     "a malformed hostname: %s. Rejecting the connection.",
+                     req->port, escaped_safe_str_client(req->address));
+            return -1;
+          }
+          if (log_sockstype)
+            log_notice(LD_APP,
+                  "Your application (using socks5 to port %d) instructed "
+                  "Tor to take care of the DNS resolution itself if "
+                  "necessary. This is good.", req->port);
+          return 1;
+        default: /* unsupported */
+          socks_request_set_socks5_error(req,
+                                         SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
+          log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.",
+                   (int) *(data+3));
+          return -1;
+      }
+      tor_assert(0);
+      break;
+    case 4: { /* socks4 */
+      enum {socks4, socks4a} socks4_prot = socks4a;
+      const char *authstart, *authend;
+      /* http://ss5.sourceforge.net/socks4.protocol.txt */
+      /* http://ss5.sourceforge.net/socks4A.protocol.txt */
+
+      req->socks_version = 4;
+      if (datalen < SOCKS4_NETWORK_LEN) {/* basic info available? */
+        *want_length_out = SOCKS4_NETWORK_LEN;
+        return 0; /* not yet */
+      }
+      // buf_pullup(buf, 1280);
+      req->command = (unsigned char) *(data+1);
+      if (req->command != SOCKS_COMMAND_CONNECT &&
+          req->command != SOCKS_COMMAND_RESOLVE) {
+        /* not a connect or resolve? we don't support it. (No resolve_ptr with
+         * socks4.) */
+        log_warn(LD_APP,"socks4: command %d not recognized. Rejecting.",
+                 req->command);
+        return -1;
+      }
+
+      req->port = ntohs(get_uint16(data+2));
+      destip = ntohl(get_uint32(data+4));
+      if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) {
+        log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting.");
+        return -1;
+      }
+      if (destip >> 8) {
+        log_debug(LD_APP,"socks4: destip not in form 0.0.0.x.");
+        in.s_addr = htonl(destip);
+        tor_inet_ntoa(&in,tmpbuf,sizeof(tmpbuf));
+        if (BUG(strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN)) {
+          /* LCOV_EXCL_START -- This branch is unreachable, given the
+           * size of tmpbuf and the actual value of MAX_SOCKS_ADDR_LEN */
+          log_debug(LD_APP,"socks4 addr (%d bytes) too long. Rejecting.",
+                    (int)strlen(tmpbuf));
+          return -1;
+          /* LCOV_EXCL_STOP */
+        }
+        log_debug(LD_APP,
+                  "socks4: successfully read destip (%s)",
+                  safe_str_client(tmpbuf));
+        socks4_prot = socks4;
+      }
+
+      authstart = data + SOCKS4_NETWORK_LEN;
+      next = memchr(authstart, 0,
+                    datalen-SOCKS4_NETWORK_LEN);
+      if (!next) {
+        if (datalen >= 1024) {
+          log_debug(LD_APP, "Socks4 user name too long; rejecting.");
+          return -1;
+        }
+        log_debug(LD_APP,"socks4: Username not here yet.");
+        *want_length_out = datalen+1024; /* More than we need, but safe */
+        return 0;
+      }
+      authend = next;
+      tor_assert(next < data+datalen);
+
+      startaddr = NULL;
+      if (socks4_prot != socks4a &&
+          !addressmap_have_mapping(tmpbuf,0)) {
+        log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks);
+
+        if (safe_socks)
+          return -1;
+      }
+      if (socks4_prot == socks4a) {
+        if (next+1 == data+datalen) {
+          log_debug(LD_APP,"socks4: No part of destaddr here yet.");
+          *want_length_out = datalen + 1024; /* More than we need, but safe */
+          return 0;
+        }
+        startaddr = next+1;
+        next = memchr(startaddr, 0, data + datalen - startaddr);
+        if (!next) {
+          if (datalen >= 1024) {
+            log_debug(LD_APP,"socks4: Destaddr too long.");
+            return -1;
+          }
+          log_debug(LD_APP,"socks4: Destaddr not all here yet.");
+          *want_length_out = datalen + 1024; /* More than we need, but safe */
+          return 0;
+        }
+        if (MAX_SOCKS_ADDR_LEN <= next-startaddr) {
+          log_warn(LD_APP,"socks4: Destaddr too long. Rejecting.");
+          return -1;
+        }
+        // tor_assert(next < buf->cur+buf_datalen(buf));
+
+        if (log_sockstype)
+          log_notice(LD_APP,
+                     "Your application (using socks4a to port %d) instructed "
+                     "Tor to take care of the DNS resolution itself if "
+                     "necessary. This is good.", req->port);
+      }
+      log_debug(LD_APP,"socks4: Everything is here. Success.");
+      strlcpy(req->address, startaddr ? startaddr : tmpbuf,
+              sizeof(req->address));
+      if (!string_is_valid_dest(req->address)) {
+        log_warn(LD_PROTOCOL,
+                 "Your application (using socks4 to port %d) gave Tor "
+                 "a malformed hostname: %s. Rejecting the connection.",
+                 req->port, escaped_safe_str_client(req->address));
+        return -1;
+      }
+      if (authend != authstart) {
+        req->got_auth = 1;
+        req->usernamelen = authend - authstart;
+        req->username = tor_memdup(authstart, authend - authstart);
+      }
+      /* next points to the final \0 on inbuf */
+      *drain_out = next - data + 1;
+      return 1;
+    }
+    case 'G': /* get */
+    case 'H': /* head */
+    case 'P': /* put/post */
+    case 'C': /* connect */
+      strlcpy((char*)req->reply, SOCKS_PROXY_IS_NOT_AN_HTTP_PROXY_MSG,
+              MAX_SOCKS_REPLY_LEN);
+      req->replylen = strlen((char*)req->reply)+1;
+      /* fall through */
+    default: /* version is not socks4 or socks5 */
+      log_warn(LD_APP,
+               "Socks version %d not recognized. (This port is not an "
+               "HTTP proxy; did you want to use HTTPTunnelPort?)",
+               *(data));
+      {
+        /* Tell the controller the first 8 bytes. */
+        char *tmp = tor_strndup(data, datalen < 8 ? datalen : 8);
+        control_event_client_status(LOG_WARN,
+                                    "SOCKS_UNKNOWN_PROTOCOL DATA=\"%s\"",
+                                    escaped(tmp));
+        tor_free(tmp);
+      }
+      return -1;
+  }
+}
+
+/** Inspect a reply from SOCKS server stored in <b>buf</b> according
+ * to <b>state</b>, removing the protocol data upon success. Return 0 on
+ * incomplete response, 1 on success and -1 on error, in which case
+ * <b>reason</b> is set to a descriptive message (free() when finished
+ * with it).
+ *
+ * As a special case, 2 is returned when user/pass is required
+ * during SOCKS5 handshake and user/pass is configured.
+ */
+int
+fetch_from_buf_socks_client(buf_t *buf, int state, char **reason)
+{
+  ssize_t drain = 0;
+  int r;
+  const char *head = NULL;
+  size_t datalen = 0;
+
+  if (buf_datalen(buf) < 2)
+    return 0;
+
+  buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, &head, &datalen);
+  tor_assert(head && datalen >= 2);
+
+  r = parse_socks_client((uint8_t*)head, datalen,
+                         state, reason, &drain);
+  if (drain > 0)
+    buf_drain(buf, drain);
+  else if (drain < 0)
+    buf_clear(buf);
+
+  return r;
+}
+
+/** Implementation logic for fetch_from_*_socks_client. */
+static int
+parse_socks_client(const uint8_t *data, size_t datalen,
+                   int state, char **reason,
+                   ssize_t *drain_out)
+{
+  unsigned int addrlen;
+  *drain_out = 0;
+  if (datalen < 2)
+    return 0;
+
+  switch (state) {
+    case PROXY_SOCKS4_WANT_CONNECT_OK:
+      /* Wait for the complete response */
+      if (datalen < 8)
+        return 0;
+
+      if (data[1] != 0x5a) {
+        *reason = tor_strdup(socks4_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      /* Success */
+      *drain_out = 8;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE:
+      /* we don't have any credentials */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("server doesn't support any of our "
+                             "available authentication methods");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: continuing without authentication");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929:
+      /* we have a username and password. return 1 if we can proceed without
+       * providing authentication, or 2 otherwise. */
+      switch (data[1]) {
+        case 0x00:
+          log_info(LD_NET, "SOCKS 5 client: we have auth details but server "
+                            "doesn't require authentication.");
+          *drain_out = -1;
+          return 1;
+        case 0x02:
+          log_info(LD_NET, "SOCKS 5 client: need authentication.");
+          *drain_out = -1;
+          return 2;
+        /* fall through */
+      }
+
+      *reason = tor_strdup("server doesn't support any of our available "
+                           "authentication methods");
+      return -1;
+
+    case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK:
+      /* handle server reply to rfc1929 authentication */
+      if (data[1] != 0x00) {
+        *reason = tor_strdup("authentication failed");
+        return -1;
+      }
+
+      log_info(LD_NET, "SOCKS 5 client: authentication successful.");
+      *drain_out = -1;
+      return 1;
+
+    case PROXY_SOCKS5_WANT_CONNECT_OK:
+      /* response is variable length. BND.ADDR, etc, isn't needed
+       * (don't bother with buf_pullup()), but make sure to eat all
+       * the data used */
+
+      /* wait for address type field to arrive */
+      if (datalen < 4)
+        return 0;
+
+      switch (data[3]) {
+        case 0x01: /* ip4 */
+          addrlen = 4;
+          break;
+        case 0x04: /* ip6 */
+          addrlen = 16;
+          break;
+        case 0x03: /* fqdn (can this happen here?) */
+          if (datalen < 5)
+            return 0;
+          addrlen = 1 + data[4];
+          break;
+        default:
+          *reason = tor_strdup("invalid response to connect request");
+          return -1;
+      }
+
+      /* wait for address and port */
+      if (datalen < 6 + addrlen)
+        return 0;
+
+      if (data[1] != 0x00) {
+        *reason = tor_strdup(socks5_response_code_to_string(data[1]));
+        return -1;
+      }
+
+      *drain_out = 6 + addrlen;
+      return 1;
+  }
+
+  /* LCOV_EXCL_START */
+  /* shouldn't get here if the input state is one we know about... */
+  tor_assert(0);
+
+  return -1;
+  /* LCOV_EXCL_STOP */
+}
+
diff --git a/src/or/proto_socks.h b/src/or/proto_socks.h
new file mode 100644
index 0000000000..02e0aca7e9
--- /dev/null
+++ b/src/or/proto_socks.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_SOCKS_H
+#define TOR_PROTO_SOCKS_H
+
+struct socks_request_t;
+struct buf_t;
+
+struct socks_request_t *socks_request_new(void);
+void socks_request_free_(struct socks_request_t *req);
+#define socks_request_free(req) \
+  FREE_AND_NULL(socks_request_t, socks_request_free_, (req))
+int fetch_from_buf_socks(struct buf_t *buf, socks_request_t *req,
+                         int log_sockstype, int safe_socks);
+int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason);
+
+#endif /* !defined(TOR_PROTO_SOCKS_H) */
+
diff --git a/src/or/protover.c b/src/or/protover.c
index 2c5d5ab1fc..c973660980 100644
--- a/src/or/protover.c
+++ b/src/or/protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,11 +28,14 @@
 #include "protover.h"
 #include "routerparse.h"
 
+#ifndef HAVE_RUST
+
 static const smartlist_t *get_supported_protocol_list(void);
 static int protocol_list_contains(const smartlist_t *protos,
                                   protocol_type_t pr, uint32_t ver);
 
 /** Mapping between protocol type string and protocol type. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `PROTOCOL_NAMES`
 static const struct {
   protocol_type_t protover_type;
   const char *name;
@@ -51,6 +54,11 @@ static const struct {
 
 #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES)
 
+/* Maximum allowed length of any single subprotocol name. */
+// C_RUST_COUPLED: src/rust/protover/protover.rs
+//                 `MAX_PROTOCOL_NAME_LENGTH`
+static const unsigned MAX_PROTOCOL_NAME_LENGTH = 100;
+
 /**
  * Given a protocol_type_t, return the corresponding string used in
  * descriptors.
@@ -94,7 +102,7 @@ str_to_protocol_type(const char *s, protocol_type_t *pr_out)
  * Release all space held by a single proto_entry_t structure
  */
 STATIC void
-proto_entry_free(proto_entry_t *entry)
+proto_entry_free_(proto_entry_t *entry)
 {
   if (!entry)
     return;
@@ -206,6 +214,16 @@ parse_single_entry(const char *s, const char *end_of_entry)
   if (equals == s)
     goto error;
 
+  /* The name must not be longer than MAX_PROTOCOL_NAME_LENGTH. */
+  if (equals - s > (int)MAX_PROTOCOL_NAME_LENGTH) {
+    log_warn(LD_NET, "When parsing a protocol entry, I got a very large "
+             "protocol name. This is possibly an attack or a bug, unless "
+             "the Tor network truly supports protocol names larger than "
+             "%ud characters. The offending string was: %s",
+             MAX_PROTOCOL_NAME_LENGTH, escaped(out->name));
+    goto error;
+  }
+
   /* The name must contain only alphanumeric characters and hyphens. */
   if (!is_valid_keyword(s, equals-s))
     goto error;
@@ -275,6 +293,21 @@ parse_protocol_list(const char *s)
 }
 
 /**
+ * Return true if the unparsed protover in <b>s</b> would contain a protocol
+ * name longer than MAX_PROTOCOL_NAME_LENGTH, and false otherwise.
+ */
+bool
+protover_contains_long_protocol_names(const char *s)
+{
+  smartlist_t *list = parse_protocol_list(s);
+  if (!list)
+    return true; /* yes, has a dangerous name */
+  SMARTLIST_FOREACH(list, proto_entry_t *, ent, proto_entry_free(ent));
+  smartlist_free(list);
+  return false; /* no, looks fine */
+}
+
+/**
  * Given a protocol type and version number, return true iff we know
  * how to speak that protocol.
  */
@@ -307,20 +340,57 @@ protocol_list_supports_protocol(const char *list, protocol_type_t tp,
   return contains;
 }
 
+/**
+ * Return true iff "list" encodes a protocol list that includes support for
+ * the indicated protocol and version, or some later version.
+ */
+int
+protocol_list_supports_protocol_or_later(const char *list,
+                                         protocol_type_t tp,
+                                         uint32_t version)
+{
+  /* NOTE: This is a pretty inefficient implementation. If it ever shows
+   * up in profiles, we should memoize it.
+   */
+  smartlist_t *protocols = parse_protocol_list(list);
+  if (!protocols) {
+    return 0;
+  }
+  const char *pr_name = protocol_type_to_str(tp);
+
+  int contains = 0;
+  SMARTLIST_FOREACH_BEGIN(protocols, proto_entry_t *, proto) {
+    if (strcasecmp(proto->name, pr_name))
+      continue;
+    SMARTLIST_FOREACH_BEGIN(proto->ranges, const proto_range_t *, range) {
+      if (range->high >= version) {
+        contains = 1;
+        goto found;
+      }
+    } SMARTLIST_FOREACH_END(range);
+  } SMARTLIST_FOREACH_END(proto);
+
+ found:
+  SMARTLIST_FOREACH(protocols, proto_entry_t *, ent, proto_entry_free(ent));
+  smartlist_free(protocols);
+  return contains;
+}
+
 /** Return the canonical string containing the list of protocols
  * that we support. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `SUPPORTED_PROTOCOLS`
 const char *
 protover_get_supported_protocols(void)
 {
   return
     "Cons=1-2 "
     "Desc=1-2 "
-    "DirCache=1 "
-    "HSDir=1 "
-    "HSIntro=3 "
+    "DirCache=1-2 "
+    "HSDir=1-2 "
+    "HSIntro=3-4 "
     "HSRend=1-2 "
-    "Link=1-4 "
-    "LinkAuth=1 "
+    "Link=1-5 "
+    "LinkAuth=1,3 "
     "Microdesc=1-2 "
     "Relay=1-2";
 }
@@ -375,7 +445,7 @@ encode_protocol_list(const smartlist_t *sl)
   const char *separator = "";
   smartlist_t *chunks = smartlist_new();
   SMARTLIST_FOREACH_BEGIN(sl, const proto_entry_t *, ent) {
-    smartlist_add(chunks, tor_strdup(separator));
+    smartlist_add_strdup(chunks, separator);
 
     proto_entry_encode_into(chunks, ent);
 
@@ -392,6 +462,8 @@ encode_protocol_list(const smartlist_t *sl)
 
 /* We treat any protocol list with more than this many subprotocols in it
  * as a DoS attempt. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs
+///                 `MAX_PROTOCOLS_TO_EXPAND`
 static const int MAX_PROTOCOLS_TO_EXPAND = (1<<16);
 
 /** Voting helper: Given a list of proto_entry_t, return a newly allocated
@@ -412,6 +484,14 @@ expand_protocol_list(const smartlist_t *protos)
 
   SMARTLIST_FOREACH_BEGIN(protos, const proto_entry_t *, ent) {
     const char *name = ent->name;
+    if (strlen(name) > MAX_PROTOCOL_NAME_LENGTH) {
+      log_warn(LD_NET, "When expanding a protocol entry, I got a very large "
+               "protocol name. This is possibly an attack or a bug, unless "
+               "the Tor network truly supports protocol names larger than "
+               "%ud characters. The offending string was: %s",
+               MAX_PROTOCOL_NAME_LENGTH, escaped(name));
+      continue;
+    }
     SMARTLIST_FOREACH_BEGIN(ent->ranges, const proto_range_t *, range) {
       uint32_t u;
       for (u = range->low; u <= range->high; ++u) {
@@ -508,7 +588,7 @@ contract_protocol_list(const smartlist_t *proto_strings)
     smartlist_sort(lst, cmp_single_ent_by_version);
 
     if (! first_entry)
-      smartlist_add(chunks, tor_strdup(" "));
+      smartlist_add_strdup(chunks, " ");
 
     /* We're going to construct this entry from the ranges. */
     proto_entry_t *entry = tor_malloc_zero(sizeof(proto_entry_t));
@@ -657,7 +737,9 @@ int
 protover_all_supported(const char *s, char **missing_out)
 {
   int all_supported = 1;
-  smartlist_t *missing;
+  smartlist_t *missing_some;
+  smartlist_t *missing_completely;
+  smartlist_t *missing_all;
 
   if (!s) {
     return 1;
@@ -670,7 +752,8 @@ protover_all_supported(const char *s, char **missing_out)
     return 1;
   }
 
-  missing = smartlist_new();
+  missing_some = smartlist_new();
+  missing_completely = smartlist_new();
 
   SMARTLIST_FOREACH_BEGIN(entries, const proto_entry_t *, ent) {
     protocol_type_t tp;
@@ -682,26 +765,86 @@ protover_all_supported(const char *s, char **missing_out)
     }
 
     SMARTLIST_FOREACH_BEGIN(ent->ranges, const proto_range_t *, range) {
+      proto_entry_t *unsupported = tor_malloc_zero(sizeof(proto_entry_t));
+      proto_range_t *versions = tor_malloc_zero(sizeof(proto_range_t));
       uint32_t i;
+
+      unsupported->name = tor_strdup(ent->name);
+      unsupported->ranges = smartlist_new();
+
       for (i = range->low; i <= range->high; ++i) {
         if (!protover_is_supported_here(tp, i)) {
-          goto unsupported;
+          if (versions->low == 0 && versions->high == 0) {
+            versions->low = i;
+            /* Pre-emptively add the high now, just in case we're in a single
+             * version range (e.g. "Link=999"). */
+            versions->high = i;
+          }
+          /* If the last one to be unsupported is one less than the current
+           * one, we're in a continuous range, so set the high field. */
+          if ((versions->high && versions->high == i - 1) ||
+              /* Similarly, if the last high wasn't set and we're currently
+               * one higher than the low, add current index as the highest
+               * known high. */
+              (!versions->high && versions->low == i - 1)) {
+            versions->high = i;
+            continue;
+          }
+        } else {
+          /* If we hit a supported version, and we previously had a range,
+           * we've hit a non-continuity. Copy the previous range and add it to
+           * the unsupported->ranges list and zero-out the previous range for
+           * the next iteration. */
+          if (versions->low != 0 && versions->high != 0) {
+            proto_range_t *versions_to_add = tor_malloc(sizeof(proto_range_t));
+
+            versions_to_add->low = versions->low;
+            versions_to_add->high = versions->high;
+            smartlist_add(unsupported->ranges, versions_to_add);
+
+            versions->low = 0;
+            versions->high = 0;
+          }
         }
       }
+      /* Once we've run out of versions to check, see if we had any unsupported
+       * ones and, if so, add them to unsupported->ranges. */
+      if (versions->low != 0 && versions->high != 0) {
+        smartlist_add(unsupported->ranges, versions);
+      }
+      /* Finally, if we had something unsupported, add it to the list of
+       * missing_some things and mark that there was something missing. */
+      if (smartlist_len(unsupported->ranges) != 0) {
+        smartlist_add(missing_some, (void*) unsupported);
+        all_supported = 0;
+      } else {
+        proto_entry_free(unsupported);
+        tor_free(versions);
+      }
     } SMARTLIST_FOREACH_END(range);
 
     continue;
 
   unsupported:
     all_supported = 0;
-    smartlist_add(missing, (void*) ent);
+    smartlist_add(missing_completely, (void*) ent);
   } SMARTLIST_FOREACH_END(ent);
 
+  /* We keep the two smartlists separate so that we can free the proto_entry_t
+   * we created and put in missing_some, so here we add them together to build
+   * the string. */
+  missing_all = smartlist_new();
+  smartlist_add_all(missing_all, missing_some);
+  smartlist_add_all(missing_all, missing_completely);
+
   if (missing_out && !all_supported) {
-    tor_assert(0 != smartlist_len(missing));
-    *missing_out = encode_protocol_list(missing);
+    tor_assert(smartlist_len(missing_all) != 0);
+    *missing_out = encode_protocol_list(missing_all);
   }
-  smartlist_free(missing);
+  SMARTLIST_FOREACH(missing_some, proto_entry_t *, ent, proto_entry_free(ent));
+  smartlist_free(missing_some);
+  smartlist_free(missing_completely);
+  smartlist_free(missing_all);
 
   SMARTLIST_FOREACH(entries, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(entries);
@@ -742,6 +885,7 @@ protocol_list_contains(const smartlist_t *protos,
  * Note that this is only used to infer protocols for Tor versions that
  * can't declare their own.
  **/
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `compute_for_old_tor`
 const char *
 protover_compute_for_old_tor(const char *version)
 {
@@ -791,3 +935,4 @@ protover_free_all(void)
   }
 }
 
+#endif /* !defined(HAVE_RUST) */
diff --git a/src/or/protover.h b/src/or/protover.h
index 5c658931ea..c46a13de66 100644
--- a/src/or/protover.h
+++ b/src/or/protover.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,9 +15,20 @@
  * descriptors.  Authorities should use this to decide whether to
  * guess proto lines. */
 /* This is a guess. */
+/// C_RUST_COUPLED: src/rust/protover/protover.rs
+///                 `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`
 #define FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS "0.2.9.3-alpha"
 
+/** The protover version number that signifies HSDir support for HSv3 */
+#define PROTOVER_HSDIR_V3 2
+/** The protover version number that signifies HSv3 intro point support */
+#define PROTOVER_HS_INTRO_V3 4
+/** The protover version number that signifies HSv3 rendezvous point support */
+#define PROTOVER_HS_RENDEZVOUS_POINT_V3 2
+
 /** List of recognized subprotocols. */
+/// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust`
+/// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto`
 typedef enum protocol_type_t {
   PRT_LINK,
   PRT_LINKAUTH,
@@ -31,6 +42,7 @@ typedef enum protocol_type_t {
   PRT_CONS,
 } protocol_type_t;
 
+bool protover_contains_long_protocol_names(const char *s);
 int protover_all_supported(const char *s, char **missing);
 int protover_is_supported_here(protocol_type_t pr, uint32_t ver);
 const char *protover_get_supported_protocols(void);
@@ -40,6 +52,9 @@ char *protover_compute_vote(const smartlist_t *list_of_proto_strings,
 const char *protover_compute_for_old_tor(const char *version);
 int protocol_list_supports_protocol(const char *list, protocol_type_t tp,
                                     uint32_t version);
+int protocol_list_supports_protocol_or_later(const char *list,
+                                             protocol_type_t tp,
+                                             uint32_t version);
 
 void protover_free_all(void);
 
@@ -63,12 +78,18 @@ typedef struct proto_entry_t {
   smartlist_t *ranges;
 } proto_entry_t;
 
+#if !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS)
 STATIC smartlist_t *parse_protocol_list(const char *s);
-STATIC void proto_entry_free(proto_entry_t *entry);
 STATIC char *encode_protocol_list(const smartlist_t *sl);
 STATIC const char *protocol_type_to_str(protocol_type_t pr);
 STATIC int str_to_protocol_type(const char *s, protocol_type_t *pr_out);
-#endif
+STATIC void proto_entry_free_(proto_entry_t *entry);
+#endif /* !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS) */
+
+#define proto_entry_free(entry) \
+  FREE_AND_NULL(proto_entry_t, proto_entry_free_, (entry))
+
+#endif /* defined(PROTOVER_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_PROTOVER_H) */
 
diff --git a/src/or/protover_rust.c b/src/or/protover_rust.c
new file mode 100644
index 0000000000..99304f8b51
--- /dev/null
+++ b/src/or/protover_rust.c
@@ -0,0 +1,34 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * \file protover_rust.c
+ * \brief Provide a C wrapper for functions exposed in /src/rust/protover,
+ * and safe translation/handling between the Rust/C boundary.
+ */
+
+#include "or.h"
+#include "protover.h"
+
+#ifdef HAVE_RUST
+
+/* Define for compatibility, used in main.c */
+void
+protover_free_all(void)
+{
+}
+
+int protover_contains_long_protocol_names_(const char *s);
+
+/**
+ * Return true if the unparsed protover in <b>s</b> would contain a protocol
+ * name longer than MAX_PROTOCOL_NAME_LENGTH, and false otherwise.
+ */
+bool
+protover_contains_long_protocol_names(const char *s)
+{
+  return protover_contains_long_protocol_names_(s) != 0;
+}
+
+#endif /* defined(HAVE_RUST) */
+
diff --git a/src/or/reasons.c b/src/or/reasons.c
index a1566e2299..ce1259b8f3 100644
--- a/src/or/reasons.c
+++ b/src/or/reasons.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -45,6 +45,8 @@ stream_end_reason_to_control_string(int reason)
     case END_STREAM_REASON_CANT_ATTACH: return "CANT_ATTACH";
     case END_STREAM_REASON_NET_UNREACHABLE: return "NET_UNREACHABLE";
     case END_STREAM_REASON_SOCKSPROTOCOL: return "SOCKS_PROTOCOL";
+    // XXXX Controlspec
+    case END_STREAM_REASON_HTTPPROTOCOL: return "HTTP_PROTOCOL";
 
     case END_STREAM_REASON_PRIVATE_ADDR: return "PRIVATE_ADDR";
 
@@ -138,6 +140,11 @@ stream_end_reason_to_socks5_response(int reason)
       return SOCKS5_NET_UNREACHABLE;
     case END_STREAM_REASON_SOCKSPROTOCOL:
       return SOCKS5_GENERAL_ERROR;
+    case END_STREAM_REASON_HTTPPROTOCOL:
+      // LCOV_EXCL_START
+      tor_assert_nonfatal_unreached();
+      return SOCKS5_GENERAL_ERROR;
+      // LCOV_EXCL_STOP
     case END_STREAM_REASON_PRIVATE_ADDR:
       return SOCKS5_GENERAL_ERROR;
 
@@ -160,7 +167,7 @@ stream_end_reason_to_socks5_response(int reason)
 #else
 #define E_CASE(s) case s
 #define S_CASE(s) case s
-#endif
+#endif /* defined(_WIN32) */
 
 /** Given an errno from a failed exit connection, return a reason code
  * appropriate for use in a RELAY END cell. */
@@ -421,7 +428,7 @@ socks5_response_code_to_string(uint8_t code)
   }
 }
 
-/** Return a string corresponding to a bandwidht_weight_rule_t */
+/** Return a string corresponding to a bandwidth_weight_rule_t */
 const char *
 bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule)
 {
@@ -442,3 +449,48 @@ bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule)
   }
 }
 
+/** Given a RELAY_END reason value, convert it to an HTTP response to be
+ * send over an HTTP tunnel connection. */
+const char *
+end_reason_to_http_connect_response_line(int endreason)
+{
+  endreason &= END_STREAM_REASON_MASK;
+  /* XXXX these are probably all wrong. Should they all be 502? */
+  switch (endreason) {
+    case 0:
+      return "HTTP/1.0 200 OK\r\n\r\n";
+    case END_STREAM_REASON_MISC:
+      return "HTTP/1.0 500 Internal Server Error\r\n\r\n";
+    case END_STREAM_REASON_RESOLVEFAILED:
+      return "HTTP/1.0 404 Not Found (resolve failed)\r\n\r\n";
+    case END_STREAM_REASON_NOROUTE:
+      return "HTTP/1.0 404 Not Found (no route)\r\n\r\n";
+    case END_STREAM_REASON_CONNECTREFUSED:
+      return "HTTP/1.0 403 Forbidden (connection refused)\r\n\r\n";
+    case END_STREAM_REASON_EXITPOLICY:
+      return "HTTP/1.0 403 Forbidden (exit policy)\r\n\r\n";
+    case END_STREAM_REASON_DESTROY:
+      return "HTTP/1.0 502 Bad Gateway (destroy cell received)\r\n\r\n";
+    case END_STREAM_REASON_DONE:
+      return "HTTP/1.0 502 Bad Gateway (unexpected close)\r\n\r\n";
+    case END_STREAM_REASON_TIMEOUT:
+      return "HTTP/1.0 504 Gateway Timeout\r\n\r\n";
+    case END_STREAM_REASON_HIBERNATING:
+      return "HTTP/1.0 502 Bad Gateway (hibernating server)\r\n\r\n";
+    case END_STREAM_REASON_INTERNAL:
+      return "HTTP/1.0 502 Bad Gateway (internal error)\r\n\r\n";
+    case END_STREAM_REASON_RESOURCELIMIT:
+      return "HTTP/1.0 502 Bad Gateway (resource limit)\r\n\r\n";
+    case END_STREAM_REASON_CONNRESET:
+      return "HTTP/1.0 403 Forbidden (connection reset)\r\n\r\n";
+    case END_STREAM_REASON_TORPROTOCOL:
+      return "HTTP/1.0 502 Bad Gateway (tor protocol violation)\r\n\r\n";
+    case END_STREAM_REASON_ENTRYPOLICY:
+      return "HTTP/1.0 403 Forbidden (entry policy violation)\r\n\r\n";
+    case END_STREAM_REASON_NOTDIRECTORY: /* Fall Through */
+    default:
+      tor_assert_nonfatal_unreached();
+      return "HTTP/1.0 500 Internal Server Error (weird end reason)\r\n\r\n";
+  }
+}
+
diff --git a/src/or/reasons.h b/src/or/reasons.h
index 2e12c93728..3d6ba8fc83 100644
--- a/src/or/reasons.h
+++ b/src/or/reasons.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,6 +26,7 @@ const char *socks4_response_code_to_string(uint8_t code);
 const char *socks5_response_code_to_string(uint8_t code);
 
 const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule);
+const char *end_reason_to_http_connect_response_line(int endreason);
 
-#endif
+#endif /* !defined(TOR_REASONS_H) */
 
diff --git a/src/or/relay.c b/src/or/relay.c
index d1c7820c7c..b6886e2bb8 100644
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -1,30 +1,70 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file relay.c
  * \brief Handle relay cell encryption/decryption, plus packaging and
  *    receiving from circuits, plus queuing on circuits.
+ *
+ * This is a core modules that makes Tor work. It's responsible for
+ * dealing with RELAY cells (the ones that travel more than one hop along a
+ * circuit), by:
+ *  <ul>
+ *   <li>constructing relays cells,
+ *   <li>encrypting relay cells,
+ *   <li>decrypting relay cells,
+ *   <li>demultiplexing relay cells as they arrive on a connection,
+ *   <li>queueing relay cells for retransmission,
+ *   <li>or handling relay cells that are for us to receive (as an exit or a
+ *   client).
+ *  </ul>
+ *
+ * RELAY cells are generated throughout the code at the client or relay side,
+ * using relay_send_command_from_edge() or one of the functions like
+ * connection_edge_send_command() that calls it.  Of particular interest is
+ * connection_edge_package_raw_inbuf(), which takes information that has
+ * arrived on an edge connection socket, and packages it as a RELAY_DATA cell
+ * -- this is how information is actually sent across the Tor network.  The
+ * cryptography for these functions is handled deep in
+ * circuit_package_relay_cell(), which either adds a single layer of
+ * encryption (if we're an exit), or multiple layers (if we're the origin of
+ * the circuit).  After construction and encryption, the RELAY cells are
+ * passed to append_cell_to_circuit_queue(), which queues them for
+ * transmission and tells the circuitmux (see circuitmux.c) that the circuit
+ * is waiting to send something.
+ *
+ * Incoming RELAY cells arrive at circuit_receive_relay_cell(), called from
+ * command.c.  There they are decrypted and, if they are for us, are passed to
+ * connection_edge_process_relay_cell(). If they're not for us, they're
+ * re-queued for retransmission again with append_cell_to_circuit_queue().
+ *
+ * The connection_edge_process_relay_cell() function handles all the different
+ * types of relay cells, launching requests or transmitting data as needed.
  **/
 
 #define RELAY_PRIVATE
 #include "or.h"
 #include "addressmap.h"
+#include "backtrace.h"
 #include "buffers.h"
 #include "channel.h"
 #include "circpathbias.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
+#include "compress.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "geoip.h"
+#include "hs_cache.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -32,20 +72,19 @@
 #include "policies.h"
 #include "reasons.h"
 #include "relay.h"
+#include "relay_crypto.h"
 #include "rendcache.h"
 #include "rendcommon.h"
 #include "router.h"
 #include "routerlist.h"
 #include "routerparse.h"
 #include "scheduler.h"
+#include "rephist.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
                                             crypt_path_t *layer_hint);
 
-static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
-                                              edge_connection_t *conn,
-                                              crypt_path_t *layer_hint);
 static void circuit_consider_sending_sendme(circuit_t *circ,
                                             crypt_path_t *layer_hint);
 static void circuit_resume_edge_reading(circuit_t *circ,
@@ -60,9 +99,6 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ,
                                                   entry_connection_t *conn,
                                                   node_t *node,
                                                   const tor_addr_t *addr);
-#if 0
-static int get_max_middle_cells(void);
-#endif
 
 /** Stop reading on edge connections when we have this many cells
  * waiting on the appropriate queue. */
@@ -79,85 +115,87 @@ uint64_t stats_n_relay_cells_relayed = 0;
  * hop?
  */
 uint64_t stats_n_relay_cells_delivered = 0;
+/** Stats: how many circuits have we closed due to the cell queue limit being
+ * reached (see append_cell_to_circuit_queue()) */
+uint64_t stats_n_circ_max_cell_reached = 0;
 
 /** Used to tell which stream to read from first on a circuit. */
 static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT;
 
-/** Update digest from the payload of cell. Assign integrity part to
- * cell.
+/**
+ * Update channel usage state based on the type of relay cell and
+ * circuit properties.
+ *
+ * This is needed to determine if a client channel is being
+ * used for application traffic, and if a relay channel is being
+ * used for multihop circuits and application traffic. The decision
+ * to pad in channelpadding.c depends upon this info (as well as
+ * consensus parameters) to decide what channels to pad.
  */
 static void
-relay_set_digest(crypto_digest_t *digest, cell_t *cell)
+circuit_update_channel_usage(circuit_t *circ, cell_t *cell)
 {
-  char integrity[4];
-  relay_header_t rh;
-
-  crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE);
-  crypto_digest_get_digest(digest, integrity, 4);
-//  log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.",
-//    integrity[0], integrity[1], integrity[2], integrity[3]);
-  relay_header_unpack(&rh, cell->payload);
-  memcpy(rh.integrity, integrity, 4);
-  relay_header_pack(cell->payload, &rh);
-}
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /*
+     * The client state was first set much earlier in
+     * circuit_send_next_onion_skin(), so we can start padding as early as
+     * possible.
+     *
+     * However, if padding turns out to be expensive, we may want to not do
+     * it until actual application traffic starts flowing (which is controlled
+     * via consensus param nf_pad_before_usage).
+     *
+     * So: If we're an origin circuit and we've created a full length circuit,
+     * then any CELL_RELAY cell means application data. Increase the usage
+     * state of the channel to indicate this.
+     *
+     * We want to wait for CELL_RELAY specifically here, so we know that
+     * the channel was definitely being used for data and not for extends.
+     * By default, we pad as soon as a channel has been used for *any*
+     * circuits, so this state is irrelevant to the padding decision in
+     * the default case. However, if padding turns out to be expensive,
+     * we would like the ability to avoid padding until we're absolutely
+     * sure that a channel is used for enough application data to be worth
+     * padding.
+     *
+     * (So it does not matter that CELL_RELAY_EARLY can actually contain
+     * application data. This is only a load reducing option and that edge
+     * case does not matter if we're desperately trying to reduce overhead
+     * anyway. See also consensus parameter nf_pad_before_usage).
+     */
+    if (BUG(!circ->n_chan))
+      return;
 
-/** Does the digest for this circuit indicate that this cell is for us?
- *
- * Update digest from the payload of cell (with the integrity part set
- * to 0). If the integrity part is valid, return 1, else restore digest
- * and cell to their original state and return 0.
- */
-static int
-relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
-{
-  uint32_t received_integrity, calculated_integrity;
-  relay_header_t rh;
-  crypto_digest_t *backup_digest=NULL;
+    if (circ->n_chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS &&
+        cell->command == CELL_RELAY) {
+      circ->n_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC;
+    }
+  } else {
+    /* If we're a relay circuit, the question is more complicated. Basically:
+     * we only want to pad connections that carry multihop (anonymous)
+     * circuits.
+     *
+     * We assume we're more than one hop if either the previous hop
+     * is not a client, or if the previous hop is a client and there's
+     * a next hop. Then, circuit traffic starts at RELAY_EARLY, and
+     * user application traffic starts when we see RELAY cells.
+     */
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
 
-  backup_digest = crypto_digest_dup(digest);
+    if (BUG(!or_circ->p_chan))
+      return;
 
-  relay_header_unpack(&rh, cell->payload);
-  memcpy(&received_integrity, rh.integrity, 4);
-  memset(rh.integrity, 0, 4);
-  relay_header_pack(cell->payload, &rh);
-
-//  log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.",
-//    received_integrity[0], received_integrity[1],
-//    received_integrity[2], received_integrity[3]);
-
-  crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE);
-  crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4);
-
-  if (calculated_integrity != received_integrity) {
-//    log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing.");
-// (%d vs %d).", received_integrity, calculated_integrity);
-    /* restore digest to its old form */
-    crypto_digest_assign(digest, backup_digest);
-    /* restore the relay header */
-    memcpy(rh.integrity, &received_integrity, 4);
-    relay_header_pack(cell->payload, &rh);
-    crypto_digest_free(backup_digest);
-    return 0;
+    if (!channel_is_client(or_circ->p_chan) ||
+        (channel_is_client(or_circ->p_chan) && circ->n_chan)) {
+      if (cell->command == CELL_RELAY_EARLY) {
+        if (or_circ->p_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) {
+          or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+        }
+      } else if (cell->command == CELL_RELAY) {
+        or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC;
+      }
+    }
   }
-  crypto_digest_free(backup_digest);
-  return 1;
-}
-
-/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b>
- * (in place).
- *
- * If <b>encrypt_mode</b> is 1 then encrypt, else decrypt.
- *
- * Returns 0.
- */
-static int
-relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in,
-                        int encrypt_mode)
-{
-  (void)encrypt_mode;
-  crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
-
-  return 0;
 }
 
 /** Receive a relay cell:
@@ -188,16 +226,22 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   if (circ->marked_for_close)
     return 0;
 
-  if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) {
-    log_warn(LD_BUG,"relay crypt failed. Dropping connection.");
+  if (relay_decrypt_cell(circ, cell, cell_direction, &layer_hint, &recognized)
+      < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "relay crypt failed. Dropping connection.");
     return -END_CIRC_REASON_INTERNAL;
   }
 
+  circuit_update_channel_usage(circ, cell);
+
   if (recognized) {
     edge_connection_t *conn = NULL;
 
     if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
-      pathbias_check_probe_response(circ, cell);
+      if (pathbias_check_probe_response(circ, cell) == -1) {
+        pathbias_count_valid_cells(circ, cell);
+      }
 
       /* We need to drop this cell no matter what to avoid code that expects
        * a certain purpose (such as the hidserv code). */
@@ -221,8 +265,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(LD_OR,"Sending to origin.");
       if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
                                                        layer_hint)) < 0) {
-        log_warn(LD_OR,
-                 "connection_edge_process_relay_cell (at origin) failed.");
+        /* If a client is trying to connect to unknown hidden service port,
+         * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
+         * Do not log warn as this is an expected behavior for a service. */
+        if (reason != END_CIRC_AT_ORIGIN) {
+          log_warn(LD_OR,
+                   "connection_edge_process_relay_cell (at origin) failed.");
+        }
         return reason;
       }
     }
@@ -285,92 +334,6 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   return 0;
 }
 
-/** Do the appropriate en/decryptions for <b>cell</b> arriving on
- * <b>circ</b> in direction <b>cell_direction</b>.
- *
- * If cell_direction == CELL_DIRECTION_IN:
- *   - If we're at the origin (we're the OP), for hops 1..N,
- *     decrypt cell. If recognized, stop.
- *   - Else (we're not the OP), encrypt one hop. Cell is not recognized.
- *
- * If cell_direction == CELL_DIRECTION_OUT:
- *   - decrypt one hop. Check if recognized.
- *
- * If cell is recognized, set *recognized to 1, and set
- * *layer_hint to the hop that recognized it.
- *
- * Return -1 to indicate that we should mark the circuit for close,
- * else return 0.
- */
-int
-relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction,
-            crypt_path_t **layer_hint, char *recognized)
-{
-  relay_header_t rh;
-
-  tor_assert(circ);
-  tor_assert(cell);
-  tor_assert(recognized);
-  tor_assert(cell_direction == CELL_DIRECTION_IN ||
-             cell_direction == CELL_DIRECTION_OUT);
-
-  if (cell_direction == CELL_DIRECTION_IN) {
-    if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit.
-                                    * We'll want to do layered decrypts. */
-      crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath;
-      thishop = cpath;
-      if (thishop->state != CPATH_STATE_OPEN) {
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "Relay cell before first created cell? Closing.");
-        return -1;
-      }
-      do { /* Remember: cpath is in forward order, that is, first hop first. */
-        tor_assert(thishop);
-
-        if (relay_crypt_one_payload(thishop->b_crypto, cell->payload, 0) < 0)
-          return -1;
-
-        relay_header_unpack(&rh, cell->payload);
-        if (rh.recognized == 0) {
-          /* it's possibly recognized. have to check digest to be sure. */
-          if (relay_digest_matches(thishop->b_digest, cell)) {
-            *recognized = 1;
-            *layer_hint = thishop;
-            return 0;
-          }
-        }
-
-        thishop = thishop->next;
-      } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN);
-      log_fn(LOG_PROTOCOL_WARN, LD_OR,
-             "Incoming cell at client not recognized. Closing.");
-      return -1;
-    } else { /* we're in the middle. Just one crypt. */
-      if (relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->p_crypto,
-                                  cell->payload, 1) < 0)
-        return -1;
-//      log_fn(LOG_DEBUG,"Skipping recognized check, because we're not "
-//             "the client.");
-    }
-  } else /* cell_direction == CELL_DIRECTION_OUT */ {
-    /* we're in the middle. Just one crypt. */
-
-    if (relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->n_crypto,
-                                cell->payload, 0) < 0)
-      return -1;
-
-    relay_header_unpack(&rh, cell->payload);
-    if (rh.recognized == 0) {
-      /* it's possibly recognized. have to check digest to be sure. */
-      if (relay_digest_matches(TO_OR_CIRCUIT(circ)->n_digest, cell)) {
-        *recognized = 1;
-        return 0;
-      }
-    }
-  }
-  return 0;
-}
-
 /** Package a relay cell from an edge:
  *  - Encrypt it to the right layer
  *  - Append it to the appropriate cell_queue on <b>circ</b>.
@@ -389,36 +352,36 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
   }
 
   if (cell_direction == CELL_DIRECTION_OUT) {
-    crypt_path_t *thishop; /* counter for repeated crypts */
     chan = circ->n_chan;
     if (!chan) {
       log_warn(LD_BUG,"outgoing relay cell sent from %s:%d has n_chan==NULL."
-               " Dropping.", filename, lineno);
+               " Dropping. Circuit is in state %s (%d), and is "
+               "%smarked for close. (%s:%d, %d)", filename, lineno,
+               circuit_state_to_string(circ->state), circ->state,
+               circ->marked_for_close ? "" : "not ",
+               circ->marked_for_close_file?circ->marked_for_close_file:"",
+               circ->marked_for_close, circ->marked_for_close_reason);
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_log_path(LOG_WARN, LD_BUG, TO_ORIGIN_CIRCUIT(circ));
+      }
+      log_backtrace(LOG_WARN,LD_BUG,"");
       return 0; /* just drop it */
     }
     if (!CIRCUIT_IS_ORIGIN(circ)) {
       log_warn(LD_BUG,"outgoing relay cell sent from %s:%d on non-origin "
                "circ. Dropping.", filename, lineno);
+      log_backtrace(LOG_WARN,LD_BUG,"");
       return 0; /* just drop it */
     }
 
-    relay_set_digest(layer_hint->f_digest, cell);
+    relay_encrypt_cell_outbound(cell, TO_ORIGIN_CIRCUIT(circ), layer_hint);
 
-    thishop = layer_hint;
-    /* moving from farthest to nearest hop */
-    do {
-      tor_assert(thishop);
-      /* XXXX RD This is a bug, right? */
-      log_debug(LD_OR,"crypting a layer of the relay cell.");
-      if (relay_crypt_one_payload(thishop->f_crypto, cell->payload, 1) < 0) {
-        return -1;
-      }
-
-      thishop = thishop->prev;
-    } while (thishop != TO_ORIGIN_CIRCUIT(circ)->cpath->prev);
+    /* Update circ written totals for control port */
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = tor_add_u32_nowrap(ocirc->n_written_circ_bw,
+                                                  CELL_PAYLOAD_SIZE);
 
   } else { /* incoming cell */
-    or_circuit_t *or_circ;
     if (CIRCUIT_IS_ORIGIN(circ)) {
       /* We should never package an _incoming_ cell from the circuit
        * origin; that means we messed up somewhere. */
@@ -426,11 +389,9 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
       assert_circuit_ok(circ);
       return 0; /* just drop it */
     }
-    or_circ = TO_OR_CIRCUIT(circ);
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    relay_encrypt_cell_inbound(cell, or_circ);
     chan = or_circ->p_chan;
-    relay_set_digest(or_circ->p_digest, cell);
-    if (relay_crypt_one_payload(or_circ->p_crypto, cell->payload, 1) < 0)
-      return -1;
   }
   ++stats_n_relay_cells_relayed;
 
@@ -564,11 +525,11 @@ relay_command_to_string(uint8_t command)
  * If you can't send the cell, mark the circuit for close and return -1. Else
  * return 0.
  */
-int
-relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
-                              uint8_t relay_command, const char *payload,
-                              size_t payload_len, crypt_path_t *cpath_layer,
-                              const char *filename, int lineno)
+MOCK_IMPL(int,
+relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
+                               uint8_t relay_command, const char *payload,
+                               size_t payload_len, crypt_path_t *cpath_layer,
+                               const char *filename, int lineno))
 {
   cell_t cell;
   relay_header_t rh;
@@ -580,14 +541,14 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
 
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_RELAY;
-  if (cpath_layer) {
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(cpath_layer);
     cell.circ_id = circ->n_circ_id;
     cell_direction = CELL_DIRECTION_OUT;
-  } else if (! CIRCUIT_IS_ORIGIN(circ)) {
+  } else {
+    tor_assert(! cpath_layer);
     cell.circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
     cell_direction = CELL_DIRECTION_IN;
-  } else {
-    return -1;
   }
 
   memset(&rh, 0, sizeof(rh));
@@ -601,6 +562,9 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
+  if (relay_command == RELAY_COMMAND_DROP)
+    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
   if (relay_command == RELAY_COMMAND_END && circ->dirreq_id)
@@ -648,6 +612,10 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
       tor_free(commands);
       smartlist_free(commands_list);
     }
+
+    /* Let's assume we're well-behaved: Anything that we decide to send is
+     * valid, delivered data. */
+    circuit_sent_valid_data(origin_circ, rh.length);
   }
 
   if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer,
@@ -707,6 +675,16 @@ connection_edge_send_command(edge_connection_t *fromconn,
     return -1;
   }
 
+#ifdef MEASUREMENTS_21206
+  /* Keep track of the number of RELAY_DATA cells sent for directory
+   * connections. */
+  connection_t *linked_conn = TO_CONN(fromconn)->linked_conn;
+
+  if (linked_conn && linked_conn->type == CONN_TYPE_DIR) {
+    ++(TO_DIR_CONN(linked_conn)->data_cells_sent);
+  }
+#endif /* defined(MEASUREMENTS_21206) */
+
   return relay_send_command_from_edge(fromconn->stream_id, circ,
                                       relay_command, payload,
                                       payload_len, cpath_layer);
@@ -767,6 +745,9 @@ connection_ap_process_end_not_open(
     }
   }
 
+  /* This end cell is now valid. */
+  circuit_read_valid_data(circ, rh->length);
+
   if (rh->length == 0) {
     reason = END_STREAM_REASON_MISC;
   }
@@ -986,7 +967,12 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr)
  * header has already been parsed into <b>rh</b>. On success, set
  * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to
  * the ttl of that address, in seconds, and return 0.  On failure, return
- * -1. */
+ * -1.
+ *
+ * Note that the resulting address can be UNSPEC if the connected cell had no
+ * address (as for a stream to an union service or a tunneled directory
+ * connection), and that the ttl can be absent (in which case <b>ttl_out</b>
+ * is set to -1). */
 STATIC int
 connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
                      tor_addr_t *addr_out, int *ttl_out)
@@ -1026,7 +1012,7 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
 
 /** Drop all storage held by <b>addr</b>. */
 STATIC void
-address_ttl_free(address_ttl_t *addr)
+address_ttl_free_(address_ttl_t *addr)
 {
   if (!addr)
     return;
@@ -1257,6 +1243,12 @@ connection_edge_process_resolved_cell(edge_connection_t *conn,
     }
   }
 
+  /* This is valid data at this point. Count it */
+  if (conn->on_circuit && CIRCUIT_IS_ORIGIN(conn->on_circuit)) {
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(conn->on_circuit),
+                            rh->length);
+  }
+
   connection_ap_handshake_socks_got_resolved_cell(entry_conn,
                                                   errcode,
                                                   resolved_addresses);
@@ -1317,7 +1309,7 @@ connection_edge_process_relay_cell_not_open(
              "after %d seconds.",
              (unsigned)circ->n_circ_id,
              rh->stream_id,
-             (int)(time(NULL) - conn->base_.timestamp_lastread));
+             (int)(time(NULL) - conn->base_.timestamp_last_read_allowed));
     if (connected_cell_parse(rh, cell, &addr, &ttl) < 0) {
       log_fn(LOG_PROTOCOL_WARN, LD_APP,
              "Got a badly formatted connected cell. Closing.");
@@ -1326,6 +1318,9 @@ connection_edge_process_relay_cell_not_open(
       return 0;
     }
     if (tor_addr_family(&addr) != AF_UNSPEC) {
+      /* The family is not UNSPEC: so we were given an address in the
+       * connected cell. (This is normal, except for BEGINDIR and onion
+       * service streams.) */
       const sa_family_t family = tor_addr_family(&addr);
       if (tor_addr_is_null(&addr) ||
           (get_options()->ClientDNSRejectInternalAddresses &&
@@ -1358,8 +1353,9 @@ connection_edge_process_relay_cell_not_open(
     circuit_log_path(LOG_INFO,LD_APP,TO_ORIGIN_CIRCUIT(circ));
     /* don't send a socks reply to transparent conns */
     tor_assert(entry_conn->socks_request != NULL);
-    if (!entry_conn->socks_request->has_finished)
+    if (!entry_conn->socks_request->has_finished) {
       connection_ap_handshake_socks_reply(entry_conn, NULL, 0, 0);
+    }
 
     /* Was it a linked dir conn? If so, a dir request just started to
      * fetch something; this could be a bootstrap status milestone. */
@@ -1391,6 +1387,9 @@ connection_edge_process_relay_cell_not_open(
       entry_conn->pending_optimistic_data = NULL;
     }
 
+    /* This is valid data at this point. Count it */
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
+
     /* handle anything that might have queued */
     if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
       /* (We already sent an end cell if possible) */
@@ -1423,7 +1422,7 @@ connection_edge_process_relay_cell_not_open(
  *
  * Return -reason if you want to warn and tear down the circuit, else 0.
  */
-static int
+STATIC int
 connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                                    edge_connection_t *conn,
                                    crypt_path_t *layer_hint)
@@ -1490,6 +1489,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
   switch (rh.command) {
     case RELAY_COMMAND_DROP:
+      rep_hist_padding_count_read(PADDING_TYPE_DROP);
 //      log_info(domain,"Got a relay-level padding cell. Dropping.");
       return 0;
     case RELAY_COMMAND_BEGIN:
@@ -1522,7 +1522,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         circ->dirreq_id = ++next_id;
         TO_OR_CIRCUIT(circ)->p_chan->dirreq_id = circ->dirreq_id;
       }
-
       return connection_exit_begin_conn(cell, circ);
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
@@ -1548,6 +1547,17 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "stream_id. Dropping.");
         return 0;
       } else if (!conn) {
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+          if (connection_half_edge_is_valid_data(ocirc->half_streams,
+                                                 rh.stream_id)) {
+            circuit_read_valid_data(ocirc, rh.length);
+            log_info(domain,
+                     "data cell on circ %u valid on half-closed "
+                     "stream id %d", ocirc->global_identifier, rh.stream_id);
+          }
+        }
+
         log_info(domain,"data cell dropped, unknown stream (streamid %d).",
                  rh.stream_id);
         return 0;
@@ -1558,11 +1568,25 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "(relay data) conn deliver_window below 0. Killing.");
         return -END_CIRC_REASON_TORPROTOCOL;
       }
+      /* Total all valid application bytes delivered */
+      if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
 
       stats_n_data_bytes_received += rh.length;
-      connection_write_to_buf((char*)(cell->payload + RELAY_HEADER_SIZE),
+      connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
                               rh.length, TO_CONN(conn));
 
+#ifdef MEASUREMENTS_21206
+      /* Count number of RELAY_DATA cells received on a linked directory
+       * connection. */
+      connection_t *linked_conn = TO_CONN(conn)->linked_conn;
+
+      if (linked_conn && linked_conn->type == CONN_TYPE_DIR) {
+        ++(TO_DIR_CONN(linked_conn)->data_cells_received);
+      }
+#endif /* defined(MEASUREMENTS_21206) */
+
       if (!optimistic_data) {
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
@@ -1575,6 +1599,20 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       reason = rh.length > 0 ?
         get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
       if (!conn) {
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+          if (connection_half_edge_is_valid_end(ocirc->half_streams,
+                                                rh.stream_id)) {
+
+            circuit_read_valid_data(ocirc, rh.length);
+            log_info(domain,
+                     "end cell (%s) on circ %u valid on half-closed "
+                     "stream id %d",
+                     stream_end_reason_to_string(reason),
+                     ocirc->global_identifier, rh.stream_id);
+            return 0;
+          }
+        }
         log_info(domain,"end cell (%s) dropped, unknown stream.",
                  stream_end_reason_to_string(reason));
         return 0;
@@ -1600,6 +1638,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         /* only mark it if not already marked. it's possible to
          * get the 'end' right around when the client hangs up on us. */
         connection_mark_and_flush(TO_CONN(conn));
+
+        /* Total all valid application bytes delivered */
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+        }
       }
       return 0;
     case RELAY_COMMAND_EXTEND:
@@ -1665,6 +1708,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         log_info(domain,"circuit_send_next_onion_skin() failed.");
         return reason;
       }
+      /* Total all valid bytes delivered. */
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      }
       return 0;
     case RELAY_COMMAND_TRUNCATE:
       if (layer_hint) {
@@ -1702,7 +1749,14 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'truncated' unsupported at non-origin. Dropping.");
         return 0;
       }
-      circuit_truncated(TO_ORIGIN_CIRCUIT(circ), layer_hint,
+
+      /* Count the truncated as valid, for completeness. The
+       * circuit is being torn down anyway, though.  */
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
+                                rh.length);
+      }
+      circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
                         get_uint8(cell->payload + RELAY_HEADER_SIZE));
       return 0;
     case RELAY_COMMAND_CONNECTED:
@@ -1711,6 +1765,19 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'connected' unsupported while open. Closing circ.");
         return -END_CIRC_REASON_TORPROTOCOL;
       }
+
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+        if (connection_half_edge_is_valid_connected(ocirc->half_streams,
+                                                    rh.stream_id)) {
+          circuit_read_valid_data(ocirc, rh.length);
+          log_info(domain,
+                   "connected cell on circ %u valid on half-closed "
+                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+          return 0;
+        }
+      }
+
       log_info(domain,
                "'connected' received on circid %u for streamid %d, "
                "no conn attached anymore. Ignoring.",
@@ -1731,6 +1798,15 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
           log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.",
                     layer_hint->package_window);
           circuit_resume_edge_reading(circ, layer_hint);
+
+          /* We count circuit-level sendme's as valid delivered data because
+           * they are rate limited.
+           */
+          if (CIRCUIT_IS_ORIGIN(circ)) {
+            circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
+                                    rh.length);
+          }
+
         } else {
           if (circ->package_window + CIRCWINDOW_INCREMENT >
                 CIRCWINDOW_START_MAX) {
@@ -1750,10 +1826,42 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         return 0;
       }
       if (!conn) {
+        if (CIRCUIT_IS_ORIGIN(circ)) {
+          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+          if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                                   rh.stream_id)) {
+            circuit_read_valid_data(ocirc, rh.length);
+            log_info(domain,
+                    "sendme cell on circ %u valid on half-closed "
+                    "stream id %d", ocirc->global_identifier, rh.stream_id);
+          }
+        }
+
         log_info(domain,"sendme cell dropped, unknown stream (streamid %d).",
                  rh.stream_id);
         return 0;
       }
+
+      /* Don't allow the other endpoint to request more than our maximim
+       * (ie initial) stream SENDME window worth of data. Well-behaved
+       * stock clients will not request more than this max (as per the check
+       * in the while loop of connection_edge_consider_sending_sendme()).
+       */
+      if (conn->package_window + STREAMWINDOW_INCREMENT >
+          STREAMWINDOW_START_MAX) {
+        static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
+        log_fn_ratelim(&stream_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL,
+               "Unexpected stream sendme cell. Closing circ (window %d).",
+               conn->package_window);
+        return -END_CIRC_REASON_TORPROTOCOL;
+      }
+
+      /* At this point, the stream sendme is valid */
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
+                                rh.length);
+      }
+
       conn->package_window += STREAMWINDOW_INCREMENT;
       log_debug(domain,"stream-level sendme, packagewindow now %d.",
                 conn->package_window);
@@ -1792,6 +1900,19 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "'resolved' unsupported while open. Closing circ.");
         return -END_CIRC_REASON_TORPROTOCOL;
       }
+
+      if (CIRCUIT_IS_ORIGIN(circ)) {
+        origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+        if (connection_half_edge_is_valid_resolved(ocirc->half_streams,
+                                                    rh.stream_id)) {
+          circuit_read_valid_data(ocirc, rh.length);
+          log_info(domain,
+                   "resolved cell on circ %u valid on half-closed "
+                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+          return 0;
+        }
+      }
+
       log_info(domain,
                "'resolved' received, no conn attached anymore. Ignoring.");
       return 0;
@@ -1919,13 +2040,13 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     /* XXXX We could be more efficient here by sometimes packing
      * previously-sent optimistic data in the same cell with data
      * from the inbuf. */
-    fetch_from_buf(payload, length, entry_conn->sending_optimistic_data);
+    buf_get_bytes(entry_conn->sending_optimistic_data, payload, length);
     if (!buf_datalen(entry_conn->sending_optimistic_data)) {
         buf_free(entry_conn->sending_optimistic_data);
         entry_conn->sending_optimistic_data = NULL;
     }
   } else {
-    connection_fetch_from_buf(payload, length, TO_CONN(conn));
+    connection_buf_get_bytes(payload, length, TO_CONN(conn));
   }
 
   log_debug(domain,TOR_SOCKET_T_FORMAT": Packaging %d bytes (%d waiting).",
@@ -1937,7 +2058,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
        retry */
     if (!entry_conn->pending_optimistic_data)
       entry_conn->pending_optimistic_data = buf_new();
-    write_to_buf(payload, length, entry_conn->pending_optimistic_data);
+    buf_add(entry_conn->pending_optimistic_data, payload, length);
   }
 
   if (connection_edge_send_command(conn, RELAY_COMMAND_DATA,
@@ -2255,13 +2376,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
   }
 }
 
-#ifdef ACTIVE_CIRCUITS_PARANOIA
-#define assert_cmux_ok_paranoid(chan) \
-     assert_circuit_mux_okay(chan)
-#else
-#define assert_cmux_ok_paranoid(chan)
-#endif
-
 /** The total number of cells we have allocated. */
 static size_t total_cells_allocated = 0;
 
@@ -2283,7 +2397,7 @@ packed_cell_new(void)
 
 /** Return a packed cell used outside by channel_t lower layer */
 void
-packed_cell_free(packed_cell_t *cell)
+packed_cell_free_(packed_cell_t *cell)
 {
   if (!cell)
     return;
@@ -2340,7 +2454,7 @@ cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue,
   (void)exitward;
   (void)use_stats;
 
-  copy->inserted_time = (uint32_t) monotime_coarse_absolute_msec();
+  copy->inserted_timestamp = monotime_coarse_get_stamp();
 
   cell_queue_append(queue, copy);
 }
@@ -2423,7 +2537,7 @@ destroy_cell_queue_append(destroy_cell_queue_t *queue,
   cell->circid = circid;
   cell->reason = reason;
   /* Not yet used, but will be required for OOM handling. */
-  cell->inserted_time = (uint32_t) monotime_coarse_absolute_msec();
+  cell->inserted_timestamp = monotime_coarse_get_stamp();
 
   TOR_SIMPLEQ_INSERT_TAIL(&queue->head, cell, next);
   ++queue->n;
@@ -2454,7 +2568,7 @@ packed_cell_mem_cost(void)
 }
 
 /* DOCDOC */
-STATIC size_t
+size_t
 cell_queues_get_total_allocation(void)
 {
   return total_cells_allocated * packed_cell_mem_cost();
@@ -2473,8 +2587,9 @@ cell_queues_check_size(void)
 {
   time_t now = time(NULL);
   size_t alloc = cell_queues_get_total_allocation();
+  alloc += half_streams_get_total_allocation();
   alloc += buf_get_total_allocation();
-  alloc += tor_zlib_get_total_allocation();
+  alloc += tor_compress_get_total_allocation();
   const size_t rend_cache_total = rend_cache_get_total_allocation();
   alloc += rend_cache_total;
   const size_t geoip_client_cache_total =
@@ -2489,9 +2604,7 @@ cell_queues_check_size(void)
       if (rend_cache_total > get_options()->MaxMemInQueues / 5) {
         const size_t bytes_to_remove =
           rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10);
-        rend_cache_clean_v2_descs_as_dir(now, bytes_to_remove);
-        alloc -= rend_cache_total;
-        alloc += rend_cache_get_total_allocation();
+        alloc -= hs_cache_handle_oom(now, bytes_to_remove);
       }
       if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) {
         const size_t bytes_to_remove =
@@ -2551,16 +2664,12 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
   }
   tor_assert(circuitmux_attached_circuit_direction(cmux, circ) == direction);
 
-  assert_cmux_ok_paranoid(chan);
-
   /* Update the number of cells we have for the circuit mux */
   if (direction == CELL_DIRECTION_OUT) {
     circuitmux_set_num_cells(cmux, circ, circ->n_chan_cells.n);
   } else {
     circuitmux_set_num_cells(cmux, circ, or_circ->p_chan_cells.n);
   }
-
-  assert_cmux_ok_paranoid(chan);
 }
 
 /** Remove all circuits from the cmux on <b>chan</b>.
@@ -2690,8 +2799,13 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
       tor_assert(dcell);
       /* frees dcell */
       cell = destroy_cell_to_packed_cell(dcell, chan->wide_circ_ids);
-      /* frees cell */
-      channel_write_packed_cell(chan, cell);
+      /* Send the DESTROY cell. It is very unlikely that this fails but just
+       * in case, get rid of the channel. */
+      if (channel_write_packed_cell(chan, cell) < 0) {
+        /* The cell has been freed. */
+        channel_mark_for_close(chan);
+        continue;
+      }
       /* Update the cmux destroy counter */
       circuitmux_notify_xmit_destroy(cmux);
       cell = NULL;
@@ -2700,7 +2814,6 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
     }
     /* If it returns NULL, no cells left to send */
     if (!circ) break;
-    assert_cmux_ok_paranoid(chan);
 
     if (circ->n_chan == chan) {
       queue = &circ->n_chan_cells;
@@ -2734,9 +2847,10 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
     /* Calculate the exact time that this cell has spent in the queue. */
     if (get_options()->CellStatistics ||
         get_options()->TestingEnableCellStatsEvent) {
-      uint32_t msec_waiting;
-      uint32_t msec_now = (uint32_t)monotime_coarse_absolute_msec();
-      msec_waiting = msec_now - cell->inserted_time;
+      uint32_t timestamp_now = monotime_coarse_get_stamp();
+      uint32_t msec_waiting =
+        (uint32_t) monotime_coarse_stamp_units_to_approx_msec(
+                         timestamp_now - cell->inserted_timestamp);
 
       if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) {
         or_circ = TO_OR_CIRCUIT(circ);
@@ -2767,8 +2881,13 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
                                 DIRREQ_TUNNELED,
                                 DIRREQ_CIRC_QUEUE_FLUSHED);
 
-    /* Now send the cell */
-    channel_write_packed_cell(chan, cell);
+    /* Now send the cell. It is very unlikely that this fails but just in
+     * case, get rid of the channel. */
+    if (channel_write_packed_cell(chan, cell) < 0) {
+      /* The cell has been freed at this point. */
+      channel_mark_for_close(chan);
+      continue;
+    }
     cell = NULL;
 
     /*
@@ -2798,27 +2917,70 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max))
   }
 
   /* Okay, we're done sending now */
-  assert_cmux_ok_paranoid(chan);
-
   return n_flushed;
 }
 
-#if 0
-/** Indicate the current preferred cap for middle circuits; zero disables
- * the cap.  Right now it's just a constant, ORCIRC_MAX_MIDDLE_CELLS, but
- * the logic in append_cell_to_circuit_queue() is written to be correct
- * if we want to base it on a consensus param or something that might change
- * in the future.
- */
-static int
-get_max_middle_cells(void)
+/* Minimum value is the maximum circuit window size.
+ *
+ * SENDME cells makes it that we can control how many cells can be inflight on
+ * a circuit from end to end. This logic makes it that on any circuit cell
+ * queue, we have a maximum of cells possible.
+ *
+ * Because the Tor protocol allows for a client to exit at any hop in a
+ * circuit and a circuit can be of a maximum of 8 hops, so in theory the
+ * normal worst case will be the circuit window start value times the maximum
+ * number of hops (8). Having more cells then that means something is wrong.
+ *
+ * However, because padding cells aren't counted in the package window, we set
+ * the maximum size to a reasonably large size for which we expect that we'll
+ * never reach in theory. And if we ever do because of future changes, we'll
+ * be able to control it with a consensus parameter.
+ *
+ * XXX: Unfortunately, END cells aren't accounted for in the circuit window
+ * which means that for instance if a client opens 8001 streams, the 8001
+ * following END cells will queue up in the circuit which will get closed if
+ * the max limit is 8000. Which is sad because it is allowed by the Tor
+ * protocol. But, we need an upper bound on circuit queue in order to avoid
+ * DoS memory pressure so the default size is a middle ground between not
+ * having any limit and having a very restricted one. This is why we can also
+ * control it through a consensus parameter. */
+#define RELAY_CIRC_CELL_QUEUE_SIZE_MIN CIRCWINDOW_START_MAX
+/* We can't have a consensus parameter above this value. */
+#define RELAY_CIRC_CELL_QUEUE_SIZE_MAX INT32_MAX
+/* Default value is set to a large value so we can handle padding cells
+ * properly which aren't accounted for in the SENDME window. Default is 50000
+ * allowed cells in the queue resulting in ~25MB. */
+#define RELAY_CIRC_CELL_QUEUE_SIZE_DEFAULT \
+  (50 * RELAY_CIRC_CELL_QUEUE_SIZE_MIN)
+
+/* The maximum number of cell a circuit queue can contain. This is updated at
+ * every new consensus and controlled by a parameter. */
+static int32_t max_circuit_cell_queue_size =
+  RELAY_CIRC_CELL_QUEUE_SIZE_DEFAULT;
+
+/* Called when the consensus has changed. At this stage, the global consensus
+ * object has NOT been updated. It is called from
+ * notify_before_networkstatus_changes(). */
+void
+relay_consensus_has_changed(const networkstatus_t *ns)
 {
-  return ORCIRC_MAX_MIDDLE_CELLS;
+  tor_assert(ns);
+
+  /* Update the circuit max cell queue size from the consensus. */
+  max_circuit_cell_queue_size =
+    networkstatus_get_param(ns, "circ_max_cell_queue_size",
+                            RELAY_CIRC_CELL_QUEUE_SIZE_DEFAULT,
+                            RELAY_CIRC_CELL_QUEUE_SIZE_MIN,
+                            RELAY_CIRC_CELL_QUEUE_SIZE_MAX);
 }
-#endif
 
 /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b>
- * transmitting in <b>direction</b>. */
+ * transmitting in <b>direction</b>.
+ *
+ * The given <b>cell</b> is copied onto the circuit queue so the caller must
+ * cleanup the memory.
+ *
+ * This function is part of the fast path. */
 void
 append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
                              cell_t *cell, cell_direction_t direction,
@@ -2827,10 +2989,6 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
   or_circuit_t *orcirc = NULL;
   cell_queue_t *queue;
   int streams_blocked;
-#if 0
-  uint32_t tgt_max_middle_cells, p_len, n_len, tmp, hard_max_middle_cells;
-#endif
-
   int exitward;
   if (circ->marked_for_close)
     return;
@@ -2845,93 +3003,25 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan,
     streams_blocked = circ->streams_blocked_on_p_chan;
   }
 
-  /*
-   * Disabling this for now because of a possible guard discovery attack
-   */
-#if 0
-  /* Are we a middle circuit about to exceed ORCIRC_MAX_MIDDLE_CELLS? */
-  if ((circ->n_chan != NULL) && CIRCUIT_IS_ORCIRC(circ)) {
-    orcirc = TO_OR_CIRCUIT(circ);
-    if (orcirc->p_chan) {
-      /* We are a middle circuit if we have both n_chan and p_chan */
-      /* We'll need to know the current preferred maximum */
-      tgt_max_middle_cells = get_max_middle_cells();
-      if (tgt_max_middle_cells > 0) {
-        /* Do we need to initialize middle_max_cells? */
-        if (orcirc->max_middle_cells == 0) {
-          orcirc->max_middle_cells = tgt_max_middle_cells;
-        } else {
-          if (tgt_max_middle_cells > orcirc->max_middle_cells) {
-            /* If we want to increase the cap, we can do so right away */
-            orcirc->max_middle_cells = tgt_max_middle_cells;
-          } else if (tgt_max_middle_cells < orcirc->max_middle_cells) {
-            /*
-             * If we're shrinking the cap, we can't shrink past either queue;
-             * compare tgt_max_middle_cells rather than tgt_max_middle_cells *
-             * ORCIRC_MAX_MIDDLE_KILL_THRESH so the queues don't shrink enough
-             * to generate spurious warnings, either.
-             */
-            n_len = circ->n_chan_cells.n;
-            p_len = orcirc->p_chan_cells.n;
-            tmp = tgt_max_middle_cells;
-            if (tmp < n_len) tmp = n_len;
-            if (tmp < p_len) tmp = p_len;
-            orcirc->max_middle_cells = tmp;
-          }
-          /* else no change */
-        }
-      } else {
-        /* tgt_max_middle_cells == 0 indicates we should disable the cap */
-        orcirc->max_middle_cells = 0;
-      }
-
-      /* Now we know orcirc->max_middle_cells is set correctly */
-      if (orcirc->max_middle_cells > 0) {
-        hard_max_middle_cells =
-          (uint32_t)(((double)orcirc->max_middle_cells) *
-                     ORCIRC_MAX_MIDDLE_KILL_THRESH);
-
-        if ((unsigned)queue->n + 1 >= hard_max_middle_cells) {
-          /* Queueing this cell would put queue over the kill theshold */
-          log_warn(LD_CIRC,
-                   "Got a cell exceeding the hard cap of %u in the "
-                   "%s direction on middle circ ID %u on chan ID "
-                   U64_FORMAT "; killing the circuit.",
-                   hard_max_middle_cells,
-                   (direction == CELL_DIRECTION_OUT) ? "n" : "p",
-                   (direction == CELL_DIRECTION_OUT) ?
-                     circ->n_circ_id : orcirc->p_circ_id,
-                   U64_PRINTF_ARG(
-                     (direction == CELL_DIRECTION_OUT) ?
-                        circ->n_chan->global_identifier :
-                        orcirc->p_chan->global_identifier));
-          circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
-          return;
-        } else if ((unsigned)queue->n + 1 == orcirc->max_middle_cells) {
-          /* Only use ==, not >= for this test so we don't spam the log */
-          log_warn(LD_CIRC,
-                   "While trying to queue a cell, reached the soft cap of %u "
-                   "in the %s direction on middle circ ID %u "
-                   "on chan ID " U64_FORMAT ".",
-                   orcirc->max_middle_cells,
-                   (direction == CELL_DIRECTION_OUT) ? "n" : "p",
-                   (direction == CELL_DIRECTION_OUT) ?
-                     circ->n_circ_id : orcirc->p_circ_id,
-                   U64_PRINTF_ARG(
-                     (direction == CELL_DIRECTION_OUT) ?
-                        circ->n_chan->global_identifier :
-                        orcirc->p_chan->global_identifier));
-        }
-      }
-    }
+  if (PREDICT_UNLIKELY(queue->n >= max_circuit_cell_queue_size)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "%s circuit has %d cells in its queue, maximum allowed is %d. "
+           "Closing circuit for safety reasons.",
+           (exitward) ? "Outbound" : "Inbound", queue->n,
+           max_circuit_cell_queue_size);
+    circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
+    stats_n_circ_max_cell_reached++;
+    return;
   }
-#endif
 
+  /* Very important that we copy to the circuit queue because all calls to
+   * this function use the stack for the cell memory. */
   cell_queue_append_packed_copy(circ, queue, exitward, cell,
                                 chan->wide_circ_ids, 1);
 
+  /* Check and run the OOM if needed. */
   if (PREDICT_UNLIKELY(cell_queues_check_size())) {
-    /* We ran the OOM handler */
+    /* We ran the OOM handler which might have closed this circuit. */
     if (circ->marked_for_close)
       return;
   }
@@ -3042,17 +3132,6 @@ circuit_clear_cell_queue(circuit_t *circ, channel_t *chan)
     update_circuit_on_cmux(circ, direction);
 }
 
-/** Fail with an assert if the circuit mux on chan is corrupt
- */
-void
-assert_circuit_mux_okay(channel_t *chan)
-{
-  tor_assert(chan);
-  tor_assert(chan->cmux);
-
-  circuitmux_assert_okay(chan->cmux);
-}
-
 /** Return 1 if we shouldn't restart reading on this circuit, even if
  * we get a SENDME.  Else return 0.
 */
@@ -3065,4 +3144,3 @@ circuit_queue_streams_are_blocked(circuit_t *circ)
     return circ->streams_blocked_on_p_chan;
   }
 }
-
diff --git a/src/or/relay.h b/src/or/relay.h
index c4f98d92ff..ce0969b46c 100644
--- a/src/or/relay.h
+++ b/src/or/relay.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,16 +14,22 @@
 
 extern uint64_t stats_n_relay_cells_relayed;
 extern uint64_t stats_n_relay_cells_delivered;
+extern uint64_t stats_n_circ_max_cell_reached;
 
+void relay_consensus_has_changed(const networkstatus_t *ns);
 int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
                                cell_direction_t cell_direction);
+size_t cell_queues_get_total_allocation(void);
 
 void relay_header_pack(uint8_t *dest, const relay_header_t *src);
 void relay_header_unpack(relay_header_t *dest, const uint8_t *src);
-int relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
+MOCK_DECL(int,
+relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
                                uint8_t relay_command, const char *payload,
                                size_t payload_len, crypt_path_t *cpath_layer,
-                               const char *filename, int lineno);
+                               const char *filename, int lineno));
+/* Indicates to relay_send_command_from_edge() that it is a control cell. */
+#define CONTROL_CELL_ID 0
 #define relay_send_command_from_edge(stream_id, circ, relay_command, payload, \
                                      payload_len, cpath_layer)          \
   relay_send_command_from_edge_((stream_id), (circ), (relay_command),   \
@@ -48,7 +54,9 @@ size_t packed_cell_mem_cost(void);
 int have_been_under_memory_pressure(void);
 
 /* For channeltls.c */
-void packed_cell_free(packed_cell_t *cell);
+void packed_cell_free_(packed_cell_t *cell);
+#define packed_cell_free(cell) \
+  FREE_AND_NULL(packed_cell_t, packed_cell_free_, (cell))
 
 void cell_queue_init(cell_queue_t *queue);
 void cell_queue_clear(cell_queue_t *queue);
@@ -70,7 +78,6 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue,
 void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out);
 MOCK_DECL(int, channel_flush_from_first_active_circuit,
           (channel_t *chan, int max));
-void assert_circuit_mux_okay(channel_t *chan);
 void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
                              const char *file, int lineno);
 #define update_circuit_on_cmux(circ, direction) \
@@ -84,9 +91,6 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
 void stream_choice_seed_weak_rng(void);
 
-int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction,
-                crypt_path_t **layer_hint, char *recognized);
-
 circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids);
 
 #ifdef RELAY_PRIVATE
@@ -98,7 +102,9 @@ typedef struct address_ttl_s {
   char *hostname;
   int ttl;
 } address_ttl_t;
-STATIC void address_ttl_free(address_ttl_t *addr);
+STATIC void address_ttl_free_(address_ttl_t *addr);
+#define address_ttl_free(addr) \
+  FREE_AND_NULL(address_ttl_t, address_ttl_free_, (addr))
 STATIC int resolved_cell_parse(const cell_t *cell, const relay_header_t *rh,
                                smartlist_t *addresses_out, int *errcode_out);
 STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn,
@@ -107,9 +113,12 @@ STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn,
 STATIC packed_cell_t *packed_cell_new(void);
 STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue);
 STATIC destroy_cell_t *destroy_cell_queue_pop(destroy_cell_queue_t *queue);
-STATIC size_t cell_queues_get_total_allocation(void);
 STATIC int cell_queues_check_size(void);
-#endif
+STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
+                                   edge_connection_t *conn,
+                                   crypt_path_t *layer_hint);
+
+#endif /* defined(RELAY_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_RELAY_H) */
 
diff --git a/src/or/relay_crypto.c b/src/or/relay_crypto.c
new file mode 100644
index 0000000000..530c8e5828
--- /dev/null
+++ b/src/or/relay_crypto.c
@@ -0,0 +1,327 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "config.h"
+#include "crypto_util.h"
+#include "hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN
+#include "relay.h"
+#include "relay_crypto.h"
+
+/** Update digest from the payload of cell. Assign integrity part to
+ * cell.
+ */
+static void
+relay_set_digest(crypto_digest_t *digest, cell_t *cell)
+{
+  char integrity[4];
+  relay_header_t rh;
+
+  crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE);
+  crypto_digest_get_digest(digest, integrity, 4);
+//  log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.",
+//    integrity[0], integrity[1], integrity[2], integrity[3]);
+  relay_header_unpack(&rh, cell->payload);
+  memcpy(rh.integrity, integrity, 4);
+  relay_header_pack(cell->payload, &rh);
+}
+
+/** Does the digest for this circuit indicate that this cell is for us?
+ *
+ * Update digest from the payload of cell (with the integrity part set
+ * to 0). If the integrity part is valid, return 1, else restore digest
+ * and cell to their original state and return 0.
+ */
+static int
+relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
+{
+  uint32_t received_integrity, calculated_integrity;
+  relay_header_t rh;
+  crypto_digest_checkpoint_t backup_digest;
+
+  crypto_digest_checkpoint(&backup_digest, digest);
+
+  relay_header_unpack(&rh, cell->payload);
+  memcpy(&received_integrity, rh.integrity, 4);
+  memset(rh.integrity, 0, 4);
+  relay_header_pack(cell->payload, &rh);
+
+//  log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.",
+//    received_integrity[0], received_integrity[1],
+//    received_integrity[2], received_integrity[3]);
+
+  crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE);
+  crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4);
+
+  int rv = 1;
+
+  if (calculated_integrity != received_integrity) {
+//    log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing.");
+// (%d vs %d).", received_integrity, calculated_integrity);
+    /* restore digest to its old form */
+    crypto_digest_restore(digest, &backup_digest);
+    /* restore the relay header */
+    memcpy(rh.integrity, &received_integrity, 4);
+    relay_header_pack(cell->payload, &rh);
+    rv = 0;
+  }
+
+  memwipe(&backup_digest, 0, sizeof(backup_digest));
+  return rv;
+}
+
+/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b>
+ * (in place).
+ *
+ * Note that we use the same operation for encrypting and for decrypting.
+ */
+static void
+relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in)
+{
+  crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
+}
+
+/** Do the appropriate en/decryptions for <b>cell</b> arriving on
+ * <b>circ</b> in direction <b>cell_direction</b>.
+ *
+ * If cell_direction == CELL_DIRECTION_IN:
+ *   - If we're at the origin (we're the OP), for hops 1..N,
+ *     decrypt cell. If recognized, stop.
+ *   - Else (we're not the OP), encrypt one hop. Cell is not recognized.
+ *
+ * If cell_direction == CELL_DIRECTION_OUT:
+ *   - decrypt one hop. Check if recognized.
+ *
+ * If cell is recognized, set *recognized to 1, and set
+ * *layer_hint to the hop that recognized it.
+ *
+ * Return -1 to indicate that we should mark the circuit for close,
+ * else return 0.
+ */
+int
+relay_decrypt_cell(circuit_t *circ, cell_t *cell,
+                   cell_direction_t cell_direction,
+                   crypt_path_t **layer_hint, char *recognized)
+{
+  relay_header_t rh;
+
+  tor_assert(circ);
+  tor_assert(cell);
+  tor_assert(recognized);
+  tor_assert(cell_direction == CELL_DIRECTION_IN ||
+             cell_direction == CELL_DIRECTION_OUT);
+
+  if (cell_direction == CELL_DIRECTION_IN) {
+    if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit.
+                                    * We'll want to do layered decrypts. */
+      crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath;
+      thishop = cpath;
+      if (thishop->state != CPATH_STATE_OPEN) {
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+               "Relay cell before first created cell? Closing.");
+        return -1;
+      }
+      do { /* Remember: cpath is in forward order, that is, first hop first. */
+        tor_assert(thishop);
+
+        /* decrypt one layer */
+        relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload);
+
+        relay_header_unpack(&rh, cell->payload);
+        if (rh.recognized == 0) {
+          /* it's possibly recognized. have to check digest to be sure. */
+          if (relay_digest_matches(thishop->crypto.b_digest, cell)) {
+            *recognized = 1;
+            *layer_hint = thishop;
+            return 0;
+          }
+        }
+
+        thishop = thishop->next;
+      } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN);
+      log_fn(LOG_PROTOCOL_WARN, LD_OR,
+             "Incoming cell at client not recognized. Closing.");
+      return -1;
+    } else {
+      relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto;
+      /* We're in the middle. Encrypt one layer. */
+      relay_crypt_one_payload(crypto->b_crypto, cell->payload);
+    }
+  } else /* cell_direction == CELL_DIRECTION_OUT */ {
+    /* We're in the middle. Decrypt one layer. */
+    relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto;
+
+    relay_crypt_one_payload(crypto->f_crypto, cell->payload);
+
+    relay_header_unpack(&rh, cell->payload);
+    if (rh.recognized == 0) {
+      /* it's possibly recognized. have to check digest to be sure. */
+      if (relay_digest_matches(crypto->f_digest, cell)) {
+        *recognized = 1;
+        return 0;
+      }
+    }
+  }
+  return 0;
+}
+
+/**
+ * Encrypt a cell <b>cell</b> that we are creating, and sending outbound on
+ * <b>circ</b> until the hop corresponding to <b>layer_hint</b>.
+ *
+ * The integrity field and recognized field of <b>cell</b>'s relay headers
+ * must be set to zero.
+ */
+void
+relay_encrypt_cell_outbound(cell_t *cell,
+                            origin_circuit_t *circ,
+                            crypt_path_t *layer_hint)
+{
+  crypt_path_t *thishop; /* counter for repeated crypts */
+  relay_set_digest(layer_hint->crypto.f_digest, cell);
+
+  thishop = layer_hint;
+  /* moving from farthest to nearest hop */
+  do {
+    tor_assert(thishop);
+    log_debug(LD_OR,"encrypting a layer of the relay cell.");
+    relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload);
+
+    thishop = thishop->prev;
+  } while (thishop != circ->cpath->prev);
+}
+
+/**
+ * Encrypt a cell <b>cell</b> that we are creating, and sending on
+ * <b>circuit</b> to the origin.
+ *
+ * The integrity field and recognized field of <b>cell</b>'s relay headers
+ * must be set to zero.
+ */
+void
+relay_encrypt_cell_inbound(cell_t *cell,
+                           or_circuit_t *or_circ)
+{
+  relay_set_digest(or_circ->crypto.b_digest, cell);
+  /* encrypt one layer */
+  relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload);
+}
+
+/**
+ * Release all storage held inside <b>crypto</b>, but do not free
+ * <b>crypto</b> itself: it lives inside another object.
+ */
+void
+relay_crypto_clear(relay_crypto_t *crypto)
+{
+  if (BUG(!crypto))
+    return;
+  crypto_cipher_free(crypto->f_crypto);
+  crypto_cipher_free(crypto->b_crypto);
+  crypto_digest_free(crypto->f_digest);
+  crypto_digest_free(crypto->b_digest);
+}
+
+/** Initialize <b>crypto</b> from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
+ *   - 20 to initialize f_digest
+ *   - 20 to initialize b_digest
+ *   - 16 to key f_crypto
+ *   - 16 to key b_crypto
+ *
+ * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
+ */
+int
+relay_crypto_init(relay_crypto_t *crypto,
+                  const char *key_data, size_t key_data_len,
+                  int reverse, int is_hs_v3)
+{
+  crypto_digest_t *tmp_digest;
+  crypto_cipher_t *tmp_crypto;
+  size_t digest_len = 0;
+  size_t cipher_key_len = 0;
+
+  tor_assert(crypto);
+  tor_assert(key_data);
+  tor_assert(!(crypto->f_crypto || crypto->b_crypto ||
+             crypto->f_digest || crypto->b_digest));
+
+  /* Basic key size validation */
+  if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) {
+    goto err;
+  } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) {
+    goto err;
+  }
+
+  /* If we are using this crypto for next gen onion services use SHA3-256,
+     otherwise use good ol' SHA1 */
+  if (is_hs_v3) {
+    digest_len = DIGEST256_LEN;
+    cipher_key_len = CIPHER256_KEY_LEN;
+    crypto->f_digest = crypto_digest256_new(DIGEST_SHA3_256);
+    crypto->b_digest = crypto_digest256_new(DIGEST_SHA3_256);
+  } else {
+    digest_len = DIGEST_LEN;
+    cipher_key_len = CIPHER_KEY_LEN;
+    crypto->f_digest = crypto_digest_new();
+    crypto->b_digest = crypto_digest_new();
+  }
+
+  tor_assert(digest_len != 0);
+  tor_assert(cipher_key_len != 0);
+  const int cipher_key_bits = (int) cipher_key_len * 8;
+
+  crypto_digest_add_bytes(crypto->f_digest, key_data, digest_len);
+  crypto_digest_add_bytes(crypto->b_digest, key_data+digest_len, digest_len);
+
+  crypto->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len),
+                                                cipher_key_bits);
+  if (!crypto->f_crypto) {
+    log_warn(LD_BUG,"Forward cipher initialization failed.");
+    goto err;
+  }
+
+  crypto->b_crypto = crypto_cipher_new_with_bits(
+                                        key_data+(2*digest_len)+cipher_key_len,
+                                        cipher_key_bits);
+  if (!crypto->b_crypto) {
+    log_warn(LD_BUG,"Backward cipher initialization failed.");
+    goto err;
+  }
+
+  if (reverse) {
+    tmp_digest = crypto->f_digest;
+    crypto->f_digest = crypto->b_digest;
+    crypto->b_digest = tmp_digest;
+    tmp_crypto = crypto->f_crypto;
+    crypto->f_crypto = crypto->b_crypto;
+    crypto->b_crypto = tmp_crypto;
+  }
+
+  return 0;
+ err:
+  relay_crypto_clear(crypto);
+  return -1;
+}
+
+/** Assert that <b>crypto</b> is valid and set. */
+void
+relay_crypto_assert_ok(const relay_crypto_t *crypto)
+{
+  tor_assert(crypto->f_crypto);
+  tor_assert(crypto->b_crypto);
+  tor_assert(crypto->f_digest);
+  tor_assert(crypto->b_digest);
+}
+
diff --git a/src/or/relay_crypto.h b/src/or/relay_crypto.h
new file mode 100644
index 0000000000..66ae02cee9
--- /dev/null
+++ b/src/or/relay_crypto.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file relay.h
+ * \brief Header file for relay.c.
+ **/
+
+#ifndef TOR_RELAY_CRYPTO_H
+#define TOR_RELAY_CRYPTO_H
+
+int relay_crypto_init(relay_crypto_t *crypto,
+                      const char *key_data, size_t key_data_len,
+                      int reverse, int is_hs_v3);
+
+int relay_decrypt_cell(circuit_t *circ, cell_t *cell,
+                       cell_direction_t cell_direction,
+                       crypt_path_t **layer_hint, char *recognized);
+void relay_encrypt_cell_outbound(cell_t *cell, origin_circuit_t *or_circ,
+                            crypt_path_t *layer_hint);
+void relay_encrypt_cell_inbound(cell_t *cell, or_circuit_t *or_circ);
+
+void relay_crypto_clear(relay_crypto_t *crypto);
+
+void relay_crypto_assert_ok(const relay_crypto_t *crypto);
+
+#endif /* !defined(TOR_RELAY_CRYPTO_H) */
+
diff --git a/src/or/rendcache.c b/src/or/rendcache.c
index aa69d735fe..d27e1c293f 100644
--- a/src/or/rendcache.c
+++ b/src/or/rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,12 +43,12 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL;
  * ID, that were NOT present in the descriptor are removed from this cache.
  * Which means that if at least one IP was not in this cache, thus usuable,
  * it's considered a new descriptor so we keep it. Else, if all IPs were in
- * this cache, we discard the descriptor as it's considered unsuable.
+ * this cache, we discard the descriptor as it's considered unusable.
  *
  * Once a descriptor is removed from the rend cache or expires, the entry
  * in this cache is also removed for the service ID.
  *
- * This scheme allows us to not realy on the descriptor's timestamp (which
+ * This scheme allows us to not rely on the descriptor's timestamp (which
  * is rounded down to the hour) to know if we have a newer descriptor. We
  * only rely on the usability of intro points from an internal state. */
 STATIC strmap_t *rend_cache_failure = NULL;
@@ -86,7 +86,7 @@ rend_cache_get_total_allocation(void)
 }
 
 /** Decrement the total bytes attributed to the rendezvous cache by n. */
-STATIC void
+void
 rend_cache_decrement_allocation(size_t n)
 {
   static int have_underflowed = 0;
@@ -103,7 +103,7 @@ rend_cache_decrement_allocation(size_t n)
 }
 
 /** Increase the total bytes attributed to the rendezvous cache by n. */
-STATIC void
+void
 rend_cache_increment_allocation(size_t n)
 {
   static int have_overflowed = 0;
@@ -120,7 +120,7 @@ rend_cache_increment_allocation(size_t n)
 
 /** Helper: free a rend cache failure intro object. */
 STATIC void
-rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t *entry)
+rend_cache_failure_intro_entry_free_(rend_cache_failure_intro_t *entry)
 {
   if (entry == NULL) {
     return;
@@ -129,9 +129,9 @@ rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t *entry)
 }
 
 static void
-rend_cache_failure_intro_entry_free_(void *entry)
+rend_cache_failure_intro_entry_free_void(void *entry)
 {
-  rend_cache_failure_intro_entry_free(entry);
+  rend_cache_failure_intro_entry_free_(entry);
 }
 
 /** Allocate a rend cache failure intro object and return it. <b>failure</b>
@@ -147,7 +147,7 @@ rend_cache_failure_intro_entry_new(rend_intro_point_failure_t failure)
 
 /** Helper: free a rend cache failure object. */
 STATIC void
-rend_cache_failure_entry_free(rend_cache_failure_t *entry)
+rend_cache_failure_entry_free_(rend_cache_failure_t *entry)
 {
   if (entry == NULL) {
     return;
@@ -155,7 +155,7 @@ rend_cache_failure_entry_free(rend_cache_failure_t *entry)
 
   /* Free and remove every intro failure object. */
   digestmap_free(entry->intro_failures,
-                 rend_cache_failure_intro_entry_free_);
+                 rend_cache_failure_intro_entry_free_void);
 
   tor_free(entry);
 }
@@ -163,9 +163,9 @@ rend_cache_failure_entry_free(rend_cache_failure_t *entry)
 /** Helper: deallocate a rend_cache_failure_t. (Used with strmap_free(),
  * which requires a function pointer whose argument is void*). */
 STATIC void
-rend_cache_failure_entry_free_(void *entry)
+rend_cache_failure_entry_free_void(void *entry)
 {
-  rend_cache_failure_entry_free(entry);
+  rend_cache_failure_entry_free_(entry);
 }
 
 /** Allocate a rend cache failure object and return it. This function can
@@ -201,7 +201,7 @@ rend_cache_failure_remove(rend_service_descriptor_t *desc)
 
 /** Helper: free storage held by a single service descriptor cache entry. */
 STATIC void
-rend_cache_entry_free(rend_cache_entry_t *e)
+rend_cache_entry_free_(rend_cache_entry_t *e)
 {
   if (!e)
     return;
@@ -217,19 +217,19 @@ rend_cache_entry_free(rend_cache_entry_t *e)
 /** Helper: deallocate a rend_cache_entry_t.  (Used with strmap_free(), which
  * requires a function pointer whose argument is void*). */
 static void
-rend_cache_entry_free_(void *p)
+rend_cache_entry_free_void(void *p)
 {
-  rend_cache_entry_free(p);
+  rend_cache_entry_free_(p);
 }
 
 /** Free all storage held by the service descriptor cache. */
 void
 rend_cache_free_all(void)
 {
-  strmap_free(rend_cache, rend_cache_entry_free_);
-  digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_);
-  strmap_free(rend_cache_local_service, rend_cache_entry_free_);
-  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
+  strmap_free(rend_cache, rend_cache_entry_free_void);
+  digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_void);
+  strmap_free(rend_cache_local_service, rend_cache_entry_free_void);
+  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
   rend_cache = NULL;
   rend_cache_v2_dir = NULL;
   rend_cache_local_service = NULL;
@@ -303,8 +303,8 @@ void
 rend_cache_purge(void)
 {
   if (rend_cache) {
-    log_info(LD_REND, "Purging HS descriptor cache");
-    strmap_free(rend_cache, rend_cache_entry_free_);
+    log_info(LD_REND, "Purging HS v2 descriptor cache");
+    strmap_free(rend_cache, rend_cache_entry_free_void);
   }
   rend_cache = strmap_new();
 }
@@ -315,8 +315,8 @@ void
 rend_cache_failure_purge(void)
 {
   if (rend_cache_failure) {
-    log_info(LD_REND, "Purging HS failure cache");
-    strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
+    log_info(LD_REND, "Purging HS v2 failure cache");
+    strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
   }
   rend_cache_failure = strmap_new();
 }
@@ -462,45 +462,36 @@ rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
 }
 
 /** Remove all old v2 descriptors and those for which this hidden service
- * directory is not responsible for any more.
- *
- * If at all possible, remove at least <b>force_remove</b> bytes of data.
- */
-void
-rend_cache_clean_v2_descs_as_dir(time_t now, size_t force_remove)
+ * directory is not responsible for any more. The cutoff is the time limit for
+ * which we want to keep the cache entry. In other words, any entry created
+ * before will be removed. */
+size_t
+rend_cache_clean_v2_descs_as_dir(time_t cutoff)
 {
   digestmap_iter_t *iter;
-  time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW;
-  const int LAST_SERVED_CUTOFF_STEP = 1800;
-  time_t last_served_cutoff = cutoff;
   size_t bytes_removed = 0;
-  do {
-    for (iter = digestmap_iter_init(rend_cache_v2_dir);
-         !digestmap_iter_done(iter); ) {
-      const char *key;
-      void *val;
-      rend_cache_entry_t *ent;
-      digestmap_iter_get(iter, &key, &val);
-      ent = val;
-      if (ent->parsed->timestamp < cutoff ||
-          ent->last_served < last_served_cutoff) {
-        char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-        base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
-        log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
-                 safe_str_client(key_base32));
-        bytes_removed += rend_cache_entry_allocation(ent);
-        iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
-        rend_cache_entry_free(ent);
-      } else {
-        iter = digestmap_iter_next(rend_cache_v2_dir, iter);
-      }
+
+  for (iter = digestmap_iter_init(rend_cache_v2_dir);
+       !digestmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    rend_cache_entry_t *ent;
+    digestmap_iter_get(iter, &key, &val);
+    ent = val;
+    if (ent->parsed->timestamp < cutoff) {
+      char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+      base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
+      log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
+               safe_str_client(key_base32));
+      bytes_removed += rend_cache_entry_allocation(ent);
+      iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
+      rend_cache_entry_free(ent);
+    } else {
+      iter = digestmap_iter_next(rend_cache_v2_dir, iter);
     }
+  }
 
-    /* In case we didn't remove enough bytes, advance the cutoff a little. */
-    last_served_cutoff += LAST_SERVED_CUTOFF_STEP;
-    if (last_served_cutoff > now)
-      break;
-  } while (bytes_removed < force_remove);
+  return bytes_removed;
 }
 
 /** Lookup in the client cache the given service ID <b>query</b> for
@@ -521,7 +512,7 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e)
   tor_assert(rend_cache);
   tor_assert(query);
 
-  if (!rend_valid_service_id(query)) {
+  if (!rend_valid_v2_service_id(query)) {
     ret = -EINVAL;
     goto end;
   }
@@ -567,7 +558,7 @@ rend_cache_lookup_v2_desc_as_service(const char *query, rend_cache_entry_t **e)
   tor_assert(rend_cache_local_service);
   tor_assert(query);
 
-  if (!rend_valid_service_id(query)) {
+  if (!rend_valid_v2_service_id(query)) {
     ret = -EINVAL;
     goto end;
   }
@@ -849,6 +840,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   char want_desc_id[DIGEST_LEN];
   rend_cache_entry_t *e;
   int retval = -1;
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
+
   tor_assert(rend_cache);
   tor_assert(desc);
   tor_assert(desc_id_base32);
@@ -874,11 +867,11 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     log_warn(LD_REND, "Couldn't compute service ID.");
     goto err;
   }
-  if (rend_query->onion_address[0] != '\0' &&
-      strcmp(rend_query->onion_address, service_id)) {
+  if (rend_data->onion_address[0] != '\0' &&
+      strcmp(rend_data->onion_address, service_id)) {
     log_warn(LD_REND, "Received service descriptor for service ID %s; "
              "expected descriptor for service ID %s.",
-             service_id, safe_str(rend_query->onion_address));
+             service_id, safe_str(rend_data->onion_address));
     goto err;
   }
   if (tor_memneq(desc_id, want_desc_id, DIGEST_LEN)) {
@@ -890,14 +883,14 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   /* Decode/decrypt introduction points. */
   if (intro_content && intro_size > 0) {
     int n_intro_points;
-    if (rend_query->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_query->descriptor_cookie,
-                         sizeof(rend_query->descriptor_cookie))) {
+    if (rend_data->auth_type != REND_NO_AUTH &&
+        !tor_mem_is_zero(rend_data->descriptor_cookie,
+                         sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
                                            &ipos_decrypted_size,
-                                           rend_query->descriptor_cookie,
+                                           rend_data->descriptor_cookie,
                                            intro_content,
                                            intro_size) < 0) {
         log_warn(LD_REND, "Failed to decrypt introduction points. We are "
diff --git a/src/or/rendcache.h b/src/or/rendcache.h
index 270b614c38..8b6fd5b671 100644
--- a/src/or/rendcache.h
+++ b/src/or/rendcache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,7 +36,7 @@ typedef struct rend_cache_entry_t {
 
 /* Introduction point failure type. */
 typedef struct rend_cache_failure_intro_t {
-  /* When this intro point failure occured thus we allocated this object and
+  /* When this intro point failure occurred thus we allocated this object and
    * cache it. */
   time_t created_ts;
   rend_intro_point_failure_t failure_type;
@@ -53,10 +53,17 @@ typedef enum {
   REND_CACHE_TYPE_SERVICE = 2,
 } rend_cache_type_t;
 
+/* Return maximum lifetime in seconds of a cache entry. */
+static inline time_t
+rend_cache_max_entry_lifetime(void)
+{
+  return REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW;
+}
+
 void rend_cache_init(void);
 void rend_cache_clean(time_t now, rend_cache_type_t cache_type);
 void rend_cache_failure_clean(time_t now);
-void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove);
+size_t rend_cache_clean_v2_descs_as_dir(time_t cutoff);
 void rend_cache_purge(void);
 void rend_cache_free_all(void);
 int rend_cache_lookup_entry(const char *query, int version,
@@ -77,20 +84,28 @@ void rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
                                    const uint8_t *identity,
                                    const char *service_id);
 void rend_cache_failure_purge(void);
+void rend_cache_decrement_allocation(size_t n);
+void rend_cache_increment_allocation(size_t n);
 
 #ifdef RENDCACHE_PRIVATE
 
 STATIC size_t rend_cache_entry_allocation(const rend_cache_entry_t *e);
-STATIC void rend_cache_entry_free(rend_cache_entry_t *e);
-STATIC void rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t
-                                                *entry);
-STATIC void rend_cache_failure_entry_free(rend_cache_failure_t *entry);
+STATIC void rend_cache_entry_free_(rend_cache_entry_t *e);
+#define rend_cache_entry_free(e) \
+  FREE_AND_NULL(rend_cache_entry_t, rend_cache_entry_free_, (e))
+STATIC void rend_cache_failure_intro_entry_free_(rend_cache_failure_intro_t
+                                                 *entry);
+#define rend_cache_failure_intro_entry_free(e)                          \
+  FREE_AND_NULL(rend_cache_failure_intro_t,                   \
+                          rend_cache_failure_intro_entry_free_, (e))
+STATIC void rend_cache_failure_entry_free_(rend_cache_failure_t *entry);
+#define rend_cache_failure_entry_free(e)                        \
+  FREE_AND_NULL(rend_cache_failure_t,                 \
+                          rend_cache_failure_entry_free_, (e))
 STATIC int cache_failure_intro_lookup(const uint8_t *identity,
                                       const char *service_id,
                                       rend_cache_failure_intro_t
                                       **intro_entry);
-STATIC void rend_cache_decrement_allocation(size_t n);
-STATIC void rend_cache_increment_allocation(size_t n);
 STATIC rend_cache_failure_intro_t *rend_cache_failure_intro_entry_new(
                                       rend_intro_point_failure_t failure);
 STATIC rend_cache_failure_t *rend_cache_failure_entry_new(void);
@@ -101,15 +116,15 @@ STATIC void cache_failure_intro_add(const uint8_t *identity,
 STATIC void validate_intro_point_failure(const rend_service_descriptor_t *desc,
                                         const char *service_id);
 
-STATIC void rend_cache_failure_entry_free_(void *entry);
+STATIC void rend_cache_failure_entry_free_void(void *entry);
 
 #ifdef TOR_UNIT_TESTS
 extern strmap_t *rend_cache;
 extern strmap_t *rend_cache_failure;
 extern digestmap_t *rend_cache_v2_dir;
 extern size_t rend_cache_total_allocation;
-#endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(RENDCACHE_PRIVATE) */
 
-#endif /* TOR_RENDCACHE_H */
+#endif /* !defined(TOR_RENDCACHE_H) */
 
diff --git a/src/or/rendclient.c b/src/or/rendclient.c
index f0144b076f..74118f8f37 100644
--- a/src/or/rendclient.c
+++ b/src/or/rendclient.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,13 @@
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
+#include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "directory.h"
+#include "hs_circuit.h"
+#include "hs_client.h"
+#include "hs_common.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -26,7 +32,6 @@
 #include "router.h"
 #include "routerlist.h"
 #include "routerset.h"
-#include "control.h"
 
 static extend_info_t *rend_client_get_random_intro_impl(
                           const rend_cache_entry_t *rend_query,
@@ -40,7 +45,7 @@ rend_client_purge_state(void)
   rend_cache_purge();
   rend_cache_failure_purge();
   rend_client_cancel_descriptor_fetches();
-  rend_client_purge_last_hid_serv_requests();
+  hs_purge_last_hid_serv_requests();
 }
 
 /** Called when we've established a circuit to an introduction point:
@@ -87,46 +92,6 @@ rend_client_send_establish_rendezvous(origin_circuit_t *circ)
   return 0;
 }
 
-/** Extend the introduction circuit <b>circ</b> to another valid
- * introduction point for the hidden service it is trying to connect
- * to, or mark it and launch a new circuit if we can't extend it.
- * Return 0 on success or possible success.  Return -1 and mark the
- * introduction circuit for close on permanent failure.
- *
- * On failure, the caller is responsible for marking the associated
- * rendezvous circuit for close. */
-static int
-rend_client_reextend_intro_circuit(origin_circuit_t *circ)
-{
-  extend_info_t *extend_info;
-  int result;
-  extend_info = rend_client_get_random_intro(circ->rend_data);
-  if (!extend_info) {
-    log_warn(LD_REND,
-             "No usable introduction points left for %s. Closing.",
-             safe_str_client(circ->rend_data->onion_address));
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
-    return -1;
-  }
-  // XXX: should we not re-extend if hs_circ_has_timed_out?
-  if (circ->remaining_relay_early_cells) {
-    log_info(LD_REND,
-             "Re-extending circ %u, this time to %s.",
-             (unsigned)circ->base_.n_circ_id,
-             safe_str_client(extend_info_describe(extend_info)));
-    result = circuit_extend_to_new_exit(circ, extend_info);
-  } else {
-    log_info(LD_REND,
-             "Closing intro circ %u (out of RELAY_EARLY cells).",
-             (unsigned)circ->base_.n_circ_id);
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
-    /* connection_ap_handshake_attach_circuit will launch a new intro circ. */
-    result = 0;
-  }
-  extend_info_free(extend_info);
-  return result;
-}
-
 /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell
  * down introcirc if possible.
  */
@@ -144,18 +109,19 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   off_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
+  const char *onion_address;
 
   tor_assert(introcirc->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
   tor_assert(rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY);
   tor_assert(introcirc->rend_data);
   tor_assert(rendcirc->rend_data);
-  tor_assert(!rend_cmp_service_ids(introcirc->rend_data->onion_address,
-                                   rendcirc->rend_data->onion_address));
+  tor_assert(!rend_cmp_service_ids(rend_data_get_address(introcirc->rend_data),
+                                  rend_data_get_address(rendcirc->rend_data)));
   assert_circ_anonymity_ok(introcirc, options);
   assert_circ_anonymity_ok(rendcirc, options);
+  onion_address = rend_data_get_address(introcirc->rend_data);
 
-  r = rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1,
-                              &entry);
+  r = rend_cache_lookup_entry(onion_address, -1, &entry);
   /* An invalid onion address is not possible else we have a big issue. */
   tor_assert(r != -EINVAL);
   if (r < 0 || !rend_client_any_intro_points_usable(entry)) {
@@ -164,14 +130,13 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     log_info(LD_REND,
              "query %s didn't have valid rend desc in cache. "
              "Refetching descriptor.",
-             safe_str_client(introcirc->rend_data->onion_address));
+             safe_str_client(onion_address));
     rend_client_refetch_v2_renddesc(introcirc->rend_data);
     {
       connection_t *conn;
 
       while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
-                       AP_CONN_STATE_CIRCUIT_WAIT,
-                       introcirc->rend_data->onion_address))) {
+                       AP_CONN_STATE_CIRCUIT_WAIT, onion_address))) {
         connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
         conn->state = AP_CONN_STATE_RENDDESC_WAIT;
       }
@@ -195,12 +160,12 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     log_info(LD_REND, "Could not find intro key for %s at %s; we "
              "have a v2 rend desc with %d intro points. "
              "Trying a different intro point...",
-             safe_str_client(introcirc->rend_data->onion_address),
+             safe_str_client(onion_address),
              safe_str_client(extend_info_describe(
                                    introcirc->build_state->chosen_exit)),
              smartlist_len(entry->parsed->intro_nodes));
 
-    if (rend_client_reextend_intro_circuit(introcirc)) {
+    if (hs_client_reextend_intro_circuit(introcirc)) {
       status = -2;
       goto perm_err;
     } else {
@@ -235,11 +200,12 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   /* If version is 3, write (optional) auth data and timestamp. */
   if (entry->parsed->protocols & (1<<3)) {
     tmp[0] = 3; /* version 3 of the cell format */
-    tmp[1] = (uint8_t)introcirc->rend_data->auth_type; /* auth type, if any */
+    /* auth type, if any */
+    tmp[1] = (uint8_t) TO_REND_DATA_V2(introcirc->rend_data)->auth_type;
     v3_shift = 1;
-    if (introcirc->rend_data->auth_type != REND_NO_AUTH) {
+    if (tmp[1] != REND_NO_AUTH) {
       set_uint16(tmp+2, htons(REND_DESC_COOKIE_LEN));
-      memcpy(tmp+4, introcirc->rend_data->descriptor_cookie,
+      memcpy(tmp+4, TO_REND_DATA_V2(introcirc->rend_data)->descriptor_cookie,
              REND_DESC_COOKIE_LEN);
       v3_shift += 2+REND_DESC_COOKIE_LEN;
     }
@@ -263,6 +229,11 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     klen = crypto_pk_asn1_encode(extend_info->onion_key,
                                  tmp+v3_shift+7+DIGEST_LEN+2,
                                  sizeof(tmp)-(v3_shift+7+DIGEST_LEN+2));
+    if (klen < 0) {
+      log_warn(LD_BUG,"Internal error: can't encode public key.");
+      status = -2;
+      goto perm_err;
+    }
     set_uint16(tmp+v3_shift+7+DIGEST_LEN, htons(klen));
     memcpy(tmp+v3_shift+7+DIGEST_LEN+2+klen, rendcirc->rend_data->rend_cookie,
            REND_COOKIE_LEN);
@@ -292,10 +263,9 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     goto perm_err;
   }
 
-  note_crypto_pk_op(REND_CLIENT);
-  /*XXX maybe give crypto_pk_public_hybrid_encrypt a max_len arg,
+  /*XXX maybe give crypto_pk_obsolete_public_hybrid_encrypt a max_len arg,
    * to avoid buffer overflows? */
-  r = crypto_pk_public_hybrid_encrypt(intro_key, payload+DIGEST_LEN,
+  r = crypto_pk_obsolete_public_hybrid_encrypt(intro_key, payload+DIGEST_LEN,
                                       sizeof(payload)-DIGEST_LEN,
                                       tmp,
                                       (int)(dh_offset+DH_KEY_LEN),
@@ -368,7 +338,7 @@ rend_client_rendcirc_has_opened(origin_circuit_t *circ)
  * Called to close other intro circuits we launched in parallel.
  */
 static void
-rend_client_close_other_intros(const char *onion_address)
+rend_client_close_other_intros(const uint8_t *rend_pk_digest)
 {
   /* abort parallel intro circs, if any */
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) {
@@ -377,8 +347,7 @@ rend_client_close_other_intros(const char *onion_address)
         !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(c);
       if (oc->rend_data &&
-          !rend_cmp_service_ids(onion_address,
-                                oc->rend_data->onion_address)) {
+          rend_circuit_pk_digest_eq(oc, rend_pk_digest)) {
         log_info(LD_REND|LD_CIRC, "Closing introduction circuit %d that we "
                  "built in parallel (Purpose %d).", oc->global_identifier,
                  c->purpose);
@@ -399,23 +368,11 @@ rend_client_introduction_acked(origin_circuit_t *circ,
   origin_circuit_t *rendcirc;
   (void) request; // XXXX Use this.
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
-    log_warn(LD_PROTOCOL,
-             "Received REND_INTRODUCE_ACK on unexpected circuit %u.",
-             (unsigned)circ->base_.n_circ_id);
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
   tor_assert(circ->build_state);
   tor_assert(circ->build_state->chosen_exit);
   assert_circ_anonymity_ok(circ, options);
   tor_assert(circ->rend_data);
 
-  /* For path bias: This circuit was used successfully. Valid
-   * nacks and acks count. */
-  pathbias_mark_use_success(circ);
-
   if (request_len == 0) {
     /* It's an ACK; the introduction point relayed our introduction request. */
     /* Locate the rend circ which is waiting to hear about this ack,
@@ -440,7 +397,8 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(circ->rend_data->onion_address);
+    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
+                                                           NULL));
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -449,14 +407,14 @@ rend_client_introduction_acked(origin_circuit_t *circ,
      * If none remain, refetch the service descriptor.
      */
     log_info(LD_REND, "Got nack for %s from %s...",
-        safe_str_client(circ->rend_data->onion_address),
+        safe_str_client(rend_data_get_address(circ->rend_data)),
         safe_str_client(extend_info_describe(circ->build_state->chosen_exit)));
     if (rend_client_report_intro_point_failure(circ->build_state->chosen_exit,
                                              circ->rend_data,
                                              INTRO_POINT_FAILURE_GENERIC)>0) {
       /* There are introduction points left. Re-extend the circuit to
        * another intro point and try again. */
-      int result = rend_client_reextend_intro_circuit(circ);
+      int result = hs_client_reextend_intro_circuit(circ);
       /* XXXX If that call failed, should we close the rend circuit,
        * too? */
       return result;
@@ -472,230 +430,6 @@ rend_client_introduction_acked(origin_circuit_t *circ,
   return 0;
 }
 
-/** The period for which a hidden service directory cannot be queried for
- * the same descriptor ID again. */
-#define REND_HID_SERV_DIR_REQUERY_PERIOD (15 * 60)
-/** Test networks generate a new consensus every 5 or 10 seconds.
- * So allow them to requery HSDirs much faster. */
-#define REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING (5)
-
-/** Return the period for which a hidden service directory cannot be queried
- * for the same descriptor ID again, taking TestingTorNetwork into account. */
-static time_t
-hsdir_requery_period(const or_options_t *options)
-{
-  tor_assert(options);
-
-  if (options->TestingTorNetwork) {
-    return REND_HID_SERV_DIR_REQUERY_PERIOD_TESTING;
-  } else {
-    return REND_HID_SERV_DIR_REQUERY_PERIOD;
-  }
-}
-
-/** Contains the last request times to hidden service directories for
- * certain queries; each key is a string consisting of the
- * concatenation of a base32-encoded HS directory identity digest and
- * base32-encoded HS descriptor ID; each value is a pointer to a time_t
- * holding the time of the last request for that descriptor ID to that
- * HS directory. */
-static strmap_t *last_hid_serv_requests_ = NULL;
-
-/** Returns last_hid_serv_requests_, initializing it to a new strmap if
- * necessary. */
-static strmap_t *
-get_last_hid_serv_requests(void)
-{
-  if (!last_hid_serv_requests_)
-    last_hid_serv_requests_ = strmap_new();
-  return last_hid_serv_requests_;
-}
-
-#define LAST_HID_SERV_REQUEST_KEY_LEN (REND_DESC_ID_V2_LEN_BASE32 + \
-                                       REND_DESC_ID_V2_LEN_BASE32)
-
-/** Look up the last request time to hidden service directory <b>hs_dir</b>
- * for descriptor ID <b>desc_id_base32</b>. If <b>set</b> is non-zero,
- * assign the current time <b>now</b> and return that. Otherwise, return the
- * most recent request time, or 0 if no such request has been sent before.
- */
-static time_t
-lookup_last_hid_serv_request(routerstatus_t *hs_dir,
-                             const char *desc_id_base32,
-                             time_t now, int set)
-{
-  char hsdir_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-  char hsdir_desc_comb_id[LAST_HID_SERV_REQUEST_KEY_LEN + 1];
-  time_t *last_request_ptr;
-  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
-  base32_encode(hsdir_id_base32, sizeof(hsdir_id_base32),
-                hs_dir->identity_digest, DIGEST_LEN);
-  tor_snprintf(hsdir_desc_comb_id, sizeof(hsdir_desc_comb_id), "%s%s",
-               hsdir_id_base32,
-               desc_id_base32);
-  /* XXX++?? tor_assert(strlen(hsdir_desc_comb_id) ==
-                       LAST_HID_SERV_REQUEST_KEY_LEN); */
-  if (set) {
-    time_t *oldptr;
-    last_request_ptr = tor_malloc_zero(sizeof(time_t));
-    *last_request_ptr = now;
-    oldptr = strmap_set(last_hid_serv_requests, hsdir_desc_comb_id,
-                        last_request_ptr);
-    tor_free(oldptr);
-  } else
-    last_request_ptr = strmap_get_lc(last_hid_serv_requests,
-                                     hsdir_desc_comb_id);
-  return (last_request_ptr) ? *last_request_ptr : 0;
-}
-
-/** Clean the history of request times to hidden service directories, so that
- * it does not contain requests older than REND_HID_SERV_DIR_REQUERY_PERIOD
- * seconds any more. */
-static void
-directory_clean_last_hid_serv_requests(time_t now)
-{
-  strmap_iter_t *iter;
-  time_t cutoff = now - hsdir_requery_period(get_options());
-  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
-  for (iter = strmap_iter_init(last_hid_serv_requests);
-       !strmap_iter_done(iter); ) {
-    const char *key;
-    void *val;
-    time_t *ent;
-    strmap_iter_get(iter, &key, &val);
-    ent = (time_t *) val;
-    if (*ent < cutoff) {
-      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
-      tor_free(ent);
-    } else {
-      iter = strmap_iter_next(last_hid_serv_requests, iter);
-    }
-  }
-}
-
-/** Remove all requests related to the descriptor ID <b>desc_id</b> from the
- * history of times of requests to hidden service directories.
- * <b>desc_id</b> is an unencoded descriptor ID of size DIGEST_LEN.
- *
- * This is called from rend_client_note_connection_attempt_ended(), which
- * must be idempotent, so any future changes to this function must leave it
- * idempotent too. */
-static void
-purge_hid_serv_from_last_hid_serv_requests(const char *desc_id)
-{
-  strmap_iter_t *iter;
-  strmap_t *last_hid_serv_requests = get_last_hid_serv_requests();
-  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-
-  /* Key is stored with the base32 encoded desc_id. */
-  base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                DIGEST_LEN);
-  for (iter = strmap_iter_init(last_hid_serv_requests);
-       !strmap_iter_done(iter); ) {
-    const char *key;
-    void *val;
-    strmap_iter_get(iter, &key, &val);
-    /* XXX++?? tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */
-    if (tor_memeq(key + LAST_HID_SERV_REQUEST_KEY_LEN -
-                  REND_DESC_ID_V2_LEN_BASE32,
-                  desc_id_base32,
-                  REND_DESC_ID_V2_LEN_BASE32)) {
-      iter = strmap_iter_next_rmv(last_hid_serv_requests, iter);
-      tor_free(val);
-    } else {
-      iter = strmap_iter_next(last_hid_serv_requests, iter);
-    }
-  }
-}
-
-/** Purge the history of request times to hidden service directories,
- * so that future lookups of an HS descriptor will not fail because we
- * accessed all of the HSDir relays responsible for the descriptor
- * recently. */
-void
-rend_client_purge_last_hid_serv_requests(void)
-{
-  /* Don't create the table if it doesn't exist yet (and it may very
-   * well not exist if the user hasn't accessed any HSes)... */
-  strmap_t *old_last_hid_serv_requests = last_hid_serv_requests_;
-  /* ... and let get_last_hid_serv_requests re-create it for us if
-   * necessary. */
-  last_hid_serv_requests_ = NULL;
-
-  if (old_last_hid_serv_requests != NULL) {
-    log_info(LD_REND, "Purging client last-HS-desc-request-time table");
-    strmap_free(old_last_hid_serv_requests, tor_free_);
-  }
-}
-
-/** This returns a good valid hs dir that should be used for the given
- * descriptor id.
- *
- * Return NULL on error else the hsdir node pointer. */
-static routerstatus_t *
-pick_hsdir(const char *desc_id, const char *desc_id_base32)
-{
-  smartlist_t *responsible_dirs = smartlist_new();
-  smartlist_t *usable_responsible_dirs = smartlist_new();
-  const or_options_t *options = get_options();
-  routerstatus_t *hs_dir;
-  time_t now = time(NULL);
-  int excluded_some;
-
-  tor_assert(desc_id);
-  tor_assert(desc_id_base32);
-
-  /* Determine responsible dirs. Even if we can't get all we want, work with
-   * the ones we have. If it's empty, we'll notice below. */
-  hid_serv_get_responsible_directories(responsible_dirs, desc_id);
-
-  /* Clean request history first. */
-  directory_clean_last_hid_serv_requests(now);
-
-  /* Only select those hidden service directories to which we did not send a
-   * request recently and for which we have a router descriptor here. */
-  SMARTLIST_FOREACH_BEGIN(responsible_dirs, routerstatus_t *, dir) {
-    time_t last = lookup_last_hid_serv_request(dir, desc_id_base32,
-                                               0, 0);
-    const node_t *node = node_get_by_id(dir->identity_digest);
-    if (last + hsdir_requery_period(options) >= now ||
-        !node || !node_has_descriptor(node)) {
-      SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
-      continue;
-    }
-    if (!routerset_contains_node(options->ExcludeNodes, node)) {
-      smartlist_add(usable_responsible_dirs, dir);
-    }
-  } SMARTLIST_FOREACH_END(dir);
-
-  excluded_some =
-    smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
-
-  hs_dir = smartlist_choose(usable_responsible_dirs);
-  if (!hs_dir && !options->StrictNodes) {
-    hs_dir = smartlist_choose(responsible_dirs);
-  }
-
-  smartlist_free(responsible_dirs);
-  smartlist_free(usable_responsible_dirs);
-  if (!hs_dir) {
-    log_info(LD_REND, "Could not pick one of the responsible hidden "
-                      "service directories, because we requested them all "
-                      "recently without success.");
-    if (options->StrictNodes && excluded_some) {
-      log_warn(LD_REND, "Could not pick a hidden service directory for the "
-               "requested hidden service: they are all either down or "
-               "excluded, and StrictNodes is set.");
-    }
-  } else {
-    /* Remember that we are requesting a descriptor from this hidden service
-     * directory now. */
-    lookup_last_hid_serv_request(hs_dir, desc_id_base32, now, 1);
-  }
-
-  return hs_dir;
-}
-
 /** Determine the responsible hidden service directories for <b>desc_id</b>
  * and fetch the descriptor with that ID from one of them. Only
  * send a request to a hidden service directory that we have not yet tried
@@ -703,30 +437,43 @@ pick_hsdir(const char *desc_id, const char *desc_id_base32)
  * in the case that no hidden service directory is left to ask for the
  * descriptor, return 0, and in case of a failure -1.  */
 static int
-directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
+directory_get_from_hs_dir(const char *desc_id,
+                          const rend_data_t *rend_query,
                           routerstatus_t *rs_hsdir)
 {
   routerstatus_t *hs_dir = rs_hsdir;
   char *hsdir_fp;
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
   char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64];
+  const rend_data_v2_t *rend_data;
 #ifdef ENABLE_TOR2WEB_MODE
   const int tor2web_mode = get_options()->Tor2webMode;
   const int how_to_fetch = tor2web_mode ? DIRIND_ONEHOP : DIRIND_ANONYMOUS;
 #else
   const int how_to_fetch = DIRIND_ANONYMOUS;
-#endif
+#endif /* defined(ENABLE_TOR2WEB_MODE) */
 
   tor_assert(desc_id);
+  tor_assert(rend_query);
+  rend_data = TO_REND_DATA_V2(rend_query);
 
   base32_encode(desc_id_base32, sizeof(desc_id_base32),
                 desc_id, DIGEST_LEN);
 
   /* Automatically pick an hs dir if none given. */
   if (!rs_hsdir) {
-    hs_dir = pick_hsdir(desc_id, desc_id_base32);
+    /* Determine responsible dirs. Even if we can't get all we want, work with
+     * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
+    smartlist_t *responsible_dirs = smartlist_new();
+    hid_serv_get_responsible_directories(responsible_dirs, desc_id);
+
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
+      control_event_hsv2_descriptor_failed(rend_query, NULL,
+                                           "QUERY_NO_HSDIR");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, NULL, NULL);
       return 0;
     }
   }
@@ -740,12 +487,16 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
   /* Encode descriptor cookie for logging purposes. Also, if the cookie is
    * malformed, no fetch is triggered thus this needs to be done before the
    * fetch request. */
-  if (rend_query->auth_type != REND_NO_AUTH) {
+  if (rend_data->auth_type != REND_NO_AUTH) {
     if (base64_encode(descriptor_cookie_base64,
                       sizeof(descriptor_cookie_base64),
-                      rend_query->descriptor_cookie, REND_DESC_COOKIE_LEN,
+                      rend_data->descriptor_cookie,
+                      REND_DESC_COOKIE_LEN,
                       0)<0) {
       log_warn(LD_BUG, "Could not base64-encode descriptor cookie.");
+      control_event_hsv2_descriptor_failed(rend_query, hsdir_fp, "BAD_DESC");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, hsdir_fp, NULL);
       return 0;
     }
     /* Remove == signs. */
@@ -758,36 +509,53 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
   /* Send fetch request. (Pass query and possibly descriptor cookie so that
    * they can be written to the directory connection and be referred to when
    * the response arrives. */
-  directory_initiate_command_routerstatus_rend(hs_dir,
-                                          DIR_PURPOSE_FETCH_RENDDESC_V2,
-                                          ROUTER_PURPOSE_GENERAL,
-                                          how_to_fetch,
-                                          desc_id_base32,
-                                          NULL, 0, 0,
-                                          rend_query);
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_RENDDESC_V2);
+  directory_request_set_routerstatus(req, hs_dir);
+  directory_request_set_indirection(req, how_to_fetch);
+  directory_request_set_resource(req, desc_id_base32);
+  directory_request_set_rend_query(req, rend_query);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
   log_info(LD_REND, "Sending fetch request for v2 descriptor for "
                     "service '%s' with descriptor ID '%s', auth type %d, "
                     "and descriptor cookie '%s' to hidden service "
                     "directory %s",
-           rend_query->onion_address, desc_id_base32,
-           rend_query->auth_type,
-           (rend_query->auth_type == REND_NO_AUTH ? "[none]" :
+           rend_data->onion_address, desc_id_base32,
+           rend_data->auth_type,
+           (rend_data->auth_type == REND_NO_AUTH ? "[none]" :
             escaped_safe_str_client(descriptor_cookie_base64)),
            routerstatus_describe(hs_dir));
-  control_event_hs_descriptor_requested(rend_query,
+  control_event_hs_descriptor_requested(rend_data->onion_address,
+                                        rend_data->auth_type,
                                         hs_dir->identity_digest,
-                                        desc_id_base32);
+                                        desc_id_base32, NULL);
   return 1;
 }
 
+/** Remove tracked HSDir requests from our history for this hidden service
+ *  descriptor <b>desc_id</b> (of size DIGEST_LEN) */
+static void
+purge_v2_hidserv_req(const char *desc_id)
+{
+  char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+
+  /* The hsdir request tracker stores v2 keys using the base32 encoded
+     desc_id. Do it: */
+  base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                DIGEST_LEN);
+  hs_purge_hid_serv_from_last_hid_serv_requests(desc_id_base32);
+}
+
 /** Fetch a v2 descriptor using the given descriptor id. If any hsdir(s) are
  * given, they will be used instead.
  *
  * On success, 1 is returned. If no hidden service is left to ask, return 0.
  * On error, -1 is returned. */
 static int
-fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query,
-                        smartlist_t *hsdirs)
+fetch_v2_desc_by_descid(const char *desc_id,
+                        const rend_data_t *rend_query, smartlist_t *hsdirs)
 {
   int ret;
 
@@ -814,19 +582,18 @@ fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query,
 
 /** Fetch a v2 descriptor using the onion address in the given query object.
  * This will compute the descriptor id for each replicas and fetch it on the
- * given hsdir(s) if any or the responsible ones that are choosen
+ * given hsdir(s) if any or the responsible ones that are chosen
  * automatically.
  *
  * On success, 1 is returned. If no hidden service is left to ask, return 0.
  * On error, -1 is returned. */
 static int
-fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
+fetch_v2_desc_by_addr(rend_data_t *rend_query, smartlist_t *hsdirs)
 {
   char descriptor_id[DIGEST_LEN];
   int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS];
   int i, tries_left, ret;
-
-  tor_assert(query);
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
 
   /* Randomly iterate over the replicas until a descriptor can be fetched
    * from one of the consecutive nodes, or no options are left. */
@@ -840,9 +607,10 @@ fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
     int chosen_replica = replicas_left_to_try[rand_val];
     replicas_left_to_try[rand_val] = replicas_left_to_try[--tries_left];
 
-    ret = rend_compute_v2_desc_id(descriptor_id, query->onion_address,
-                                  query->auth_type == REND_STEALTH_AUTH ?
-                                  query->descriptor_cookie : NULL,
+    ret = rend_compute_v2_desc_id(descriptor_id,
+                                  rend_data->onion_address,
+                                  rend_data->auth_type == REND_STEALTH_AUTH ?
+                                    rend_data->descriptor_cookie : NULL,
                                   time(NULL), chosen_replica);
     if (ret < 0) {
       /* Normally, on failure the descriptor_id is untouched but let's be
@@ -850,18 +618,17 @@ fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
       goto end;
     }
 
-    if (tor_memcmp(descriptor_id, query->descriptor_id[chosen_replica],
+    if (tor_memcmp(descriptor_id, rend_data->descriptor_id[chosen_replica],
                    sizeof(descriptor_id)) != 0) {
       /* Not equal from what we currently have so purge the last hid serv
        * request cache and update the descriptor ID with the new value. */
-      purge_hid_serv_from_last_hid_serv_requests(
-                                        query->descriptor_id[chosen_replica]);
-      memcpy(query->descriptor_id[chosen_replica], descriptor_id,
-             sizeof(query->descriptor_id[chosen_replica]));
+      purge_v2_hidserv_req(rend_data->descriptor_id[chosen_replica]);
+      memcpy(rend_data->descriptor_id[chosen_replica], descriptor_id,
+             sizeof(rend_data->descriptor_id[chosen_replica]));
     }
 
     /* Trigger the fetch with the computed descriptor ID. */
-    ret = fetch_v2_desc_by_descid(descriptor_id, query, hsdirs);
+    ret = fetch_v2_desc_by_descid(descriptor_id, rend_query, hsdirs);
     if (ret != 0) {
       /* Either on success or failure, as long as we tried a fetch we are
        * done here. */
@@ -889,16 +656,23 @@ int
 rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs)
 {
   int ret;
+  rend_data_v2_t *rend_data;
+  const char *onion_address;
 
   tor_assert(query);
 
+  /* Get the version 2 data structure of the query. */
+  rend_data = TO_REND_DATA_V2(query);
+  onion_address = rend_data_get_address(query);
+
   /* Depending on what's available in the rend data query object, we will
    * trigger a fetch by HS address or using a descriptor ID. */
 
-  if (query->onion_address[0] != '\0') {
+  if (onion_address[0] != '\0') {
     ret = fetch_v2_desc_by_addr(query, hsdirs);
-  } else if (!tor_digest_is_zero(query->desc_id_fetch)) {
-    ret = fetch_v2_desc_by_descid(query->desc_id_fetch, query, hsdirs);
+  } else if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
+    ret = fetch_v2_desc_by_descid(rend_data->desc_id_fetch, query,
+                                  hsdirs);
   } else {
     /* Query data is invalid. */
     ret = -1;
@@ -916,10 +690,11 @@ void
 rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
 {
   rend_cache_entry_t *e = NULL;
+  const char *onion_address = rend_data_get_address(rend_query);
 
   tor_assert(rend_query);
   /* Before fetching, check if we already have a usable descriptor here. */
-  if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 &&
+  if (rend_cache_lookup_entry(onion_address, -1, &e) == 0 &&
       rend_client_any_intro_points_usable(e)) {
     log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we "
                       "already have a usable descriptor here. Not fetching.");
@@ -932,7 +707,7 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
     return;
   }
   log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s",
-            safe_str_client(rend_query->onion_address));
+            safe_str_client(onion_address));
 
   rend_client_fetch_v2_desc(rend_query, NULL);
   /* We don't need to look the error code because either on failure or
@@ -968,7 +743,7 @@ rend_client_cancel_descriptor_fetches(void)
       } else {
         log_debug(LD_REND, "Marking for close dir conn fetching "
                   "rendezvous descriptor for service %s",
-                  safe_str(rd->onion_address));
+                  safe_str(rend_data_get_address(rd)));
       }
       connection_mark_for_close(conn);
     }
@@ -998,25 +773,26 @@ rend_client_cancel_descriptor_fetches(void)
  */
 int
 rend_client_report_intro_point_failure(extend_info_t *failed_intro,
-                                       rend_data_t *rend_query,
+                                       rend_data_t *rend_data,
                                        unsigned int failure_type)
 {
   int i, r;
   rend_cache_entry_t *ent;
   connection_t *conn;
+  const char *onion_address = rend_data_get_address(rend_data);
 
-  r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent);
+  r = rend_cache_lookup_entry(onion_address, -1, &ent);
   if (r < 0) {
     /* Either invalid onion address or cache entry not found. */
     switch (-r) {
     case EINVAL:
       log_warn(LD_BUG, "Malformed service ID %s.",
-          escaped_safe_str_client(rend_query->onion_address));
+               escaped_safe_str_client(onion_address));
       return -1;
     case ENOENT:
       log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.",
-          escaped_safe_str_client(rend_query->onion_address));
-      rend_client_refetch_v2_renddesc(rend_query);
+               escaped_safe_str_client(onion_address));
+      rend_client_refetch_v2_renddesc(rend_data);
       return 0;
     default:
       log_warn(LD_BUG, "Unknown cache lookup returned code: %d", r);
@@ -1040,7 +816,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
       case INTRO_POINT_FAILURE_GENERIC:
         rend_cache_intro_failure_note(failure_type,
                                       (uint8_t *)failed_intro->identity_digest,
-                                      rend_query->onion_address);
+                                      onion_address);
         rend_intro_point_free(intro);
         smartlist_del(ent->parsed->intro_nodes, i);
         break;
@@ -1058,8 +834,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
           if (zap_intro_point) {
             rend_cache_intro_failure_note(
                 failure_type,
-                (uint8_t *) failed_intro->identity_digest,
-                rend_query->onion_address);
+                (uint8_t *) failed_intro->identity_digest, onion_address);
             rend_intro_point_free(intro);
             smartlist_del(ent->parsed->intro_nodes, i);
           }
@@ -1073,14 +848,14 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
   if (! rend_client_any_intro_points_usable(ent)) {
     log_info(LD_REND,
              "No more intro points remain for %s. Re-fetching descriptor.",
-             escaped_safe_str_client(rend_query->onion_address));
-    rend_client_refetch_v2_renddesc(rend_query);
+             escaped_safe_str_client(onion_address));
+    rend_client_refetch_v2_renddesc(rend_data);
 
     /* move all pending streams back to renddesc_wait */
     /* NOTE: We can now do this faster, if we use pending_entry_connections */
     while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
                                    AP_CONN_STATE_CIRCUIT_WAIT,
-                                   rend_query->onion_address))) {
+                                   onion_address))) {
       connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
       conn->state = AP_CONN_STATE_RENDDESC_WAIT;
     }
@@ -1089,122 +864,28 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
   }
   log_info(LD_REND,"%d options left for %s.",
            smartlist_len(ent->parsed->intro_nodes),
-           escaped_safe_str_client(rend_query->onion_address));
+           escaped_safe_str_client(onion_address));
   return 1;
 }
 
-/** Called when we receive a RENDEZVOUS_ESTABLISHED cell; changes the state of
- * the circuit to C_REND_READY.
- */
-int
-rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request,
-                             size_t request_len)
-{
-  (void) request;
-  (void) request_len;
-  /* we just got an ack for our establish-rendezvous. switch purposes. */
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_C_ESTABLISH_REND) {
-    log_warn(LD_PROTOCOL,"Got a rendezvous ack when we weren't expecting one. "
-             "Closing circ.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-  log_info(LD_REND,"Got rendezvous ack. This circuit is now ready for "
-           "rendezvous.");
-  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_READY);
-  /* Set timestamp_dirty, because circuit_expire_building expects it
-   * to specify when a circuit entered the _C_REND_READY state. */
-  circ->base_.timestamp_dirty = time(NULL);
-
-  /* From a path bias point of view, this circuit is now successfully used.
-   * Waiting any longer opens us up to attacks from malicious hidden services.
-   * They could induce the client to attempt to connect to their hidden
-   * service and never reply to the client's rend requests */
-  pathbias_mark_use_success(circ);
-
-  /* XXXX++ This is a pretty brute-force approach. It'd be better to
-   * attach only the connections that are waiting on this circuit, rather
-   * than trying to attach them all. See comments bug 743. */
-  /* If we already have the introduction circuit built, make sure we send
-   * the INTRODUCE cell _now_ */
-  connection_ap_attach_pending(1);
-  return 0;
-}
-
 /** The service sent us a rendezvous cell; join the circuits. */
 int
 rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request,
                                size_t request_len)
 {
-  crypt_path_t *hop;
-  char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
-
-  if ((circ->base_.purpose != CIRCUIT_PURPOSE_C_REND_READY &&
-       circ->base_.purpose != CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED)
-      || !circ->build_state->pending_final_cpath) {
-    log_warn(LD_PROTOCOL,"Got rendezvous2 cell from hidden service, but not "
-             "expecting it. Closing.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
   if (request_len != DH_KEY_LEN+DIGEST_LEN) {
     log_warn(LD_PROTOCOL,"Incorrect length (%d) on RENDEZVOUS2 cell.",
              (int)request_len);
     goto err;
   }
 
-  log_info(LD_REND,"Got RENDEZVOUS2 cell from hidden service.");
-
-  /* first DH_KEY_LEN bytes are g^y from the service. Finish the dh
-   * handshake...*/
-  tor_assert(circ->build_state);
-  tor_assert(circ->build_state->pending_final_cpath);
-  hop = circ->build_state->pending_final_cpath;
-  tor_assert(hop->rend_dh_handshake_state);
-  if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN,
-                               hop->rend_dh_handshake_state, (char*)request,
-                               DH_KEY_LEN,
-                               keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) {
-    log_warn(LD_GENERAL, "Couldn't complete DH handshake.");
-    goto err;
-  }
-  /* ... and set up cpath. */
-  if (circuit_init_cpath_crypto(hop, keys+DIGEST_LEN, 0)<0)
-    goto err;
-
-  /* Check whether the digest is right... */
-  if (tor_memneq(keys, request+DH_KEY_LEN, DIGEST_LEN)) {
-    log_warn(LD_PROTOCOL, "Incorrect digest of key material.");
+  if (hs_circuit_setup_e2e_rend_circ_legacy_client(circ, request) < 0) {
+    log_warn(LD_GENERAL, "Failed to setup circ");
     goto err;
   }
-
-  crypto_dh_free(hop->rend_dh_handshake_state);
-  hop->rend_dh_handshake_state = NULL;
-
-  /* All is well. Extend the circuit. */
-  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_JOINED);
-  hop->state = CPATH_STATE_OPEN;
-  /* set the windows to default. these are the windows
-   * that the client thinks the service has.
-   */
-  hop->package_window = circuit_initial_package_window();
-  hop->deliver_window = CIRCWINDOW_START;
-
-  /* Now that this circuit has finished connecting to its destination,
-   * make sure circuit_get_open_circ_or_launch is willing to return it
-   * so we can actually use it. */
-  circ->hs_circ_has_timed_out = 0;
-
-  onion_append_to_cpath(&circ->cpath, hop);
-  circ->build_state->pending_final_cpath = NULL; /* prevent double-free */
-
-  circuit_try_attaching_streams(circ);
-
-  memwipe(keys, 0, sizeof(keys));
   return 0;
+
  err:
-  memwipe(keys, 0, sizeof(keys));
   circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
   return -1;
 }
@@ -1230,10 +911,11 @@ rend_client_desc_trynow(const char *query)
     rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
     if (!rend_data)
       continue;
-    if (rend_cmp_service_ids(query, rend_data->onion_address))
+    const char *onion_address = rend_data_get_address(rend_data);
+    if (rend_cmp_service_ids(query, onion_address))
       continue;
     assert_connection_ok(base_conn, now);
-    if (rend_cache_lookup_entry(rend_data->onion_address, -1,
+    if (rend_cache_lookup_entry(onion_address, -1,
                                 &entry) == 0 &&
         rend_client_any_intro_points_usable(entry)) {
       /* either this fetch worked, or it failed but there was a
@@ -1244,8 +926,8 @@ rend_client_desc_trynow(const char *query)
       /* restart their timeout values, so they get a fair shake at
        * connecting to the hidden service. */
       base_conn->timestamp_created = now;
-      base_conn->timestamp_lastread = now;
-      base_conn->timestamp_lastwritten = now;
+      base_conn->timestamp_last_read_allowed = now;
+      base_conn->timestamp_last_write_allowed = now;
 
       connection_ap_mark_as_pending_circuit(conn);
     } else { /* 404, or fetch didn't get that far */
@@ -1268,11 +950,12 @@ rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
 {
   unsigned int have_onion = 0;
   rend_cache_entry_t *cache_entry = NULL;
+  const char *onion_address = rend_data_get_address(rend_data);
+  rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
 
-  if (*rend_data->onion_address != '\0') {
+  if (onion_address[0] != '\0') {
     /* Ignore return value; we find an entry, or we don't. */
-    (void) rend_cache_lookup_entry(rend_data->onion_address, -1,
-                                   &cache_entry);
+    (void) rend_cache_lookup_entry(onion_address, -1, &cache_entry);
     have_onion = 1;
   }
 
@@ -1286,17 +969,17 @@ rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
   /* Remove the HS's entries in last_hid_serv_requests. */
   if (have_onion) {
     unsigned int replica;
-    for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id);
+    for (replica = 0; replica < ARRAY_LENGTH(rend_data_v2->descriptor_id);
          replica++) {
-      const char *desc_id = rend_data->descriptor_id[replica];
-      purge_hid_serv_from_last_hid_serv_requests(desc_id);
+      const char *desc_id = rend_data_v2->descriptor_id[replica];
+      purge_v2_hidserv_req(desc_id);
     }
     log_info(LD_REND, "Connection attempt for %s has ended; "
              "cleaning up temporary state.",
-             safe_str_client(rend_data->onion_address));
+             safe_str_client(onion_address));
   } else {
     /* We only have an ID for a fetch. Probably used by HSFETCH. */
-    purge_hid_serv_from_last_hid_serv_requests(rend_data->desc_id_fetch);
+    purge_v2_hidserv_req(rend_data_v2->desc_id_fetch);
   }
 }
 
@@ -1310,12 +993,13 @@ rend_client_get_random_intro(const rend_data_t *rend_query)
   int ret;
   extend_info_t *result;
   rend_cache_entry_t *entry;
+  const char *onion_address = rend_data_get_address(rend_query);
 
-  ret = rend_cache_lookup_entry(rend_query->onion_address, -1, &entry);
+  ret = rend_cache_lookup_entry(onion_address, -1, &entry);
   if (ret < 0 || !rend_client_any_intro_points_usable(entry)) {
     log_warn(LD_REND,
              "Query '%s' didn't have valid rend desc in cache. Failing.",
-             safe_str_client(rend_query->onion_address));
+             safe_str_client(onion_address));
     /* XXX: Should we refetch the descriptor here if the IPs are not usable
      * anymore ?. */
     return NULL;
@@ -1431,18 +1115,22 @@ rend_client_lookup_service_authorization(const char *onion_address)
   return strmap_get(auth_hid_servs, onion_address);
 }
 
+#define rend_service_authorization_free(val)                    \
+  FREE_AND_NULL(rend_service_authorization_t,                   \
+                rend_service_authorization_free_, (val))
+
 /** Helper: Free storage held by rend_service_authorization_t. */
 static void
-rend_service_authorization_free(rend_service_authorization_t *auth)
+rend_service_authorization_free_(rend_service_authorization_t *auth)
 {
   tor_free(auth);
 }
 
 /** Helper for strmap_free. */
 static void
-rend_service_authorization_strmap_item_free(void *service_auth)
+rend_service_authorization_free_void(void *service_auth)
 {
-  rend_service_authorization_free(service_auth);
+  rend_service_authorization_free_(service_auth);
 }
 
 /** Release all the storage held in auth_hid_servs.
@@ -1453,7 +1141,7 @@ rend_service_authorization_free_all(void)
   if (!auth_hid_servs) {
     return;
   }
-  strmap_free(auth_hid_servs, rend_service_authorization_strmap_item_free);
+  strmap_free(auth_hid_servs, rend_service_authorization_free_void);
   auth_hid_servs = NULL;
 }
 
@@ -1494,7 +1182,7 @@ rend_parse_service_authorization(const or_options_t *options,
       goto err;
     }
     strlcpy(auth->onion_address, onion_address, REND_SERVICE_ID_LEN_BASE32+1);
-    if (!rend_valid_service_id(auth->onion_address)) {
+    if (!rend_valid_v2_service_id(auth->onion_address)) {
       log_warn(LD_CONFIG, "Onion address has wrong format: '%s'",
                onion_address);
       goto err;
@@ -1528,7 +1216,7 @@ rend_parse_service_authorization(const or_options_t *options,
     rend_service_authorization_free_all();
     auth_hid_servs = parsed;
   } else {
-    strmap_free(parsed, rend_service_authorization_strmap_item_free);
+    strmap_free(parsed, rend_service_authorization_free_void);
   }
   return res;
 }
@@ -1547,7 +1235,7 @@ rend_client_allow_non_anonymous_connection(const or_options_t *options)
 #else
   (void)options;
   return 0;
-#endif
+#endif /* defined(NON_ANONYMOUS_MODE_ENABLED) */
 }
 
 /* At compile-time, was non-anonymous mode enabled via
@@ -1562,6 +1250,6 @@ rend_client_non_anonymous_mode_enabled(const or_options_t *options)
   return 1;
 #else
   return 0;
-#endif
+#endif /* defined(NON_ANONYMOUS_MODE_ENABLED) */
 }
 
diff --git a/src/or/rendclient.h b/src/or/rendclient.h
index b8f8c2f871..e8495ce09c 100644
--- a/src/or/rendclient.h
+++ b/src/or/rendclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,15 +24,11 @@ int rend_client_introduction_acked(origin_circuit_t *circ,
 void rend_client_refetch_v2_renddesc(rend_data_t *rend_query);
 int rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs);
 void rend_client_cancel_descriptor_fetches(void);
-void rend_client_purge_last_hid_serv_requests(void);
 
 int rend_client_report_intro_point_failure(extend_info_t *failed_intro,
-                                           rend_data_t *rend_query,
+                                           rend_data_t *rend_data,
                                            unsigned int failure_type);
 
-int rend_client_rendezvous_acked(origin_circuit_t *circ,
-                                 const uint8_t *request,
-                                 size_t request_len);
 int rend_client_receive_rendezvous(origin_circuit_t *circ,
                                    const uint8_t *request,
                                    size_t request_len);
@@ -54,5 +50,5 @@ void rend_service_authorization_free_all(void);
 int rend_client_allow_non_anonymous_connection(const or_options_t *options);
 int rend_client_non_anonymous_mode_enabled(const or_options_t *options);
 
-#endif
+#endif /* !defined(TOR_RENDCLIENT_H) */
 
diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c
index d9d39b1f19..f3fa2f64d1 100644
--- a/src/or/rendcommon.c
+++ b/src/or/rendcommon.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,10 +8,19 @@
  * introducers, services, clients, and rendezvous points.
  **/
 
+#define RENDCOMMON_PRIVATE
+
 #include "or.h"
 #include "circuitbuild.h"
+#include "circuituse.h"
 #include "config.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
+#include "hs_client.h"
+#include "hs_common.h"
+#include "hs_intropoint.h"
+#include "networkstatus.h"
 #include "rendclient.h"
 #include "rendcommon.h"
 #include "rendmid.h"
@@ -20,7 +29,6 @@
 #include "router.h"
 #include "routerlist.h"
 #include "routerparse.h"
-#include "networkstatus.h"
 
 /** Return 0 if one and two are the same service ids, else -1 or 1 */
 int
@@ -32,7 +40,7 @@ rend_cmp_service_ids(const char *one, const char *two)
 /** Free the storage held by the service descriptor <b>desc</b>.
  */
 void
-rend_service_descriptor_free(rend_service_descriptor_t *desc)
+rend_service_descriptor_free_(rend_service_descriptor_t *desc)
 {
   if (!desc)
     return;
@@ -393,7 +401,7 @@ rend_encrypt_v2_intro_points_stealth(char **encrypted_out,
 
 /** Attempt to parse the given <b>desc_str</b> and return true if this
  * succeeds, false otherwise. */
-static int
+STATIC int
 rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc)
 {
   rend_service_descriptor_t *test_parsed = NULL;
@@ -414,7 +422,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc)
 
 /** Free the storage held by an encoded v2 service descriptor. */
 void
-rend_encoded_v2_service_descriptor_free(
+rend_encoded_v2_service_descriptor_free_(
   rend_encoded_v2_service_descriptor_t *desc)
 {
   if (!desc)
@@ -425,7 +433,7 @@ rend_encoded_v2_service_descriptor_free(
 
 /** Free the storage held by an introduction point info. */
 void
-rend_intro_point_free(rend_intro_point_t *intro)
+rend_intro_point_free_(rend_intro_point_t *intro)
 {
   if (!intro)
     return;
@@ -475,7 +483,10 @@ rend_encode_v2_descriptors(smartlist_t *descs_out,
     tor_assert(descriptor_cookie);
   }
   /* Obtain service_id from public key. */
-  crypto_pk_get_digest(service_key, service_id);
+  if (crypto_pk_get_digest(service_key, service_id) < 0) {
+    log_warn(LD_BUG, "Couldn't compute service key digest.");
+    return -1;
+  }
   /* Calculate current time-period. */
   time_period = get_time_period(now, period, service_id);
   /* Determine how many seconds the descriptor will be valid. */
@@ -691,7 +702,7 @@ rend_get_service_id(crypto_pk_t *pk, char *out)
 /** Return true iff <b>query</b> is a syntactically valid service ID (as
  * generated by rend_get_service_id).  */
 int
-rend_valid_service_id(const char *query)
+rend_valid_v2_service_id(const char *query)
 {
   if (strlen(query) != REND_SERVICE_ID_LEN_BASE32)
     return 0;
@@ -761,7 +772,7 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = rend_mid_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ,payload,length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
@@ -769,15 +780,15 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = rend_mid_introduce(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
-        r = rend_service_receive_introduction(origin_circ,payload,length);
+        r = hs_service_receive_introduce2(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRODUCE_ACK:
       if (origin_circ)
-        r = rend_client_introduction_acked(origin_circ,payload,length);
+        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_RENDEZVOUS1:
       if (or_circ)
@@ -785,143 +796,30 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
       break;
     case RELAY_COMMAND_RENDEZVOUS2:
       if (origin_circ)
-        r = rend_client_receive_rendezvous(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRO_ESTABLISHED:
       if (origin_circ)
-        r = rend_service_intro_established(origin_circ,payload,length);
+        r = hs_service_receive_intro_established(origin_circ,payload,length);
       break;
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       if (origin_circ)
-        r = rend_client_rendezvous_acked(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
       break;
     default:
       tor_fragile_assert();
   }
 
+  if (r == 0 && origin_circ) {
+    /* This was a valid cell. Count it as delivered + overhead. */
+    circuit_read_valid_data(origin_circ, length);
+  }
+
   if (r == -2)
     log_info(LD_PROTOCOL, "Dropping cell (type %d) for wrong circuit type.",
              command);
 }
 
-/** Allocate and return a new rend_data_t with the same
- * contents as <b>query</b>. */
-rend_data_t *
-rend_data_dup(const rend_data_t *data)
-{
-  rend_data_t *data_dup;
-  tor_assert(data);
-  data_dup = tor_memdup(data, sizeof(rend_data_t));
-  data_dup->hsdirs_fp = smartlist_new();
-  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
-                    smartlist_add(data_dup->hsdirs_fp,
-                                  tor_memdup(fp, DIGEST_LEN)));
-  return data_dup;
-}
-
-/** Compute descriptor ID for each replicas and save them. A valid onion
- * address must be present in the <b>rend_data</b>.
- *
- * Return 0 on success else -1. */
-static int
-compute_desc_id(rend_data_t *rend_data)
-{
-  int ret = 0;
-  unsigned replica;
-  time_t now = time(NULL);
-
-  tor_assert(rend_data);
-
-  /* Compute descriptor ID for each replicas. */
-  for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id);
-       replica++) {
-    ret = rend_compute_v2_desc_id(rend_data->descriptor_id[replica],
-                                  rend_data->onion_address,
-                                  rend_data->descriptor_cookie,
-                                  now, replica);
-    if (ret < 0) {
-      goto end;
-    }
-  }
-
- end:
-  return ret;
-}
-
-/** Allocate and initialize a rend_data_t object for a service using the
- * given arguments. Only the <b>onion_address</b> is not optional.
- *
- * Return a valid rend_data_t pointer. */
-rend_data_t *
-rend_data_service_create(const char *onion_address, const char *pk_digest,
-                         const uint8_t *cookie, rend_auth_type_t auth_type)
-{
-  rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data));
-
-  /* We need at least one else the call is wrong. */
-  tor_assert(onion_address != NULL);
-
-  if (pk_digest) {
-    memcpy(rend_data->rend_pk_digest, pk_digest,
-           sizeof(rend_data->rend_pk_digest));
-  }
-  if (cookie) {
-    memcpy(rend_data->rend_cookie, cookie,
-           sizeof(rend_data->rend_cookie));
-  }
-
-  strlcpy(rend_data->onion_address, onion_address,
-          sizeof(rend_data->onion_address));
-  rend_data->auth_type = auth_type;
-  /* Won't be used but still need to initialize it for rend_data dup and
-   * free. */
-  rend_data->hsdirs_fp = smartlist_new();
-
-  return rend_data;
-}
-
-/** Allocate and initialize a rend_data_t object for a client request using
- * the given arguments.  Either an onion address or a descriptor ID is
- * needed. Both can be given but only the onion address will be used to make
- * the descriptor fetch.
- *
- * Return a valid rend_data_t pointer or NULL on error meaning the
- * descriptor IDs couldn't be computed from the given data. */
-rend_data_t *
-rend_data_client_create(const char *onion_address, const char *desc_id,
-                        const char *cookie, rend_auth_type_t auth_type)
-{
-  rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data));
-
-  /* We need at least one else the call is wrong. */
-  tor_assert(onion_address != NULL || desc_id != NULL);
-
-  if (cookie) {
-    memcpy(rend_data->descriptor_cookie, cookie,
-           sizeof(rend_data->descriptor_cookie));
-  }
-  if (desc_id) {
-    memcpy(rend_data->desc_id_fetch, desc_id,
-           sizeof(rend_data->desc_id_fetch));
-  }
-  if (onion_address) {
-    strlcpy(rend_data->onion_address, onion_address,
-            sizeof(rend_data->onion_address));
-    if (compute_desc_id(rend_data) < 0) {
-      goto error;
-    }
-  }
-
-  rend_data->auth_type = auth_type;
-  rend_data->hsdirs_fp = smartlist_new();
-
-  return rend_data;
-
- error:
-  rend_data_free(rend_data);
-  return NULL;
-}
-
 /** Determine the routers that are responsible for <b>id</b> (binary) and
  * add pointers to those routers' routerstatus_t to <b>responsible_dirs</b>.
  * Return -1 if we're returning an empty smartlist, else return 0.
@@ -1104,7 +1002,7 @@ rend_non_anonymous_mode_enabled(const or_options_t *options)
  * service.
  */
 void
-assert_circ_anonymity_ok(origin_circuit_t *circ,
+assert_circ_anonymity_ok(const origin_circuit_t *circ,
                          const or_options_t *options)
 {
   tor_assert(options);
@@ -1116,3 +1014,32 @@ assert_circ_anonymity_ok(origin_circuit_t *circ,
   }
 }
 
+/* Return 1 iff the given <b>digest</b> of a permenanent hidden service key is
+ * equal to the digest in the origin circuit <b>ocirc</b> of its rend data .
+ * If the rend data doesn't exist, 0 is returned. This function is agnostic to
+ * the rend data version. */
+int
+rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                          const uint8_t *digest)
+{
+  size_t rend_pk_digest_len;
+  const uint8_t *rend_pk_digest;
+
+  tor_assert(ocirc);
+  tor_assert(digest);
+
+  if (ocirc->rend_data == NULL) {
+    goto no_match;
+  }
+
+  rend_pk_digest = rend_data_get_pk_digest(ocirc->rend_data,
+                                           &rend_pk_digest_len);
+  if (tor_memeq(rend_pk_digest, digest, rend_pk_digest_len)) {
+    goto match;
+  }
+ no_match:
+  return 0;
+ match:
+  return 1;
+}
+
diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h
index 090e6f25e0..1ed0f62609 100644
--- a/src/or/rendcommon.h
+++ b/src/or/rendcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,32 +18,27 @@ typedef enum rend_intro_point_failure_t {
   INTRO_POINT_FAILURE_UNREACHABLE = 2,
 } rend_intro_point_failure_t;
 
-/** Free all storage associated with <b>data</b> */
-static inline void
-rend_data_free(rend_data_t *data)
-{
-  if (!data) {
-    return;
-  }
-  /* Cleanup the HSDir identity digest. */
-  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
-  smartlist_free(data->hsdirs_fp);
-  tor_free(data);
-}
-
 int rend_cmp_service_ids(const char *one, const char *two);
 
 void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
                              int command, size_t length,
                              const uint8_t *payload);
 
-void rend_service_descriptor_free(rend_service_descriptor_t *desc);
+void rend_service_descriptor_free_(rend_service_descriptor_t *desc);
+#define rend_service_descriptor_free(desc) \
+  FREE_AND_NULL(rend_service_descriptor_t, rend_service_descriptor_free_, \
+                (desc))
 int rend_get_service_id(crypto_pk_t *pk, char *out);
-void rend_encoded_v2_service_descriptor_free(
+void rend_encoded_v2_service_descriptor_free_(
                                rend_encoded_v2_service_descriptor_t *desc);
-void rend_intro_point_free(rend_intro_point_t *intro);
+#define rend_encoded_v2_service_descriptor_free(desc) \
+  FREE_AND_NULL(rend_encoded_v2_service_descriptor_t, \
+                rend_encoded_v2_service_descriptor_free_, (desc))
+void rend_intro_point_free_(rend_intro_point_t *intro);
+#define rend_intro_point_free(intro) \
+  FREE_AND_NULL(rend_intro_point_t, rend_intro_point_free_, (intro))
 
-int rend_valid_service_id(const char *query);
+int rend_valid_v2_service_id(const char *query);
 int rend_valid_descriptor_id(const char *query);
 int rend_valid_client_name(const char *client_name);
 int rend_encode_v2_descriptors(smartlist_t *descs_out,
@@ -60,15 +55,8 @@ void rend_get_descriptor_id_bytes(char *descriptor_id_out,
 int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
                                          const char *id);
 
-rend_data_t *rend_data_dup(const rend_data_t *data);
-rend_data_t *rend_data_client_create(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *cookie,
-                                     rend_auth_type_t auth_type);
-rend_data_t *rend_data_service_create(const char *onion_address,
-                                      const char *pk_digest,
-                                      const uint8_t *cookie,
-                                      rend_auth_type_t auth_type);
+int rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                              const uint8_t *digest);
 
 char *rend_auth_encode_cookie(const uint8_t *cookie_in,
                               rend_auth_type_t auth_type);
@@ -80,8 +68,15 @@ int rend_auth_decode_cookie(const char *cookie_in,
 int rend_allow_non_anonymous_connection(const or_options_t* options);
 int rend_non_anonymous_mode_enabled(const or_options_t *options);
 
-void assert_circ_anonymity_ok(origin_circuit_t *circ,
+void assert_circ_anonymity_ok(const origin_circuit_t *circ,
                               const or_options_t *options);
 
-#endif
+#ifdef RENDCOMMON_PRIVATE
+
+STATIC int
+rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc);
+
+#endif /* defined(RENDCOMMON_PRIVATE) */
+
+#endif /* !defined(TOR_RENDCOMMON_H) */
 
diff --git a/src/or/rendmid.c b/src/or/rendmid.c
index 08b2e8923a..521023179a 100644
--- a/src/or/rendmid.c
+++ b/src/or/rendmid.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,17 +12,20 @@
 #include "circuitlist.h"
 #include "circuituse.h"
 #include "config.h"
+#include "crypto.h"
 #include "dos.h"
 #include "relay.h"
 #include "rendmid.h"
 #include "rephist.h"
+#include "hs_circuitmap.h"
+#include "hs_intropoint.h"
 
 /** Respond to an ESTABLISH_INTRO cell by checking the signed data and
  * setting the circuit's purpose and service pk digest.
  */
 int
-rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
-                         size_t request_len)
+rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                size_t request_len)
 {
   crypto_pk_t *pk = NULL;
   char buf[DIGEST_LEN+9];
@@ -34,15 +37,14 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   int reason = END_CIRC_REASON_INTERNAL;
 
   log_info(LD_REND,
-           "Received an ESTABLISH_INTRO request on circuit %u",
+           "Received a legacy ESTABLISH_INTRO request on circuit %u",
            (unsigned) circ->p_circ_id);
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-         "Rejecting ESTABLISH_INTRO on non-OR or non-edge circuit.");
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
   }
+
   if (request_len < 2+DIGEST_LEN)
     goto truncated;
   /* First 2 bytes: length of asn1-encoded key. */
@@ -71,7 +73,6 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
   /* Rest of body: signature of previous data */
-  note_crypto_pk_op(REND_MID);
   if (crypto_pk_public_checksig_digest(pk,
                                        (char*)request, 2+asn1len+DIGEST_LEN,
                                        (char*)(request+2+DIGEST_LEN+asn1len),
@@ -96,7 +97,8 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
 
   /* Close any other intro circuits with the same pk. */
   c = NULL;
-  while ((c = circuit_get_intro_point((const uint8_t *)pk_digest))) {
+  while ((c = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                (const uint8_t *)pk_digest))) {
     log_info(LD_REND, "Replacing old circuit for service %s",
              safe_str(serviceid));
     circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED);
@@ -104,16 +106,14 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   }
 
   /* Acknowledge the request. */
-  if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
-                                   RELAY_COMMAND_INTRO_ESTABLISHED,
-                                   "", 0, NULL)<0) {
+  if (hs_intro_send_intro_established_cell(circ) < 0) {
     log_info(LD_GENERAL, "Couldn't send INTRO_ESTABLISHED cell.");
-    goto err;
+    goto err_no_close;
   }
 
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
-  circuit_set_intro_point_digest(circ, (uint8_t *)pk_digest);
+  hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -124,8 +124,9 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   log_warn(LD_PROTOCOL, "Rejecting truncated ESTABLISH_INTRO cell.");
   reason = END_CIRC_REASON_TORPROTOCOL;
  err:
-  if (pk) crypto_pk_free(pk);
   circuit_mark_for_close(TO_CIRCUIT(circ), reason);
+ err_no_close:
+  if (pk) crypto_pk_free(pk);
   return -1;
 }
 
@@ -134,8 +135,8 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
  * INTRODUCE2 cell.
  */
 int
-rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
-                   size_t request_len)
+rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                          size_t request_len)
 {
   or_circuit_t *intro_circ;
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
@@ -144,26 +145,10 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
   log_info(LD_REND, "Received an INTRODUCE1 request on circuit %u",
            (unsigned)circ->p_circ_id);
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_warn(LD_PROTOCOL,
-             "Rejecting INTRODUCE1 on non-OR or non-edge circuit %u.",
-             (unsigned)circ->p_circ_id);
-    goto err;
-  }
-
-  /* We have already done an introduction on this circuit but we just
-     received a request for another one. We block it since this might
-     be an attempt to DoS a hidden service (#15515). */
-  if (circ->already_received_introduce1) {
-    log_fn(LOG_PROTOCOL_WARN, LD_REND,
-           "Blocking multiple introductions on the same circuit. "
-           "Someone might be trying to attack a hidden service through "
-           "this relay.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
-  circ->already_received_introduce1 = 1;
+  /* At this point, we know that the circuit is valid for an INTRODUCE1
+   * because the validation has been made before calling this function. */
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_OR);
+  tor_assert(!circ->base_.n_chan);
 
   /* We could change this to MAX_HEX_NICKNAME_LEN now that 0.0.9.x is
    * obsolete; however, there isn't much reason to do so, and we're going
@@ -182,7 +167,8 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
 
   /* The first 20 bytes are all we look at: they have a hash of the service's
    * PK. */
-  intro_circ = circuit_get_intro_point((const uint8_t*)request);
+  intro_circ = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                      (const uint8_t*)request);
   if (!intro_circ) {
     log_info(LD_REND,
              "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; "
@@ -203,14 +189,15 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
                                    (char*)request, request_len, NULL)) {
     log_warn(LD_GENERAL,
              "Unable to send INTRODUCE2 cell to Tor client.");
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
   /* And send an ack down the client's circuit.  Empty body means succeeded. */
   if (relay_send_command_from_edge(0,TO_CIRCUIT(circ),
                                    RELAY_COMMAND_INTRODUCE_ACK,
                                    NULL,0,NULL)) {
     log_warn(LD_GENERAL, "Unable to send INTRODUCE_ACK cell to Tor client.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    /* Stop right now, the circuit has been closed. */
     return -1;
   }
 
@@ -222,8 +209,6 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
                                    RELAY_COMMAND_INTRODUCE_ACK,
                                    nak_body, 1, NULL)) {
     log_warn(LD_GENERAL, "Unable to send NAK to Tor client.");
-    /* Is this right? */
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
   }
   return -1;
 }
@@ -270,7 +255,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  if (circuit_get_rendezvous(request)) {
+  if (hs_circuitmap_get_rend_circ_relay_side(request)) {
     log_warn(LD_PROTOCOL,
              "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
     goto err;
@@ -281,12 +266,12 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
                                    RELAY_COMMAND_RENDEZVOUS_ESTABLISHED,
                                    "", 0, NULL)<0) {
     log_warn(LD_PROTOCOL, "Couldn't send RENDEZVOUS_ESTABLISHED cell.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
 
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING);
-  circuit_set_rendezvous_cookie(circ, request);
+  hs_circuitmap_register_rend_circ_relay_side(circ, request);
 
   base16_encode(hexid,9,(char*)request,4);
 
@@ -335,7 +320,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
            "Got request for rendezvous from circuit %u to cookie %s.",
            (unsigned)circ->p_circ_id, hexid);
 
-  rend_circ = circuit_get_rendezvous(request);
+  rend_circ = hs_circuitmap_get_rend_circ_relay_side(request);
   if (!rend_circ) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.",
@@ -358,7 +343,8 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
     log_warn(LD_GENERAL,
              "Unable to send RENDEZVOUS2 cell to client on circuit %u.",
              (unsigned)rend_circ->p_circ_id);
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
 
   /* Join the circuits. */
@@ -369,7 +355,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED);
   circuit_change_purpose(TO_CIRCUIT(rend_circ),
                          CIRCUIT_PURPOSE_REND_ESTABLISHED);
-  circuit_set_rendezvous_cookie(circ, NULL);
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
 
   rend_circ->rend_splice = circ;
   circ->rend_splice = rend_circ;
diff --git a/src/or/rendmid.h b/src/or/rendmid.h
index 10d1287085..6cc1fc8d95 100644
--- a/src/or/rendmid.h
+++ b/src/or/rendmid.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,14 +12,14 @@
 #ifndef TOR_RENDMID_H
 #define TOR_RENDMID_H
 
-int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
-                             size_t request_len);
-int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
-                       size_t request_len);
+int rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                    size_t request_len);
+int rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                              size_t request_len);
 int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
                                   size_t request_len);
 int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
                         size_t request_len);
 
-#endif
+#endif /* !defined(TOR_RENDMID_H) */
 
diff --git a/src/or/rendservice.c b/src/or/rendservice.c
index 32b856452d..92c323b10d 100644
--- a/src/or/rendservice.c
+++ b/src/or/rendservice.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,11 @@
 #include "circuituse.h"
 #include "config.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "directory.h"
+#include "hs_common.h"
+#include "hs_config.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -75,58 +79,11 @@ static ssize_t rend_service_parse_intro_for_v3(
 static int rend_service_check_private_dir(const or_options_t *options,
                                           const rend_service_t *s,
                                           int create);
-static int rend_service_check_private_dir_impl(const or_options_t *options,
-                                               const rend_service_t *s,
-                                               int create);
-
-/** Represents the mapping from a virtual port of a rendezvous service to
- * a real port on some IP.
- */
-struct rend_service_port_config_s {
-  /* The incoming HS virtual port we're mapping */
-  uint16_t virtual_port;
-  /* Is this an AF_UNIX port? */
-  unsigned int is_unix_addr:1;
-  /* The outgoing TCP port to use, if !is_unix_addr */
-  uint16_t real_port;
-  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
-  tor_addr_t real_addr;
-  /* The socket path to connect to, if is_unix_addr */
-  char unix_addr[FLEXIBLE_ARRAY_MEMBER];
-};
-
-/** Try to maintain this many intro points per service by default. */
-#define NUM_INTRO_POINTS_DEFAULT 3
-/** Maximum number of intro points per service. */
-#define NUM_INTRO_POINTS_MAX 10
-/** Number of extra intro points we launch if our set of intro nodes is
- * empty. See proposal 155, section 4. */
-#define NUM_INTRO_POINTS_EXTRA 2
-
-/** If we can't build our intro circuits, don't retry for this long. */
-#define INTRO_CIRC_RETRY_PERIOD (60*5)
-/** Don't try to build more than this many circuits before giving up
- * for a while.*/
-#define MAX_INTRO_CIRCS_PER_PERIOD 10
-/** How many seconds should we spend trying to connect to a requested
- * rendezvous point before giving up? */
-#define MAX_REND_TIMEOUT 30
-/* Default, minimum and maximum values for the maximum rendezvous failures
- * consensus parameter. */
-#define MAX_REND_FAILURES_DEFAULT 2
-#define MAX_REND_FAILURES_MIN 1
-#define MAX_REND_FAILURES_MAX 10
-
-/** How many times will a hidden service operator attempt to connect to
- * a requested rendezvous point before giving up? */
-static int
-get_max_rend_failures(void)
-{
-  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
-                                 MAX_REND_FAILURES_DEFAULT,
-                                 MAX_REND_FAILURES_MIN,
-                                 MAX_REND_FAILURES_MAX);
-}
+static const smartlist_t* rend_get_service_list(
+                                  const smartlist_t* substitute_service_list);
+static smartlist_t* rend_get_service_list_mutable(
+                                  smartlist_t* substitute_service_list);
+static int rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted);
 
 /* Hidden service directory file names:
  * new file names should be added to rend_service_add_filenames_to_list()
@@ -136,21 +93,64 @@ static const char *hostname_fname = "hostname";
 static const char *client_keys_fname = "client_keys";
 static const char *sos_poison_fname = "onion_service_non_anonymous";
 
+/** A list of rend_service_t's for services run on this OP. */
+static smartlist_t *rend_service_list = NULL;
+/** A list of rend_service_t's for services run on this OP which is used as a
+ * staging area before they are put in the main list in order to prune dying
+ * service on config reload. */
+static smartlist_t *rend_service_staging_list = NULL;
+
+/* Like rend_get_service_list_mutable, but returns a read-only list. */
+static const smartlist_t*
+rend_get_service_list(const smartlist_t* substitute_service_list)
+{
+  /* It is safe to cast away the const here, because
+   * rend_get_service_list_mutable does not actually modify the list */
+  return rend_get_service_list_mutable((smartlist_t*)substitute_service_list);
+}
+
+/* Return a mutable list of hidden services.
+ * If substitute_service_list is not NULL, return it.
+ * Otherwise, check if the global rend_service_list is non-NULL, and if so,
+ * return it.
+ * Otherwise, log a BUG message and return NULL.
+ * */
+static smartlist_t*
+rend_get_service_list_mutable(smartlist_t* substitute_service_list)
+{
+  if (substitute_service_list) {
+    return substitute_service_list;
+  }
+
+  /* If no special service list is provided, then just use the global one. */
+
+  if (BUG(!rend_service_list)) {
+    /* No global HS list, which is a programmer error. */
+    return NULL;
+  }
+
+  return rend_service_list;
+}
+
+/** Tells if onion service <b>s</b> is ephemeral.
+ */
+static unsigned int
+rend_service_is_ephemeral(const struct rend_service_t *s)
+{
+  return (s->directory == NULL);
+}
+
 /** Returns a escaped string representation of the service, <b>s</b>.
  */
 static const char *
 rend_service_escaped_dir(const struct rend_service_t *s)
 {
-  return (s->directory) ? escaped(s->directory) : "[EPHEMERAL]";
+  return rend_service_is_ephemeral(s) ? "[EPHEMERAL]" : escaped(s->directory);
 }
 
-/** A list of rend_service_t's for services run on this OP.
- */
-static smartlist_t *rend_service_list = NULL;
-
 /** Return the number of rendezvous services we have configured. */
 int
-num_rend_services(void)
+rend_num_services(void)
 {
   if (!rend_service_list)
     return 0;
@@ -159,7 +159,7 @@ num_rend_services(void)
 
 /** Helper: free storage held by a single service authorized client entry. */
 void
-rend_authorized_client_free(rend_authorized_client_t *client)
+rend_authorized_client_free_(rend_authorized_client_t *client)
 {
   if (!client)
     return;
@@ -174,15 +174,15 @@ rend_authorized_client_free(rend_authorized_client_t *client)
 
 /** Helper for strmap_free. */
 static void
-rend_authorized_client_strmap_item_free(void *authorized_client)
+rend_authorized_client_free_void(void *authorized_client)
 {
-  rend_authorized_client_free(authorized_client);
+  rend_authorized_client_free_(authorized_client);
 }
 
 /** Release the storage held by <b>service</b>.
  */
 STATIC void
-rend_service_free(rend_service_t *service)
+rend_service_free_(rend_service_t *service)
 {
   if (!service)
     return;
@@ -218,137 +218,146 @@ rend_service_free(rend_service_t *service)
   tor_free(service);
 }
 
-/** Release all the storage held in rend_service_list.
- */
+/* Release all the storage held in rend_service_staging_list. */
 void
-rend_service_free_all(void)
+rend_service_free_staging_list(void)
 {
-  if (!rend_service_list)
-    return;
-
-  SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr,
-                    rend_service_free(ptr));
-  smartlist_free(rend_service_list);
-  rend_service_list = NULL;
+  if (rend_service_staging_list) {
+    SMARTLIST_FOREACH(rend_service_staging_list, rend_service_t*, ptr,
+                      rend_service_free(ptr));
+    smartlist_free(rend_service_staging_list);
+    rend_service_staging_list = NULL;
+  }
 }
 
-/** Validate <b>service</b> and add it to <b>service_list</b>, or to
- * the global rend_service_list if <b>service_list</b> is NULL.
- * Return 0 on success.  On failure, free <b>service</b> and return -1.
- * Takes ownership of <b>service</b>.
- */
-static int
-rend_add_service(smartlist_t *service_list, rend_service_t *service)
+/** Release all the storage held in both rend_service_list and
+ * rend_service_staging_list. */
+void
+rend_service_free_all(void)
 {
-  int i;
-  rend_service_port_config_t *p;
+  if (rend_service_list) {
+    SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr,
+                      rend_service_free(ptr));
+    smartlist_free(rend_service_list);
+    rend_service_list = NULL;
+  }
+  rend_service_free_staging_list();
+}
 
-  smartlist_t *s_list;
-  /* If no special service list is provided, then just use the global one. */
-  if (!service_list) {
-    if (BUG(!rend_service_list)) {
-      /* No global HS list, which is a failure. */
-      return -1;
-    }
+/* Initialize the subsystem. */
+void
+rend_service_init(void)
+{
+  tor_assert(!rend_service_list);
+  tor_assert(!rend_service_staging_list);
 
-    s_list = rend_service_list;
-  } else {
-    s_list = service_list;
-  }
+  rend_service_list = smartlist_new();
+  rend_service_staging_list = smartlist_new();
+}
 
-  service->intro_nodes = smartlist_new();
-  service->expiring_nodes = smartlist_new();
+/* Validate a <b>service</b>. Use the <b>service_list</b> to make sure there
+ * is no duplicate entry for the given service object. Return 0 if valid else
+ * -1 if not.*/
+static int
+rend_validate_service(const smartlist_t *service_list,
+                      const rend_service_t *service)
+{
+  tor_assert(service_list);
+  tor_assert(service);
 
   if (service->max_streams_per_circuit < 0) {
     log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max "
-                        "streams per circuit; ignoring.",
+                        "streams per circuit.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (service->max_streams_close_circuit < 0 ||
       service->max_streams_close_circuit > 1) {
     log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid "
-                        "max streams handling; ignoring.",
+                        "max streams handling.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (service->auth_type != REND_NO_AUTH &&
-      (!service->clients ||
-       smartlist_len(service->clients) == 0)) {
-    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but no "
-                        "clients; ignoring.",
+      (!service->clients || smartlist_len(service->clients) == 0)) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but "
+                        "no clients.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (!service->ports || !smartlist_len(service->ports)) {
-    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured; "
-             "ignoring.",
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
              rend_service_escaped_dir(service));
+    goto invalid;
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/** Add it to <b>service_list</b>, or to the global rend_service_list if
+ * <b>service_list</b> is NULL.  Return 0 on success.  On failure, free
+ * <b>service</b> and return -1. Takes ownership of <b>service</b>. */
+static int
+rend_add_service(smartlist_t *service_list, rend_service_t *service)
+{
+  int i;
+  rend_service_port_config_t *p;
+
+  tor_assert(service);
+
+  smartlist_t *s_list = rend_get_service_list_mutable(service_list);
+  /* We must have a service list, even if it's a temporary one, so we can
+   * check for duplicate services */
+  if (BUG(!s_list)) {
     rend_service_free(service);
     return -1;
-  } else {
-    int dupe = 0;
-    /* XXX This duplicate check has two problems:
-     *
-     * a) It's O(n^2), but the same comment from the bottom of
-     *    rend_config_services() should apply.
-     *
-     * b) We only compare directory paths as strings, so we can't
-     *    detect two distinct paths that specify the same directory
-     *    (which can arise from symlinks, case-insensitivity, bind
-     *    mounts, etc.).
-     *
-     * It also can't detect that two separate Tor instances are trying
-     * to use the same HiddenServiceDir; for that, we would need a
-     * lock file.  But this is enough to detect a simple mistake that
-     * at least one person has actually made.
-     */
-    if (service->directory != NULL) {
-      /* Skip dupe for ephemeral services. */
-      SMARTLIST_FOREACH(s_list, rend_service_t*, ptr,
-                        dupe = dupe ||
-                               !strcmp(ptr->directory, service->directory));
-      if (dupe) {
-        log_warn(LD_REND, "Another hidden service is already configured for "
-                 "directory %s, ignoring.",
-                 rend_service_escaped_dir(service));
-        rend_service_free(service);
-        return -1;
-      }
-    }
-    smartlist_add(s_list, service);
-    log_debug(LD_REND,"Configuring service with directory \"%s\"",
-              service->directory);
-    for (i = 0; i < smartlist_len(service->ports); ++i) {
-      p = smartlist_get(service->ports, i);
-      if (!(p->is_unix_addr)) {
-        log_debug(LD_REND,
-                  "Service maps port %d to %s",
-                  p->virtual_port,
-                  fmt_addrport(&p->real_addr, p->real_port));
-      } else {
+  }
+
+  service->intro_nodes = smartlist_new();
+  service->expiring_nodes = smartlist_new();
+
+  log_debug(LD_REND,"Configuring service with directory %s",
+            rend_service_escaped_dir(service));
+  for (i = 0; i < smartlist_len(service->ports); ++i) {
+    p = smartlist_get(service->ports, i);
+    if (!(p->is_unix_addr)) {
+      log_debug(LD_REND,
+                "Service maps port %d to %s",
+                p->virtual_port,
+                fmt_addrport(&p->real_addr, p->real_port));
+    } else {
 #ifdef HAVE_SYS_UN_H
-        log_debug(LD_REND,
-                  "Service maps port %d to socket at \"%s\"",
-                  p->virtual_port, p->unix_addr);
+      log_debug(LD_REND,
+                "Service maps port %d to socket at \"%s\"",
+                p->virtual_port, p->unix_addr);
 #else
-        log_debug(LD_REND,
-                  "Service maps port %d to an AF_UNIX socket, but we "
-                  "have no AF_UNIX support on this platform.  This is "
-                  "probably a bug.",
-                  p->virtual_port);
+      log_warn(LD_BUG,
+               "Service maps port %d to an AF_UNIX socket, but we "
+               "have no AF_UNIX support on this platform.  This is "
+               "probably a bug.",
+               p->virtual_port);
+      rend_service_free(service);
+      return -1;
 #endif /* defined(HAVE_SYS_UN_H) */
-      }
     }
-    return 0;
   }
-  /* NOTREACHED */
+  /* The service passed all the checks */
+  tor_assert(s_list);
+  smartlist_add(s_list, service);
+
+  /* Notify that our global service list has changed only if this new service
+   * went into our global list. If not, when we move service from the staging
+   * list to the new list, a notify is triggered. */
+  if (s_list == rend_service_list) {
+    hs_service_map_has_changed();
+  }
+  return 0;
 }
 
 /** Return a new rend_service_port_config_t with its path set to
@@ -367,9 +376,9 @@ rend_service_port_config_new(const char *socket_path)
   return conf;
 }
 
-/** Parses a real-port to virtual-port mapping separated by the provided
- * separator and returns a new rend_service_port_config_t, or NULL and an
- * optional error string on failure.
+/** Parses a virtual-port to real-port/socket mapping separated by
+ * the provided separator and returns a new rend_service_port_config_t,
+ * or NULL and an optional error string on failure.
  *
  * The format is: VirtualPort SEP (IP|RealPort|IP:RealPort|'socket':path)?
  *
@@ -394,14 +403,12 @@ rend_service_parse_port_config(const char *string, const char *sep,
   smartlist_split_string(sl, string, sep,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
   if (smartlist_len(sl) < 1 || BUG(smartlist_len(sl) > 2)) {
-    if (err_msg_out)
-      err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
+    err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
     goto err;
   }
   virtport = (int)tor_parse_long(smartlist_get(sl,0), 10, 1, 65535, NULL,NULL);
   if (!virtport) {
-    if (err_msg_out)
-      tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
+    tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
                    "port configuration", escaped(smartlist_get(sl,0)));
 
     goto err;
@@ -429,10 +436,8 @@ rend_service_parse_port_config(const char *string, const char *sep,
     } else if (strchr(addrport, ':') || strchr(addrport, '.')) {
       /* else try it as an IP:port pair if it has a : or . in it */
       if (tor_addr_port_lookup(addrport, &addr, &p)<0) {
-        if (err_msg_out)
-          err_msg = tor_strdup("Unparseable address in hidden service port "
-                               "configuration.");
-
+        err_msg = tor_strdup("Unparseable address in hidden service port "
+                             "configuration.");
         goto err;
       }
       realport = p?p:virtport;
@@ -440,11 +445,9 @@ rend_service_parse_port_config(const char *string, const char *sep,
       /* No addr:port, no addr -- must be port. */
       realport = (int)tor_parse_long(addrport, 10, 1, 65535, NULL, NULL);
       if (!realport) {
-        if (err_msg_out)
-          tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
-                       "hidden service port configuration.",
-                       escaped(addrport));
-
+        tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
+                     "hidden service port configuration.",
+                     escaped(addrport));
         goto err;
       }
       tor_addr_from_ipv4h(&addr, 0x7F000001u); /* Default to 127.0.0.1 */
@@ -463,7 +466,11 @@ rend_service_parse_port_config(const char *string, const char *sep,
 
  err:
   tor_free(addrport);
-  if (err_msg_out) *err_msg_out = err_msg;
+  if (err_msg_out != NULL) {
+    *err_msg_out = err_msg;
+  } else {
+    tor_free(err_msg);
+  }
   SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
   smartlist_free(sl);
 
@@ -472,66 +479,11 @@ rend_service_parse_port_config(const char *string, const char *sep,
 
 /** Release all storage held in a rend_service_port_config_t. */
 void
-rend_service_port_config_free(rend_service_port_config_t *p)
+rend_service_port_config_free_(rend_service_port_config_t *p)
 {
   tor_free(p);
 }
 
-/* Check the directory for <b>service</b>, and add the service to
- * <b>service_list</b>, or to the global list if <b>service_list</b> is NULL.
- * Only add the service to the list if <b>validate_only</b> is false.
- * If <b>validate_only</b> is true, free the service.
- * If <b>service</b> is NULL, ignore it, and return 0.
- * Returns 0 on success, and -1 on failure.
- * Takes ownership of <b>service</b>, either freeing it, or adding it to the
- * global service list.
- */
-STATIC int
-rend_service_check_dir_and_add(smartlist_t *service_list,
-                               const or_options_t *options,
-                               rend_service_t *service,
-                               int validate_only)
-{
-  if (!service) {
-    /* It is ok for a service to be NULL, this means there are no services */
-    return 0;
-  }
-
-  if (rend_service_check_private_dir(options, service, !validate_only)
-      < 0) {
-    rend_service_free(service);
-    return -1;
-  }
-
-  if (validate_only) {
-    rend_service_free(service);
-    return 0;
-  } else {
-    /* Use service_list for unit tests */
-    smartlist_t *s_list = NULL;
-    /* If no special service list is provided, then just use the global one. */
-    if (!service_list) {
-      if (BUG(!rend_service_list)) {
-        /* No global HS list, which is a failure, because we plan on adding to
-         * it */
-        return -1;
-      }
-      s_list = rend_service_list;
-    } else {
-      s_list = service_list;
-    }
-    /* s_list can not be NULL here - if both service_list and rend_service_list
-     * are NULL, and validate_only is false, we exit earlier in the function
-     */
-    if (BUG(!s_list)) {
-      return -1;
-    }
-    /* Ignore service failures until 030 */
-    rend_add_service(s_list, service);
-    return 0;
-  }
-}
-
 /* Copy relevant data from service src to dst while pruning the service lists.
  * This should only be called during the pruning process which takes existing
  * services and copy their data to the newly configured services. The src
@@ -556,141 +508,238 @@ copy_service_on_prunning(rend_service_t *dst, rend_service_t *src)
   dst->n_intro_points_wanted = src->n_intro_points_wanted;
 }
 
-/** Set up rend_service_list, based on the values of HiddenServiceDir and
- * HiddenServicePort in <b>options</b>.  Return 0 on success and -1 on
- * failure.  (If <b>validate_only</b> is set, parse, warn and return as
- * normal, but don't actually change the configured services.)
- */
+/* Helper: Actual implementation of the pruning on reload which we've
+ * decoupled in order to make the unit test workeable without ugly hacks.
+ * Furthermore, this function does NOT free any memory but will nullify the
+ * temporary list pointer whatever happens. */
+STATIC void
+rend_service_prune_list_impl_(void)
+{
+  origin_circuit_t *ocirc = NULL;
+  smartlist_t *surviving_services, *old_service_list, *new_service_list;
+
+  /* When pruning our current service list, we must have a staging list that
+   * contains what we want to check else it's a code flow error. */
+  tor_assert(rend_service_staging_list);
+
+  /* We are about to prune the current list of its dead service so set the
+   * semantic for that list to be the "old" one. */
+  old_service_list = rend_service_list;
+  /* The staging list is now the "new" list so set this semantic. */
+  new_service_list = rend_service_staging_list;
+  /* After this, whatever happens, we'll use our new list. */
+  rend_service_list = new_service_list;
+  /* Finally, nullify the staging list pointer as we don't need it anymore
+   * and it needs to be NULL before the next reload. */
+  rend_service_staging_list = NULL;
+  /* Nothing to prune if we have no service list so stop right away. */
+  if (!old_service_list) {
+    return;
+  }
+
+  /* This contains all _existing_ services that survives the relaod that is
+   * that haven't been removed from the configuration. The difference between
+   * this list and the new service list is that the new list can possibly
+   * contain newly configured service that have no introduction points opened
+   * yet nor key material loaded or generated. */
+  surviving_services = smartlist_new();
+
+  /* Preserve the existing ephemeral services.
+   *
+   * This is the ephemeral service equivalent of the "Copy introduction
+   * points to new services" block, except there's no copy required since
+   * the service structure isn't regenerated.
+   *
+   * After this is done, all ephemeral services will be:
+   *  * Removed from old_service_list, so the equivalent non-ephemeral code
+   *    will not attempt to preserve them.
+   *  * Added to the new_service_list (that previously only had the
+   *    services listed in the configuration).
+   *  * Added to surviving_services, which is the list of services that
+   *    will NOT have their intro point closed.
+   */
+  SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+    if (rend_service_is_ephemeral(old)) {
+      SMARTLIST_DEL_CURRENT(old_service_list, old);
+      smartlist_add(surviving_services, old);
+      smartlist_add(new_service_list, old);
+    }
+  } SMARTLIST_FOREACH_END(old);
+
+  /* Copy introduction points to new services. This is O(n^2), but it's only
+   * called on reconfigure, so it's ok performance wise. */
+  SMARTLIST_FOREACH_BEGIN(new_service_list, rend_service_t *, new) {
+    SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+      /* Skip ephemeral services as we only want to copy introduction points
+       * from current services to newly configured one that already exists.
+       * The same directory means it's the same service. */
+      if (rend_service_is_ephemeral(new) || rend_service_is_ephemeral(old) ||
+          strcmp(old->directory, new->directory)) {
+        continue;
+      }
+      smartlist_add_all(new->intro_nodes, old->intro_nodes);
+      smartlist_clear(old->intro_nodes);
+      smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
+      smartlist_clear(old->expiring_nodes);
+
+      /* Copy needed information from old to new. */
+      copy_service_on_prunning(new, old);
+
+      /* This regular service will survive the closing IPs step after. */
+      smartlist_add(surviving_services, old);
+      break;
+    } SMARTLIST_FOREACH_END(old);
+  } SMARTLIST_FOREACH_END(new);
+
+  /* For every service introduction circuit we can find, see if we have a
+   * matching surviving configured service. If not, close the circuit. */
+  while ((ocirc = circuit_get_next_service_intro_circ(ocirc))) {
+    int keep_it = 0;
+    if (ocirc->rend_data == NULL) {
+      /* This is a v3 circuit, ignore it. */
+      continue;
+    }
+    SMARTLIST_FOREACH_BEGIN(surviving_services, const rend_service_t *, s) {
+      if (rend_circuit_pk_digest_eq(ocirc, (uint8_t *) s->pk_digest)) {
+        /* Keep this circuit as we have a matching configured service. */
+        keep_it = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(s);
+    if (keep_it) {
+      continue;
+    }
+    log_info(LD_REND, "Closing intro point %s for service %s.",
+             safe_str_client(extend_info_describe(
+                                        ocirc->build_state->chosen_exit)),
+             safe_str_client(rend_data_get_address(ocirc->rend_data)));
+    /* Reason is FINISHED because service has been removed and thus the
+     * circuit is considered old/uneeded. */
+    circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+  }
+  smartlist_free(surviving_services);
+  /* Notify that our global service list has changed. */
+  hs_service_map_has_changed();
+}
+
+/* Try to prune our main service list using the temporary one that we just
+ * loaded and parsed successfully. The pruning process decides which onion
+ * services to keep and which to discard after a reload. */
+void
+rend_service_prune_list(void)
+{
+  smartlist_t *old_service_list = rend_service_list;
+
+  if (!rend_service_staging_list) {
+    rend_service_staging_list = smartlist_new();
+  }
+
+  rend_service_prune_list_impl_();
+  if (old_service_list) {
+    /* Every remaining service in the old list have been removed from the
+     * configuration so clean them up safely. */
+    SMARTLIST_FOREACH(old_service_list, rend_service_t *, s,
+                      rend_service_free(s));
+    smartlist_free(old_service_list);
+  }
+}
+
+/* Copy all the relevant data that the hs_service object contains over to the
+ * rend_service_t object. The reason to do so is because when configuring a
+ * service, we go through a generic handler that creates an hs_service_t
+ * object which so we have to copy the parsed values to a rend service object
+ * which is version 2 specific. */
+static void
+service_config_shadow_copy(rend_service_t *service,
+                           hs_service_config_t *config)
+{
+  tor_assert(service);
+  tor_assert(config);
+
+  service->directory = tor_strdup(config->directory_path);
+  service->dir_group_readable = config->dir_group_readable;
+  service->allow_unknown_ports = config->allow_unknown_ports;
+  /* This value can't go above HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT (65535)
+   * if the code flow is right so this cast is safe. But just in case, we'll
+   * check it. */
+  service->max_streams_per_circuit = (int) config->max_streams_per_rdv_circuit;
+  if (BUG(config->max_streams_per_rdv_circuit >
+          HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT)) {
+    service->max_streams_per_circuit = HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT;
+  }
+  service->max_streams_close_circuit = config->max_streams_close_circuit;
+  service->n_intro_points_wanted = config->num_intro_points;
+  /* Switching ownership of the ports to the rend service object. */
+  smartlist_add_all(service->ports, config->ports);
+  smartlist_free(config->ports);
+  config->ports = NULL;
+}
+
+/* Parse the hidden service configuration starting at <b>line_</b> using the
+ * already configured generic service configuration in <b>config</b>. This
+ * function will translate the config object to a rend_service_t and add it to
+ * the temporary list if valid. If <b>validate_only</b> is set, parse, warn
+ * and return as normal but don't actually add the service to the list. */
 int
-rend_config_services(const or_options_t *options, int validate_only)
+rend_config_service(const config_line_t *line_,
+                    const or_options_t *options,
+                    hs_service_config_t *config)
 {
-  config_line_t *line;
+  const config_line_t *line;
   rend_service_t *service = NULL;
-  rend_service_port_config_t *portcfg;
-  smartlist_t *old_service_list = NULL;
-  int ok = 0;
 
-  if (!validate_only) {
-    old_service_list = rend_service_list;
-    rend_service_list = smartlist_new();
+  /* line_ can be NULL which would mean that the service configuration only
+   * have one line that is the directory directive. */
+  tor_assert(options);
+  tor_assert(config);
+
+  /* Use the staging service list so that we can check then do the pruning
+   * process using the main list at the end. */
+  if (rend_service_staging_list == NULL) {
+    rend_service_staging_list = smartlist_new();
   }
 
-  for (line = options->RendConfigLines; line; line = line->next) {
+  /* Initialize service. */
+  service = tor_malloc_zero(sizeof(rend_service_t));
+  service->intro_period_started = time(NULL);
+  service->ports = smartlist_new();
+  /* From the hs_service object which has been used to load the generic
+   * options, we'll copy over the useful data to the rend_service_t object. */
+  service_config_shadow_copy(service, config);
+
+  for (line = line_; line; line = line->next) {
     if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* register the service we just finished parsing
-       * this code registers every service except the last one parsed,
-       * which is registered below the loop */
-      if (rend_service_check_dir_and_add(NULL, options, service,
-                                         validate_only) < 0) {
-          return -1;
-      }
-      service = tor_malloc_zero(sizeof(rend_service_t));
-      service->directory = tor_strdup(line->value);
-      service->ports = smartlist_new();
-      service->intro_period_started = time(NULL);
-      service->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT;
-      continue;
-    }
-    if (!service) {
-      log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
-               line->key);
-      rend_service_free(service);
-      return -1;
+      /* We just hit the next hidden service, stop right now. */
+      break;
     }
-    if (!strcasecmp(line->key, "HiddenServicePort")) {
-      char *err_msg = NULL;
-      portcfg = rend_service_parse_port_config(line->value, " ", &err_msg);
-      if (!portcfg) {
-        if (err_msg)
-          log_warn(LD_CONFIG, "%s", err_msg);
-        tor_free(err_msg);
-        rend_service_free(service);
-        return -1;
-      }
-      tor_assert(!err_msg);
-      smartlist_add(service->ports, portcfg);
-    } else if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
-      service->allow_unknown_ports = (int)tor_parse_long(line->value,
-                                                        10, 0, 1, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceAllowUnknownPorts should be 0 or 1, not %s",
-                 line->value);
-        rend_service_free(service);
-        return -1;
-      }
-      log_info(LD_CONFIG,
-               "HiddenServiceAllowUnknownPorts=%d for %s",
-               (int)service->allow_unknown_ports, service->directory);
-    } else if (!strcasecmp(line->key,
-                           "HiddenServiceDirGroupReadable")) {
-        service->dir_group_readable = (int)tor_parse_long(line->value,
-                                                        10, 0, 1, &ok, NULL);
-        if (!ok) {
-            log_warn(LD_CONFIG,
-                     "HiddenServiceDirGroupReadable should be 0 or 1, not %s",
-                     line->value);
-            rend_service_free(service);
-            return -1;
-        }
-        log_info(LD_CONFIG,
-                 "HiddenServiceDirGroupReadable=%d for %s",
-                 service->dir_group_readable, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
-      service->max_streams_per_circuit = (int)tor_parse_long(line->value,
-                                                    10, 0, 65535, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceMaxStreams should be between 0 and %d, not %s",
-                 65535, line->value);
-        rend_service_free(service);
-        return -1;
-      }
-      log_info(LD_CONFIG,
-               "HiddenServiceMaxStreams=%d for %s",
-               service->max_streams_per_circuit, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
-      service->max_streams_close_circuit = (int)tor_parse_long(line->value,
-                                                        10, 0, 1, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceMaxStreamsCloseCircuit should be 0 or 1, "
-                 "not %s",
-                 line->value);
-        rend_service_free(service);
-        return -1;
-      }
-      log_info(LD_CONFIG,
-               "HiddenServiceMaxStreamsCloseCircuit=%d for %s",
-               (int)service->max_streams_close_circuit, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+    /* Number of introduction points. */
+    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
+      int ok = 0;
+      /* Those are specific defaults for version 2. */
       service->n_intro_points_wanted =
         (unsigned int) tor_parse_long(line->value, 10,
-                                      NUM_INTRO_POINTS_DEFAULT,
-                                      NUM_INTRO_POINTS_MAX, &ok, NULL);
+                                      0, NUM_INTRO_POINTS_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_CONFIG,
                  "HiddenServiceNumIntroductionPoints "
                  "should be between %d and %d, not %s",
-                 NUM_INTRO_POINTS_DEFAULT, NUM_INTRO_POINTS_MAX,
-                 line->value);
-        rend_service_free(service);
-        return -1;
+                 0, NUM_INTRO_POINTS_MAX, line->value);
+        goto err;
       }
       log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
-               service->n_intro_points_wanted, service->directory);
-    } else if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+               service->n_intro_points_wanted, escaped(service->directory));
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
       /* Parse auth type and comma-separated list of client names and add a
        * rend_authorized_client_t for each client to the service's list
        * of authorized clients. */
       smartlist_t *type_names_split, *clients;
       const char *authname;
-      int num_clients;
       if (service->auth_type != REND_NO_AUTH) {
         log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient "
                  "lines for a single service.");
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
       type_names_split = smartlist_new();
       smartlist_split_string(type_names_split, line->value, " ", 0, 2);
@@ -699,8 +748,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                          "should have been prevented when parsing the "
                          "configuration.");
         smartlist_free(type_names_split);
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
       authname = smartlist_get(type_names_split, 0);
       if (!strcasecmp(authname, "basic")) {
@@ -714,8 +762,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                  (char *) smartlist_get(type_names_split, 0));
         SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
         smartlist_free(type_names_split);
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
       service->clients = smartlist_new();
       if (smartlist_len(type_names_split) < 2) {
@@ -732,14 +779,15 @@ rend_config_services(const or_options_t *options, int validate_only)
       SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
       smartlist_free(type_names_split);
       /* Remove duplicate client names. */
-      num_clients = smartlist_len(clients);
-      smartlist_sort_strings(clients);
-      smartlist_uniq_strings(clients);
-      if (smartlist_len(clients) < num_clients) {
-        log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
-                            "duplicate client name(s); removing.",
-                 num_clients - smartlist_len(clients));
-        num_clients = smartlist_len(clients);
+      {
+        int num_clients = smartlist_len(clients);
+        smartlist_sort_strings(clients);
+        smartlist_uniq_strings(clients);
+        if (smartlist_len(clients) < num_clients) {
+          log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                   "duplicate client name(s); removing.",
+                   num_clients - smartlist_len(clients));
+        }
       }
       SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name)
       {
@@ -752,8 +800,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                    client_name, REND_CLIENTNAME_MAX_LEN);
           SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
           smartlist_free(clients);
-          rend_service_free(service);
-          return -1;
+          goto err;
         }
         client = tor_malloc_zero(sizeof(rend_authorized_client_t));
         client->client_name = tor_strdup(client_name);
@@ -775,112 +822,29 @@ rend_config_services(const or_options_t *options, int validate_only)
                  smartlist_len(service->clients),
                  service->auth_type == REND_BASIC_AUTH ? 512 : 16,
                  service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
-        rend_service_free(service);
-        return -1;
-      }
-    } else {
-      tor_assert(!strcasecmp(line->key, "HiddenServiceVersion"));
-      if (strcmp(line->value, "2")) {
-        log_warn(LD_CONFIG,
-                 "The only supported HiddenServiceVersion is 2.");
-        rend_service_free(service);
-        return -1;
+        goto err;
       }
+      continue;
     }
   }
-  /* register the final service after we have finished parsing all services
-   * this code only registers the last service, other services are registered
-   * within the loop. It is ok for this service to be NULL, it is ignored. */
-  if (rend_service_check_dir_and_add(NULL, options, service,
-                                     validate_only) < 0) {
-    return -1;
+  /* Validate the service just parsed. */
+  if (rend_validate_service(rend_service_staging_list, service) < 0) {
+    /* Service is in the staging list so don't try to free it. */
+    goto err;
   }
 
-  /* If this is a reload and there were hidden services configured before,
-   * keep the introduction points that are still needed and close the
-   * other ones. */
-  if (old_service_list && !validate_only) {
-    smartlist_t *surviving_services = smartlist_new();
-
-    /* Preserve the existing ephemeral services.
-     *
-     * This is the ephemeral service equivalent of the "Copy introduction
-     * points to new services" block, except there's no copy required since
-     * the service structure isn't regenerated.
-     *
-     * After this is done, all ephemeral services will be:
-     *  * Removed from old_service_list, so the equivalent non-ephemeral code
-     *    will not attempt to preserve them.
-     *  * Added to the new rend_service_list (that previously only had the
-     *    services listed in the configuration).
-     *  * Added to surviving_services, which is the list of services that
-     *    will NOT have their intro point closed.
-     */
-    SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, {
-      if (!old->directory) {
-        SMARTLIST_DEL_CURRENT(old_service_list, old);
-        smartlist_add(surviving_services, old);
-        smartlist_add(rend_service_list, old);
-      }
-    });
-
-    /* Copy introduction points to new services. */
-    /* XXXX This is O(n^2), but it's only called on reconfigure, so it's
-     * probably ok? */
-    SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, new) {
-      SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
-        if (new->directory && old->directory &&
-            !strcmp(old->directory, new->directory)) {
-          smartlist_add_all(new->intro_nodes, old->intro_nodes);
-          smartlist_clear(old->intro_nodes);
-          smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
-          smartlist_clear(old->expiring_nodes);
-          smartlist_add(surviving_services, old);
-
-          /* Copy service flags to the new service object. */
-          copy_service_on_prunning(new, old);
-          break;
-        }
-      } SMARTLIST_FOREACH_END(old);
-    } SMARTLIST_FOREACH_END(new);
-
-    /* Close introduction circuits of services we don't serve anymore. */
-    /* XXXX it would be nicer if we had a nicer abstraction to use here,
-     * so we could just iterate over the list of services to close, but
-     * once again, this isn't critical-path code. */
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-      if (!circ->marked_for_close &&
-          circ->state == CIRCUIT_STATE_OPEN &&
-          (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
-           circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
-        origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
-        int keep_it = 0;
-        tor_assert(oc->rend_data);
-        SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, {
-          if (tor_memeq(ptr->pk_digest, oc->rend_data->rend_pk_digest,
-                      DIGEST_LEN)) {
-            keep_it = 1;
-            break;
-          }
-        });
-        if (keep_it)
-          continue;
-        log_info(LD_REND, "Closing intro point %s for service %s.",
-                 safe_str_client(extend_info_describe(
-                                            oc->build_state->chosen_exit)),
-                 oc->rend_data->onion_address);
-        circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
-        /* XXXX Is there another reason we should use here? */
-      }
-    }
-    SMARTLIST_FOREACH_END(circ);
-    smartlist_free(surviving_services);
-    SMARTLIST_FOREACH(old_service_list, rend_service_t *, ptr,
-                      rend_service_free(ptr));
-    smartlist_free(old_service_list);
+  /* Add it to the temporary list which we will use to prune our current
+   * list if any after configuring all services. */
+  if (rend_add_service(rend_service_staging_list, service) < 0) {
+    /* The object has been freed on error already. */
+    service = NULL;
+    goto err;
   }
 
   return 0;
+ err:
+  rend_service_free(service);
+  return -1;
 }
 
 /** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, using
@@ -895,9 +859,9 @@ rend_config_services(const or_options_t *options, int validate_only)
  * after calling this routine, and may assume that correct cleanup has
  * been done on failure.
  *
- * Return an appropriate rend_service_add_ephemeral_status_t.
+ * Return an appropriate hs_service_add_ephemeral_status_t.
  */
-rend_service_add_ephemeral_status_t
+hs_service_add_ephemeral_status_t
 rend_service_add_ephemeral(crypto_pk_t *pk,
                            smartlist_t *ports,
                            int max_streams_per_circuit,
@@ -969,7 +933,7 @@ int
 rend_service_del_ephemeral(const char *service_id)
 {
   rend_service_t *s;
-  if (!rend_valid_service_id(service_id)) {
+  if (!rend_valid_v2_service_id(service_id)) {
     log_warn(LD_CONFIG, "Requested malformed Onion Service id for removal.");
     return -1;
   }
@@ -978,7 +942,7 @@ rend_service_del_ephemeral(const char *service_id)
              "removal.");
     return -1;
   }
-  if (s->directory) {
+  if (!rend_service_is_ephemeral(s)) {
     log_warn(LD_CONFIG, "Requested non-ephemeral Onion Service for removal.");
     return -1;
   }
@@ -994,17 +958,20 @@ rend_service_del_ephemeral(const char *service_id)
         (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
          circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
-      tor_assert(oc->rend_data);
-      if (!tor_memeq(s->pk_digest, oc->rend_data->rend_pk_digest, DIGEST_LEN))
+      if (oc->rend_data == NULL ||
+          !rend_circuit_pk_digest_eq(oc, (uint8_t *) s->pk_digest)) {
         continue;
+      }
       log_debug(LD_REND, "Closing intro point %s for service %s.",
                 safe_str_client(extend_info_describe(
                                           oc->build_state->chosen_exit)),
-                oc->rend_data->onion_address);
+                rend_data_get_address(oc->rend_data));
       circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
     }
   } SMARTLIST_FOREACH_END(circ);
   smartlist_remove(rend_service_list, s);
+  /* Notify that we just removed a service from our global list. */
+  hs_service_map_has_changed();
   rend_service_free(s);
 
   log_debug(LD_CONFIG, "Removed ephemeral Onion Service: %s", service_id);
@@ -1012,6 +979,45 @@ rend_service_del_ephemeral(const char *service_id)
   return 0;
 }
 
+/* There can be 1 second's delay due to second_elapsed_callback, and perhaps
+ * another few seconds due to blocking calls. */
+#define INTRO_CIRC_RETRY_PERIOD_SLOP 10
+
+/** Log information about the intro point creation rate and current intro
+ * points for service, upgrading the log level from min_severity to warn if
+ * we have stopped launching new intro point circuits. */
+static void
+rend_log_intro_limit(const rend_service_t *service, int min_severity)
+{
+  int exceeded_limit = (service->n_intro_circuits_launched >=
+                        rend_max_intro_circs_per_period(
+                                              service->n_intro_points_wanted));
+  int severity = min_severity;
+  /* We stopped creating circuits */
+  if (exceeded_limit) {
+    severity = LOG_WARN;
+  }
+  time_t intro_period_elapsed = time(NULL) - service->intro_period_started;
+  tor_assert_nonfatal(intro_period_elapsed >= 0);
+  {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_fn(severity, LD_REND,
+             "Hidden service %s %s %d intro points in the last %d seconds. "
+             "Intro circuit launches are limited to %d per %d seconds.%s",
+             service->service_id,
+             exceeded_limit ? "exceeded launch limit with" : "launched",
+             service->n_intro_circuits_launched,
+             (int)intro_period_elapsed,
+             rend_max_intro_circs_per_period(service->n_intro_points_wanted),
+             INTRO_CIRC_RETRY_PERIOD, msg);
+      rend_service_dump_stats(severity);
+      tor_free(msg);
+    }
+  }
+}
+
 /** Replace the old value of <b>service</b>-\>desc with one that reflects
  * the other fields in service.
  */
@@ -1019,7 +1025,6 @@ static void
 rend_service_update_descriptor(rend_service_t *service)
 {
   rend_service_descriptor_t *d;
-  origin_circuit_t *circ;
   int i;
 
   rend_service_descriptor_free(service->desc);
@@ -1040,9 +1045,10 @@ rend_service_update_descriptor(rend_service_t *service)
     /* This intro point won't be listed in the descriptor... */
     intro_svc->listed_in_last_desc = 0;
 
-    circ = find_intro_circuit(intro_svc, service->pk_digest);
-    if (!circ || circ->base_.purpose != CIRCUIT_PURPOSE_S_INTRO) {
-      /* This intro point's circuit isn't finished yet.  Don't list it. */
+    /* circuit_established is set in rend_service_intro_established(), and
+     * checked every second in rend_consider_services_intro_points(), so it's
+     * safe to use it here */
+    if (!intro_svc->circuit_established) {
       continue;
     }
 
@@ -1064,6 +1070,26 @@ rend_service_update_descriptor(rend_service_t *service)
       intro_svc->time_published = time(NULL);
     }
   }
+
+  /* Check that we have the right number of intro points */
+  unsigned int have_intro = (unsigned int)smartlist_len(d->intro_nodes);
+  if (have_intro != service->n_intro_points_wanted) {
+    int severity;
+    /* Getting less than we wanted or more than we're allowed is serious */
+    if (have_intro < service->n_intro_points_wanted ||
+        have_intro > NUM_INTRO_POINTS_MAX) {
+      severity = LOG_WARN;
+    } else {
+      /* Getting more than we wanted is weird, but less of a problem */
+      severity = LOG_NOTICE;
+    }
+    log_fn(severity, LD_REND, "Hidden service %s wanted %d intro points, but "
+           "descriptor was updated with %d instead.",
+           service->service_id,
+           service->n_intro_points_wanted, have_intro);
+    /* Now log an informative message about how we might have got here. */
+    rend_log_intro_limit(service, severity);
+  }
 }
 
 /* Allocate and return a string containing the path to file_name in
@@ -1073,15 +1099,8 @@ rend_service_update_descriptor(rend_service_t *service)
 static char *
 rend_service_path(const rend_service_t *service, const char *file_name)
 {
-  char *file_path = NULL;
-
   tor_assert(service->directory);
-
-  /* Can never fail: asserts rather than leaving file_path NULL. */
-  tor_asprintf(&file_path, "%s%s%s",
-               service->directory, PATH_SEPARATOR, file_name);
-
-  return file_path;
+  return hs_path_from_filename(service->directory, file_name);
 }
 
 /* Allocate and return a string containing the path to the single onion
@@ -1094,7 +1113,7 @@ rend_service_sos_poison_path(const rend_service_t *service)
   return rend_service_path(service, sos_poison_fname);
 }
 
-/** Return True if hidden services <b>service> has been poisoned by single
+/** Return True if hidden services <b>service</b> has been poisoned by single
  * onion mode. */
 static int
 service_is_single_onion_poisoned(const rend_service_t *service)
@@ -1107,7 +1126,7 @@ service_is_single_onion_poisoned(const rend_service_t *service)
     return 0;
   }
 
-  if (!service->directory) {
+  if (rend_service_is_ephemeral(service)) {
     return 0;
   }
 
@@ -1159,8 +1178,13 @@ rend_service_verify_single_onion_poison(const rend_service_t* s,
   }
 
   /* Ephemeral services are checked at ADD_ONION time */
-  if (!s->directory) {
-    return 0;
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
   }
 
   /* Services without keys are always ok - their keys will only ever be used
@@ -1203,7 +1227,7 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   int retval = -1;
   char *poison_fname = NULL;
 
-  if (!service->directory) {
+  if (rend_service_is_ephemeral(service)) {
     log_info(LD_REND, "Ephemeral HS started in non-anonymous mode.");
     return 0;
   }
@@ -1216,7 +1240,8 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   }
 
   /* Make sure the directory was created before calling this function. */
-  if (BUG(rend_service_check_private_dir_impl(options, service, 0) < 0))
+  if (BUG(hs_check_service_private_dir(options->User, service->directory,
+                                       service->dir_group_readable, 0) < 0))
     return -1;
 
   poison_fname = rend_service_sos_poison_path(service);
@@ -1253,7 +1278,7 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   return retval;
 }
 
-/** We just got launched in Single Onion Mode. That's a non-anoymous mode for
+/** We just got launched in Single Onion Mode. That's a non-anonymous mode for
  * hidden services. If s is new, we should mark its hidden service
  * directory appropriately so that it is never launched as a location-private
  * hidden service. (New directories don't have private key files.)
@@ -1270,6 +1295,16 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
   /* We must only poison directories if we're in Single Onion mode */
   tor_assert(rend_service_non_anonymous_mode_enabled(options));
 
+  /* Ephemeral services aren't allowed in non-anonymous mode */
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
+  }
+
   if (!rend_service_private_key_exists(s)) {
     if (poison_new_single_onion_hidden_service_dir_impl(s, options)
         < 0) {
@@ -1289,22 +1324,17 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
 int
 rend_service_load_all_keys(const smartlist_t *service_list)
 {
-  const smartlist_t *s_list = NULL;
-  /* If no special service list is provided, then just use the global one. */
-  if (!service_list) {
-    if (BUG(!rend_service_list)) {
-      return -1;
-    }
-    s_list = rend_service_list;
-  } else {
-    s_list = service_list;
+  /* Use service_list for unit tests */
+  const smartlist_t *s_list = rend_get_service_list(service_list);
+  if (BUG(!s_list)) {
+    return -1;
   }
 
   SMARTLIST_FOREACH_BEGIN(s_list, rend_service_t *, s) {
     if (s->private_key)
       continue;
-    log_info(LD_REND, "Loading hidden-service keys from \"%s\"",
-             s->directory);
+    log_info(LD_REND, "Loading hidden-service keys from %s",
+             rend_service_escaped_dir(s));
 
     if (rend_service_load_keys(s) < 0)
       return -1;
@@ -1336,15 +1366,15 @@ rend_services_add_filenames_to_lists(smartlist_t *open_lst,
   if (!rend_service_list)
     return;
   SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) {
-    if (s->directory) {
+    if (!rend_service_is_ephemeral(s)) {
       rend_service_add_filenames_to_list(open_lst, s);
-      smartlist_add(stat_lst, tor_strdup(s->directory));
+      smartlist_add_strdup(stat_lst, s->directory);
     }
   } SMARTLIST_FOREACH_END(s);
 }
 
 /** Derive all rend_service_t internal material based on the service's key.
- * Returns 0 on sucess, -1 on failure.
+ * Returns 0 on success, -1 on failure.
  */
 static int
 rend_service_derive_key_digests(struct rend_service_t *s)
@@ -1361,32 +1391,6 @@ rend_service_derive_key_digests(struct rend_service_t *s)
   return 0;
 }
 
-/* Implements the directory check from rend_service_check_private_dir,
- * without doing the single onion poison checks. */
-static int
-rend_service_check_private_dir_impl(const or_options_t *options,
-                                    const rend_service_t *s,
-                                    int create)
-{
-  cpd_check_t  check_opts = CPD_NONE;
-  if (create) {
-    check_opts |= CPD_CREATE;
-  } else {
-    check_opts |= CPD_CHECK_MODE_ONLY;
-    check_opts |= CPD_CHECK;
-  }
-  if (s->dir_group_readable) {
-    check_opts |= CPD_GROUP_READ;
-  }
-  /* Check/create directory */
-  if (check_private_dir(s->directory, check_opts, options->User) < 0) {
-    log_warn(LD_REND, "Checking service directory %s failed.", s->directory);
-    return -1;
-  }
-
-  return 0;
-}
-
 /** Make sure that the directory for <b>s</b> is private, using the config in
  * <b>options</b>.
  * If <b>create</b> is true:
@@ -1407,7 +1411,8 @@ rend_service_check_private_dir(const or_options_t *options,
   }
 
   /* Check/create directory */
-  if (rend_service_check_private_dir_impl(options, s, create) < 0) {
+  if (hs_check_service_private_dir(options->User, s->directory,
+                                   s->dir_group_readable, create) < 0) {
     return -1;
   }
 
@@ -1465,9 +1470,9 @@ rend_service_load_keys(rend_service_t *s)
   char *fname = NULL;
   char buf[128];
 
-  /* Make sure the directory was created and single onion poisoning was
-   * checked before calling this function */
-  if (BUG(rend_service_check_private_dir(get_options(), s, 0) < 0))
+  /* Create the directory if needed which will also poison it in case of
+   * single onion service. */
+  if (rend_service_check_private_dir(get_options(), s, 1) < 0)
     goto err;
 
   /* Load key */
@@ -1496,7 +1501,7 @@ rend_service_load_keys(rend_service_t *s)
       log_warn(LD_FS,"Unable to make hidden hostname file %s group-readable.",
                fname);
   }
-#endif
+#endif /* !defined(_WIN32) */
 
   /* If client authorization is configured, load or generate keys. */
   if (s->auth_type != REND_NO_AUTH) {
@@ -1684,7 +1689,7 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname)
     memwipe(client_keys_str, 0, strlen(client_keys_str));
     tor_free(client_keys_str);
   }
-  strmap_free(parsed_clients, rend_authorized_client_strmap_item_free);
+  strmap_free(parsed_clients, rend_authorized_client_free_void);
 
   if (cfname) {
     memwipe(cfname, 0, strlen(cfname));
@@ -1725,24 +1730,6 @@ rend_service_get_by_service_id(const char *id)
   return NULL;
 }
 
-/** Return 1 if any virtual port in <b>service</b> wants a circuit
- * to have good uptime. Else return 0.
- */
-static int
-rend_service_requires_uptime(rend_service_t *service)
-{
-  int i;
-  rend_service_port_config_t *p;
-
-  for (i=0; i < smartlist_len(service->ports); ++i) {
-    p = smartlist_get(service->ports, i);
-    if (smartlist_contains_int_as_string(get_options()->LongLivedPorts,
-                                  p->virtual_port))
-      return 1;
-  }
-  return 0;
-}
-
 /** Check client authorization of a given <b>descriptor_cookie</b> of
  * length <b>cookie_len</b> for <b>service</b>. Return 1 for success
  * and 0 for failure. */
@@ -1833,7 +1820,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   const or_options_t *options = get_options();
   char *err_msg = NULL;
   int err_msg_severity = LOG_WARN;
-  const char *stage_descr = NULL;
+  const char *stage_descr = NULL, *rend_pk_digest;
   int reason = END_CIRC_REASON_TORPROTOCOL;
   /* Service/circuit/key stuff we can learn before parsing */
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
@@ -1868,14 +1855,15 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   assert_circ_anonymity_ok(circuit, options);
   tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   /* We'll use this in a bazillion log messages */
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
   /* look up service depending on circuit. */
-  service =
-    rend_service_get_by_pk_digest(circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_BUG,
              "Internal error: Got an INTRODUCE2 cell on an intro "
@@ -2061,14 +2049,14 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
     goto err;
   }
 
-  circ_needs_uptime = rend_service_requires_uptime(service);
+  circ_needs_uptime = hs_service_requires_uptime_circ(service->ports);
 
   /* help predict this next time */
   rep_hist_note_used_internal(now, circ_needs_uptime, 1);
 
   /* Launch a circuit to the client's chosen rendezvous point.
    */
-  int max_rend_failures=get_max_rend_failures();
+  int max_rend_failures=hs_get_service_max_rend_failures();
   for (i=0;i<max_rend_failures;i++) {
     int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
     if (circ_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME;
@@ -2100,8 +2088,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   /* Fill in the circuit's state. */
 
   launched->rend_data =
-    rend_data_service_create(service->service_id,
-                             circuit->rend_data->rend_pk_digest,
+    rend_data_service_create(service->service_id, rend_pk_digest,
                              parsed_req->rc, service->auth_type);
 
   launched->build_state->service_pending_final_cpath_ref =
@@ -2115,7 +2102,9 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   cpath->rend_dh_handshake_state = dh;
   dh = NULL;
-  if (circuit_init_cpath_crypto(cpath,keys+DIGEST_LEN,1)<0)
+  if (circuit_init_cpath_crypto(cpath,
+                                keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
+                                1, 0)<0)
     goto err;
   memcpy(cpath->rend_circ_nonce, keys, DIGEST_LEN);
 
@@ -2178,7 +2167,7 @@ find_rp_for_intro(const rend_intro_cell_t *intro,
   if (intro->version == 0 || intro->version == 1) {
     rp_nickname = (const char *)(intro->u.v0_v1.rp);
 
-    node = node_get_by_nickname(rp_nickname, 0);
+    node = node_get_by_nickname(rp_nickname, NNF_NO_WARN_UNNAMED);
     if (!node) {
       if (err_msg_out) {
         tor_asprintf(&err_msg,
@@ -2249,7 +2238,7 @@ find_rp_for_intro(const rend_intro_cell_t *intro,
  * rend_service_parse_intro().
  */
 void
-rend_service_free_intro(rend_intro_cell_t *request)
+rend_service_free_intro_(rend_intro_cell_t *request)
 {
   if (!request) {
     return;
@@ -2732,7 +2721,14 @@ rend_service_decrypt_intro(
   /* Check that this cell actually matches this service key */
 
   /* first DIGEST_LEN bytes of request is intro or service pk digest */
-  crypto_pk_get_digest(key, (char *)key_digest);
+  if (crypto_pk_get_digest(key, (char *)key_digest) < 0) {
+    if (err_msg_out)
+      *err_msg_out = tor_strdup("Couldn't compute RSA digest.");
+    log_warn(LD_BUG, "Couldn't compute key digest.");
+    status = -7;
+    goto err;
+  }
+
   if (tor_memneq(key_digest, intro->pk, DIGEST_LEN)) {
     if (err_msg_out) {
       base32_encode(service_id, REND_SERVICE_ID_LEN_BASE32 + 1,
@@ -2763,10 +2759,8 @@ rend_service_decrypt_intro(
   }
 
   /* Decrypt the encrypted part */
-
-  note_crypto_pk_op(REND_SERVER);
   result =
-    crypto_pk_private_hybrid_decrypt(
+    crypto_pk_obsolete_private_hybrid_decrypt(
        key, (char *)buf, sizeof(buf),
        (const char *)(intro->ciphertext), intro->ciphertext_len,
        PK_PKCS1_OAEP_PADDING, 1);
@@ -2959,35 +2953,6 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   cpath_build_state_t *newstate, *oldstate;
 
   tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
-
-  /* Don't relaunch the same rend circ twice. */
-  if (oldcirc->hs_service_side_rend_circ_has_been_relaunched) {
-    log_info(LD_REND, "Rendezvous circuit to %s has already been relaunched; "
-             "not relaunching it again.",
-             oldcirc->build_state ?
-             safe_str(extend_info_describe(oldcirc->build_state->chosen_exit))
-             : "*unknown*");
-    return;
-  }
-  oldcirc->hs_service_side_rend_circ_has_been_relaunched = 1;
-
-  /* We check failure_count >= get_max_rend_failures()-1 below, and the -1
-   * is because we increment the failure count for our current failure
-   * *after* this clause. */
-  int max_rend_failures = get_max_rend_failures() - 1;
-
-  if (!oldcirc->build_state ||
-      oldcirc->build_state->failure_count >= max_rend_failures ||
-      oldcirc->build_state->expiry_time < time(NULL)) {
-    log_info(LD_REND,
-             "Attempt to build circuit to %s for rendezvous has failed "
-             "too many times or expired; giving up.",
-             oldcirc->build_state ?
-             safe_str(extend_info_describe(oldcirc->build_state->chosen_exit))
-             : "*unknown*");
-    return;
-  }
-
   oldstate = oldcirc->build_state;
   tor_assert(oldstate);
 
@@ -3140,15 +3105,67 @@ count_intro_point_circuits(const rend_service_t *service)
          circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
       if (oc->rend_data &&
-          !rend_cmp_service_ids(service->service_id,
-                                oc->rend_data->onion_address))
+          rend_circuit_pk_digest_eq(oc, (uint8_t *) service->pk_digest)) {
         num_ipos++;
+      }
     }
   }
   SMARTLIST_FOREACH_END(circ);
   return num_ipos;
 }
 
+/* Given a buffer of at least RELAY_PAYLOAD_SIZE bytes in <b>cell_body_out</b>,
+   write the body of a legacy ESTABLISH_INTRO cell in it. Use <b>intro_key</b>
+   as the intro point auth key, and <b>rend_circ_nonce</b> as the circuit
+   crypto material. On success, fill <b>cell_body_out</b> and return the number
+   of bytes written. On fail, return -1.
+ */
+ssize_t
+rend_service_encode_establish_intro_cell(char *cell_body_out,
+                                         size_t cell_body_out_len,
+                                         crypto_pk_t *intro_key,
+                                         const char *rend_circ_nonce)
+{
+  int retval = -1;
+  int r;
+  int len = 0;
+  char auth[DIGEST_LEN + 9];
+
+  tor_assert(intro_key);
+  tor_assert(rend_circ_nonce);
+
+  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
+  r = crypto_pk_asn1_encode(intro_key, cell_body_out+2,
+                            RELAY_PAYLOAD_SIZE-2);
+  if (r < 0) {
+    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
+    goto err;
+  }
+  len = r;
+  set_uint16(cell_body_out, htons((uint16_t)len));
+  len += 2;
+  memcpy(auth, rend_circ_nonce, DIGEST_LEN);
+  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
+  if (crypto_digest(cell_body_out+len, auth, DIGEST_LEN+9))
+    goto err;
+  len += 20;
+  r = crypto_pk_private_sign_digest(intro_key, cell_body_out+len,
+                                    cell_body_out_len - len,
+                                    cell_body_out, len);
+  if (r<0) {
+    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
+    goto err;
+  }
+  len += r;
+
+  retval = len;
+
+ err:
+  memwipe(auth, 0, sizeof(auth));
+
+  return retval;
+}
+
 /** Called when we're done building a circuit to an introduction point:
  *  sends a RELAY_ESTABLISH_INTRO cell.
  */
@@ -3156,23 +3173,23 @@ void
 rend_service_intro_has_opened(origin_circuit_t *circuit)
 {
   rend_service_t *service;
-  size_t len;
-  int r;
   char buf[RELAY_PAYLOAD_SIZE];
-  char auth[DIGEST_LEN + 9];
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  unsigned int expiring_nodes_len, num_ip_circuits, valid_ip_circuits = 0;
   int reason = END_CIRC_REASON_TORPROTOCOL;
+  const char *rend_pk_digest;
 
   tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
   assert_circ_anonymity_ok(circuit, get_options());
   tor_assert(circuit->cpath);
   tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only on supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %u.",
              safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id);
@@ -3180,13 +3197,22 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
     goto err;
   }
 
+  /* Take the current amount of expiring nodes and the current amount of IP
+   * circuits and compute how many valid IP circuits we have. */
+  expiring_nodes_len = (unsigned int) smartlist_len(service->expiring_nodes);
+  num_ip_circuits = count_intro_point_circuits(service);
+  /* Let's avoid an underflow. The valid_ip_circuits is initialized to 0 in
+   * case this condition turns out false because it means that all circuits
+   * are expiring so we need to keep this circuit. */
+  if (num_ip_circuits > expiring_nodes_len) {
+    valid_ip_circuits = num_ip_circuits - expiring_nodes_len;
+  }
+
   /* If we already have enough introduction circuits for this service,
    * redefine this one as a general circuit or close it, depending.
-   * Substract the amount of expiring nodes here since the circuits are
+   * Subtract the amount of expiring nodes here because the circuits are
    * still opened. */
-  if ((count_intro_point_circuits(service) -
-       smartlist_len(service->expiring_nodes)) >
-      service->n_intro_points_wanted) {
+  if (valid_ip_circuits > service->n_intro_points_wanted) {
     const or_options_t *options = get_options();
     /* Remove the intro point associated with this circuit, it's being
      * repurposed or closed thus cleanup memory. */
@@ -3210,12 +3236,16 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
                "circuit, but we already have enough. Redefining purpose to "
                "general; leaving as internal.");
 
-      circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_C_GENERAL);
+      if (circuit_should_use_vanguards(TO_CIRCUIT(circuit)->purpose)) {
+        circuit_change_purpose(TO_CIRCUIT(circuit),
+                CIRCUIT_PURPOSE_HS_VANGUARDS);
+      } else {
+        circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_C_GENERAL);
+      }
 
       {
-        rend_data_t *rend_data = circuit->rend_data;
+        rend_data_free(circuit->rend_data);
         circuit->rend_data = NULL;
-        rend_data_free(rend_data);
       }
       {
         crypto_pk_t *intro_key = circuit->intro_key;
@@ -3233,42 +3263,25 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
            (unsigned)circuit->base_.n_circ_id, serviceid);
   circuit_log_path(LOG_INFO, LD_REND, circuit);
 
-  /* Use the intro key instead of the service key in ESTABLISH_INTRO. */
-  crypto_pk_t *intro_key = circuit->intro_key;
-  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
-  r = crypto_pk_asn1_encode(intro_key, buf+2,
-                            RELAY_PAYLOAD_SIZE-2);
-  if (r < 0) {
-    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
-  }
-  len = r;
-  set_uint16(buf, htons((uint16_t)len));
-  len += 2;
-  memcpy(auth, circuit->cpath->prev->rend_circ_nonce, DIGEST_LEN);
-  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
-  if (crypto_digest(buf+len, auth, DIGEST_LEN+9))
-    goto err;
-  len += 20;
-  note_crypto_pk_op(REND_SERVER);
-  r = crypto_pk_private_sign_digest(intro_key, buf+len, sizeof(buf)-len,
-                                    buf, len);
-  if (r<0) {
-    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
-  }
-  len += r;
+  /* Send the ESTABLISH_INTRO cell */
+  {
+    ssize_t len;
+    len = rend_service_encode_establish_intro_cell(buf, sizeof(buf),
+                                      circuit->intro_key,
+                                      circuit->cpath->prev->rend_circ_nonce);
+    if (len < 0) {
+      reason = END_CIRC_REASON_INTERNAL;
+      goto err;
+    }
 
-  if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
-                                   RELAY_COMMAND_ESTABLISH_INTRO,
-                                   buf, len, circuit->cpath->prev)<0) {
-    log_info(LD_GENERAL,
+    if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
+                                     RELAY_COMMAND_ESTABLISH_INTRO,
+                                     buf, len, circuit->cpath->prev)<0) {
+      log_info(LD_GENERAL,
              "Couldn't send introduction request for service %s on circuit %u",
              serviceid, (unsigned)circuit->base_.n_circ_id);
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+      goto done;
+    }
   }
 
   /* We've attempted to use this circuit */
@@ -3280,7 +3293,6 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
   circuit_mark_for_close(TO_CIRCUIT(circuit), reason);
  done:
   memwipe(buf, 0, sizeof(buf));
-  memwipe(auth, 0, sizeof(auth));
   memwipe(serviceid, 0, sizeof(serviceid));
 
   return;
@@ -3299,22 +3311,24 @@ rend_service_intro_established(origin_circuit_t *circuit,
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
   (void) request;
   (void) request_len;
+  tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only supported one for now). */
+  const char *rend_pk_digest =
+    (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   if (circuit->base_.purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
     log_warn(LD_PROTOCOL,
              "received INTRO_ESTABLISHED cell on non-intro circuit.");
     goto err;
   }
-  tor_assert(circuit->rend_data);
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Unknown service on introduction circuit %u.",
              (unsigned)circuit->base_.n_circ_id);
     goto err;
   }
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32 + 1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
   /* We've just successfully established a intro circuit to one of our
    * introduction point, account for it. */
   intro = find_intro_point(circuit);
@@ -3357,6 +3371,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
   char hexcookie[9];
   int reason;
+  const char *rend_cookie, *rend_pk_digest;
 
   tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   tor_assert(circuit->cpath);
@@ -3364,18 +3379,24 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   assert_circ_anonymity_ok(circuit, get_options());
   tor_assert(circuit->rend_data);
 
-  /* Declare the circuit dirty to avoid reuse, and for path-bias */
-  if (!circuit->base_.timestamp_dirty)
-    circuit->base_.timestamp_dirty = time(NULL);
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data,
+                                                    NULL);
+  rend_cookie = circuit->rend_data->rend_cookie;
+
+  /* Declare the circuit dirty to avoid reuse, and for path-bias. We set the
+   * timestamp regardless of its content because that circuit could have been
+   * cannibalized so in any cases, we are about to use that circuit more. */
+  circuit->base_.timestamp_dirty = time(NULL);
 
   /* This may be redundant */
   pathbias_count_use_attempt(circuit);
 
   hop = circuit->build_state->service_pending_final_cpath_ref->cpath;
 
-  base16_encode(hexcookie,9,circuit->rend_data->rend_cookie,4);
+  base16_encode(hexcookie,9, rend_cookie,4);
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
   log_info(LD_REND,
            "Done building circuit %u to rendezvous with "
@@ -3404,8 +3425,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   circuit->build_state->pending_final_cpath = hop;
   circuit->build_state->service_pending_final_cpath_ref->cpath = NULL;
 
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_GENERAL, "Internal error: unrecognized service ID on "
              "rendezvous circuit.");
@@ -3414,7 +3434,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   }
 
   /* All we need to do is send a RELAY_RENDEZVOUS1 cell... */
-  memcpy(buf, circuit->rend_data->rend_cookie, REND_COOKIE_LEN);
+  memcpy(buf, rend_cookie, REND_COOKIE_LEN);
   if (crypto_dh_get_public(hop->rend_dh_handshake_state,
                            buf+REND_COOKIE_LEN, DH_KEY_LEN)<0) {
     log_warn(LD_GENERAL,"Couldn't get DH public key.");
@@ -3427,11 +3447,10 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   /* Send the cell */
   if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
                                    RELAY_COMMAND_RENDEZVOUS1,
-                                   buf, REND_COOKIE_LEN+DH_KEY_LEN+DIGEST_LEN,
+                                   buf, HS_LEGACY_RENDEZVOUS_CELL_SIZE,
                                    circuit->cpath->prev)<0) {
     log_warn(LD_GENERAL, "Couldn't send RENDEZVOUS1 cell.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+    goto done;
   }
 
   crypto_dh_free(hop->rend_dh_handshake_state);
@@ -3478,8 +3497,8 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
   origin_circuit_t *circ = NULL;
 
   tor_assert(intro);
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest,
-                                                  CIRCUIT_PURPOSE_S_INTRO))) {
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest, CIRCUIT_PURPOSE_S_INTRO))) {
     if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
                 intro->extend_info->identity_digest, DIGEST_LEN) &&
         circ->rend_data) {
@@ -3488,8 +3507,9 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
   }
 
   circ = NULL;
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest,
-                                        CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest,
+                         CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
     if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
                 intro->extend_info->identity_digest, DIGEST_LEN) &&
         circ->rend_data) {
@@ -3528,7 +3548,7 @@ find_intro_point(origin_circuit_t *circ)
   tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
              TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
   tor_assert(circ->rend_data);
-  serviceid = circ->rend_data->onion_address;
+  serviceid = rend_data_get_address(circ->rend_data);
 
   SMARTLIST_FOREACH(rend_service_list, rend_service_t *, s,
     if (tor_memeq(s->service_id, serviceid, REND_SERVICE_ID_LEN_BASE32)) {
@@ -3575,7 +3595,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
                           "directories to post descriptors to.");
         control_event_hs_descriptor_upload(service_id,
                                            "UNKNOWN",
-                                           "UNKNOWN");
+                                           "UNKNOWN", NULL);
         goto done;
       }
     }
@@ -3590,7 +3610,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
         /* Don't upload descriptor if we succeeded in doing so last time. */
         continue;
       node = node_get_by_id(hs_dir->identity_digest);
-      if (!node || !node_has_descriptor(node)) {
+      if (!node || !node_has_preferred_descriptor(node,0)) {
         log_info(LD_REND, "Not launching upload for for v2 descriptor to "
                           "hidden service directory %s; we don't have its "
                           "router descriptor. Queuing for later upload.",
@@ -3604,13 +3624,16 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
        * request. Lookup is made in rend_service_desc_has_uploaded(). */
       rend_data = rend_data_client_create(service_id, desc->desc_id, NULL,
                                           REND_NO_AUTH);
-      directory_initiate_command_routerstatus_rend(hs_dir,
-                                              DIR_PURPOSE_UPLOAD_RENDDESC_V2,
-                                                   ROUTER_PURPOSE_GENERAL,
-                                                   DIRIND_ANONYMOUS, NULL,
-                                                   desc->desc_str,
-                                                   strlen(desc->desc_str),
-                                                   0, rend_data);
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+      directory_request_set_routerstatus(req, hs_dir);
+      directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+      directory_request_set_payload(req,
+                                    desc->desc_str, strlen(desc->desc_str));
+      directory_request_set_rend_query(req, rend_data);
+      directory_initiate_request(req);
+      directory_request_free(req);
+
       rend_data_free(rend_data);
       base32_encode(desc_id_base32, sizeof(desc_id_base32),
                     desc->desc_id, DIGEST_LEN);
@@ -3627,7 +3650,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
                hs_dir->or_port);
       control_event_hs_descriptor_upload(service_id,
                                          hs_dir->identity_digest,
-                                         desc_id_base32);
+                                         desc_id_base32, NULL);
       tor_free(hs_dir_ip);
       /* Remember successful upload to this router for next time. */
       if (!smartlist_contains_digest(successful_uploads,
@@ -3724,7 +3747,7 @@ upload_service_descriptor(rend_service_t *service)
       }
       /* Free memory for descriptors. */
       for (i = 0; i < smartlist_len(descs); i++)
-        rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+        rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
       smartlist_clear(descs);
       /* Update next upload time. */
       if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS
@@ -3755,7 +3778,7 @@ upload_service_descriptor(rend_service_t *service)
         }
         /* Free memory for descriptors. */
         for (i = 0; i < smartlist_len(descs); i++)
-          rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+          rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
         smartlist_clear(descs);
       }
     }
@@ -3842,6 +3865,19 @@ remove_invalid_intro_points(rend_service_t *service,
 {
   tor_assert(service);
 
+  /* Remove any expired nodes that doesn't have a circuit. */
+  SMARTLIST_FOREACH_BEGIN(service->expiring_nodes, rend_intro_point_t *,
+                          intro) {
+    origin_circuit_t *intro_circ =
+      find_intro_circuit(intro, service->pk_digest);
+    if (intro_circ) {
+      continue;
+    }
+    /* No more circuit, cleanup the into point object. */
+    SMARTLIST_DEL_CURRENT(service->expiring_nodes, intro);
+    rend_intro_point_free(intro);
+  } SMARTLIST_FOREACH_END(intro);
+
   SMARTLIST_FOREACH_BEGIN(service->intro_nodes, rend_intro_point_t *,
                           intro) {
     /* Find the introduction point node object. */
@@ -3917,10 +3953,13 @@ void
 rend_service_desc_has_uploaded(const rend_data_t *rend_data)
 {
   rend_service_t *service;
+  const char *onion_address;
 
   tor_assert(rend_data);
 
-  service = rend_service_get_by_service_id(rend_data->onion_address);
+  onion_address = rend_data_get_address(rend_data);
+
+  service = rend_service_get_by_service_id(onion_address);
   if (service == NULL) {
     return;
   }
@@ -3938,6 +3977,23 @@ rend_service_desc_has_uploaded(const rend_data_t *rend_data)
   } SMARTLIST_FOREACH_END(intro);
 }
 
+/** Don't try to build more than this many circuits before giving up
+ * for a while. Dynamically calculated based on the configured number of
+ * introduction points for the service, n_intro_points_wanted. */
+static int
+rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted)
+{
+  /* Allow all but one of the initial connections to fail and be
+   * retried. (If all fail, we *want* to wait, because something is broken.) */
+  tor_assert(n_intro_points_wanted <= NUM_INTRO_POINTS_MAX);
+
+  /* For the normal use case, 3 intro points plus 2 extra for performance and
+   * allow that twice because once every 24h or so, we can do it twice for two
+   * descriptors that is the current one and the next one. So (3 + 2) * 2 ==
+   * 12 allowed attempts for one period. */
+  return ((n_intro_points_wanted + NUM_INTRO_POINTS_EXTRA) * 2);
+}
+
 /** For every service, check how many intro points it currently has, and:
  *  - Invalidate introdution points based on specific criteria, see
  *  remove_invalid_intro_points comments.
@@ -3947,10 +4003,9 @@ rend_service_desc_has_uploaded(const rend_data_t *rend_data)
  * This is called once a second by the main loop.
  */
 void
-rend_consider_services_intro_points(void)
+rend_consider_services_intro_points(time_t now)
 {
   int i;
-  time_t now;
   const or_options_t *options = get_options();
   /* Are we in single onion mode? */
   const int allow_direct = rend_service_allow_non_anonymous_connection(
@@ -3967,7 +4022,6 @@ rend_consider_services_intro_points(void)
 
   exclude_nodes = smartlist_new();
   retry_nodes = smartlist_new();
-  now = time(NULL);
 
   SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, service) {
     int r;
@@ -3982,23 +4036,29 @@ rend_consider_services_intro_points(void)
     smartlist_clear(exclude_nodes);
     smartlist_clear(retry_nodes);
 
+    /* Cleanup the invalid intro points and save the node objects, if any,
+     * in the exclude_nodes and retry_nodes lists. */
+    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
+
     /* This retry period is important here so we don't stress circuit
      * creation. */
+
     if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) {
-      /* One period has elapsed; we can try building circuits again. */
+      /* One period has elapsed:
+       *  - if we stopped, we can try building circuits again,
+       *  - if we haven't, we reset the circuit creation counts. */
+      rend_log_intro_limit(service, LOG_INFO);
       service->intro_period_started = now;
       service->n_intro_circuits_launched = 0;
     } else if (service->n_intro_circuits_launched >=
-               MAX_INTRO_CIRCS_PER_PERIOD) {
+               rend_max_intro_circs_per_period(
+                                      service->n_intro_points_wanted)) {
       /* We have failed too many times in this period; wait for the next
-       * one before we try again. */
+       * one before we try to initiate any more connections. */
+      rend_log_intro_limit(service, LOG_WARN);
       continue;
     }
 
-    /* Cleanup the invalid intro points and save the node objects, if apply,
-     * in the exclude_nodes and retry_nodes list. */
-    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
-
     /* Let's try to rebuild circuit on the nodes we want to retry on. */
     SMARTLIST_FOREACH_BEGIN(retry_nodes, rend_intro_point_t *, intro) {
       r = rend_service_launch_establish_intro(service, intro);
@@ -4018,17 +4078,17 @@ rend_consider_services_intro_points(void)
     /* Avoid mismatched signed comparaison below. */
     intro_nodes_len = (unsigned int) smartlist_len(service->intro_nodes);
 
-    /* Quiescent state, no node expiring and we have more or the amount of
-     * wanted node for this service. Proceed to the next service. Could be
-     * more because we launch two preemptive circuits if our intro nodes
-     * list is empty. */
-    if (smartlist_len(service->expiring_nodes) == 0 &&
-        intro_nodes_len >= service->n_intro_points_wanted) {
+    /* Quiescent state, we have more or the equal amount of wanted node for
+     * this service. Proceed to the next service. We can have more nodes
+     * because we launch extra preemptive circuits if our intro nodes list was
+     * originally empty for performance reasons. */
+    if (intro_nodes_len >= service->n_intro_points_wanted) {
       continue;
     }
 
-    /* Number of intro points we want to open which is the wanted amount
-     * minus the current amount of valid nodes. */
+    /* Number of intro points we want to open which is the wanted amount minus
+     * the current amount of valid nodes. We know that this won't underflow
+     * because of the check above. */
     n_intro_points_to_open = service->n_intro_points_wanted - intro_nodes_len;
     if (intro_nodes_len == 0) {
       /* We want to end up with n_intro_points_wanted intro points, but if
@@ -4048,8 +4108,6 @@ rend_consider_services_intro_points(void)
       const node_t *node;
       rend_intro_point_t *intro;
       router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC;
-      if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION)
-        flags |= CRN_ALLOW_INVALID;
       router_crn_flags_t direct_flags = flags;
       direct_flags |= CRN_PREF_ADDR;
       direct_flags |= CRN_DIRECT_CONN;
@@ -4077,7 +4135,7 @@ rend_consider_services_intro_points(void)
                  n_intro_points_to_open);
         break;
       }
-      /* Add the choosen node to the exclusion list in order to avoid picking
+      /* Add the chosen node to the exclusion list in order to avoid picking
        * it again in the next iteration. */
       smartlist_add(exclude_nodes, (void*)node);
       intro = tor_malloc_zero(sizeof(rend_intro_point_t));
@@ -4085,6 +4143,9 @@ rend_consider_services_intro_points(void)
        * even if we are a single onion service and intend to connect to it
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
+      if (BUG(intro->extend_info == NULL)) {
+        break;
+      }
       intro->intro_key = crypto_pk_new();
       const int fail = crypto_pk_generate_key(intro->intro_key);
       tor_assert(!fail);
@@ -4220,8 +4281,8 @@ rend_service_dump_stats(int severity)
 
   for (i=0; i < smartlist_len(rend_service_list); ++i) {
     service = smartlist_get(rend_service_list, i);
-    tor_log(severity, LD_GENERAL, "Service configured in \"%s\":",
-        service->directory);
+    tor_log(severity, LD_GENERAL, "Service configured in %s:",
+            rend_service_escaped_dir(service));
     for (j=0; j < smartlist_len(service->intro_nodes); ++j) {
       intro = smartlist_get(service->intro_nodes, j);
       safe_name = safe_str_client(intro->extend_info->nickname);
@@ -4238,62 +4299,8 @@ rend_service_dump_stats(int severity)
   }
 }
 
-#ifdef HAVE_SYS_UN_H
-
-/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t,
- * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success
- * else return -ENOSYS if AF_UNIX is not supported (see function in the
- * #else statement below). */
-static int
-add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
-{
-  tor_assert(ports);
-  tor_assert(p);
-  tor_assert(p->is_unix_addr);
-
-  smartlist_add(ports, p);
-  return 0;
-}
-
-/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0
- * on success else return -ENOSYS if AF_UNIX is not supported (see function
- * in the #else statement below). */
-static int
-set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
-{
-  tor_assert(conn);
-  tor_assert(p);
-  tor_assert(p->is_unix_addr);
-
-  conn->base_.socket_family = AF_UNIX;
-  tor_addr_make_unspec(&conn->base_.addr);
-  conn->base_.port = 1;
-  conn->base_.address = tor_strdup(p->unix_addr);
-  return 0;
-}
-
-#else /* defined(HAVE_SYS_UN_H) */
-
-static int
-set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
-{
-  (void) conn;
-  (void) p;
-  return -ENOSYS;
-}
-
-static int
-add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
-{
-  (void) ports;
-  (void) p;
-  return -ENOSYS;
-}
-
-#endif /* HAVE_SYS_UN_H */
-
 /** Given <b>conn</b>, a rendezvous exit stream, look up the hidden service for
- * 'circ', and look up the port and address based on conn-\>port.
+ * <b>circ</b>, and look up the port and address based on conn-\>port.
  * Assign the actual conn-\>addr and conn-\>port. Return -2 on failure
  * for which the circuit should be closed, -1 on other failure,
  * or 0 for success.
@@ -4304,17 +4311,15 @@ rend_service_set_connection_addr_port(edge_connection_t *conn,
 {
   rend_service_t *service;
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
-  smartlist_t *matching_ports;
-  rend_service_port_config_t *chosen_port;
-  unsigned int warn_once = 0;
+  const char *rend_pk_digest;
 
   tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
   tor_assert(circ->rend_data);
   log_debug(LD_REND,"beginning to hunt for addr/port");
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circ->rend_data, NULL);
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
-  service = rend_service_get_by_pk_digest(
-                circ->rend_data->rend_pk_digest);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Couldn't find any service associated with pk %s on "
              "rendezvous circuit %u; closing.",
@@ -4340,41 +4345,9 @@ rend_service_set_connection_addr_port(edge_connection_t *conn,
       return service->max_streams_close_circuit ? -2 : -1;
     }
   }
-  matching_ports = smartlist_new();
-  SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p,
-  {
-    if (conn->base_.port != p->virtual_port) {
-      continue;
-    }
-    if (!(p->is_unix_addr)) {
-      smartlist_add(matching_ports, p);
-    } else {
-      if (add_unix_port(matching_ports, p)) {
-        if (!warn_once) {
-         /* Unix port not supported so warn only once. */
-          log_warn(LD_REND,
-              "Saw AF_UNIX virtual port mapping for port %d on service "
-              "%s, which is unsupported on this platform. Ignoring it.",
-              conn->base_.port, serviceid);
-        }
-        warn_once++;
-      }
-    }
-  });
-  chosen_port = smartlist_choose(matching_ports);
-  smartlist_free(matching_ports);
-  if (chosen_port) {
-    if (!(chosen_port->is_unix_addr)) {
-      /* Get a non-AF_UNIX connection ready for connection_exit_connect() */
-      tor_addr_copy(&conn->base_.addr, &chosen_port->real_addr);
-      conn->base_.port = chosen_port->real_port;
-    } else {
-      if (set_unix_port(conn, chosen_port)) {
-        /* Simply impossible to end up here else we were able to add a Unix
-         * port without AF_UNIX support... ? */
-        tor_assert(0);
-      }
-    }
+
+  if (hs_set_conn_addr_port(service->ports, conn) == 0) {
+    /* Successfully set the port to the connection. We are done. */
     return 0;
   }
 
@@ -4436,3 +4409,19 @@ rend_service_non_anonymous_mode_enabled(const or_options_t *options)
   return options->HiddenServiceNonAnonymousMode ? 1 : 0;
 }
 
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+set_rend_service_list(smartlist_t *new_list)
+{
+  rend_service_list = new_list;
+}
+
+STATIC void
+set_rend_rend_service_staging_list(smartlist_t *new_list)
+{
+  rend_service_staging_list = new_list;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/or/rendservice.h b/src/or/rendservice.h
index 3b185672f6..cc872ab575 100644
--- a/src/or/rendservice.h
+++ b/src/or/rendservice.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,11 +13,9 @@
 #define TOR_RENDSERVICE_H
 
 #include "or.h"
+#include "hs_service.h"
 
 typedef struct rend_intro_cell_s rend_intro_cell_t;
-typedef struct rend_service_port_config_s rend_service_port_config_t;
-
-#ifdef RENDSERVICE_PRIVATE
 
 /* This can be used for both INTRODUCE1 and INTRODUCE2 */
 
@@ -63,6 +61,8 @@ struct rend_intro_cell_s {
   uint8_t dh[DH_KEY_LEN];
 };
 
+#ifdef RENDSERVICE_PRIVATE
+
 /** Represents a single hidden service running at this OP. */
 typedef struct rend_service_t {
   /* Fields specified in config file */
@@ -108,7 +108,7 @@ typedef struct rend_service_t {
   /** If true, we don't close circuits for making requests to unsupported
    * ports. */
   int allow_unknown_ports;
-  /** The maximum number of simultanious streams-per-circuit that are allowed
+  /** The maximum number of simultaneous streams-per-circuit that are allowed
    * to be established, or 0 if no limit is set.
    */
   int max_streams_per_circuit;
@@ -117,26 +117,36 @@ typedef struct rend_service_t {
   int max_streams_close_circuit;
 } rend_service_t;
 
-STATIC void rend_service_free(rend_service_t *service);
+STATIC void rend_service_free_(rend_service_t *service);
+#define rend_service_free(s) \
+  FREE_AND_NULL(rend_service_t, rend_service_free_, (s))
 STATIC char *rend_service_sos_poison_path(const rend_service_t *service);
-STATIC int rend_service_check_dir_and_add(smartlist_t *service_list,
-                                          const or_options_t *options,
-                                          rend_service_t *service,
-                                          int validate_only);
 STATIC int rend_service_verify_single_onion_poison(
                                                   const rend_service_t *s,
                                                   const or_options_t *options);
 STATIC int rend_service_poison_new_single_onion_dir(
                                                   const rend_service_t *s,
                                                   const or_options_t* options);
-#endif
+#ifdef TOR_UNIT_TESTS
+
+STATIC void set_rend_service_list(smartlist_t *new_list);
+STATIC void set_rend_rend_service_staging_list(smartlist_t *new_list);
+STATIC void rend_service_prune_list_impl_(void);
 
-int num_rend_services(void);
-int rend_config_services(const or_options_t *options, int validate_only);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(RENDSERVICE_PRIVATE) */
+
+int rend_num_services(void);
+int rend_config_service(const config_line_t *line_,
+                        const or_options_t *options,
+                        hs_service_config_t *config);
+void rend_service_prune_list(void);
+void rend_service_free_staging_list(void);
 int rend_service_load_all_keys(const smartlist_t *service_list);
 void rend_services_add_filenames_to_lists(smartlist_t *open_lst,
                                           smartlist_t *stat_lst);
-void rend_consider_services_intro_points(void);
+void rend_consider_services_intro_points(time_t now);
 void rend_consider_services_upload(time_t now);
 void rend_hsdir_routers_changed(void);
 void rend_consider_descriptor_republication(void);
@@ -152,13 +162,21 @@ int rend_service_receive_introduction(origin_circuit_t *circuit,
 int rend_service_decrypt_intro(rend_intro_cell_t *request,
                                crypto_pk_t *key,
                                char **err_msg_out);
-void rend_service_free_intro(rend_intro_cell_t *request);
+void rend_service_free_intro_(rend_intro_cell_t *request);
+#define rend_service_free_intro(req) do {       \
+    rend_service_free_intro_(req);              \
+    (req) = NULL;                               \
+  } while (0)
 rend_intro_cell_t * rend_service_begin_parse_intro(const uint8_t *request,
                                                    size_t request_len,
                                                    uint8_t type,
                                                    char **err_msg_out);
 int rend_service_parse_intro_plaintext(rend_intro_cell_t *intro,
                                        char **err_msg_out);
+ssize_t rend_service_encode_establish_intro_cell(char *cell_body_out,
+                                                 size_t cell_body_out_len,
+                                                 crypto_pk_t *intro_key,
+                                                 const char *rend_circ_nonce);
 int rend_service_validate_intro_late(const rend_intro_cell_t *intro,
                                      char **err_msg_out);
 void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc);
@@ -166,24 +184,22 @@ int rend_service_set_connection_addr_port(edge_connection_t *conn,
                                           origin_circuit_t *circ);
 void rend_service_dump_stats(int severity);
 void rend_service_free_all(void);
+void rend_service_init(void);
 
 rend_service_port_config_t *rend_service_parse_port_config(const char *string,
                                                            const char *sep,
                                                            char **err_msg_out);
-void rend_service_port_config_free(rend_service_port_config_t *p);
-
-void rend_authorized_client_free(rend_authorized_client_t *client);
-
-/** Return value from rend_service_add_ephemeral. */
-typedef enum {
-  RSAE_BADAUTH = -5, /**< Invalid auth_type/auth_clients */
-  RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */
-  RSAE_ADDREXISTS = -3, /**< Onion address collision */
-  RSAE_BADPRIVKEY = -2, /**< Invalid public key */
-  RSAE_INTERNAL = -1, /**< Internal error */
-  RSAE_OKAY = 0 /**< Service added as expected */
-} rend_service_add_ephemeral_status_t;
-rend_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk,
+void rend_service_port_config_free_(rend_service_port_config_t *p);
+#define rend_service_port_config_free(p) \
+  FREE_AND_NULL(rend_service_port_config_t, rend_service_port_config_free_, \
+                (p))
+
+void rend_authorized_client_free_(rend_authorized_client_t *client);
+#define rend_authorized_client_free(client) \
+  FREE_AND_NULL(rend_authorized_client_t, rend_authorized_client_free_, \
+                (client))
+
+hs_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk,
                                smartlist_t *ports,
                                int max_streams_per_circuit,
                                int max_streams_close_circuit,
@@ -201,5 +217,5 @@ int rend_service_allow_non_anonymous_connection(const or_options_t *options);
 int rend_service_reveal_startup_time(const or_options_t *options);
 int rend_service_non_anonymous_mode_enabled(const or_options_t *options);
 
-#endif
+#endif /* !defined(TOR_RENDSERVICE_H) */
 
diff --git a/src/or/rephist.c b/src/or/rephist.c
index dc86fad1d0..aa31ef973c 100644
--- a/src/or/rephist.c
+++ b/src/or/rephist.c
@@ -78,15 +78,19 @@
 #include "circuitlist.h"
 #include "circuituse.h"
 #include "config.h"
+#include "crypto_rand.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "rephist.h"
 #include "router.h"
 #include "routerlist.h"
 #include "ht.h"
+#include "channelpadding.h"
+#include "connection_or.h"
+#include "statefile.h"
 
 static void bw_arrays_init(void);
-static void predicted_ports_init(void);
+static void predicted_ports_alloc(void);
 
 typedef struct bw_array_t bw_array_t;
 STATIC uint64_t find_largest_max(bw_array_t *b);
@@ -111,38 +115,12 @@ uint32_t rephist_total_num=0;
  * 20X as much as one that ended a month ago, and routers that have had no
  * uptime data for about half a year will get forgotten.) */
 
-/** History of an OR-\>OR link. */
-typedef struct link_history_t {
-  /** When did we start tracking this list? */
-  time_t since;
-  /** When did we most recently note a change to this link */
-  time_t changed;
-  /** How many times did extending from OR1 to OR2 succeed? */
-  unsigned long n_extend_ok;
-  /** How many times did extending from OR1 to OR2 fail? */
-  unsigned long n_extend_fail;
-} link_history_t;
-
 /** History of an OR. */
 typedef struct or_history_t {
   /** When did we start tracking this OR? */
   time_t since;
   /** When did we most recently note a change to this OR? */
   time_t changed;
-  /** How many times did we successfully connect? */
-  unsigned long n_conn_ok;
-  /** How many times did we try to connect and fail?*/
-  unsigned long n_conn_fail;
-  /** How many seconds have we been connected to this OR before
-   * 'up_since'? */
-  unsigned long uptime;
-  /** How many seconds have we been unable to connect to this OR before
-   * 'down_since'? */
-  unsigned long downtime;
-  /** If nonzero, we have been connected since this time. */
-  time_t up_since;
-  /** If nonzero, we have been unable to connect since this time. */
-  time_t down_since;
 
   /** The address at which we most recently connected to this OR
    * successfully. */
@@ -164,12 +142,46 @@ typedef struct or_history_t {
   time_t start_of_downtime;
   unsigned long weighted_uptime;
   unsigned long total_weighted_time;
-
-  /** Map from hex OR2 identity digest to a link_history_t for the link
-   * from this OR to OR2. */
-  digestmap_t *link_history_map;
 } or_history_t;
 
+/**
+ * This structure holds accounting needed to calculate the padding overhead.
+ */
+typedef struct padding_counts_t {
+  /** Total number of cells we have received, including padding */
+  uint64_t read_cell_count;
+  /** Total number of cells we have sent, including padding */
+  uint64_t write_cell_count;
+  /** Total number of CELL_PADDING cells we have received */
+  uint64_t read_pad_cell_count;
+  /** Total number of CELL_PADDING cells we have sent */
+  uint64_t write_pad_cell_count;
+  /** Total number of read cells on padding-enabled conns */
+  uint64_t enabled_read_cell_count;
+  /** Total number of sent cells on padding-enabled conns */
+  uint64_t enabled_write_cell_count;
+  /** Total number of read CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_read_pad_cell_count;
+  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_write_pad_cell_count;
+  /** Total number of RELAY_DROP cells we have received */
+  uint64_t read_drop_cell_count;
+  /** Total number of RELAY_DROP cells we have sent */
+  uint64_t write_drop_cell_count;
+  /** The maximum number of padding timers we've seen in 24 hours */
+  uint64_t maximum_chanpad_timers;
+  /** When did we first copy padding_current into padding_published? */
+  char first_published_at[ISO_TIME_LEN+1];
+} padding_counts_t;
+
+/** Holds the current values of our padding statistics.
+ * It is not published until it is transferred to padding_published. */
+static padding_counts_t padding_current;
+
+/** Remains fixed for a 24 hour period, and then is replaced
+ * by a redacted copy of padding_current */
+static padding_counts_t padding_published;
+
 /** When did we last multiply all routers' weighted_run_length and
  * total_run_weights by STABILITY_ALPHA? */
 static time_t stability_last_downrated = 0;
@@ -195,7 +207,6 @@ get_or_history(const char* id)
     hist = tor_malloc_zero(sizeof(or_history_t));
     rephist_total_alloc += sizeof(or_history_t);
     rephist_total_num++;
-    hist->link_history_map = digestmap_new();
     hist->since = hist->changed = time(NULL);
     tor_addr_make_unspec(&hist->last_reached_addr);
     digestmap_set(history_map, id, hist);
@@ -203,166 +214,23 @@ get_or_history(const char* id)
   return hist;
 }
 
-/** Return the link_history_t for the link from the first named OR to
- * the second, creating it if necessary. (ORs are identified by
- * identity digest.)
- */
-static link_history_t *
-get_link_history(const char *from_id, const char *to_id)
-{
-  or_history_t *orhist;
-  link_history_t *lhist;
-  orhist = get_or_history(from_id);
-  if (!orhist)
-    return NULL;
-  if (tor_digest_is_zero(to_id))
-    return NULL;
-  lhist = digestmap_get(orhist->link_history_map, to_id);
-  if (!lhist) {
-    lhist = tor_malloc_zero(sizeof(link_history_t));
-    rephist_total_alloc += sizeof(link_history_t);
-    lhist->since = lhist->changed = time(NULL);
-    digestmap_set(orhist->link_history_map, to_id, lhist);
-  }
-  return lhist;
-}
-
-/** Helper: free storage held by a single link history entry. */
-static void
-free_link_history_(void *val)
-{
-  rephist_total_alloc -= sizeof(link_history_t);
-  tor_free(val);
-}
-
 /** Helper: free storage held by a single OR history entry. */
 static void
 free_or_history(void *_hist)
 {
   or_history_t *hist = _hist;
-  digestmap_free(hist->link_history_map, free_link_history_);
   rephist_total_alloc -= sizeof(or_history_t);
   rephist_total_num--;
   tor_free(hist);
 }
 
-/** Update an or_history_t object <b>hist</b> so that its uptime/downtime
- * count is up-to-date as of <b>when</b>.
- */
-static void
-update_or_history(or_history_t *hist, time_t when)
-{
-  tor_assert(hist);
-  if (hist->up_since) {
-    tor_assert(!hist->down_since);
-    hist->uptime += (when - hist->up_since);
-    hist->up_since = when;
-  } else if (hist->down_since) {
-    hist->downtime += (when - hist->down_since);
-    hist->down_since = when;
-  }
-}
-
 /** Initialize the static data structures for tracking history. */
 void
 rep_hist_init(void)
 {
   history_map = digestmap_new();
   bw_arrays_init();
-  predicted_ports_init();
-}
-
-/** Helper: note that we are no longer connected to the router with history
- * <b>hist</b>.  If <b>failed</b>, the connection failed; otherwise, it was
- * closed correctly. */
-static void
-mark_or_down(or_history_t *hist, time_t when, int failed)
-{
-  if (hist->up_since) {
-    hist->uptime += (when - hist->up_since);
-    hist->up_since = 0;
-  }
-  if (failed && !hist->down_since) {
-    hist->down_since = when;
-  }
-}
-
-/** Helper: note that we are connected to the router with history
- * <b>hist</b>. */
-static void
-mark_or_up(or_history_t *hist, time_t when)
-{
-  if (hist->down_since) {
-    hist->downtime += (when - hist->down_since);
-    hist->down_since = 0;
-  }
-  if (!hist->up_since) {
-    hist->up_since = when;
-  }
-}
-
-/** Remember that an attempt to connect to the OR with identity digest
- * <b>id</b> failed at <b>when</b>.
- */
-void
-rep_hist_note_connect_failed(const char* id, time_t when)
-{
-  or_history_t *hist;
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  ++hist->n_conn_fail;
-  mark_or_down(hist, when, 1);
-  hist->changed = when;
-}
-
-/** Remember that an attempt to connect to the OR with identity digest
- * <b>id</b> succeeded at <b>when</b>.
- */
-void
-rep_hist_note_connect_succeeded(const char* id, time_t when)
-{
-  or_history_t *hist;
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  ++hist->n_conn_ok;
-  mark_or_up(hist, when);
-  hist->changed = when;
-}
-
-/** Remember that we intentionally closed our connection to the OR
- * with identity digest <b>id</b> at <b>when</b>.
- */
-void
-rep_hist_note_disconnect(const char* id, time_t when)
-{
-  or_history_t *hist;
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  mark_or_down(hist, when, 0);
-  hist->changed = when;
-}
-
-/** Remember that our connection to the OR with identity digest
- * <b>id</b> had an error and stopped working at <b>when</b>.
- */
-void
-rep_hist_note_connection_died(const char* id, time_t when)
-{
-  or_history_t *hist;
-  if (!id) {
-    /* If conn has no identity, it didn't complete its handshake, or something
-     * went wrong.  Ignore it.
-     */
-    return;
-  }
-  hist = get_or_history(id);
-  if (!hist)
-    return;
-  mark_or_down(hist, when, 1);
-  hist->changed = when;
+  predicted_ports_alloc();
 }
 
 /** We have just decided that this router with identity digest <b>id</b> is
@@ -501,7 +369,6 @@ rep_hist_make_router_pessimal(const char *id, time_t when)
   tor_assert(hist);
 
   rep_hist_note_router_unreachable(id, when);
-  mark_or_down(hist, when, 1);
 
   hist->weighted_run_length = 0;
   hist->weighted_uptime = 0;
@@ -678,57 +545,17 @@ rep_hist_have_measured_enough_stability(void)
   return started_tracking_stability < time(NULL) - 4*60*60;
 }
 
-/** Remember that we successfully extended from the OR with identity
- * digest <b>from_id</b> to the OR with identity digest
- * <b>to_name</b>.
- */
-void
-rep_hist_note_extend_succeeded(const char *from_id, const char *to_id)
-{
-  link_history_t *hist;
-  /* log_fn(LOG_WARN, "EXTEND SUCCEEDED: %s->%s",from_name,to_name); */
-  hist = get_link_history(from_id, to_id);
-  if (!hist)
-    return;
-  ++hist->n_extend_ok;
-  hist->changed = time(NULL);
-}
-
-/** Remember that we tried to extend from the OR with identity digest
- * <b>from_id</b> to the OR with identity digest <b>to_name</b>, but
- * failed.
- */
-void
-rep_hist_note_extend_failed(const char *from_id, const char *to_id)
-{
-  link_history_t *hist;
-  /* log_fn(LOG_WARN, "EXTEND FAILED: %s->%s",from_name,to_name); */
-  hist = get_link_history(from_id, to_id);
-  if (!hist)
-    return;
-  ++hist->n_extend_fail;
-  hist->changed = time(NULL);
-}
-
 /** Log all the reliability data we have remembered, with the chosen
  * severity.
  */
 void
 rep_hist_dump_stats(time_t now, int severity)
 {
-  digestmap_iter_t *lhist_it;
   digestmap_iter_t *orhist_it;
-  const char *name1, *name2, *digest1, *digest2;
+  const char *name1, *digest1;
   char hexdigest1[HEX_DIGEST_LEN+1];
-  char hexdigest2[HEX_DIGEST_LEN+1];
   or_history_t *or_history;
-  link_history_t *link_history;
-  void *or_history_p, *link_history_p;
-  double uptime;
-  char buffer[2048];
-  size_t len;
-  int ret;
-  unsigned long upt, downt;
+  void *or_history_p;
   const node_t *node;
 
   rep_history_clean(now - get_options()->RephistTrackTime);
@@ -748,52 +575,12 @@ rep_hist_dump_stats(time_t now, int severity)
     else
       name1 = "(unknown)";
     base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
-    update_or_history(or_history, now);
-    upt = or_history->uptime;
-    downt = or_history->downtime;
     s = get_stability(or_history, now);
     stability = (long)s;
-    if (upt+downt) {
-      uptime = ((double)upt) / (upt+downt);
-    } else {
-      uptime=1.0;
-    }
     tor_log(severity, LD_HIST,
-        "OR %s [%s]: %ld/%ld good connections; uptime %ld/%ld sec (%.2f%%); "
-        "wmtbf %lu:%02lu:%02lu",
+        "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
         name1, hexdigest1,
-        or_history->n_conn_ok, or_history->n_conn_fail+or_history->n_conn_ok,
-        upt, upt+downt, uptime*100.0,
         stability/3600, (stability/60)%60, stability%60);
-
-    if (!digestmap_isempty(or_history->link_history_map)) {
-      strlcpy(buffer, "    Extend attempts: ", sizeof(buffer));
-      len = strlen(buffer);
-      for (lhist_it = digestmap_iter_init(or_history->link_history_map);
-           !digestmap_iter_done(lhist_it);
-           lhist_it = digestmap_iter_next(or_history->link_history_map,
-                                          lhist_it)) {
-        digestmap_iter_get(lhist_it, &digest2, &link_history_p);
-        if ((node = node_get_by_id(digest2)) && node_get_nickname(node))
-          name2 = node_get_nickname(node);
-        else
-          name2 = "(unknown)";
-
-        link_history = (link_history_t*) link_history_p;
-
-        base16_encode(hexdigest2, sizeof(hexdigest2), digest2, DIGEST_LEN);
-        ret = tor_snprintf(buffer+len, 2048-len, "%s [%s](%ld/%ld); ",
-                        name2,
-                        hexdigest2,
-                        link_history->n_extend_ok,
-                        link_history->n_extend_ok+link_history->n_extend_fail);
-        if (ret<0)
-          break;
-        else
-          len += ret;
-      }
-      tor_log(severity, LD_HIST, "%s", buffer);
-    }
   }
 }
 
@@ -805,10 +592,9 @@ rep_history_clean(time_t before)
 {
   int authority = authdir_mode(get_options());
   or_history_t *or_history;
-  link_history_t *link_history;
-  void *or_history_p, *link_history_p;
-  digestmap_iter_t *orhist_it, *lhist_it;
-  const char *d1, *d2;
+  void *or_history_p;
+  digestmap_iter_t *orhist_it;
+  const char *d1;
 
   orhist_it = digestmap_iter_init(history_map);
   while (!digestmap_iter_done(orhist_it)) {
@@ -825,19 +611,6 @@ rep_history_clean(time_t before)
       free_or_history(or_history);
       continue;
     }
-    for (lhist_it = digestmap_iter_init(or_history->link_history_map);
-         !digestmap_iter_done(lhist_it); ) {
-      digestmap_iter_get(lhist_it, &d2, &link_history_p);
-      link_history = link_history_p;
-      if (link_history->changed < before) {
-        lhist_it = digestmap_iter_next_rmv(or_history->link_history_map,
-                                           lhist_it);
-        rephist_total_alloc -= sizeof(link_history_t);
-        tor_free(link_history);
-        continue;
-      }
-      lhist_it = digestmap_iter_next(or_history->link_history_map,lhist_it);
-    }
     orhist_it = digestmap_iter_next(history_map, orhist_it);
   }
 }
@@ -910,9 +683,9 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down)
     base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
 
     if (missing_means_down && hist->start_of_run &&
-        !router_get_by_id_digest(digest)) {
+        !connection_or_digest_is_known_relay(digest)) {
       /* We think this relay is running, but it's not listed in our
-       * routerlist. Somehow it fell out without telling us it went
+       * consensus. Somehow it fell out without telling us it went
        * down. Complain and also correct it. */
       log_info(LD_HIST,
                "Relay '%s' is listed as up in rephist, but it's not in "
@@ -1321,9 +1094,12 @@ bw_array_new(void)
   return b;
 }
 
+#define bw_array_free(val) \
+  FREE_AND_NULL(bw_array_t, bw_array_free_, (val))
+
 /** Free storage held by bandwidth array <b>b</b>. */
 static void
-bw_array_free(bw_array_t *b)
+bw_array_free_(bw_array_t *b)
 {
   if (!b) {
     return;
@@ -1763,6 +1539,42 @@ typedef struct predicted_port_t {
 
 /** A list of port numbers that have been used recently. */
 static smartlist_t *predicted_ports_list=NULL;
+/** How long do we keep predicting circuits? */
+static int prediction_timeout=0;
+/** When was the last time we added a prediction entry (HS or port) */
+static time_t last_prediction_add_time=0;
+
+/**
+ * How much time left until we stop predicting circuits?
+ */
+int
+predicted_ports_prediction_time_remaining(time_t now)
+{
+  time_t idle_delta;
+
+  /* Protect against overflow of return value. This can happen if the clock
+   * jumps backwards in time. Update the last prediction time (aka last
+   * active time) to prevent it. This update is preferable to using monotonic
+   * time because it prevents clock jumps into the past from simply causing
+   * very long idle timeouts while the monotonic time stands still. */
+  if (last_prediction_add_time > now) {
+    last_prediction_add_time = now;
+    idle_delta = 0;
+  } else {
+    idle_delta = now - last_prediction_add_time;
+  }
+
+  /* Protect against underflow of the return value. This can happen for very
+   * large periods of inactivity/system sleep. */
+  if (idle_delta > prediction_timeout)
+    return 0;
+
+  if (BUG((prediction_timeout - idle_delta) > INT_MAX)) {
+    return INT_MAX;
+  }
+
+  return (int)(prediction_timeout - idle_delta);
+}
 
 /** We just got an application request for a connection with
  * port <b>port</b>. Remember it for the future, so we can keep
@@ -1772,28 +1584,47 @@ static void
 add_predicted_port(time_t now, uint16_t port)
 {
   predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t));
+
+  //  If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
   pp->port = port;
   pp->time = now;
   rephist_total_alloc += sizeof(*pp);
   smartlist_add(predicted_ports_list, pp);
 }
 
-/** Initialize whatever memory and structs are needed for predicting
+/**
+ * Allocate whatever memory and structs are needed for predicting
  * which ports will be used. Also seed it with port 80, so we'll build
  * circuits on start-up.
  */
 static void
-predicted_ports_init(void)
+predicted_ports_alloc(void)
 {
   predicted_ports_list = smartlist_new();
-  add_predicted_port(time(NULL), 80); /* add one to kickstart us */
+}
+
+void
+predicted_ports_init(void)
+{
+  add_predicted_port(time(NULL), 443); // Add a port to get us started
 }
 
 /** Free whatever memory is needed for predicting which ports will
  * be used.
  */
 static void
-predicted_ports_free(void)
+predicted_ports_free_all(void)
 {
   rephist_total_alloc -=
     smartlist_len(predicted_ports_list)*sizeof(predicted_port_t);
@@ -1817,6 +1648,12 @@ rep_hist_note_used_port(time_t now, uint16_t port)
   SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
     if (pp->port == port) {
       pp->time = now;
+
+      last_prediction_add_time = now;
+      log_info(LD_CIRC,
+               "New port prediction added. Will continue predictive circ "
+               "building for %d more seconds.",
+               predicted_ports_prediction_time_remaining(now));
       return;
     }
   } SMARTLIST_FOREACH_END(pp);
@@ -1833,7 +1670,8 @@ rep_hist_get_predicted_ports(time_t now)
   int predicted_circs_relevance_time;
   smartlist_t *out = smartlist_new();
   tor_assert(predicted_ports_list);
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
+
+  predicted_circs_relevance_time = prediction_timeout;
 
   /* clean out obsolete entries */
   SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
@@ -1893,6 +1731,18 @@ static time_t predicted_internal_capacity_time = 0;
 void
 rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity)
 {
+  // If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
   predicted_internal_time = now;
   if (need_uptime)
     predicted_internal_uptime_time = now;
@@ -1906,7 +1756,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime,
                                 int *need_capacity)
 {
   int predicted_circs_relevance_time;
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
+
+  predicted_circs_relevance_time = prediction_timeout;
 
   if (!predicted_internal_time) { /* initialize it */
     predicted_internal_time = now;
@@ -1928,7 +1779,7 @@ int
 any_predicted_circuits(time_t now)
 {
   int predicted_circs_relevance_time;
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
+  predicted_circs_relevance_time = prediction_timeout;
 
   return smartlist_len(predicted_ports_list) ||
          predicted_internal_time + predicted_circs_relevance_time >= now;
@@ -1954,105 +1805,6 @@ rep_hist_circbuilding_dormant(time_t now)
   return 1;
 }
 
-/** Structure to track how many times we've done each public key operation. */
-static struct {
-  /** How many directory objects have we signed? */
-  unsigned long n_signed_dir_objs;
-  /** How many routerdescs have we signed? */
-  unsigned long n_signed_routerdescs;
-  /** How many directory objects have we verified? */
-  unsigned long n_verified_dir_objs;
-  /** How many routerdescs have we verified */
-  unsigned long n_verified_routerdescs;
-  /** How many onionskins have we encrypted to build circuits? */
-  unsigned long n_onionskins_encrypted;
-  /** How many onionskins have we decrypted to do circuit build requests? */
-  unsigned long n_onionskins_decrypted;
-  /** How many times have we done the TLS handshake as a client? */
-  unsigned long n_tls_client_handshakes;
-  /** How many times have we done the TLS handshake as a server? */
-  unsigned long n_tls_server_handshakes;
-  /** How many PK operations have we done as a hidden service client? */
-  unsigned long n_rend_client_ops;
-  /** How many PK operations have we done as a hidden service midpoint? */
-  unsigned long n_rend_mid_ops;
-  /** How many PK operations have we done as a hidden service provider? */
-  unsigned long n_rend_server_ops;
-} pk_op_counts = {0,0,0,0,0,0,0,0,0,0,0};
-
-/** Increment the count of the number of times we've done <b>operation</b>. */
-void
-note_crypto_pk_op(pk_op_t operation)
-{
-  switch (operation)
-    {
-    case SIGN_DIR:
-      pk_op_counts.n_signed_dir_objs++;
-      break;
-    case SIGN_RTR:
-      pk_op_counts.n_signed_routerdescs++;
-      break;
-    case VERIFY_DIR:
-      pk_op_counts.n_verified_dir_objs++;
-      break;
-    case VERIFY_RTR:
-      pk_op_counts.n_verified_routerdescs++;
-      break;
-    case ENC_ONIONSKIN:
-      pk_op_counts.n_onionskins_encrypted++;
-      break;
-    case DEC_ONIONSKIN:
-      pk_op_counts.n_onionskins_decrypted++;
-      break;
-    case TLS_HANDSHAKE_C:
-      pk_op_counts.n_tls_client_handshakes++;
-      break;
-    case TLS_HANDSHAKE_S:
-      pk_op_counts.n_tls_server_handshakes++;
-      break;
-    case REND_CLIENT:
-      pk_op_counts.n_rend_client_ops++;
-      break;
-    case REND_MID:
-      pk_op_counts.n_rend_mid_ops++;
-      break;
-    case REND_SERVER:
-      pk_op_counts.n_rend_server_ops++;
-      break;
-    default:
-      log_warn(LD_BUG, "Unknown pk operation %d", operation);
-  }
-}
-
-/** Log the number of times we've done each public/private-key operation. */
-void
-dump_pk_ops(int severity)
-{
-  tor_log(severity, LD_HIST,
-      "PK operations: %lu directory objects signed, "
-      "%lu directory objects verified, "
-      "%lu routerdescs signed, "
-      "%lu routerdescs verified, "
-      "%lu onionskins encrypted, "
-      "%lu onionskins decrypted, "
-      "%lu client-side TLS handshakes, "
-      "%lu server-side TLS handshakes, "
-      "%lu rendezvous client operations, "
-      "%lu rendezvous middle operations, "
-      "%lu rendezvous server operations.",
-      pk_op_counts.n_signed_dir_objs,
-      pk_op_counts.n_verified_dir_objs,
-      pk_op_counts.n_signed_routerdescs,
-      pk_op_counts.n_verified_routerdescs,
-      pk_op_counts.n_onionskins_encrypted,
-      pk_op_counts.n_onionskins_decrypted,
-      pk_op_counts.n_tls_client_handshakes,
-      pk_op_counts.n_tls_server_handshakes,
-      pk_op_counts.n_rend_client_ops,
-      pk_op_counts.n_rend_mid_ops,
-      pk_op_counts.n_rend_server_ops);
-}
-
 /*** Exit port statistics ***/
 
 /* Some constants */
@@ -2541,7 +2293,7 @@ rep_hist_format_buffer_stats(time_t now)
                processed_cells_string,
                queued_cells_string,
                time_in_queue_string,
-               (number_of_circuits + SHARES - 1) / SHARES);
+               CEIL_DIV(number_of_circuits, SHARES));
   tor_free(processed_cells_string);
   tor_free(queued_cells_string);
   tor_free(time_in_queue_string);
@@ -2720,7 +2472,7 @@ rep_hist_desc_stats_write(time_t now)
 }
 
 /** Called to note that we've served a given descriptor (by
- * digest). Incrememnts the count of descriptors served, and the number
+ * digest). Increments the count of descriptors served, and the number
  * of times we've served this descriptor. */
 void
 rep_hist_note_desc_served(const char * desc)
@@ -2817,7 +2569,7 @@ HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
 
 /* DOCDOC bidi_map_free */
 static void
-bidi_map_free(void)
+bidi_map_free_all(void)
 {
   bidi_map_entry_t **ptr, **next, *ent;
   for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
@@ -2837,7 +2589,7 @@ rep_hist_reset_conn_stats(time_t now)
   mostly_read = 0;
   mostly_written = 0;
   both_read_and_written = 0;
-  bidi_map_free();
+  bidi_map_free_all();
 }
 
 /** Stop collecting connection stats in a way that we can re-start doing
@@ -3026,9 +2778,12 @@ hs_stats_new(void)
   return new_hs_stats;
 }
 
+#define hs_stats_free(val) \
+  FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val))
+
 /** Free an hs_stats_t structure. */
 static void
-hs_stats_free(hs_stats_t *victim_hs_stats)
+hs_stats_free_(hs_stats_t *victim_hs_stats)
 {
   if (!victim_hs_stats) {
     return;
@@ -3215,8 +2970,7 @@ rep_hist_hs_stats_write(time_t now)
   return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
 }
 
-#define MAX_LINK_PROTO_TO_LOG 4
-static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2];
+static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
 
 /** Note that we negotiated link protocol version <b>link_proto</b>, on
  * a connection that started here iff <b>started_here</b> is true.
@@ -3225,7 +2979,7 @@ void
 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
 {
   started_here = !!started_here; /* force to 0 or 1 */
-  if (link_proto > MAX_LINK_PROTO_TO_LOG) {
+  if (link_proto > MAX_LINK_PROTO) {
     log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
     return;
   }
@@ -3233,6 +2987,165 @@ rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
   link_proto_count[link_proto][started_here]++;
 }
 
+/**
+ * Update the maximum count of total pending channel padding timers
+ * in this period.
+ */
+void
+rep_hist_padding_count_timers(uint64_t num_timers)
+{
+  if (num_timers > padding_current.maximum_chanpad_timers) {
+    padding_current.maximum_chanpad_timers = num_timers;
+  }
+}
+
+/**
+ * Count a cell that we sent for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_write(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.write_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.write_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_write_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Count a cell that we've received for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_read(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.read_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.read_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_read_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Reset our current padding statistics. Called once every 24 hours.
+ */
+void
+rep_hist_reset_padding_counts(void)
+{
+  memset(&padding_current, 0, sizeof(padding_current));
+}
+
+/**
+ * Copy our current cell counts into a structure for listing in our
+ * extra-info descriptor. Also perform appropriate rounding and redaction.
+ *
+ * This function is called once every 24 hours.
+ */
+#define MIN_CELL_COUNTS_TO_PUBLISH 1
+#define ROUND_CELL_COUNTS_TO 10000
+void
+rep_hist_prep_published_padding_counts(time_t now)
+{
+  memcpy(&padding_published, &padding_current, sizeof(padding_published));
+
+  if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH ||
+      padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) {
+    memset(&padding_published, 0, sizeof(padding_published));
+    return;
+  }
+
+  format_iso_time(padding_published.first_published_at, now);
+#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
+                                      ROUND_CELL_COUNTS_TO)
+  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
+#undef ROUND_AND_SET_COUNT
+}
+
+/**
+ * Returns an allocated string for extra-info documents for publishing
+ * padding statistics from the last 24 hour interval.
+ */
+char *
+rep_hist_get_padding_count_lines(void)
+{
+  char *result = NULL;
+
+  if (!padding_published.read_cell_count ||
+          !padding_published.write_cell_count) {
+    return NULL;
+  }
+
+  tor_asprintf(&result, "padding-counts %s (%d s)"
+                        " bin-size="U64_FORMAT
+                        " write-drop="U64_FORMAT
+                        " write-pad="U64_FORMAT
+                        " write-total="U64_FORMAT
+                        " read-drop="U64_FORMAT
+                        " read-pad="U64_FORMAT
+                        " read-total="U64_FORMAT
+                        " enabled-read-pad="U64_FORMAT
+                        " enabled-read-total="U64_FORMAT
+                        " enabled-write-pad="U64_FORMAT
+                        " enabled-write-total="U64_FORMAT
+                        " max-chanpad-timers="U64_FORMAT
+                        "\n",
+               padding_published.first_published_at,
+               REPHIST_CELL_PADDING_COUNTS_INTERVAL,
+               U64_PRINTF_ARG(ROUND_CELL_COUNTS_TO),
+               U64_PRINTF_ARG(padding_published.write_drop_cell_count),
+               U64_PRINTF_ARG(padding_published.write_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.write_cell_count),
+               U64_PRINTF_ARG(padding_published.read_drop_cell_count),
+               U64_PRINTF_ARG(padding_published.read_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.read_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_read_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_read_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_write_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_write_cell_count),
+               U64_PRINTF_ARG(padding_published.maximum_chanpad_timers)
+               );
+
+  return result;
+}
+
 /** Log a heartbeat message explaining how many connections of each link
  * protocol version we have used.
  */
@@ -3282,8 +3195,8 @@ rep_hist_free_all(void)
   tor_free(exit_bytes_read);
   tor_free(exit_bytes_written);
   tor_free(exit_streams);
-  predicted_ports_free();
-  bidi_map_free();
+  predicted_ports_free_all();
+  bidi_map_free_all();
 
   if (circuits_for_buffer_stats) {
     SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s,
diff --git a/src/or/rephist.h b/src/or/rephist.h
index c464b34f7c..9ef49d19c0 100644
--- a/src/or/rephist.h
+++ b/src/or/rephist.h
@@ -13,13 +13,6 @@
 #define TOR_REPHIST_H
 
 void rep_hist_init(void);
-void rep_hist_note_connect_failed(const char* nickname, time_t when);
-void rep_hist_note_connect_succeeded(const char* nickname, time_t when);
-void rep_hist_note_disconnect(const char* nickname, time_t when);
-void rep_hist_note_connection_died(const char* nickname, time_t when);
-void rep_hist_note_extend_succeeded(const char *from_name,
-                                    const char *to_name);
-void rep_hist_note_extend_failed(const char *from_name, const char *to_name);
 void rep_hist_dump_stats(time_t now, int severity);
 void rep_hist_note_bytes_read(uint64_t num_bytes, time_t when);
 void rep_hist_note_bytes_written(uint64_t num_bytes, time_t when);
@@ -48,6 +41,7 @@ double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when);
 long rep_hist_get_weighted_time_known(const char *id, time_t when);
 int rep_hist_have_measured_enough_stability(void);
 
+void predicted_ports_init(void);
 void rep_hist_note_used_port(time_t now, uint16_t port);
 smartlist_t *rep_hist_get_predicted_ports(time_t now);
 void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports);
@@ -59,9 +53,7 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime,
 
 int any_predicted_circuits(time_t now);
 int rep_hist_circbuilding_dormant(time_t now);
-
-void note_crypto_pk_op(pk_op_t operation);
-void dump_pk_ops(int severity);
+int predicted_ports_prediction_time_remaining(time_t now);
 
 void rep_hist_exit_stats_init(time_t now);
 void rep_hist_reset_exit_stats(time_t now);
@@ -121,5 +113,29 @@ typedef struct bw_array_t bw_array_t;
 extern bw_array_t *write_array;
 #endif
 
-#endif
-
+/**
+ * Represents the type of a cell for padding accounting
+ */
+typedef enum padding_type_t {
+    /** A RELAY_DROP cell */
+    PADDING_TYPE_DROP,
+    /** A CELL_PADDING cell */
+    PADDING_TYPE_CELL,
+    /** Total counts of padding and non-padding together */
+    PADDING_TYPE_TOTAL,
+    /** Total cell counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_TOTAL,
+    /** CELL_PADDING counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_CELL
+} padding_type_t;
+
+/** The amount of time over which the padding cell counts were counted */
+#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60)
+void rep_hist_padding_count_read(padding_type_t type);
+void rep_hist_padding_count_write(padding_type_t type);
+char *rep_hist_get_padding_count_lines(void);
+void rep_hist_reset_padding_counts(void);
+void rep_hist_prep_published_padding_counts(time_t now);
+void rep_hist_padding_count_timers(uint64_t num_timers);
+
+#endif /* !defined(TOR_REPHIST_H) */
diff --git a/src/or/replaycache.c b/src/or/replaycache.c
index 8290fa6964..a9a6709937 100644
--- a/src/or/replaycache.c
+++ b/src/or/replaycache.c
@@ -1,4 +1,4 @@
- /* Copyright (c) 2012-2016, The Tor Project, Inc. */
+ /* Copyright (c) 2012-2017, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 /**
@@ -26,9 +26,8 @@
 
 /** Free the replaycache r and all of its entries.
  */
-
 void
-replaycache_free(replaycache_t *r)
+replaycache_free_(replaycache_t *r)
 {
   if (!r) {
     log_info(LD_BUG, "replaycache_free() called on NULL");
@@ -44,7 +43,6 @@ replaycache_free(replaycache_t *r)
  * for entries to age out and interval is the time after which the cache
  * should be scrubbed for old entries.
  */
-
 replaycache_t *
 replaycache_new(time_t horizon, time_t interval)
 {
@@ -72,9 +70,8 @@ replaycache_new(time_t horizon, time_t interval)
   return r;
 }
 
-/** See documentation for replaycache_add_and_test()
+/** See documentation for replaycache_add_and_test().
  */
-
 STATIC int
 replaycache_add_and_test_internal(
     time_t present, replaycache_t *r, const void *data, size_t len,
@@ -136,9 +133,8 @@ replaycache_add_and_test_internal(
   return rv;
 }
 
-/** See documentation for replaycache_scrub_if_needed()
+/** See documentation for replaycache_scrub_if_needed().
  */
-
 STATIC void
 replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r)
 {
@@ -186,7 +182,6 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r)
  * and the function will return 1 if it was already seen within the cache's
  * horizon, or 0 otherwise.
  */
-
 int
 replaycache_add_and_test(replaycache_t *r, const void *data, size_t len)
 {
@@ -196,7 +191,6 @@ replaycache_add_and_test(replaycache_t *r, const void *data, size_t len)
 /** Like replaycache_add_and_test(), but if it's a hit also return the time
  * elapsed since this digest was last seen.
  */
-
 int
 replaycache_add_test_and_elapsed(
     replaycache_t *r, const void *data, size_t len, time_t *elapsed)
@@ -207,7 +201,6 @@ replaycache_add_test_and_elapsed(
 /** Scrub aged entries out of r if sufficiently long has elapsed since r was
  * last scrubbed.
  */
-
 void
 replaycache_scrub_if_needed(replaycache_t *r)
 {
diff --git a/src/or/replaycache.h b/src/or/replaycache.h
index 64a6caf5f5..81a8d907fd 100644
--- a/src/or/replaycache.h
+++ b/src/or/replaycache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,11 +29,13 @@ struct replaycache_s {
   digest256map_t *digests_seen;
 };
 
-#endif /* REPLAYCACHE_PRIVATE */
+#endif /* defined(REPLAYCACHE_PRIVATE) */
 
 /* replaycache_t free/new */
 
-void replaycache_free(replaycache_t *r);
+void replaycache_free_(replaycache_t *r);
+#define replaycache_free(r) \
+  FREE_AND_NULL(replaycache_t, replaycache_free_, (r))
 replaycache_t * replaycache_new(time_t horizon, time_t interval);
 
 #ifdef REPLAYCACHE_PRIVATE
@@ -51,7 +53,7 @@ STATIC int replaycache_add_and_test_internal(
 STATIC void replaycache_scrub_if_needed_internal(
     time_t present, replaycache_t *r);
 
-#endif /* REPLAYCACHE_PRIVATE */
+#endif /* defined(REPLAYCACHE_PRIVATE) */
 
 /*
  * replaycache_t methods
@@ -62,5 +64,5 @@ int replaycache_add_test_and_elapsed(
     replaycache_t *r, const void *data, size_t len, time_t *elapsed);
 void replaycache_scrub_if_needed(replaycache_t *r);
 
-#endif
+#endif /* !defined(TOR_REPLAYCACHE_H) */
 
diff --git a/src/or/router.c b/src/or/router.c
index c416474226..2f7d8349f9 100644
--- a/src/or/router.c
+++ b/src/or/router.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTER_PRIVATE
@@ -13,6 +13,8 @@
 #include "config.h"
 #include "connection.h"
 #include "control.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "crypto_curve25519.h"
 #include "directory.h"
 #include "dirserv.h"
@@ -35,6 +37,8 @@
 #include "transports.h"
 #include "routerset.h"
 
+#include "dirauth/mode.h"
+
 /**
  * \file router.c
  * \brief Miscellaneous relay functionality, including RSA key maintenance,
@@ -103,6 +107,64 @@ static authority_cert_t *legacy_key_certificate = NULL;
  * used by tor-gencert to sign new signing keys and make new key
  * certificates. */
 
+const char *format_node_description(char *buf,
+                                    const char *id_digest,
+                                    int is_named,
+                                    const char *nickname,
+                                    const tor_addr_t *addr,
+                                    uint32_t addr32h);
+
+/** Return a readonly string with human readable description
+ * of <b>err</b>.
+ */
+const char *
+routerinfo_err_to_string(int err)
+{
+  switch (err) {
+    case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+      return "No known exit address yet";
+    case TOR_ROUTERINFO_ERROR_CANNOT_PARSE:
+      return "Cannot parse descriptor";
+    case TOR_ROUTERINFO_ERROR_NOT_A_SERVER:
+      return "Not running in server mode";
+    case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+      return "Key digest failed";
+    case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+      return "Cannot generate descriptor";
+    case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
+      return "Descriptor still rebuilding - not ready yet";
+  }
+
+  log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
+  tor_assert_unreached();
+
+  return "Unknown error";
+}
+
+/** Return true if we expect given error to be transient.
+ * Return false otherwise.
+ */
+int
+routerinfo_err_is_transient(int err)
+{
+  switch (err) {
+    case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+      return 1;
+    case TOR_ROUTERINFO_ERROR_CANNOT_PARSE:
+      return 1;
+    case TOR_ROUTERINFO_ERROR_NOT_A_SERVER:
+      return 0;
+    case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+      return 0; // XXX: bug?
+    case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+      return 1;
+    case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
+      return 1;
+  }
+
+  return 0;
+}
+
 /** Replace the current onion key with <b>k</b>.  Does not affect
  * lastonionkey; to update lastonionkey correctly, call rotate_onion_key().
  */
@@ -131,7 +193,8 @@ get_onion_key(void)
 }
 
 /** Store a full copy of the current onion key into *<b>key</b>, and a full
- * copy of the most recent onion key into *<b>last</b>.
+ * copy of the most recent onion key into *<b>last</b>.  Store NULL into
+ * a pointer if the corresponding key does not exist.
  */
 void
 dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
@@ -139,8 +202,10 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
   tor_assert(key);
   tor_assert(last);
   tor_mutex_acquire(key_lock);
-  tor_assert(onionkey);
-  *key = crypto_pk_copy_full(onionkey);
+  if (onionkey)
+    *key = crypto_pk_copy_full(onionkey);
+  else
+    *key = NULL;
   if (lastonionkey)
     *last = crypto_pk_copy_full(lastonionkey);
   else
@@ -148,6 +213,51 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
   tor_mutex_release(key_lock);
 }
 
+/** Expire our old set of onion keys. This is done by setting
+ * last_curve25519_onion_key and lastonionkey to all zero's and NULL
+ * respectively.
+ *
+ * This function does not perform any grace period checks for the old onion
+ * keys.
+ */
+void
+expire_old_onion_keys(void)
+{
+  char *fname = NULL;
+
+  tor_mutex_acquire(key_lock);
+
+  /* Free lastonionkey and set it to NULL. */
+  if (lastonionkey) {
+    crypto_pk_free(lastonionkey);
+    lastonionkey = NULL;
+  }
+
+  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+   * construct_ntor_key_map() below. */
+  memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+  tor_mutex_release(key_lock);
+
+  fname = get_keydir_fname("secret_onion_key.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+
+  fname = get_keydir_fname("secret_onion_key_ntor.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+}
+
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
 static const curve25519_keypair_t *
@@ -162,10 +272,14 @@ construct_ntor_key_map(void)
 {
   di_digest256_map_t *m = NULL;
 
-  dimap_add_entry(&m,
-                  curve25519_onion_key.pubkey.public_key,
-                  tor_memdup(&curve25519_onion_key,
-                             sizeof(curve25519_keypair_t)));
+  if (!tor_mem_is_zero((const char*)
+                       curve25519_onion_key.pubkey.public_key,
+                       CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m,
+                    curve25519_onion_key.pubkey.public_key,
+                    tor_memdup(&curve25519_onion_key,
+                               sizeof(curve25519_keypair_t)));
+  }
   if (!tor_mem_is_zero((const char*)
                           last_curve25519_onion_key.pubkey.public_key,
                        CURVE25519_PUBKEY_LEN)) {
@@ -188,7 +302,7 @@ ntor_key_map_free_helper(void *arg)
 }
 /** Release all storage from a keymap returned by construct_ntor_key_map. */
 void
-ntor_key_map_free(di_digest256_map_t *map)
+ntor_key_map_free_(di_digest256_map_t *map)
 {
   if (!map)
     return;
@@ -212,7 +326,11 @@ set_server_identity_key(crypto_pk_t *k)
 {
   crypto_pk_free(server_identitykey);
   server_identitykey = k;
-  crypto_pk_get_digest(server_identitykey, server_identitykey_digest);
+  if (crypto_pk_get_digest(server_identitykey,
+                           server_identitykey_digest) < 0) {
+    log_err(LD_BUG, "Couldn't compute our own identity key digest.");
+    tor_assert(0);
+  }
 }
 
 /** Make sure that we have set up our identity keys to match or not match as
@@ -329,8 +447,8 @@ rotate_onion_key(void)
   or_state_t *state = get_or_state();
   curve25519_keypair_t new_curve25519_keypair;
   time_t now;
-  fname = get_datadir_fname2("keys", "secret_onion_key");
-  fname_prev = get_datadir_fname2("keys", "secret_onion_key.old");
+  fname = get_keydir_fname("secret_onion_key");
+  fname_prev = get_keydir_fname("secret_onion_key.old");
   /* There isn't much point replacing an old key with an empty file */
   if (file_status(fname) == FN_FILE) {
     if (replace_file(fname, fname_prev))
@@ -350,8 +468,8 @@ rotate_onion_key(void)
   }
   tor_free(fname);
   tor_free(fname_prev);
-  fname = get_datadir_fname2("keys", "secret_onion_key_ntor");
-  fname_prev = get_datadir_fname2("keys", "secret_onion_key_ntor.old");
+  fname = get_keydir_fname("secret_onion_key_ntor");
+  fname_prev = get_keydir_fname("secret_onion_key_ntor.old");
   if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0)
     goto error;
   /* There isn't much point replacing an old key with an empty file */
@@ -575,7 +693,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
   crypto_pk_t *signing_key = NULL;
   authority_cert_t *parsed = NULL;
 
-  fname = get_datadir_fname2("keys",
+  fname = get_keydir_fname(
                  legacy ? "legacy_signing_key" : "authority_signing_key");
   signing_key = init_key_from_file(fname, 0, LOG_ERR, 0);
   if (!signing_key) {
@@ -583,7 +701,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
     goto done;
   }
   tor_free(fname);
-  fname = get_datadir_fname2("keys",
+  fname = get_keydir_fname(
                legacy ? "legacy_certificate" : "authority_certificate");
   cert = read_file_to_str(fname, 0, NULL);
   if (!cert) {
@@ -683,6 +801,47 @@ v3_authority_check_key_expiry(void)
   last_warned = now;
 }
 
+/** Get the lifetime of an onion key in days. This value is defined by the
+ * network consesus parameter "onion-key-rotation-days". Always returns a value
+ * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
+ */
+static int
+get_onion_key_rotation_days_(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "onion-key-rotation-days",
+                                 DEFAULT_ONION_KEY_LIFETIME_DAYS,
+                                 MIN_ONION_KEY_LIFETIME_DAYS,
+                                 MAX_ONION_KEY_LIFETIME_DAYS);
+}
+
+/** Get the current lifetime of an onion key in seconds. This value is defined
+ * by the network consesus parameter "onion-key-rotation-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_lifetime(void)
+{
+  return get_onion_key_rotation_days_()*24*60*60;
+}
+
+/** Get the grace period of an onion key in seconds. This value is defined by
+ * the network consesus parameter "onion-key-grace-period-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_grace_period(void)
+{
+  int grace_period;
+  grace_period = networkstatus_get_param(NULL,
+                                         "onion-key-grace-period-days",
+                                         DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         MIN_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         get_onion_key_rotation_days_());
+  return grace_period*24*60*60;
+}
+
 /** Set up Tor's TLS contexts, based on our configuration and keys. Return 0
  * on success, and -1 on failure. */
 int
@@ -693,12 +852,6 @@ router_initialize_tls_context(void)
   int lifetime = options->SSLKeyLifetime;
   if (public_server_mode(options))
     flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER;
-  if (options->TLSECGroup) {
-    if (!strcasecmp(options->TLSECGroup, "P256"))
-      flags |= TOR_TLS_CTX_USE_ECDHE_P256;
-    else if (!strcasecmp(options->TLSECGroup, "P224"))
-      flags |= TOR_TLS_CTX_USE_ECDHE_P224;
-  }
   if (!lifetime) { /* we should guess a good ssl cert lifetime */
 
     /* choose between 5 and 365 days, and round to the day */
@@ -848,17 +1001,9 @@ init_keys(void)
   }
   if (init_keys_common() < 0)
     return -1;
-  /* Make sure DataDirectory exists, and is private. */
-  if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) {
-    return -1;
-  }
-  /* Check the key directory. */
-  keydir = get_datadir_fname("keys");
-  if (check_private_dir(keydir, CPD_CREATE, options->User)) {
-    tor_free(keydir);
+
+  if (create_keys_directory(options) < 0)
     return -1;
-  }
-  tor_free(keydir);
 
   /* 1a. Read v3 directory authority key/cert information. */
   memset(v3_digest, 0, sizeof(v3_digest));
@@ -871,14 +1016,18 @@ init_keys(void)
     }
     cert = get_my_v3_authority_cert();
     if (cert) {
-      crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
-                           v3_digest);
+      if (crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
+                               v3_digest) < 0) {
+        log_err(LD_BUG, "Couldn't compute my v3 authority identity key "
+                "digest.");
+        return -1;
+      }
       v3_digest_set = 1;
     }
   }
 
   /* 1b. Read identity key. Make it if none is found. */
-  keydir = get_datadir_fname2("keys", "secret_id_key");
+  keydir = get_keydir_fname("secret_id_key");
   log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir);
   prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
   tor_free(keydir);
@@ -901,11 +1050,12 @@ init_keys(void)
   }
 
   /* 1d. Load all ed25519 keys */
-  if (load_ed_keys(options,now) < 0)
+  const int new_signing_key = load_ed_keys(options,now);
+  if (new_signing_key < 0)
     return -1;
 
   /* 2. Read onion key.  Make it if none is found. */
-  keydir = get_datadir_fname2("keys", "secret_onion_key");
+  keydir = get_keydir_fname("secret_onion_key");
   log_info(LD_GENERAL,"Reading/making onion key \"%s\"...",keydir);
   prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
   tor_free(keydir);
@@ -923,14 +1073,14 @@ init_keys(void)
       /* We have no LastRotatedOnionKey set; either we just created the key
        * or it's a holdover from 0.1.2.4-alpha-dev or earlier.  In either case,
        * start the clock ticking now so that we will eventually rotate it even
-       * if we don't stay up for a full MIN_ONION_KEY_LIFETIME. */
+       * if we don't stay up for the full lifetime of an onion key. */
       state->LastRotatedOnionKey = onionkey_set_at = now;
       or_state_mark_dirty(state, options->AvoidDiskWrites ?
                                    time(NULL)+3600 : 0);
     }
   }
 
-  keydir = get_datadir_fname2("keys", "secret_onion_key.old");
+  keydir = get_keydir_fname("secret_onion_key.old");
   if (!lastonionkey && file_status(keydir) == FN_FILE) {
     /* Load keys from non-empty files only.
      * Missing old keys won't be replaced with freshly generated keys. */
@@ -943,14 +1093,14 @@ init_keys(void)
   {
     /* 2b. Load curve25519 onion keys. */
     int r;
-    keydir = get_datadir_fname2("keys", "secret_onion_key_ntor");
+    keydir = get_keydir_fname("secret_onion_key_ntor");
     r = init_curve25519_keypair_from_file(&curve25519_onion_key,
                                           keydir, 1, LOG_ERR, "onion");
     tor_free(keydir);
     if (r<0)
       return -1;
 
-    keydir = get_datadir_fname2("keys", "secret_onion_key_ntor.old");
+    keydir = get_keydir_fname("secret_onion_key_ntor.old");
     if (tor_mem_is_zero((const char *)
                            last_curve25519_onion_key.pubkey.public_key,
                         CURVE25519_PUBKEY_LEN) &&
@@ -971,7 +1121,7 @@ init_keys(void)
 
   /* 3b. Get an ed25519 link certificate.  Note that we need to do this
    * after we set up the TLS context */
-  if (generate_ed_link_cert(options, now) < 0) {
+  if (generate_ed_link_cert(options, now, new_signing_key > 0) < 0) {
     log_err(LD_GENERAL,"Couldn't make link cert");
     return -1;
   }
@@ -979,7 +1129,7 @@ init_keys(void)
   /* 4. Build our router descriptor. */
   /* Must be called after keys are initialized. */
   mydesc = router_get_my_descriptor();
-  if (authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)) {
+  if (authdir_mode_v3(options)) {
     const char *m = NULL;
     routerinfo_t *ri;
     /* We need to add our own fingerprint so it gets recognized. */
@@ -1146,7 +1296,8 @@ check_whether_dirport_reachable(const or_options_t *options)
 /* XXX Should this be increased? */
 #define MIN_BW_TO_ADVERTISE_DIRSERVER 51200
 
-/** Return true iff we have enough configured bandwidth to cache directory
+/** Return true iff we have enough configured bandwidth to advertise or
+ * automatically provide directory services from cache directory
  * information. */
 static int
 router_has_bandwidth_to_be_dirserver(const or_options_t *options)
@@ -1169,7 +1320,7 @@ router_has_bandwidth_to_be_dirserver(const or_options_t *options)
  * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests.
  */
 static int
-router_should_be_directory_server(const or_options_t *options, int dir_port)
+router_should_be_dirserver(const or_options_t *options, int dir_port)
 {
   static int advertising=1; /* start out assuming we will advertise */
   int new_choice=1;
@@ -1178,9 +1329,9 @@ router_should_be_directory_server(const or_options_t *options, int dir_port)
   if (accounting_is_enabled(options) &&
     get_options()->AccountingRule != ACCT_IN) {
     /* Don't spend bytes for directory traffic if we could end up hibernating,
-     * but allow DirPort otherwise. Some people set AccountingMax because
-     * they're confused or to get statistics. Directory traffic has a much
-     * larger effect on output than input so there is no reason to turn it
+     * but allow DirPort otherwise. Some relay operators set AccountingMax
+     * because they're confused or to get statistics. Directory traffic has a
+     * much larger effect on output than input so there is no reason to turn it
      * off if using AccountingRule in. */
     int interval_length = accounting_get_interval_length();
     uint32_t effective_bw = get_effective_bwrate(options);
@@ -1274,7 +1425,7 @@ decide_to_advertise_dir_impl(const or_options_t *options,
 
   /* Part two: consider config options that could make us choose to
    * publish or not publish that the user might find surprising. */
-  return router_should_be_directory_server(options, dir_port);
+  return router_should_be_dirserver(options, dir_port);
 }
 
 /** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to
@@ -1282,7 +1433,7 @@ decide_to_advertise_dir_impl(const or_options_t *options,
  * DirPort we want to advertise.
  */
 static int
-decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
+router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port)
 {
   /* supports_tunnelled_dir_requests is not relevant, pass 0 */
   return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0;
@@ -1292,7 +1443,7 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port)
  * advertise the fact that we support begindir requests, else return 1.
  */
 static int
-decide_to_advertise_begindir(const or_options_t *options,
+router_should_advertise_begindir(const or_options_t *options,
                              int supports_tunnelled_dir_requests)
 {
   /* dir_port is not relevant, pass 0 */
@@ -1312,32 +1463,30 @@ extend_info_from_router(const routerinfo_t *r)
   /* Make sure we don't need to check address reachability */
   tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0));
 
+  const ed25519_public_key_t *ed_id_key;
+  if (r->cache_info.signing_key_cert)
+    ed_id_key = &r->cache_info.signing_key_cert->signing_key;
+  else
+    ed_id_key = NULL;
+
   router_get_prim_orport(r, &ap);
   return extend_info_new(r->nickname, r->cache_info.identity_digest,
+                         ed_id_key,
                          r->onion_pkey, r->onion_curve25519_pkey,
                          &ap.addr, ap.port);
 }
 
-/** Some time has passed, or we just got new directory information.
- * See if we currently believe our ORPort or DirPort to be
- * unreachable. If so, launch a new test for it.
- *
- * For ORPort, we simply try making a circuit that ends at ourselves.
- * Success is noticed in onionskin_answer().
- *
- * For DirPort, we make a connection via Tor to our DirPort and ask
- * for our own server descriptor.
- * Success is noticed in connection_dir_client_reached_eof().
+/**See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, return 1 else return 0.
  */
-void
-consider_testing_reachability(int test_or, int test_dir)
+static int
+router_should_check_reachability(int test_or, int test_dir)
 {
   const routerinfo_t *me = router_get_my_routerinfo();
   const or_options_t *options = get_options();
-  int orport_reachable = check_whether_orport_reachable(options);
-  tor_addr_t addr;
+
   if (!me)
-    return;
+    return 0;
 
   if (routerset_contains_router(options->ExcludeNodes, me, -1) &&
       options->StrictNodes) {
@@ -1352,33 +1501,66 @@ consider_testing_reachability(int test_or, int test_dir)
                  "We cannot learn whether we are usable, and will not "
                  "be able to advertise ourself.");
     }
-    return;
+    return 0;
   }
+  return 1;
+}
+
+/** Some time has passed, or we just got new directory information.
+ * See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, launch a new test for it.
+ *
+ * For ORPort, we simply try making a circuit that ends at ourselves.
+ * Success is noticed in onionskin_answer().
+ *
+ * For DirPort, we make a connection via Tor to our DirPort and ask
+ * for our own server descriptor.
+ * Success is noticed in connection_dir_client_reached_eof().
+ */
+void
+router_do_reachability_checks(int test_or, int test_dir)
+{
+  const routerinfo_t *me = router_get_my_routerinfo();
+  const or_options_t *options = get_options();
+  int orport_reachable = check_whether_orport_reachable(options);
+  tor_addr_t addr;
+
+  if (router_should_check_reachability(test_or, test_dir)) {
+    if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) {
+      extend_info_t *ei = extend_info_from_router(me);
+      /* XXX IPv6 self testing */
+      log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.",
+               !orport_reachable ? "reachability" : "bandwidth",
+               fmt_addr32(me->addr), me->or_port);
+      circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei,
+                              CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+      extend_info_free(ei);
+    }
 
-  if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) {
-    extend_info_t *ei = extend_info_from_router(me);
     /* XXX IPv6 self testing */
-    log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.",
-             !orport_reachable ? "reachability" : "bandwidth",
-             fmt_addr32(me->addr), me->or_port);
-    circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei,
-                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
-    extend_info_free(ei);
-  }
-
-  /* XXX IPv6 self testing */
-  tor_addr_from_ipv4h(&addr, me->addr);
-  if (test_dir && !check_whether_dirport_reachable(options) &&
-      !connection_get_by_type_addr_port_purpose(
-                CONN_TYPE_DIR, &addr, me->dir_port,
-                DIR_PURPOSE_FETCH_SERVERDESC)) {
-    /* ask myself, via tor, for my server descriptor. */
-    directory_initiate_command(&addr, me->or_port,
-                               &addr, me->dir_port,
-                               me->cache_info.identity_digest,
-                               DIR_PURPOSE_FETCH_SERVERDESC,
-                               ROUTER_PURPOSE_GENERAL,
-                               DIRIND_ANON_DIRPORT, "authority.z", NULL, 0, 0);
+    tor_addr_from_ipv4h(&addr, me->addr);
+    if (test_dir && !check_whether_dirport_reachable(options) &&
+        !connection_get_by_type_addr_port_purpose(
+                  CONN_TYPE_DIR, &addr, me->dir_port,
+                  DIR_PURPOSE_FETCH_SERVERDESC)) {
+      tor_addr_port_t my_orport, my_dirport;
+      memcpy(&my_orport.addr, &addr, sizeof(addr));
+      memcpy(&my_dirport.addr, &addr, sizeof(addr));
+      my_orport.port = me->or_port;
+      my_dirport.port = me->dir_port;
+      /* ask myself, via tor, for my server descriptor. */
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+      directory_request_set_or_addr_port(req, &my_orport);
+      directory_request_set_dir_addr_port(req, &my_dirport);
+      directory_request_set_directory_id_digest(req,
+                                              me->cache_info.identity_digest);
+      // ask via an anon circuit, connecting to our dirport.
+      directory_request_set_indirection(req, DIRIND_ANON_DIRPORT);
+      directory_request_set_resource(req, "authority.z");
+      directory_initiate_request(req);
+      directory_request_free(req);
+    }
   }
 }
 
@@ -1423,7 +1605,7 @@ router_dirport_found_reachable(void)
                && check_whether_orport_reachable(options) ?
                " Publishing server descriptor." : "");
     can_reach_dir_port = 1;
-    if (decide_to_advertise_dirport(options, me->dir_port)) {
+    if (router_should_advertise_dirport(options, me->dir_port)) {
       mark_my_descriptor_dirty("DirPort found reachable");
       /* This is a significant enough change to upload immediately,
        * at least in a test network */
@@ -1468,57 +1650,45 @@ router_perform_bandwidth_test(int num_circs, time_t now)
   }
 }
 
-/** Return true iff our network is in some sense disabled: either we're
- * hibernating, entering hibernation, or the network is turned off with
- * DisableNetwork. */
+/** Return true iff our network is in some sense disabled or shutting down:
+ * either we're hibernating, entering hibernation, or the network is turned
+ * off with DisableNetwork. */
 int
 net_is_disabled(void)
 {
   return get_options()->DisableNetwork || we_are_hibernating();
 }
 
-/** Return true iff we believe ourselves to be an authoritative
- * directory server.
- */
+/** Return true iff our network is in some sense "completely disabled" either
+ * we're fully hibernating or the network is turned off with
+ * DisableNetwork. */
 int
-authdir_mode(const or_options_t *options)
+net_is_completely_disabled(void)
 {
-  return options->AuthoritativeDir != 0;
+  return get_options()->DisableNetwork || we_are_fully_hibernating();
 }
-/** Return true iff we believe ourselves to be a v3 authoritative
+
+/** Return true iff we believe ourselves to be an authoritative
  * directory server.
  */
 int
-authdir_mode_v3(const or_options_t *options)
-{
-  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
-}
-/** Return true iff we are a v3 directory authority. */
-int
-authdir_mode_any_main(const or_options_t *options)
-{
-  return options->V3AuthoritativeDir;
-}
-/** Return true if we believe ourselves to be any kind of
- * authoritative directory beyond just a hidserv authority. */
-int
-authdir_mode_any_nonhidserv(const or_options_t *options)
+authdir_mode(const or_options_t *options)
 {
-  return options->BridgeAuthoritativeDir ||
-         authdir_mode_any_main(options);
+  return options->AuthoritativeDir != 0;
 }
 /** Return true iff we are an authoritative directory server that is
  * authoritative about receiving and serving descriptors of type
- * <b>purpose</b> on its dirport.  Use -1 for "any purpose". */
+ * <b>purpose</b> on its dirport.
+ */
 int
 authdir_mode_handles_descs(const or_options_t *options, int purpose)
 {
-  if (purpose < 0)
-    return authdir_mode_any_nonhidserv(options);
+  if (BUG(purpose < 0)) /* Deprecated. */
+    return authdir_mode(options);
   else if (purpose == ROUTER_PURPOSE_GENERAL)
-    return authdir_mode_any_main(options);
+    return authdir_mode_v3(options);
   else if (purpose == ROUTER_PURPOSE_BRIDGE)
-    return (options->BridgeAuthoritativeDir);
+    return authdir_mode_bridge(options);
   else
     return 0;
 }
@@ -1530,7 +1700,7 @@ authdir_mode_publishes_statuses(const or_options_t *options)
 {
   if (authdir_mode_bridge(options))
     return 0;
-  return authdir_mode_any_nonhidserv(options);
+  return authdir_mode(options);
 }
 /** Return true iff we are an authoritative directory server that
  * tests reachability of the descriptors it learns about.
@@ -1538,7 +1708,7 @@ authdir_mode_publishes_statuses(const or_options_t *options)
 int
 authdir_mode_tests_reachability(const or_options_t *options)
 {
-  return authdir_mode_handles_descs(options, -1);
+  return authdir_mode(options);
 }
 /** Return true iff we believe ourselves to be a bridge authoritative
  * directory server.
@@ -1555,8 +1725,7 @@ MOCK_IMPL(int,
 server_mode,(const or_options_t *options))
 {
   if (options->ClientOnly) return 0;
-  /* XXXX I believe we can kill off ORListenAddress here.*/
-  return (options->ORPort_set || options->ORListenAddress);
+  return (options->ORPort_set);
 }
 
 /** Return true iff we are trying to be a non-bridge server.
@@ -1762,12 +1931,12 @@ static routerinfo_t *desc_routerinfo = NULL;
 static extrainfo_t *desc_extrainfo = NULL;
 /** Why did we most recently decide to regenerate our descriptor?  Used to
  * tell the authorities why we're sending it to them. */
-static const char *desc_gen_reason = NULL;
+static const char *desc_gen_reason = "uninitialized reason";
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
 static time_t desc_clean_since = 0;
 /** Why did we mark the descriptor dirty? */
-static const char *desc_dirty_reason = NULL;
+static const char *desc_dirty_reason = "Tor just started";
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1848,7 +2017,7 @@ router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port)
       desc_routerinfo->ipv6_exit_policy &&
       compare_tor_addr_to_short_policy(addr, port,
                                me->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED;
-#endif
+#endif /* 0 */
   } else {
     return -1;
   }
@@ -1908,10 +2077,43 @@ router_is_me(const routerinfo_t *router)
 MOCK_IMPL(const routerinfo_t *,
 router_get_my_routerinfo,(void))
 {
-  if (!server_mode(get_options()))
+  return router_get_my_routerinfo_with_err(NULL);
+}
+
+/** Return routerinfo of this OR. Rebuild it from
+ * scratch if needed. Set <b>*err</b> to 0 on success or to
+ * appropriate TOR_ROUTERINFO_ERROR_* value on failure.
+ */
+MOCK_IMPL(const routerinfo_t *,
+router_get_my_routerinfo_with_err,(int *err))
+{
+  if (!server_mode(get_options())) {
+    if (err)
+      *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+
     return NULL;
-  if (router_rebuild_descriptor(0))
+  }
+
+  if (!desc_clean_since) {
+    int rebuild_err = router_rebuild_descriptor(0);
+    if (rebuild_err < 0) {
+      if (err)
+        *err = rebuild_err;
+
+      return NULL;
+    }
+  }
+
+  if (!desc_routerinfo) {
+    if (err)
+      *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+
     return NULL;
+  }
+
+  if (err)
+    *err = 0;
+
   return desc_routerinfo;
 }
 
@@ -2088,7 +2290,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
 
   if (router_pick_published_address(options, &addr, 0) < 0) {
     log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
-    return -1;
+    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
   }
 
   /* Log a message if the address in the descriptor doesn't match the ORPort
@@ -2144,7 +2346,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   if (crypto_pk_get_digest(ri->identity_pkey,
                            ri->cache_info.identity_digest)<0) {
     routerinfo_free(ri);
-    return -1;
+    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
   }
   ri->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
@@ -2160,6 +2362,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   /* and compute ri->bandwidthburst similarly */
   ri->bandwidthburst = get_effective_bwburst(options);
 
+  /* Report bandwidth, unless we're hibernating or shutting down */
   ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess();
 
   if (dns_seems_to_be_broken() || has_dns_init_failed()) {
@@ -2181,19 +2384,17 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   }
 
   if (options->MyFamily && ! options->BridgeRelay) {
-    smartlist_t *family;
     if (!warned_nonexistent_family)
       warned_nonexistent_family = smartlist_new();
-    family = smartlist_new();
     ri->declared_family = smartlist_new();
-    smartlist_split_string(family, options->MyFamily, ",",
-      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
-    SMARTLIST_FOREACH_BEGIN(family, char *, name) {
+    config_line_t *family;
+    for (family = options->MyFamily; family; family = family->next) {
+       char *name = family->value;
        const node_t *member;
        if (!strcasecmp(name, options->Nickname))
-         goto skip; /* Don't list ourself, that's redundant */
+         continue; /* Don't list ourself, that's redundant */
        else
-         member = node_get_by_nickname(name, 1);
+         member = node_get_by_nickname(name, 0);
        if (!member) {
          int is_legal = is_legal_nickname_or_hexdigest(name);
          if (!smartlist_contains_string(warned_nonexistent_family, name) &&
@@ -2207,11 +2408,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
              log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
                       "declared family, but that isn't a legal nickname. "
                       "Skipping it.", escaped(name));
-           smartlist_add(warned_nonexistent_family, tor_strdup(name));
+           smartlist_add_strdup(warned_nonexistent_family, name);
          }
          if (is_legal) {
-           smartlist_add(ri->declared_family, name);
-           name = NULL;
+           smartlist_add_strdup(ri->declared_family, name);
          }
        } else if (router_digest_is_me(member->identity)) {
          /* Don't list ourself in our own family; that's redundant */
@@ -2225,15 +2425,11 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
          if (smartlist_contains_string(warned_nonexistent_family, name))
            smartlist_string_remove(warned_nonexistent_family, name);
        }
-    skip:
-       tor_free(name);
-    } SMARTLIST_FOREACH_END(name);
+    }
 
     /* remove duplicates from the list */
     smartlist_sort_strings(ri->declared_family);
     smartlist_uniq_strings(ri->declared_family);
-
-    smartlist_free(family);
   }
 
   /* Now generate the extrainfo. */
@@ -2284,7 +2480,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
     log_warn(LD_BUG, "Couldn't generate router descriptor.");
     routerinfo_free(ri);
     extrainfo_free(ei);
-    return -1;
+    return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
   }
   ri->cache_info.signed_descriptor_len =
     strlen(ri->cache_info.signed_descriptor_body);
@@ -2327,6 +2523,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
 int
 router_rebuild_descriptor(int force)
 {
+  int err = 0;
   routerinfo_t *ri;
   extrainfo_t *ei;
   uint32_t addr;
@@ -2341,13 +2538,14 @@ router_rebuild_descriptor(int force)
      * learn that it's time to try again when ip_address_changed()
      * marks it dirty. */
     desc_clean_since = time(NULL);
-    return -1;
+    return TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
   }
 
   log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : "");
 
-  if (router_build_fresh_descriptor(&ri, &ei) < 0) {
-    return -1;
+  err = router_build_fresh_descriptor(&ri, &ei);
+  if (err < 0) {
+    return err;
   }
 
   routerinfo_free(desc_routerinfo);
@@ -2358,6 +2556,9 @@ router_rebuild_descriptor(int force)
   desc_clean_since = time(NULL);
   desc_needs_upload = 1;
   desc_gen_reason = desc_dirty_reason;
+  if (BUG(desc_gen_reason == NULL)) {
+    desc_gen_reason = "descriptor was marked dirty earlier, for no reason.";
+  }
   desc_dirty_reason = NULL;
   control_event_my_descriptor_changed();
   return 0;
@@ -2414,6 +2615,9 @@ void
 mark_my_descriptor_dirty(const char *reason)
 {
   const or_options_t *options = get_options();
+  if (BUG(reason == NULL)) {
+    reason = "marked descriptor dirty for unspecified reason";
+  }
   if (server_mode(options) && options->PublishServerDescriptor_)
     log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason);
   desc_clean_since = 0;
@@ -2454,6 +2658,9 @@ check_descriptor_bandwidth_changed(time_t now)
     return;
 
   prev = my_ri->bandwidthcapacity;
+
+  /* Consider ourselves to have zero bandwidth if we're hibernating or
+   * shutting down. */
   cur = hibernating ? 0 : rep_hist_bandwidth_assess();
   if ((prev != cur && (!prev || !cur)) ||
       cur > (prev * BANDWIDTH_CHANGE_FACTOR) ||
@@ -2767,7 +2974,7 @@ router_dump_router_to_string(routerinfo_t *router,
       make_ntor_onion_key_crosscert(ntor_keypair,
                          &router->cache_info.signing_key_cert->signing_key,
                          router->cache_info.published_on,
-                         MIN_ONION_KEY_LIFETIME, &sign);
+                         get_onion_key_lifetime(), &sign);
     if (!cert) {
       log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!");
       goto err;
@@ -2853,18 +3060,18 @@ router_dump_router_to_string(routerinfo_t *router,
                     "onion-key\n%s"
                     "signing-key\n%s"
                     "%s%s"
-                    "%s%s%s%s",
+                    "%s%s%s",
     router->nickname,
     address,
     router->or_port,
-    decide_to_advertise_dirport(options, router->dir_port),
+    router_should_advertise_dirport(options, router->dir_port),
     ed_cert_line ? ed_cert_line : "",
     extra_or_address ? extra_or_address : "",
     router->platform,
     proto_line,
     published,
     fingerprint,
-    stats_n_seconds_working,
+    get_uptime(),
     (int) router->bandwidthrate,
     (int) router->bandwidthburst,
     (int) router->bandwidthcapacity,
@@ -2876,8 +3083,7 @@ router_dump_router_to_string(routerinfo_t *router,
     ntor_cc_line ? ntor_cc_line : "",
     family_line,
     we_are_hibernating() ? "hibernating 1\n" : "",
-    "hidden-service-dir\n",
-    options->AllowSingleHopExits ? "allow-single-hop-exits\n" : "");
+    "hidden-service-dir\n");
 
   if (options->ContactInfo && strlen(options->ContactInfo)) {
     const char *ci = options->ContactInfo;
@@ -2888,10 +3094,13 @@ router_dump_router_to_string(routerinfo_t *router,
 
   if (options->BridgeRelay) {
     const char *bd;
-    if (options->PublishServerDescriptor_ & BRIDGE_DIRINFO)
+    if (options->BridgeDistribution && strlen(options->BridgeDistribution)) {
+      bd = options->BridgeDistribution;
+    } else {
       bd = "any";
-    else
-      bd = "none";
+    }
+    if (strchr(bd, '\n') || strchr(bd, '\r'))
+      bd = escaped(bd);
     smartlist_add_asprintf(chunks, "bridge-distribution-request %s\n", bd);
   }
 
@@ -2909,7 +3118,7 @@ router_dump_router_to_string(routerinfo_t *router,
 
   /* Write the exit policy to the end of 's'. */
   if (!router->exit_policy || !smartlist_len(router->exit_policy)) {
-    smartlist_add(chunks, tor_strdup("reject *:*\n"));
+    smartlist_add_strdup(chunks, "reject *:*\n");
   } else if (router->exit_policy) {
     char *exit_policy = router_dump_exit_policy_to_string(router,1,0);
 
@@ -2929,14 +3138,14 @@ router_dump_router_to_string(routerinfo_t *router,
     tor_free(p6);
   }
 
-  if (decide_to_advertise_begindir(options,
+  if (router_should_advertise_begindir(options,
                                    router->supports_tunnelled_dir_requests)) {
-    smartlist_add(chunks, tor_strdup("tunnelled-dir-server\n"));
+    smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
   }
 
   /* Sign the descriptor with Ed25519 */
   if (emit_ed_sigs)  {
-    smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
     crypto_digest_smartlist_prefix(digest, DIGEST256_LEN,
                                    ED_DESC_SIGNATURE_PREFIX,
                                    chunks, "", DIGEST_SHA256);
@@ -2952,11 +3161,10 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* Sign the descriptor with RSA */
-  smartlist_add(chunks, tor_strdup("router-signature\n"));
+  smartlist_add_strdup(chunks, "router-signature\n");
 
   crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1);
 
-  note_crypto_pk_op(SIGN_RTR);
   {
     char *sig;
     if (!(sig = router_get_dirobj_signature(digest, DIGEST_LEN, ident_key))) {
@@ -2967,7 +3175,7 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* include a last '\n' */
-  smartlist_add(chunks, tor_strdup("\n"));
+  smartlist_add_strdup(chunks, "\n");
 
   output = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -2987,7 +3195,7 @@ router_dump_router_to_string(routerinfo_t *router,
     tor_free(s_dup);
     routerinfo_free(ri_tmp);
   }
-#endif
+#endif /* defined(DEBUG_ROUTER_DUMP_ROUTER_TO_STRING) */
 
   goto done;
 
@@ -3213,6 +3421,11 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
                         "conn-bi-direct", now, &contents) > 0) {
       smartlist_add(chunks, contents);
     }
+    if (options->PaddingStatistics) {
+      contents = rep_hist_get_padding_count_lines();
+      if (contents)
+        smartlist_add(chunks, contents);
+    }
   }
 
   /* Add information about the pluggable transports we support. */
@@ -3225,13 +3438,13 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
     const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
     if (bridge_stats) {
-      smartlist_add(chunks, tor_strdup(bridge_stats));
+      smartlist_add_strdup(chunks, bridge_stats);
     }
   }
 
   if (emit_ed_sigs) {
     char sha256_digest[DIGEST256_LEN];
-    smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
     crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
                                    ED_DESC_SIGNATURE_PREFIX,
                                    chunks, "", DIGEST_SHA256);
@@ -3246,7 +3459,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
-  smartlist_add(chunks, tor_strdup("router-signature\n"));
+  smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
@@ -3281,7 +3494,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
                      "descriptor.");
     goto err;
   }
-  smartlist_add(chunks, tor_strdup(sig));
+  smartlist_add_strdup(chunks, sig);
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3373,6 +3586,15 @@ is_legal_hexdigest(const char *s)
           strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN);
 }
 
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+
 /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
  * hold a human-readable description of a node with identity digest
  * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
@@ -3418,33 +3640,35 @@ format_node_description(char *buf,
   return buf;
 }
 
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>ri</b>.
- *
+/** Return a human-readable description of the routerinfo_t <b>ri</b>.
  *
- * Return a pointer to the front of <b>buf</b>.
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
  */
 const char *
-router_get_description(char *buf, const routerinfo_t *ri)
+router_describe(const routerinfo_t *ri)
 {
+  static char buf[NODE_DESC_BUF_LEN];
+
   if (!ri)
     return "<null>";
   return format_node_description(buf,
                                  ri->cache_info.identity_digest,
-                                 router_is_named(ri),
+                                 0,
                                  ri->nickname,
                                  NULL,
                                  ri->addr);
 }
 
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>node</b>.
+/** Return a human-readable description of the node_t <b>node</b>.
  *
- * Return a pointer to the front of <b>buf</b>.
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
  */
 const char *
-node_get_description(char *buf, const node_t *node)
+node_describe(const node_t *node)
 {
+  static char buf[NODE_DESC_BUF_LEN];
   const char *nickname = NULL;
   uint32_t addr32h = 0;
   int is_named = 0;
@@ -3469,14 +3693,16 @@ node_get_description(char *buf, const node_t *node)
                                  addr32h);
 }
 
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>rs</b>.
+/** Return a human-readable description of the routerstatus_t <b>rs</b>.
  *
- * Return a pointer to the front of <b>buf</b>.
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
  */
 const char *
-routerstatus_get_description(char *buf, const routerstatus_t *rs)
+routerstatus_describe(const routerstatus_t *rs)
 {
+  static char buf[NODE_DESC_BUF_LEN];
+
   if (!rs)
     return "<null>";
   return format_node_description(buf,
@@ -3487,14 +3713,16 @@ routerstatus_get_description(char *buf, const routerstatus_t *rs)
                                  rs->addr);
 }
 
-/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
- * hold a human-readable description of <b>ei</b>.
+/** Return a human-readable description of the extend_info_t <b>ei</b>.
  *
- * Return a pointer to the front of <b>buf</b>.
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
  */
 const char *
-extend_info_get_description(char *buf, const extend_info_t *ei)
+extend_info_describe(const extend_info_t *ei)
 {
+  static char buf[NODE_DESC_BUF_LEN];
+
   if (!ei)
     return "<null>";
   return format_node_description(buf,
@@ -3505,73 +3733,20 @@ extend_info_get_description(char *buf, const extend_info_t *ei)
                                  0);
 }
 
-/** Return a human-readable description of the routerinfo_t <b>ri</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-router_describe(const routerinfo_t *ri)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return router_get_description(buf, ri);
-}
-
-/** Return a human-readable description of the node_t <b>node</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-node_describe(const node_t *node)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return node_get_description(buf, node);
-}
-
-/** Return a human-readable description of the routerstatus_t <b>rs</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-routerstatus_describe(const routerstatus_t *rs)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return routerstatus_get_description(buf, rs);
-}
-
-/** Return a human-readable description of the extend_info_t <b>ri</b>.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-const char *
-extend_info_describe(const extend_info_t *ei)
-{
-  static char buf[NODE_DESC_BUF_LEN];
-  return extend_info_get_description(buf, ei);
-}
-
 /** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the
  * verbose representation of the identity of <b>router</b>.  The format is:
  *  A dollar sign.
  *  The upper-case hexadecimal encoding of the SHA1 hash of router's identity.
- *  A "=" if the router is named; a "~" if it is not.
+ *  A "=" if the router is named (no longer implemented); a "~" if it is not.
  *  The router's nickname.
  **/
 void
 router_get_verbose_nickname(char *buf, const routerinfo_t *router)
 {
-  const char *good_digest = networkstatus_get_router_digest_by_nickname(
-                                                         router->nickname);
-  int is_named = good_digest && tor_memeq(good_digest,
-                                        router->cache_info.identity_digest,
-                                        DIGEST_LEN);
   buf[0] = '$';
   base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
                 DIGEST_LEN);
-  buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
+  buf[1+HEX_DIGEST_LEN] = '~';
   strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
 }
 
@@ -3625,6 +3800,7 @@ router_free_all(void)
   crypto_pk_free(lastonionkey);
   crypto_pk_free(server_identitykey);
   crypto_pk_free(client_identitykey);
+
   tor_mutex_free(key_lock);
   routerinfo_free(desc_routerinfo);
   extrainfo_free(desc_extrainfo);
diff --git a/src/or/router.h b/src/or/router.h
index c30a0301b7..752f2f2dbe 100644
--- a/src/or/router.h
+++ b/src/or/router.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,13 @@
 
 #include "testsupport.h"
 
+#define TOR_ROUTERINFO_ERROR_NO_EXT_ADDR     (-1)
+#define TOR_ROUTERINFO_ERROR_CANNOT_PARSE    (-2)
+#define TOR_ROUTERINFO_ERROR_NOT_A_SERVER    (-3)
+#define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
+#define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
+#define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+
 crypto_pk_t *get_onion_key(void);
 time_t get_onion_key_set_at(void);
 void set_server_identity_key(crypto_pk_t *k);
@@ -27,13 +34,18 @@ crypto_pk_t *get_my_v3_authority_signing_key(void);
 authority_cert_t *get_my_v3_legacy_cert(void);
 crypto_pk_t *get_my_v3_legacy_signing_key(void);
 void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last);
+void expire_old_onion_keys(void);
 void rotate_onion_key(void);
 crypto_pk_t *init_key_from_file(const char *fname, int generate,
                                     int severity, int log_greeting);
 void v3_authority_check_key_expiry(void);
+int get_onion_key_lifetime(void);
+int get_onion_key_grace_period(void);
 
 di_digest256_map_t *construct_ntor_key_map(void);
-void ntor_key_map_free(di_digest256_map_t *map);
+void ntor_key_map_free_(di_digest256_map_t *map);
+#define ntor_key_map_free(map) \
+  FREE_AND_NULL(di_digest256_map_t, ntor_key_map_free_, (map))
 
 int router_initialize_tls_context(void);
 int init_keys(void);
@@ -42,17 +54,15 @@ int init_keys_client(void);
 int check_whether_orport_reachable(const or_options_t *options);
 int check_whether_dirport_reachable(const or_options_t *options);
 int dir_server_mode(const or_options_t *options);
-void consider_testing_reachability(int test_or, int test_dir);
+void router_do_reachability_checks(int test_or, int test_dir);
 void router_orport_found_reachable(void);
 void router_dirport_found_reachable(void);
 void router_perform_bandwidth_test(int num_circs, time_t now);
 
 int net_is_disabled(void);
+int net_is_completely_disabled(void);
 
 int authdir_mode(const or_options_t *options);
-int authdir_mode_v3(const or_options_t *options);
-int authdir_mode_any_main(const or_options_t *options);
-int authdir_mode_any_nonhidserv(const or_options_t *options);
 int authdir_mode_handles_descs(const or_options_t *options, int purpose);
 int authdir_mode_publishes_statuses(const or_options_t *options);
 int authdir_mode_tests_reachability(const or_options_t *options);
@@ -83,6 +93,7 @@ void router_new_address_suggestion(const char *suggestion,
 int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port);
 MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void));
 MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo_with_err,(int *err));
 extrainfo_t *router_get_my_extrainfo(void);
 const char *router_get_my_descriptor(void);
 const char *router_get_descriptor_gen_reason(void);
@@ -120,29 +131,14 @@ int is_legal_nickname(const char *s);
 int is_legal_nickname_or_hexdigest(const char *s);
 int is_legal_hexdigest(const char *s);
 
-/**
- * Longest allowed output of format_node_description, plus 1 character for
- * NUL.  This allows space for:
- * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
- * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
- * plus a terminating NUL.
- */
-#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
-const char *format_node_description(char *buf,
-                                    const char *id_digest,
-                                    int is_named,
-                                    const char *nickname,
-                                    const tor_addr_t *addr,
-                                    uint32_t addr32h);
-const char *router_get_description(char *buf, const routerinfo_t *ri);
-const char *node_get_description(char *buf, const node_t *node);
-const char *routerstatus_get_description(char *buf, const routerstatus_t *rs);
-const char *extend_info_get_description(char *buf, const extend_info_t *ei);
 const char *router_describe(const routerinfo_t *ri);
 const char *node_describe(const node_t *node);
 const char *routerstatus_describe(const routerstatus_t *ri);
 const char *extend_info_describe(const extend_info_t *ei);
 
+const char *routerinfo_err_to_string(int err);
+int routerinfo_err_is_transient(int err);
+
 void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
 void router_reset_warnings(void);
 void router_reset_reachability(void);
@@ -159,5 +155,5 @@ STATIC void get_platform_str(char *platform, size_t len);
 STATIC int router_write_fingerprint(int hashed);
 #endif
 
-#endif
+#endif /* !defined(TOR_ROUTER_H) */
 
diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c
index ca32228fc7..43460da8cc 100644
--- a/src/or/routerkeys.c
+++ b/src/or/routerkeys.c
@@ -1,16 +1,22 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file routerkeys.c
  *
  * \brief Functions and structures to handle generating and maintaining the
- *  set of keypairs necessary to be an OR. (Some of the code in router.c
- *  belongs here.)
+ *  set of keypairs necessary to be an OR.
+ *
+ * The keys handled here now are the Ed25519 keys that Tor relays use to sign
+ * descriptors, authenticate themselves on links, and identify one another
+ * uniquely.  Other keys are maintained in router.c and rendservice.c.
+ *
+ * (TODO: The keys in router.c should go here too.)
  */
 
 #include "or.h"
 #include "config.h"
+#include "crypto_util.h"
 #include "router.h"
 #include "crypto_pwbox.h"
 #include "routerkeys.h"
@@ -19,6 +25,7 @@
 #define ENC_KEY_HEADER "Boxed Ed25519 key"
 #define ENC_KEY_TAG "master"
 
+/* DOCDOC */
 static ssize_t
 do_getpass(const char *prompt, char *buf, size_t buflen,
            int twice, const or_options_t *options)
@@ -85,6 +92,7 @@ do_getpass(const char *prompt, char *buf, size_t buflen,
   return length;
 }
 
+/* DOCDOC */
 int
 read_encrypted_secret_key(ed25519_secret_key_t *out,
                           const char *fname)
@@ -157,6 +165,7 @@ read_encrypted_secret_key(ed25519_secret_key_t *out,
   return r;
 }
 
+/* DOCDOC */
 int
 write_encrypted_secret_key(const ed25519_secret_key_t *key,
                            const char *fname)
@@ -200,6 +209,7 @@ write_encrypted_secret_key(const ed25519_secret_key_t *key,
   return r;
 }
 
+/* DOCDOC */
 static int
 write_secret_key(const ed25519_secret_key_t *key, int encrypted,
                  const char *fname,
@@ -527,7 +537,8 @@ ed_key_init_from_file(const char *fname, uint32_t flags,
     bad_cert = 1;
   } else if (signing_key &&
              tor_cert_checksig(cert, &signing_key->pubkey, now) < 0) {
-    tor_log(severity, LD_OR, "Can't check certificate");
+    tor_log(severity, LD_OR, "Can't check certificate: %s",
+            tor_cert_describe_signature_status(cert));
     bad_cert = 1;
   } else if (cert->cert_expired) {
     tor_log(severity, LD_OR, "Certificate is expired");
@@ -659,10 +670,14 @@ static tor_cert_t *auth_key_cert = NULL;
 
 static uint8_t *rsa_ed_crosscert = NULL;
 static size_t rsa_ed_crosscert_len = 0;
+static time_t rsa_ed_crosscert_expiration = 0;
 
 /**
  * Running as a server: load, reload, or refresh our ed25519 keys and
  * certificates, creating and saving new ones as needed.
+ *
+ * Return -1 on failure; 0 on success if the signing key was not replaced;
+ * and 1 on success if the signing key was replaced.
  */
 int
 load_ed_keys(const or_options_t *options, time_t now)
@@ -675,6 +690,11 @@ load_ed_keys(const or_options_t *options, time_t now)
   const tor_cert_t *check_signing_cert = NULL;
   tor_cert_t *sign_cert = NULL;
   tor_cert_t *auth_cert = NULL;
+  int signing_key_changed = 0;
+
+  // It is later than 1972, since otherwise there would be no C compilers.
+  // (Try to diagnose #22466.)
+  tor_assert_nonfatal(now >= 2 * 365 * 86400);
 
 #define FAIL(msg) do {                          \
     log_warn(LD_OR, (msg));                     \
@@ -690,15 +710,17 @@ load_ed_keys(const or_options_t *options, time_t now)
       tor_cert_free(cert);                      \
     cert = (newval);                            \
   } while (0)
+#define HAPPENS_SOON(when, interval)            \
+  ((when) < now + (interval))
 #define EXPIRES_SOON(cert, interval)            \
-  (!(cert) || (cert)->valid_until < now + (interval))
+  (!(cert) || HAPPENS_SOON((cert)->valid_until, (interval)))
 
   /* XXXX support encrypted identity keys fully */
 
   /* First try to get the signing key to see how it is. */
   {
     char *fname =
-      options_get_datadir_fname2(options, "keys", "ed25519_signing");
+      options_get_keydir_fname(options, "ed25519_signing");
     sign = ed_key_init_from_file(
                fname,
                INIT_ED_KEY_NEEDCERT|
@@ -710,7 +732,23 @@ load_ed_keys(const or_options_t *options, time_t now)
     use_signing = sign;
   }
 
+  if (use_signing) {
+    /* We loaded a signing key with its certificate.  */
+    if (! master_signing_key) {
+      /* We didn't know one before! */
+      signing_key_changed = 1;
+    } else if (! ed25519_pubkey_eq(&use_signing->pubkey,
+                                   &master_signing_key->pubkey) ||
+               ! tor_memeq(use_signing->seckey.seckey,
+                           master_signing_key->seckey.seckey,
+                           ED25519_SECKEY_LEN)) {
+      /* We loaded a different signing key than the one we knew before. */
+      signing_key_changed = 1;
+    }
+  }
+
   if (!use_signing && master_signing_key) {
+    /* We couldn't load a signing key, but we already had one loaded */
     check_signing_cert = signing_key_cert;
     use_signing = master_signing_key;
   }
@@ -733,8 +771,12 @@ load_ed_keys(const or_options_t *options, time_t now)
 
   if (need_new_signing_key) {
     log_notice(LD_OR, "It looks like I need to generate and sign a new "
-               "medium-term signing key, because %s. To do that, I need to "
-               "load%s the permanent master identity key.",
+               "medium-term signing key, because %s. To do that, I "
+               "need to load%s the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.",
             (NULL == use_signing) ? "I don't have one" :
             EXPIRES_SOON(check_signing_cert, 0) ? "the one I have is expired" :
                "you asked me to make one with --keygen",
@@ -742,15 +784,19 @@ load_ed_keys(const or_options_t *options, time_t now)
   } else if (want_new_signing_key && !offline_master) {
     log_notice(LD_OR, "It looks like I should try to generate and sign a "
                "new medium-term signing key, because the one I have is "
-               "going to expire soon. To do that, I'm going to have to try to "
-               "load the permanent master identity key.");
+               "going to expire soon. To do that, I'm going to have to "
+               "try to load the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.");
   } else if (want_new_signing_key) {
     log_notice(LD_OR, "It looks like I should try to generate and sign a "
                "new medium-term signing key, because the one I have is "
                "going to expire soon. But OfflineMasterKey is set, so I "
-               "won't try to load a permanent master identity key is set. "
-               "You will need to use 'tor --keygen' make a new signing key "
-               "and certificate.");
+               "won't try to load a permanent master identity key. You "
+               "will need to use 'tor --keygen' to make a new signing "
+               "key and certificate.");
   }
 
   {
@@ -768,24 +814,16 @@ load_ed_keys(const or_options_t *options, time_t now)
     if (options->command == CMD_KEYGEN)
       flags |= INIT_ED_KEY_TRY_ENCRYPTED;
 
-    /* Check the key directory */
-    if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) {
-      log_err(LD_OR, "Can't create/check datadirectory %s",
-              options->DataDirectory);
-      goto err;
-    }
-    char *fname = get_datadir_fname("keys");
-    if (check_private_dir(fname, CPD_CREATE, options->User) < 0) {
-      log_err(LD_OR, "Problem creating/checking key directory %s", fname);
-      tor_free(fname);
-      goto err;
-    }
-    tor_free(fname);
+    /* Check/Create the key directory */
+    if (create_keys_directory(options) < 0)
+      return -1;
+
+    char *fname;
     if (options->master_key_fname) {
       fname = tor_strdup(options->master_key_fname);
       flags |= INIT_ED_KEY_EXPLICIT_FNAME;
     } else {
-      fname = options_get_datadir_fname2(options, "keys", "ed25519_master_id");
+      fname = options_get_keydir_fname(options, "ed25519_master_id");
     }
     id = ed_key_init_from_file(
              fname,
@@ -805,8 +843,8 @@ load_ed_keys(const or_options_t *options, time_t now)
         id = tor_malloc_zero(sizeof(*id));
         memcpy(&id->pubkey, &check_signing_cert->signing_key,
                sizeof(ed25519_public_key_t));
-        fname = options_get_datadir_fname2(options, "keys",
-                                           "ed25519_master_id_public_key");
+        fname = options_get_keydir_fname(options,
+                                         "ed25519_master_id_public_key");
         if (ed25519_pubkey_write_to_file(&id->pubkey, fname, "type0") < 0) {
           log_warn(LD_OR, "Error while attempting to write master public key "
                    "to disk");
@@ -836,8 +874,12 @@ load_ed_keys(const or_options_t *options, time_t now)
     if (! ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey))
       FAIL("The signing cert we have was not signed with the master key "
            "we loaded!");
-    if (tor_cert_checksig(sign_cert, &id->pubkey, 0) < 0)
-      FAIL("The signing cert we loaded was not signed correctly!");
+    if (tor_cert_checksig(sign_cert, &id->pubkey, 0) < 0) {
+      log_warn(LD_OR, "The signing cert we loaded was not signed "
+               "correctly: %s!",
+               tor_cert_describe_signature_status(sign_cert));
+      goto err;
+    }
   }
 
   if (want_new_signing_key && sign_signing_key_with_id) {
@@ -847,7 +889,7 @@ load_ed_keys(const or_options_t *options, time_t now)
                       INIT_ED_KEY_NEEDCERT|
                       INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT);
     char *fname =
-      options_get_datadir_fname2(options, "keys", "ed25519_signing");
+      options_get_keydir_fname(options, "ed25519_signing");
     ed25519_keypair_free(sign);
     tor_cert_free(sign_cert);
     sign = ed_key_init_from_file(fname,
@@ -859,6 +901,7 @@ load_ed_keys(const or_options_t *options, time_t now)
     if (!sign)
       FAIL("Missing signing key");
     use_signing = sign;
+    signing_key_changed = 1;
 
     tor_assert(sign_cert->signing_key_included);
     tor_assert(ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey));
@@ -879,17 +922,23 @@ load_ed_keys(const or_options_t *options, time_t now)
   if (options->command == CMD_KEYGEN)
     goto end;
 
-  if (!rsa_ed_crosscert && server_mode(options)) {
+  if (server_mode(options) &&
+      (!rsa_ed_crosscert ||
+       HAPPENS_SOON(rsa_ed_crosscert_expiration, 30*86400))) {
     uint8_t *crosscert;
+    time_t expiration = now+6*30*86400; /* 6 months in the future. */
     ssize_t crosscert_len = tor_make_rsa_ed25519_crosscert(&id->pubkey,
                                                    get_server_identity_key(),
-                                                   now+10*365*86400,/*XXXX*/
+                                                   expiration,
                                                    &crosscert);
+    tor_free(rsa_ed_crosscert);
     rsa_ed_crosscert_len = crosscert_len;
     rsa_ed_crosscert = crosscert;
+    rsa_ed_crosscert_expiration = expiration;
   }
 
   if (!current_auth_key ||
+      signing_key_changed ||
       EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop)) {
     auth = ed_key_new(use_signing, INIT_ED_KEY_NEEDCERT,
                       now,
@@ -917,7 +966,7 @@ load_ed_keys(const or_options_t *options, time_t now)
     SET_CERT(auth_key_cert, auth_cert);
   }
 
-  return 0;
+  return signing_key_changed;
  err:
   ed25519_keypair_free(id);
   ed25519_keypair_free(sign);
@@ -927,21 +976,39 @@ load_ed_keys(const or_options_t *options, time_t now)
   return -1;
 }
 
-/* DOCDOC */
+/**
+ * Retrieve our currently-in-use Ed25519 link certificate and id certificate,
+ * and, if they would expire soon (based on the time <b>now</b>, generate new
+ * certificates (without embedding the public part of the signing key inside).
+ * If <b>force</b> is true, always generate a new certificate.
+ *
+ * The signed_key from the current id->signing certificate will be used to
+ * sign the new key within newly generated X509 certificate.
+ *
+ * Returns -1 upon error.  Otherwise, returns 0 upon success (either when the
+ * current certificate is still valid, or when a new certificate was
+ * successfully generated, or no certificate was needed).
+ */
 int
-generate_ed_link_cert(const or_options_t *options, time_t now)
+generate_ed_link_cert(const or_options_t *options, time_t now,
+                      int force)
 {
   const tor_x509_cert_t *link_ = NULL, *id = NULL;
   tor_cert_t *link_cert = NULL;
 
   if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL) {
+    if (!server_mode(options)) {
+        /* No need to make an Ed25519->Link cert: we are a client */
+      return 0;
+    }
     log_warn(LD_OR, "Can't get my x509 link cert.");
     return -1;
   }
 
   const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
 
-  if (link_cert_cert &&
+  if (force == 0 &&
+      link_cert_cert &&
       ! EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop) &&
       fast_memeq(digests->d[DIGEST_SHA256], link_cert_cert->signed_key.pubkey,
                  DIGEST256_LEN)) {
@@ -967,6 +1034,17 @@ generate_ed_link_cert(const or_options_t *options, time_t now)
 #undef SET_KEY
 #undef SET_CERT
 
+/**
+ * Return 1 if any of the following are true:
+ *
+ *   - if one of our Ed25519 signing, auth, or link certificates would expire
+ *     soon w.r.t. the time <b>now</b>,
+ *   - if we do not currently have a link certificate, or
+ *   - if our cached Ed25519 link certificate is not same as the one we're
+ *     currently using.
+ *
+ * Otherwise, returns 0.
+ */
 int
 should_make_new_ed_keys(const or_options_t *options, const time_t now)
 {
@@ -996,6 +1074,164 @@ should_make_new_ed_keys(const or_options_t *options, const time_t now)
 }
 
 #undef EXPIRES_SOON
+#undef HAPPENS_SOON
+
+#ifdef TOR_UNIT_TESTS
+/* Helper for unit tests: populate the ed25519 keys without saving or
+ * loading */
+void
+init_mock_ed_keys(const crypto_pk_t *rsa_identity_key)
+{
+  routerkeys_free_all();
+
+#define MAKEKEY(k)                                      \
+  k = tor_malloc_zero(sizeof(*k));                      \
+  if (ed25519_keypair_generate(k, 0) < 0) {             \
+    log_warn(LD_BUG, "Couldn't make a keypair");        \
+    goto err;                                           \
+  }
+  MAKEKEY(master_identity_key);
+  MAKEKEY(master_signing_key);
+  MAKEKEY(current_auth_key);
+#define MAKECERT(cert, signing, signed_, type, flags)            \
+  cert = tor_cert_create(signing,                                \
+                         type,                                   \
+                         &signed_->pubkey,                       \
+                         time(NULL), 86400,                      \
+                         flags);                                 \
+  if (!cert) {                                                   \
+    log_warn(LD_BUG, "Couldn't make a %s certificate!", #cert);  \
+    goto err;                                                    \
+  }
+
+  MAKECERT(signing_key_cert,
+           master_identity_key, master_signing_key, CERT_TYPE_ID_SIGNING,
+           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  MAKECERT(auth_key_cert,
+           master_signing_key, current_auth_key, CERT_TYPE_SIGNING_AUTH, 0);
+
+  if (generate_ed_link_cert(get_options(), time(NULL), 0) < 0) {
+    log_warn(LD_BUG, "Couldn't make link certificate");
+    goto err;
+  }
+
+  rsa_ed_crosscert_len = tor_make_rsa_ed25519_crosscert(
+                                     &master_identity_key->pubkey,
+                                     rsa_identity_key,
+                                     time(NULL)+86400,
+                                     &rsa_ed_crosscert);
+
+  return;
+
+ err:
+  routerkeys_free_all();
+  tor_assert_nonfatal_unreached();
+}
+#undef MAKEKEY
+#undef MAKECERT
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Print the ISO8601-formated <b>expiration</b> for a certificate with
+ * some <b>description</b> to stdout.
+ *
+ * For example, for a signing certificate, this might print out:
+ * signing-cert-expiry: 2017-07-25 08:30:15 UTC
+ */
+static void
+print_cert_expiration(const char *expiration,
+                      const char *description)
+{
+  fprintf(stderr, "%s-cert-expiry: %s\n", description, expiration);
+}
+
+/**
+ * Log when a certificate, <b>cert</b>, with some <b>description</b> and
+ * stored in a file named <b>fname</b>, is going to expire.
+ */
+static void
+log_ed_cert_expiration(const tor_cert_t *cert,
+                       const char *description,
+                       const char *fname) {
+  char expiration[ISO_TIME_LEN+1];
+
+  if (BUG(!cert)) { /* If the specified key hasn't been loaded */
+    log_warn(LD_OR, "No %s key loaded; can't get certificate expiration.",
+             description);
+  } else {
+    format_local_iso_time(expiration, cert->valid_until);
+    log_notice(LD_OR, "The %s certificate stored in %s is valid until %s.",
+               description, fname, expiration);
+    print_cert_expiration(expiration, description);
+  }
+}
+
+/**
+ * Log when our master signing key certificate expires.  Used when tor is given
+ * the --key-expiration command-line option.
+ *
+ * Returns 0 on success and 1 on failure.
+ */
+static int
+log_master_signing_key_cert_expiration(const or_options_t *options)
+{
+  const tor_cert_t *signing_key;
+  char *fn = NULL;
+  int failed = 0;
+  time_t now = approx_time();
+
+  fn = options_get_keydir_fname(options, "ed25519_signing_cert");
+
+  /* Try to grab our cached copy of the key. */
+  signing_key = get_master_signing_key_cert();
+
+  tor_assert(server_identity_key_is_set());
+
+  /* Load our keys from disk, if necessary. */
+  if (!signing_key) {
+    failed = load_ed_keys(options, now) < 0;
+    signing_key = get_master_signing_key_cert();
+  }
+
+  /* If we do have a signing key, log the expiration time. */
+  if (signing_key) {
+    log_ed_cert_expiration(signing_key, "signing", fn);
+  } else {
+    log_warn(LD_OR, "Could not load signing key certificate from %s, so " \
+             "we couldn't learn anything about certificate expiration.", fn);
+  }
+
+  tor_free(fn);
+
+  return failed;
+}
+
+/**
+ * Log when a key certificate expires.  Used when tor is given the
+ * --key-expiration command-line option.
+ *
+ * If an command argument is given, which should specify the type of
+ * key to get expiry information about (currently supported arguments
+ * are "sign"), get info about that type of certificate.  Otherwise,
+ * print info about the supported arguments.
+ *
+ * Returns 0 on success and -1 on failure.
+ */
+int
+log_cert_expiration(void)
+{
+  const or_options_t *options = get_options();
+  const char *arg = options->command_arg;
+
+  if (!strcmp(arg, "sign")) {
+    return log_master_signing_key_cert_expiration(options);
+  } else {
+    fprintf(stderr, "No valid argument to --key-expiration found!\n");
+    fprintf(stderr, "Currently recognised arguments are: 'sign'\n");
+
+    return -1;
+  }
+}
 
 const ed25519_public_key_t *
 get_master_identity_key(void)
@@ -1005,6 +1241,24 @@ get_master_identity_key(void)
   return &master_identity_key->pubkey;
 }
 
+/** Return true iff <b>id</b> is our Ed25519 master identity key. */
+int
+router_ed25519_id_is_me(const ed25519_public_key_t *id)
+{
+  return id && master_identity_key &&
+    ed25519_pubkey_eq(id, &master_identity_key->pubkey);
+}
+
+#ifdef TOR_UNIT_TESTS
+/* only exists for the unit tests, since otherwise the identity key
+ * should be used to sign nothing but the signing key. */
+const ed25519_keypair_t *
+get_master_identity_keypair(void)
+{
+  return master_identity_key;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 const ed25519_keypair_t *
 get_master_signing_keypair(void)
 {
@@ -1079,7 +1333,9 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
   uint8_t signed_data[DIGEST_LEN + ED25519_PUBKEY_LEN];
 
   *len_out = 0;
-  crypto_pk_get_digest(rsa_id_key, (char*)signed_data);
+  if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    return NULL;
+  }
   memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
 
   int r = crypto_pk_private_sign(onion_key,
@@ -1095,12 +1351,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
 
 /** Check whether an RSA-TAP cross-certification is correct. Return 0 if it
  * is, -1 if it isn't. */
-int
-check_tap_onion_key_crosscert(const uint8_t *crosscert,
-                              int crosscert_len,
-                              const crypto_pk_t *onion_pkey,
-                              const ed25519_public_key_t *master_id_pkey,
-                              const uint8_t *rsa_id_digest)
+MOCK_IMPL(int,
+check_tap_onion_key_crosscert,(const uint8_t *crosscert,
+                               int crosscert_len,
+                               const crypto_pk_t *onion_pkey,
+                               const ed25519_public_key_t *master_id_pkey,
+                               const uint8_t *rsa_id_digest))
 {
   uint8_t *cc = tor_malloc(crypto_pk_keysize(onion_pkey));
   int cc_len =
@@ -1139,9 +1395,12 @@ routerkeys_free_all(void)
   tor_cert_free(signing_key_cert);
   tor_cert_free(link_cert_cert);
   tor_cert_free(auth_key_cert);
+  tor_free(rsa_ed_crosscert);
 
   master_identity_key = master_signing_key = NULL;
   current_auth_key = NULL;
   signing_key_cert = link_cert_cert = auth_key_cert = NULL;
+  rsa_ed_crosscert = NULL; // redundant
+  rsa_ed_crosscert_len = 0;
 }
 
diff --git a/src/or/routerkeys.h b/src/or/routerkeys.h
index be9b19aea8..3e67952ea0 100644
--- a/src/or/routerkeys.h
+++ b/src/or/routerkeys.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ROUTERKEYS_H
@@ -45,6 +45,8 @@ const struct tor_cert_st *get_current_auth_key_cert(void);
 void get_master_rsa_crosscert(const uint8_t **cert_out,
                               size_t *size_out);
 
+int router_ed25519_id_is_me(const ed25519_public_key_t *id);
+
 struct tor_cert_st *make_ntor_onion_key_crosscert(
                                   const curve25519_keypair_t *onion_key,
                                   const ed25519_public_key_t *master_id_key,
@@ -55,16 +57,17 @@ uint8_t *make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
                                   const crypto_pk_t *rsa_id_key,
                                   int *len_out);
 
-int check_tap_onion_key_crosscert(const uint8_t *crosscert,
+MOCK_DECL(int, check_tap_onion_key_crosscert,(const uint8_t *crosscert,
                                   int crosscert_len,
                                   const crypto_pk_t *onion_pkey,
                                   const ed25519_public_key_t *master_id_pkey,
-                                  const uint8_t *rsa_id_digest);
+                                  const uint8_t *rsa_id_digest));
 
+int log_cert_expiration(void);
 int load_ed_keys(const or_options_t *options, time_t now);
 int should_make_new_ed_keys(const or_options_t *options, const time_t now);
 
-int generate_ed_link_cert(const or_options_t *options, time_t now);
+int generate_ed_link_cert(const or_options_t *options, time_t now, int force);
 
 int read_encrypted_secret_key(ed25519_secret_key_t *out,
                               const char *fname);
@@ -73,5 +76,10 @@ int write_encrypted_secret_key(const ed25519_secret_key_t *out,
 
 void routerkeys_free_all(void);
 
+#ifdef TOR_UNIT_TESTS
+const ed25519_keypair_t *get_master_identity_keypair(void);
+void init_mock_ed_keys(const crypto_pk_t *rsa_identity_key);
 #endif
 
+#endif /* !defined(TOR_ROUTERKEYS_H) */
+
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index f73ec9baa1..2f27af7f06 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -93,14 +93,15 @@
 #define ROUTERLIST_PRIVATE
 #include "or.h"
 #include "backtrace.h"
+#include "bridges.h"
 #include "crypto_ed25519.h"
 #include "circuitstats.h"
 #include "config.h"
 #include "connection.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
 #include "entrynodes.h"
 #include "fp_pair.h"
 #include "geoip.h"
@@ -121,6 +122,9 @@
 #include "sandbox.h"
 #include "torcert.h"
 
+#include "dirauth/dirvote.h"
+#include "dirauth/mode.h"
+
 // #define DEBUG_ROUTERLIST
 
 /****************************************************************************/
@@ -142,6 +146,10 @@ DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
 #define DSMAP_FOREACH(map, keyvar, valvar) \
   DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
                     valvar)
+#define eimap_free(map, fn) MAP_FREE_AND_NULL(eimap, (map), (fn))
+#define rimap_free(map, fn) MAP_FREE_AND_NULL(rimap, (map), (fn))
+#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
+#define sdmap_free(map, fn) MAP_FREE_AND_NULL(sdmap, (map), (fn))
 
 /* Forward declaration for cert_list_t */
 typedef struct cert_list_t cert_list_t;
@@ -158,7 +166,6 @@ static const routerstatus_t *router_pick_dirserver_generic(
                               smartlist_t *sourcelist,
                               dirinfo_type_t type, int flags);
 static void mark_all_dirservers_up(smartlist_t *server_list);
-static void dir_server_free(dir_server_t *ds);
 static int signed_desc_digest_is_recognized(signed_descriptor_t *desc);
 static const char *signed_descriptor_get_body_impl(
                                               const signed_descriptor_t *desc,
@@ -173,7 +180,7 @@ static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
                                               const char *digest);
 static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
                                                 const char *digest,
-                                                time_t now, int max_failures);
+                                                time_t now);
 
 /****************************************************************************/
 
@@ -231,7 +238,7 @@ get_n_authorities(dirinfo_type_t type)
   return n;
 }
 
-/** Initialise schedule, want_authority, and increment on in the download
+/** Initialise schedule, want_authority, and increment_on in the download
  * status dlstatus, then call download_status_reset() on it.
  * It is safe to call this function or download_status_reset() multiple times
  * on a new dlstatus. But it should *not* be called after a dlstatus has been
@@ -242,7 +249,6 @@ download_status_cert_init(download_status_t *dlstatus)
   dlstatus->schedule = DL_SCHED_CONSENSUS;
   dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
   dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
-  dlstatus->backoff = DL_SCHED_RANDOM_EXPONENTIAL;
   dlstatus->last_backoff_position = 0;
   dlstatus->last_delay_used = 0;
 
@@ -284,7 +290,7 @@ download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
 static int
 download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
                                      const char *digest,
-                                     time_t now, int max_failures)
+                                     time_t now)
 {
   int rv = 0;
   download_status_t *dlstatus = NULL;
@@ -301,7 +307,7 @@ download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
   /* Got one? */
   if (dlstatus) {
     /* Use download_status_is_ready() */
-    rv = download_status_is_ready(dlstatus, now, max_failures);
+    rv = download_status_is_ready(dlstatus, now);
   } else {
     /*
      * If we don't know anything about it, return 1, since we haven't
@@ -363,7 +369,7 @@ list_authority_ids_with_downloads, (void))
       smartlist_add(ids, tmp);
     }
   }
-  /* else definitely no downlaods going since nothing even has a cert list */
+  /* else definitely no downloads going since nothing even has a cert list */
 
   return ids;
 }
@@ -426,8 +432,8 @@ list_sk_digests_for_authority_id, (const char *digest))
  * download_status_t or NULL if none exists. */
 
 MOCK_IMPL(download_status_t *,
-  download_status_for_authority_id_and_sk,
-  (const char *id_digest, const char *sk_digest))
+download_status_for_authority_id_and_sk,(const char *id_digest,
+                                         const char *sk_digest))
 {
   download_status_t *dl = NULL;
   cert_list_t *cl = NULL;
@@ -442,9 +448,12 @@ MOCK_IMPL(download_status_t *,
   return dl;
 }
 
+#define cert_list_free(val) \
+  FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
+
 /** Release all space held by a cert_list_t */
 static void
-cert_list_free(cert_list_t *cl)
+cert_list_free_(cert_list_t *cl)
 {
   if (!cl)
     return;
@@ -458,9 +467,9 @@ cert_list_free(cert_list_t *cl)
 
 /** Wrapper for cert_list_free so we can pass it to digestmap_free */
 static void
-cert_list_free_(void *cl)
+cert_list_free_void(void *cl)
 {
-  cert_list_free(cl);
+  cert_list_free_(cl);
 }
 
 /** Reload the cached v3 key certificates from the cached-certs file in
@@ -472,7 +481,7 @@ trusted_dirs_reload_certs(void)
   char *contents;
   int r;
 
-  filename = get_datadir_fname("cached-certs");
+  filename = get_cachedir_fname("cached-certs");
   contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
   tor_free(filename);
   if (!contents)
@@ -587,7 +596,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
                "signing key %s", from_store ? "cached" : "downloaded",
                ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
     } else {
-      int adding = directory_caches_unknown_auth_certs(get_options());
+      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
       log_info(LD_DIR, "%s %s certificate for unrecognized directory "
                "authority with signing key %s",
                adding ? "Adding" : "Not adding",
@@ -661,7 +670,7 @@ trusted_dirs_flush_certs_to_disk(void)
           });
   } DIGESTMAP_FOREACH_END;
 
-  filename = get_datadir_fname("cached-certs");
+  filename = get_cachedir_fname("cached-certs");
   if (write_chunks_to_file(filename, chunks, 0, 0)) {
     log_warn(LD_FS, "Error writing certificates to disk.");
   }
@@ -930,7 +939,8 @@ authority_certs_fetch_resource_impl(const char *resource,
                                     const routerstatus_t *rs)
 {
   const or_options_t *options = get_options();
-  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0);
+  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                                            resource);
 
   /* Make sure bridge clients never connect to anything but a bridge */
   if (options->UseBridges) {
@@ -946,31 +956,34 @@ authority_certs_fetch_resource_impl(const char *resource,
   const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
                                                     : DIRIND_ONEHOP;
 
+  directory_request_t *req = NULL;
   /* If we've just downloaded a consensus from a bridge, re-use that
    * bridge */
-  if (options->UseBridges && node && !get_via_tor) {
+  if (options->UseBridges && node && node->ri && !get_via_tor) {
     /* clients always make OR connections to bridges */
     tor_addr_port_t or_ap;
     /* we are willing to use a non-preferred address if we need to */
     fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
                                          &or_ap);
-    directory_initiate_command(&or_ap.addr, or_ap.port,
-                               NULL, 0, /*no dirport*/
-                               dir_hint,
-                               DIR_PURPOSE_FETCH_CERTIFICATE,
-                               0,
-                               indirection,
-                               resource, NULL, 0, 0);
-    return;
-  }
 
-  if (rs) {
-    /* If we've just downloaded a consensus from a directory, re-use that
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_or_addr_port(req, &or_ap);
+    if (dir_hint)
+      directory_request_set_directory_id_digest(req, dir_hint);
+  } else if (rs) {
+    /* And if we've just downloaded a consensus from a directory, re-use that
      * directory */
-    directory_initiate_command_routerstatus(rs,
-                                            DIR_PURPOSE_FETCH_CERTIFICATE,
-                                            0, indirection, resource, NULL,
-                                            0, 0);
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_routerstatus(req, rs);
+  }
+
+  if (req) {
+    /* We've set up a request object -- fill in the other request fields, and
+     * send the request.  */
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
     return;
   }
 
@@ -1012,7 +1025,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
   char *resource = NULL;
   cert_list_t *cl;
   const or_options_t *options = get_options();
-  const int cache = directory_caches_unknown_auth_certs(options);
+  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
   fp_pair_t *fp_tmp = NULL;
   char id_digest_str[2*DIGEST_LEN+1];
   char sk_digest_str[2*DIGEST_LEN+1];
@@ -1057,8 +1070,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
       }
     } SMARTLIST_FOREACH_END(cert);
     if (!found &&
-        download_status_is_ready(&(cl->dl_status_by_id), now,
-                                 options->TestingCertMaxDownloadTries) &&
+        download_status_is_ready(&(cl->dl_status_by_id), now) &&
         !digestmap_get(pending_id, ds->v3_identity_digest)) {
       log_info(LD_DIR,
                "No current certificate known for authority %s "
@@ -1084,9 +1096,10 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
       if (!smartlist_len(voter->sigs))
         continue; /* This authority never signed this consensus, so don't
                    * go looking for a cert with key digest 0000000000. */
-      if (!cache &&
+      if (!keep_unknown &&
           !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
-        continue; /* We are not a cache, and we don't know this authority.*/
+        continue; /* We don't want unknown certs, and we don't know this
+                   * authority.*/
 
       /*
        * If we don't know *any* cert for this authority, and a download by ID
@@ -1120,8 +1133,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
           continue;
         }
         if (download_status_is_ready_by_sk_in_cl(
-              cl, sig->signing_key_digest,
-              now, options->TestingCertMaxDownloadTries) &&
+              cl, sig->signing_key_digest, now) &&
             !fp_pair_map_get_by_digests(pending_cert,
                                         voter->identity_digest,
                                         sig->signing_key_digest)) {
@@ -1158,7 +1170,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
     } SMARTLIST_FOREACH_END(voter);
   }
 
-  /* Bridge clients look up the node for the dir_hint  */
+  /* Bridge clients look up the node for the dir_hint */
   const node_t *node = NULL;
   /* All clients, including bridge clients, look up the routerstatus for the
    * dir_hint */
@@ -1203,7 +1215,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
     int need_plus = 0;
     smartlist_t *fps = smartlist_new();
 
-    smartlist_add(fps, tor_strdup("fp/"));
+    smartlist_add_strdup(fps, "fp/");
 
     SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
       char *fp = NULL;
@@ -1243,7 +1255,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
     int need_plus = 0;
     smartlist_t *fp_pairs = smartlist_new();
 
-    smartlist_add(fp_pairs, tor_strdup("fp-sk/"));
+    smartlist_add_strdup(fp_pairs, "fp-sk/");
 
     SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
       char *fp_pair = NULL;
@@ -1333,7 +1345,7 @@ static int
 signed_desc_append_to_journal(signed_descriptor_t *desc,
                               desc_store_t *store)
 {
-  char *fname = get_datadir_fname_suffix(store->fname_base, ".new");
+  char *fname = get_cachedir_fname_suffix(store->fname_base, ".new");
   const char *body = signed_descriptor_get_body_impl(desc,1);
   size_t len = desc->signed_descriptor_len + desc->annotations_len;
 
@@ -1404,8 +1416,8 @@ router_rebuild_store(int flags, desc_store_t *store)
 
   log_info(LD_DIR, "Rebuilding %s cache", store->description);
 
-  fname = get_datadir_fname(store->fname_base);
-  fname_tmp = get_datadir_fname_suffix(store->fname_base, ".tmp");
+  fname = get_cachedir_fname(store->fname_base);
+  fname_tmp = get_cachedir_fname_suffix(store->fname_base, ".tmp");
 
   chunk_list = smartlist_new();
 
@@ -1502,7 +1514,7 @@ router_rebuild_store(int flags, desc_store_t *store)
   } SMARTLIST_FOREACH_END(sd);
 
   tor_free(fname);
-  fname = get_datadir_fname_suffix(store->fname_base, ".new");
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
   write_str_to_file(fname, "", 1);
 
   r = 0;
@@ -1532,7 +1544,7 @@ router_reload_router_list_impl(desc_store_t *store)
   int extrainfo = (store->type == EXTRAINFO_STORE);
   store->journal_len = store->store_len = 0;
 
-  fname = get_datadir_fname(store->fname_base);
+  fname = get_cachedir_fname(store->fname_base);
 
   if (store->mmap) {
     /* get rid of it first */
@@ -1559,7 +1571,7 @@ router_reload_router_list_impl(desc_store_t *store)
   }
 
   tor_free(fname);
-  fname = get_datadir_fname_suffix(store->fname_base, ".new");
+  fname = get_cachedir_fname_suffix(store->fname_base, ".new");
   /* don't load empty files - we wouldn't get any data, even if we tried */
   if (file_status(fname) == FN_FILE)
     contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st);
@@ -1824,43 +1836,24 @@ router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
   return 0;
 }
 
-/* Check if we already have a directory fetch from ds, for serverdesc
- * (including extrainfo) or microdesc documents.
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
  * If so, return 1, if not, return 0.
  */
 static int
-router_is_already_dir_fetching_ds(const dir_server_t *ds,
-                                  int serverdesc,
-                                  int microdesc)
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+                                const tor_addr_t *ipv6_addr,
+                                uint16_t dir_port,
+                                int serverdesc,
+                                int microdesc)
 {
   tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
 
   /* Assume IPv6 DirPort is the same as IPv4 DirPort */
-  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ds->addr);
-  ipv4_dir_ap.port = ds->dir_port;
-  tor_addr_copy(&ipv6_dir_ap.addr, &ds->ipv6_addr);
-  ipv6_dir_ap.port = ds->dir_port;
-
-  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
-       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
-}
-
-/* Check if we already have a directory fetch from rs, for serverdesc
- * (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- */
-static int
-router_is_already_dir_fetching_rs(const routerstatus_t *rs,
-                                  int serverdesc,
-                                  int microdesc)
-{
-  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
-
-  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
-  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, rs->addr);
-  ipv4_dir_ap.port = rs->dir_port;
-  tor_addr_copy(&ipv6_dir_ap.addr, &rs->ipv6_addr);
-  ipv6_dir_ap.port = rs->dir_port;
+  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+  ipv4_dir_ap.port = dir_port;
+  tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+  ipv6_dir_ap.port = dir_port;
 
   return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
        || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
@@ -1886,7 +1879,7 @@ router_picked_poor_directory_log(const routerstatus_t *rs)
       || !router_have_minimum_dir_info()) {
     return;
   }
-#endif
+#endif /* !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP */
 
   /* We couldn't find a node, or the one we have doesn't fit our preferences.
    * Sometimes this is normal, sometimes it can be a reachability issue. */
@@ -1952,6 +1945,21 @@ router_picked_poor_directory_log(const routerstatus_t *rs)
     }                                                                         \
   STMT_END
 
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity)                        \
+  STMT_BEGIN                                                                  \
+    int is_trusted_extrainfo = router_digest_is_trusted_dir_type(             \
+                               (identity), EXTRAINFO_DIRINFO);                \
+    if (((type) & EXTRAINFO_DIRINFO) &&                                       \
+        !router_supports_extrainfo((identity), is_trusted_extrainfo))         \
+      continue;                                                               \
+  STMT_END
+
 /* When iterating through the routerlist, can OR address/port preference
  * and reachability checks be skipped?
  */
@@ -1998,7 +2006,6 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
   const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
   const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
   const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
-  const int for_guard = (flags & PDS_FOR_GUARD);
   int try_excluding = 1, n_excluded = 0, n_busy = 0;
   int try_ip_pref = 1;
 
@@ -2020,7 +2027,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
 
   /* Find all the running dirservers we know about. */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
-    int is_trusted, is_trusted_extrainfo;
+    int is_trusted;
     int is_overloaded;
     const routerstatus_t *status = node->rs;
     const country_t country = node->country;
@@ -2031,20 +2038,9 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
       continue;
     if (requireother && router_digest_is_me(node->identity))
       continue;
-    is_trusted = router_digest_is_trusted_dir(node->identity);
-    is_trusted_extrainfo = router_digest_is_trusted_dir_type(
-                           node->identity, EXTRAINFO_DIRINFO);
-    if ((type & EXTRAINFO_DIRINFO) &&
-        !router_supports_extrainfo(node->identity, is_trusted_extrainfo))
-      continue;
-    /* Don't make the same node a guard twice */
-    if (for_guard && node->using_as_guard) {
-      continue;
-    }
-    /* Ensure that a directory guard is actually a guard node. */
-    if (for_guard && !node->is_possible_guard) {
-      continue;
-    }
+
+    SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
     if (try_excluding &&
         routerset_contains_routerstatus(options->ExcludeNodes, status,
                                         country)) {
@@ -2052,14 +2048,17 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
       continue;
     }
 
-    if (router_is_already_dir_fetching_rs(status,
-                                          no_serverdesc_fetching,
-                                          no_microdesc_fetching)) {
+    if (router_is_already_dir_fetching_(status->addr,
+                                        &status->ipv6_addr,
+                                        status->dir_port,
+                                        no_serverdesc_fetching,
+                                        no_microdesc_fetching)) {
       ++n_busy;
       continue;
     }
 
     is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+    is_trusted = router_digest_is_trusted_dir(node->identity);
 
     /* Clients use IPv6 addresses if the server has one and the client
      * prefers IPv6.
@@ -2191,11 +2190,9 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
       if (!d->is_running) continue;
       if ((type & d->type) == 0)
         continue;
-      int is_trusted_extrainfo = router_digest_is_trusted_dir_type(
-                                 d->digest, EXTRAINFO_DIRINFO);
-      if ((type & EXTRAINFO_DIRINFO) &&
-          !router_supports_extrainfo(d->digest, is_trusted_extrainfo))
-        continue;
+
+      SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
       if (requireother && me && router_digest_is_me(d->digest))
         continue;
       if (try_excluding &&
@@ -2205,8 +2202,11 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
         continue;
       }
 
-      if (router_is_already_dir_fetching_ds(d, no_serverdesc_fetching,
-                                            no_microdesc_fetching)) {
+      if (router_is_already_dir_fetching_(d->addr,
+                                          &d->ipv6_addr,
+                                          d->dir_port,
+                                          no_serverdesc_fetching,
+                                          no_microdesc_fetching)) {
         ++n_busy;
         continue;
       }
@@ -2313,7 +2313,7 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
 {
   /* XXXX MOVE ? */
   node_t fake_node;
-  const node_t *node = node_get_by_id(router->cache_info.identity_digest);;
+  const node_t *node = node_get_by_id(router->cache_info.identity_digest);
   if (node == NULL) {
     memset(&fake_node, 0, sizeof(fake_node));
     fake_node.ri = (routerinfo_t *)router;
@@ -2327,19 +2327,18 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
  * we can pick a node for a circuit.
  */
 void
-router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid,
-                                      int need_uptime, int need_capacity,
-                                      int need_guard, int need_desc,
-                                      int pref_addr, int direct_conn)
+router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                      int need_capacity, int need_guard,
+                                      int need_desc, int pref_addr,
+                                      int direct_conn)
 {
   const int check_reach = !router_skip_or_reachability(get_options(),
                                                        pref_addr);
   /* XXXX MOVE */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
-    if (!node->is_running ||
-        (!node->is_valid && !allow_invalid))
+    if (!node->is_running || !node->is_valid)
       continue;
-    if (need_desc && !(node->ri || (node->rs && node->md)))
+    if (need_desc && !node_has_preferred_descriptor(node, direct_conn))
       continue;
     if (node->ri && node->ri->purpose != ROUTER_PURPOSE_GENERAL)
       continue;
@@ -2653,7 +2652,7 @@ compute_weighted_bandwidths(const smartlist_t *sl,
     is_dir = node_is_dir(node);
     if (node->rs) {
       if (!node->rs->has_bandwidth) {
-        /* This should never happen, unless all the authorites downgrade
+        /* This should never happen, unless all the authorities downgrade
          * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */
         if (! warned_missing_bw) {
           log_warn(LD_BUG,
@@ -2762,17 +2761,16 @@ frac_nodes_with_descriptors(const smartlist_t *sl,
       total <= 0.0) {
     int n_with_descs = 0;
     SMARTLIST_FOREACH(sl, const node_t *, node, {
-      if (node_has_descriptor(node))
+      if (node_has_any_descriptor(node))
         n_with_descs++;
     });
-
     tor_free(bandwidths);
-    return ((double)n_with_descs) / (double)smartlist_len(sl);
+    return ((double)n_with_descs) / smartlist_len(sl);
   }
 
   present = 0.0;
   SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
-    if (node_has_descriptor(node))
+    if (node_has_any_descriptor(node))
       present += bandwidths[node_sl_idx];
   } SMARTLIST_FOREACH_END(node);
 
@@ -2798,8 +2796,6 @@ node_sl_choose_by_bandwidth(const smartlist_t *sl,
  * a minimum uptime, return one of those.
  * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
  * advertised capacity of each router.
- * If <b>CRN_ALLOW_INVALID</b> is not set in flags, consider only Valid
- * routers.
  * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
  * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
  * picking an exit node, otherwise we weight bandwidths for picking a relay
@@ -2820,11 +2816,11 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
   const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
   const int need_guard = (flags & CRN_NEED_GUARD) != 0;
-  const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0;
   const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
   const int need_desc = (flags & CRN_NEED_DESC) != 0;
   const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
+  const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
 
   smartlist_t *sl=smartlist_new(),
     *excludednodes=smartlist_new();
@@ -2836,14 +2832,19 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   rule = weight_for_exit ? WEIGHT_FOR_EXIT :
     (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
 
-  /* Exclude relays that allow single hop exit circuits, if the user
-   * wants to (such relays might be risky) */
-  if (get_options()->ExcludeSingleHopRelays) {
-    SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node,
-      if (node_allows_single_hop_exits(node)) {
-        smartlist_add(excludednodes, node);
-      });
-  }
+  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+    if (node_allows_single_hop_exits(node)) {
+      /* Exclude relays that allow single hop exit circuits. This is an
+       * obsolete option since 0.2.9.2-alpha and done by default in
+       * 0.3.1.0-alpha. */
+      smartlist_add(excludednodes, node);
+    } else if (rendezvous_v3 &&
+               !node_supports_v3_rendezvous_point(node)) {
+      /* Exclude relays that do not support to rendezvous for a hidden service
+       * version 3. */
+      smartlist_add(excludednodes, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
 
   /* If the node_t is not found we won't be to exclude ourself but we
    * won't be able to pick ourself in router_choose_random_node() so
@@ -2851,8 +2852,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   if ((r = router_get_my_routerinfo()))
     routerlist_add_node_and_family(excludednodes, r);
 
-  router_add_running_nodes_to_smartlist(sl, allow_invalid,
-                                        need_uptime, need_capacity,
+  router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
                                         need_guard, need_desc, pref_addr,
                                         direct_conn);
   log_debug(LD_CIRC,
@@ -2958,7 +2958,7 @@ hex_digest_nickname_decode(const char *hexdigest,
  * <b>hexdigest</b> is malformed, or it doesn't match.  */
 int
 hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
-                            const char *nickname, int is_named)
+                            const char *nickname)
 {
   char digest[DIGEST_LEN];
   char nn_char='\0';
@@ -2967,30 +2967,20 @@ hex_digest_nickname_matches(const char *hexdigest, const char *identity_digest,
   if (hex_digest_nickname_decode(hexdigest, digest, &nn_char, nn_buf) == -1)
     return 0;
 
-  if (nn_char == '=' || nn_char == '~') {
-    if (!nickname)
+  if (nn_char == '=') {
+    return 0;
+  }
+
+  if (nn_char == '~') {
+    if (!nickname)     // XXX This seems wrong. -NM
       return 0;
     if (strcasecmp(nn_buf, nickname))
       return 0;
-    if (nn_char == '=' && !is_named)
-      return 0;
   }
 
   return tor_memeq(digest, identity_digest, DIGEST_LEN);
 }
 
-/** Return true iff <b>router</b> is listed as named in the current
- * consensus. */
-int
-router_is_named(const routerinfo_t *router)
-{
-  const char *digest =
-    networkstatus_get_router_digest_by_nickname(router->nickname);
-
-  return (digest &&
-          tor_memeq(digest, router->cache_info.identity_digest, DIGEST_LEN));
-}
-
 /** Return true iff <b>digest</b> is the digest of the identity key of a
  * trusted directory matching at least one bit of <b>type</b>.  If <b>type</b>
  * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */
@@ -3008,20 +2998,6 @@ router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
   return 0;
 }
 
-/** Return true iff <b>addr</b> is the address of one of our trusted
- * directory authorities. */
-int
-router_addr_is_trusted_dir(uint32_t addr)
-{
-  if (!trusted_dir_servers)
-    return 0;
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
-    if (ent->addr == addr)
-      return 1;
-    );
-  return 0;
-}
-
 /** If hexdigest is correctly formed, base16_decode it into
  * digest, which must have DIGEST_LEN space in it.
  * Return 0 on success, -1 on failure.
@@ -3087,8 +3063,8 @@ router_get_by_extrainfo_digest,(const char *digest))
 /** Return the signed descriptor for the extrainfo_t in our routerlist whose
  * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info
  * document is known. */
-signed_descriptor_t *
-extrainfo_get_by_descriptor_digest(const char *digest)
+MOCK_IMPL(signed_descriptor_t *,
+extrainfo_get_by_descriptor_digest,(const char *digest))
 {
   extrainfo_t *ei;
   tor_assert(digest);
@@ -3130,7 +3106,7 @@ signed_descriptor_get_body_impl(const signed_descriptor_t *desc,
       log_err(LD_DIR, "We couldn't read a descriptor that is supposedly "
               "mmaped in our cache.  Is another process running in our data "
               "directory?  Exiting.");
-      exit(1);
+      exit(1); // XXXX bad exit: should recover.
     }
   }
   if (!r) /* no mmap, or not in cache. */
@@ -3144,7 +3120,7 @@ signed_descriptor_get_body_impl(const signed_descriptor_t *desc,
       log_err(LD_DIR, "descriptor at %p begins with unexpected string %s.  "
               "Is another process running in our data directory?  Exiting.",
               desc, escaped(cp));
-      exit(1);
+      exit(1); // XXXX bad exit: should recover.
     }
   }
 
@@ -3198,7 +3174,7 @@ router_get_routerlist(void)
 
 /** Free all storage held by <b>router</b>. */
 void
-routerinfo_free(routerinfo_t *router)
+routerinfo_free_(routerinfo_t *router)
 {
   if (!router)
     return;
@@ -3228,7 +3204,7 @@ routerinfo_free(routerinfo_t *router)
 
 /** Release all storage held by <b>extrainfo</b> */
 void
-extrainfo_free(extrainfo_t *extrainfo)
+extrainfo_free_(extrainfo_t *extrainfo)
 {
   if (!extrainfo)
     return;
@@ -3240,9 +3216,12 @@ extrainfo_free(extrainfo_t *extrainfo)
   tor_free(extrainfo);
 }
 
+#define signed_descriptor_free(val) \
+  FREE_AND_NULL(signed_descriptor_t, signed_descriptor_free_, (val))
+
 /** Release storage held by <b>sd</b>. */
 static void
-signed_descriptor_free(signed_descriptor_t *sd)
+signed_descriptor_free_(signed_descriptor_t *sd)
 {
   if (!sd)
     return;
@@ -3296,21 +3275,21 @@ signed_descriptor_from_routerinfo(routerinfo_t *ri)
 
 /** Helper: free the storage held by the extrainfo_t in <b>e</b>. */
 static void
-extrainfo_free_(void *e)
+extrainfo_free_void(void *e)
 {
-  extrainfo_free(e);
+  extrainfo_free_(e);
 }
 
 /** Free all storage held by a routerlist <b>rl</b>. */
 void
-routerlist_free(routerlist_t *rl)
+routerlist_free_(routerlist_t *rl)
 {
   if (!rl)
     return;
   rimap_free(rl->identity_map, NULL);
   sdmap_free(rl->desc_digest_map, NULL);
   sdmap_free(rl->desc_by_eid_map, NULL);
-  eimap_free(rl->extra_info_map, extrainfo_free_);
+  eimap_free(rl->extra_info_map, extrainfo_free_void);
   SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
                     routerinfo_free(r));
   SMARTLIST_FOREACH(rl->old_routers, signed_descriptor_t *, sd,
@@ -3802,7 +3781,7 @@ routerlist_free_all(void)
   smartlist_free(fallback_dir_servers);
   trusted_dir_servers = fallback_dir_servers = NULL;
   if (trusted_dir_certs) {
-    digestmap_free(trusted_dir_certs, cert_list_free_);
+    digestmap_free(trusted_dir_certs, cert_list_free_void);
     trusted_dir_certs = NULL;
   }
 }
@@ -3926,7 +3905,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
                router_describe(router));
       *msg = "Router descriptor is not referenced by any network-status.";
 
-      /* Only journal this desc if we'll be serving it. */
+      /* Only journal this desc if we want to keep old descriptors */
       if (!from_cache && should_cache_old_descriptors())
         signed_desc_append_to_journal(&router->cache_info,
                                       &routerlist->desc_store);
@@ -4100,7 +4079,7 @@ routerlist_remove_old_cached_routers_with_id(time_t now,
     signed_descriptor_t *r = smartlist_get(lst, i);
     tor_assert(tor_memeq(ident, r->identity_digest, DIGEST_LEN));
   }
-#endif
+#endif /* 1 */
   /* Check whether we need to do anything at all. */
   {
     int mdpr = directory_caches_dir_info(get_options()) ? 2 : 1;
@@ -4513,7 +4492,7 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
                         ei->cache_info.identity_digest,
                       DIGEST_LEN);
         smartlist_string_remove(requested_fingerprints, fp);
-        /* We silently let people stuff us with extrainfos we didn't ask for,
+        /* We silently let relays stuff us with extrainfos we didn't ask for,
          * so long as we would have wanted them anyway.  Since we always fetch
          * all the extrainfos we want, and we never actually act on them
          * inside Tor, this should be harmless. */
@@ -4556,13 +4535,14 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
   smartlist_free(extrainfo_list);
 }
 
-/** Return true iff any networkstatus includes a descriptor whose digest
- * is that of <b>desc</b>. */
+/** Return true iff the latest ns-flavored consensus includes a descriptor
+ * whose digest is that of <b>desc</b>. */
 static int
 signed_desc_digest_is_recognized(signed_descriptor_t *desc)
 {
   const routerstatus_t *rs;
-  networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(
+                                                                      FLAV_NS);
 
   if (consensus) {
     rs = networkstatus_vote_find_entry(consensus, desc->identity_digest);
@@ -4578,7 +4558,7 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc)
 void
 update_all_descriptor_downloads(time_t now)
 {
-  if (get_options()->DisableNetwork)
+  if (should_delay_dir_fetches(get_options(), NULL))
     return;
   update_router_descriptor_downloads(now);
   update_microdesc_downloads(now);
@@ -4608,7 +4588,7 @@ router_exit_policy_rejects_all(const routerinfo_t *router)
   return router->policy_is_reject_star;
 }
 
-/** Create an directory server at <b>address</b>:<b>port</b>, with OR identity
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
  * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
  * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
  * the new directory server entry on success or NULL on failure. */
@@ -4769,7 +4749,7 @@ dir_server_add(dir_server_t *ent)
 
 /** Free storage held in <b>cert</b>. */
 void
-authority_cert_free(authority_cert_t *cert)
+authority_cert_free_(authority_cert_t *cert)
 {
   if (!cert)
     return;
@@ -4781,9 +4761,12 @@ authority_cert_free(authority_cert_t *cert)
   tor_free(cert);
 }
 
+#define dir_server_free(val) \
+  FREE_AND_NULL(dir_server_t, dir_server_free_, (val))
+
 /** Free storage held in <b>ds</b>. */
 static void
-dir_server_free(dir_server_t *ds)
+dir_server_free_(dir_server_t *ds)
 {
   if (!ds)
     return;
@@ -4919,9 +4902,10 @@ list_pending_fpsk_downloads(fp_pair_map_t *result)
  * range.)  If <b>source</b> is given, download from <b>source</b>;
  * otherwise, download from an appropriate random directory server.
  */
-MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
-          (const routerstatus_t *source, int purpose, smartlist_t *digests,
-           int lo, int hi, int pds_flags))
+MOCK_IMPL(STATIC void,
+initiate_descriptor_downloads,(const routerstatus_t *source,
+                               int purpose, smartlist_t *digests,
+                               int lo, int hi, int pds_flags))
 {
   char *resource, *cp;
   int digest_len, enc_digest_len;
@@ -4973,10 +4957,11 @@ MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
 
   if (source) {
     /* We know which authority or directory mirror we want. */
-    directory_initiate_command_routerstatus(source, purpose,
-                                            ROUTER_PURPOSE_GENERAL,
-                                            DIRIND_ONEHOP,
-                                            resource, NULL, 0, 0);
+    directory_request_t *req = directory_request_new(purpose);
+    directory_request_set_routerstatus(req, source);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
   } else {
     directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource,
                                  pds_flags, DL_WANT_ANY_DIRSERVER);
@@ -5014,8 +4999,9 @@ max_dl_per_request(const or_options_t *options, int purpose)
 }
 
 /** Don't split our requests so finely that we are requesting fewer than
- * this number per server. */
-#define MIN_DL_PER_REQUEST 4
+ * this number per server. (Grouping more than this at once leads to
+ * diminishing returns.) */
+#define MIN_DL_PER_REQUEST 32
 /** To prevent a single screwy cache from confusing us by selective reply,
  * try to split our requests into at least this many requests. */
 #define MIN_REQUESTS 3
@@ -5081,7 +5067,7 @@ launch_descriptor_downloads(int purpose,
     }
   }
 
-  if (!authdir_mode_any_nonhidserv(options)) {
+  if (!authdir_mode(options)) {
     /* If we wind up going to the authorities, we want to only open one
      * connection to each authority at a time, so that we don't overload
      * them.  We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH
@@ -5103,8 +5089,9 @@ launch_descriptor_downloads(int purpose,
   if (n_per_request > max_dl_per_req)
     n_per_request = max_dl_per_req;
 
-  if (n_per_request < MIN_DL_PER_REQUEST)
-    n_per_request = MIN_DL_PER_REQUEST;
+  if (n_per_request < MIN_DL_PER_REQUEST) {
+    n_per_request = MIN(MIN_DL_PER_REQUEST, n_downloadable);
+  }
 
   if (n_downloadable > n_per_request)
     req_plural = rtr_plural = "s";
@@ -5181,8 +5168,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_inprogress;
         continue; /* We have an in-progress download. */
       }
-      if (!download_status_is_ready(&rs->dl_status, now,
-                          options->TestingDescriptorMaxDownloadTries)) {
+      if (!download_status_is_ready(&rs->dl_status, now)) {
         ++n_delayed; /* Not ready for retry. */
         continue;
       }
@@ -5190,8 +5176,8 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_would_reject;
         continue; /* We would throw it out immediately. */
       }
-      if (!directory_caches_dir_info(options) &&
-          !client_would_use_router(rs, now, options)) {
+      if (!we_want_to_fetch_flavor(options, consensus->flavor) &&
+          !client_would_use_router(rs, now)) {
         ++n_wouldnt_use;
         continue; /* We would never use it ourself. */
       }
@@ -5212,7 +5198,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
       smartlist_add(downloadable, rs->descriptor_digest);
   } SMARTLIST_FOREACH_END(rsp);
 
-  if (!authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)
+  if (!authdir_mode_v3(options)
       && smartlist_len(no_longer_old)) {
     routerlist_t *rl = router_get_routerlist();
     log_info(LD_DIR, "%d router descriptors listed in consensus are "
@@ -5221,15 +5207,30 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
     SMARTLIST_FOREACH_BEGIN(no_longer_old, signed_descriptor_t *, sd) {
         const char *msg;
         was_router_added_t r;
+        time_t tmp_cert_expiration_time;
         routerinfo_t *ri = routerlist_reparse_old(rl, sd);
         if (!ri) {
           log_warn(LD_BUG, "Failed to re-parse a router.");
           continue;
         }
+        /* need to remember for below, since add_to_routerlist may free. */
+        tmp_cert_expiration_time = ri->cert_expiration_time;
+
         r = router_add_to_routerlist(ri, &msg, 1, 0);
         if (WRA_WAS_OUTDATED(r)) {
-          log_warn(LD_DIR, "Couldn't add re-parsed router: %s",
+          log_warn(LD_DIR, "Couldn't add re-parsed router: %s. This isn't "
+                   "usually a big deal, but you should make sure that your "
+                   "clock and timezone are set correctly.",
                    msg?msg:"???");
+          if (r == ROUTER_CERTS_EXPIRED) {
+            char time_cons[ISO_TIME_LEN+1];
+            char time_cert_expires[ISO_TIME_LEN+1];
+            format_iso_time(time_cons, consensus->valid_after);
+            format_iso_time(time_cert_expires, tmp_cert_expiration_time);
+            log_warn(LD_DIR, "  (I'm looking at a consensus from %s; This "
+                     "router's certificates began expiring at %s.)",
+                     time_cons, time_cert_expires);
+          }
         }
     } SMARTLIST_FOREACH_END(sd);
     routerlist_assert_ok(rl);
@@ -5343,8 +5344,7 @@ update_extrainfo_downloads(time_t now)
         ++n_have;
         continue;
       }
-      if (!download_status_is_ready(&sd->ei_dl_status, now,
-                          options->TestingDescriptorMaxDownloadTries)) {
+      if (!download_status_is_ready(&sd->ei_dl_status, now)) {
         ++n_delay;
         continue;
       }
@@ -5406,8 +5406,10 @@ update_extrainfo_downloads(time_t now)
   smartlist_free(wanted);
 }
 
-/** Reset the descriptor download failure count on all routers, so that we
- * can retry any long-failed routers immediately.
+/** Reset the consensus and extra-info download failure count on all routers.
+ * When we get a new consensus,
+ * routers_update_status_from_consensus_networkstatus() will reset the
+ * download statuses on the descriptors in that consensus.
  */
 void
 router_reset_descriptor_download_failures(void)
@@ -5419,6 +5421,8 @@ router_reset_descriptor_download_failures(void)
   last_descriptor_download_attempted = 0;
   if (!routerlist)
     return;
+  /* We want to download *all* extra-info descriptors, not just those in
+   * the consensus we currently have (or are about to have) */
   SMARTLIST_FOREACH(routerlist->routers, routerinfo_t *, ri,
   {
     download_status_reset(&ri->cache_info.ei_dl_status);
@@ -5660,11 +5664,11 @@ routerstatus_version_supports_extend2_cells(const routerstatus_t *rs,
     return allow_unknown_versions;
   }
 
-  if (!rs->protocols_known) {
+  if (!rs->pv.protocols_known) {
     return allow_unknown_versions;
   }
 
-  return rs->supports_extend2_cells;
+  return rs->pv.supports_extend2_cells;
 }
 
 /** Assert that the internal representation of <b>rl</b> is
diff --git a/src/or/routerlist.h b/src/or/routerlist.h
index 606e9085ce..83f4d1002f 100644
--- a/src/or/routerlist.h
+++ b/src/or/routerlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -62,10 +62,10 @@ int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
 int router_get_my_share_of_directory_requests(double *v3_share_out);
 void router_reset_status_download_failures(void);
 int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
-void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid,
-                                           int need_uptime, int need_capacity,
-                                           int need_guard, int need_desc,
-                                           int pref_addr, int direct_conn);
+void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                           int need_capacity, int need_guard,
+                                           int need_desc, int pref_addr,
+                                           int direct_conn);
 
 const routerinfo_t *routerlist_find_my_routerinfo(void);
 uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
@@ -80,26 +80,29 @@ const node_t *router_choose_random_node(smartlist_t *excludedsmartlist,
                                         struct routerset_t *excludedset,
                                         router_crn_flags_t flags);
 
-int router_is_named(const routerinfo_t *router);
 int router_digest_is_trusted_dir_type(const char *digest,
                                       dirinfo_type_t type);
 #define router_digest_is_trusted_dir(d) \
   router_digest_is_trusted_dir_type((d), NO_DIRINFO)
 
-int router_addr_is_trusted_dir(uint32_t addr);
 int hexdigest_to_digest(const char *hexdigest, char *digest);
 const routerinfo_t *router_get_by_id_digest(const char *digest);
 routerinfo_t *router_get_mutable_by_digest(const char *digest);
 signed_descriptor_t *router_get_by_descriptor_digest(const char *digest);
 MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest,
           (const char *digest));
-signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest);
+MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest,
+          (const char *digest));
 const char *signed_descriptor_get_body(const signed_descriptor_t *desc);
 const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc);
 routerlist_t *router_get_routerlist(void);
-void routerinfo_free(routerinfo_t *router);
-void extrainfo_free(extrainfo_t *extrainfo);
-void routerlist_free(routerlist_t *rl);
+void routerinfo_free_(routerinfo_t *router);
+#define routerinfo_free(router) \
+  FREE_AND_NULL(routerinfo_t, routerinfo_free_, (router))
+void extrainfo_free_(extrainfo_t *extrainfo);
+#define extrainfo_free(ei) FREE_AND_NULL(extrainfo_t, extrainfo_free_, (ei))
+void routerlist_free_(routerlist_t *rl);
+#define routerlist_free(rl) FREE_AND_NULL(routerlist_t, routerlist_free_, (rl))
 void dump_routerlist_mem_usage(int severity);
 void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old,
                        time_t now);
@@ -124,7 +127,7 @@ static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s);
  */
 static inline int
 WRA_WAS_ADDED(was_router_added_t s) {
-  return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR;
+  return s == ROUTER_ADDED_SUCCESSFULLY;
 }
 /** Return true iff the outcome code in <b>s</b> indicates that the descriptor
  * was not added because it was either:
@@ -192,7 +195,9 @@ dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
                                       const char *id_digest, double weight);
 void dir_server_add(dir_server_t *ent);
 
-void authority_cert_free(authority_cert_t *cert);
+void authority_cert_free_(authority_cert_t *cert);
+#define authority_cert_free(cert) \
+  FREE_AND_NULL(authority_cert_t, authority_cert_free_, (cert))
 void clear_dir_servers(void);
 void update_consensus_router_descriptor_downloads(time_t now, int is_vote,
                                                   networkstatus_t *consensus);
@@ -228,7 +233,7 @@ int hex_digest_nickname_decode(const char *hexdigest,
                                char *nickname_out);
 int hex_digest_nickname_matches(const char *hexdigest,
                                 const char *identity_digest,
-                                const char *nickname, int is_named);
+                                const char *nickname);
 
 #ifdef ROUTERLIST_PRIVATE
 STATIC int choose_array_element_by_weight(const uint64_t *entries,
@@ -252,7 +257,7 @@ MOCK_DECL(STATIC void, initiate_descriptor_downloads,
 STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
                                           int serverdesc, int microdesc);
 
-#endif
+#endif /* defined(ROUTERLIST_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_ROUTERLIST_H) */
 
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index 37d2d975fc..e82ecec5b7 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,291 +56,35 @@
 #define ROUTERPARSE_PRIVATE
 
 #include "or.h"
-#include "config.h"
 #include "circuitstats.h"
+#include "config.h"
+#include "crypto_util.h"
+#include "dirauth/shared_random.h"
 #include "dirserv.h"
-#include "dirvote.h"
-#include "policies.h"
-#include "protover.h"
-#include "rendcommon.h"
-#include "router.h"
-#include "routerlist.h"
+#include "entrynodes.h"
 #include "memarea.h"
 #include "microdesc.h"
 #include "networkstatus.h"
+#include "parsecommon.h"
+#include "policies.h"
+#include "protover.h"
+#include "rendcommon.h"
 #include "rephist.h"
+#include "router.h"
 #include "routerkeys.h"
+#include "routerlist.h"
 #include "routerparse.h"
-#include "entrynodes.h"
-#include "torcert.h"
 #include "sandbox.h"
-#include "shared_random.h"
+#include "shared_random_client.h"
+#include "torcert.h"
+#include "voting_schedule.h"
 
 #undef log
 #include <math.h>
 
-/****************************************************************************/
-
-/** Enumeration of possible token types.  The ones starting with K_ correspond
- * to directory 'keywords'. A_ is for an annotation, R or C is related to
- * hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
- * end-of-file marker, and NIL_ is used to encode not-a-token.
- */
-typedef enum {
-  K_ACCEPT = 0,
-  K_ACCEPT6,
-  K_DIRECTORY_SIGNATURE,
-  K_RECOMMENDED_SOFTWARE,
-  K_REJECT,
-  K_REJECT6,
-  K_ROUTER,
-  K_SIGNED_DIRECTORY,
-  K_SIGNING_KEY,
-  K_ONION_KEY,
-  K_ONION_KEY_NTOR,
-  K_ROUTER_SIGNATURE,
-  K_PUBLISHED,
-  K_RUNNING_ROUTERS,
-  K_ROUTER_STATUS,
-  K_PLATFORM,
-  K_PROTO,
-  K_OPT,
-  K_BANDWIDTH,
-  K_CONTACT,
-  K_NETWORK_STATUS,
-  K_UPTIME,
-  K_DIR_SIGNING_KEY,
-  K_FAMILY,
-  K_FINGERPRINT,
-  K_HIBERNATING,
-  K_READ_HISTORY,
-  K_WRITE_HISTORY,
-  K_NETWORK_STATUS_VERSION,
-  K_DIR_SOURCE,
-  K_DIR_OPTIONS,
-  K_CLIENT_VERSIONS,
-  K_SERVER_VERSIONS,
-  K_RECOMMENDED_CLIENT_PROTOCOLS,
-  K_RECOMMENDED_RELAY_PROTOCOLS,
-  K_REQUIRED_CLIENT_PROTOCOLS,
-  K_REQUIRED_RELAY_PROTOCOLS,
-  K_OR_ADDRESS,
-  K_ID,
-  K_P,
-  K_P6,
-  K_R,
-  K_A,
-  K_S,
-  K_V,
-  K_W,
-  K_M,
-  K_EXTRA_INFO,
-  K_EXTRA_INFO_DIGEST,
-  K_CACHES_EXTRA_INFO,
-  K_HIDDEN_SERVICE_DIR,
-  K_ALLOW_SINGLE_HOP_EXITS,
-  K_IPV6_POLICY,
-  K_ROUTER_SIG_ED25519,
-  K_IDENTITY_ED25519,
-  K_MASTER_KEY_ED25519,
-  K_ONION_KEY_CROSSCERT,
-  K_NTOR_ONION_KEY_CROSSCERT,
-
-  K_DIRREQ_END,
-  K_DIRREQ_V2_IPS,
-  K_DIRREQ_V3_IPS,
-  K_DIRREQ_V2_REQS,
-  K_DIRREQ_V3_REQS,
-  K_DIRREQ_V2_SHARE,
-  K_DIRREQ_V3_SHARE,
-  K_DIRREQ_V2_RESP,
-  K_DIRREQ_V3_RESP,
-  K_DIRREQ_V2_DIR,
-  K_DIRREQ_V3_DIR,
-  K_DIRREQ_V2_TUN,
-  K_DIRREQ_V3_TUN,
-  K_ENTRY_END,
-  K_ENTRY_IPS,
-  K_CELL_END,
-  K_CELL_PROCESSED,
-  K_CELL_QUEUED,
-  K_CELL_TIME,
-  K_CELL_CIRCS,
-  K_EXIT_END,
-  K_EXIT_WRITTEN,
-  K_EXIT_READ,
-  K_EXIT_OPENED,
-
-  K_DIR_KEY_CERTIFICATE_VERSION,
-  K_DIR_IDENTITY_KEY,
-  K_DIR_KEY_PUBLISHED,
-  K_DIR_KEY_EXPIRES,
-  K_DIR_KEY_CERTIFICATION,
-  K_DIR_KEY_CROSSCERT,
-  K_DIR_ADDRESS,
-  K_DIR_TUNNELLED,
-
-  K_VOTE_STATUS,
-  K_VALID_AFTER,
-  K_FRESH_UNTIL,
-  K_VALID_UNTIL,
-  K_VOTING_DELAY,
-
-  K_KNOWN_FLAGS,
-  K_PARAMS,
-  K_BW_WEIGHTS,
-  K_VOTE_DIGEST,
-  K_CONSENSUS_DIGEST,
-  K_ADDITIONAL_DIGEST,
-  K_ADDITIONAL_SIGNATURE,
-  K_CONSENSUS_METHODS,
-  K_CONSENSUS_METHOD,
-  K_LEGACY_DIR_KEY,
-  K_DIRECTORY_FOOTER,
-  K_SIGNING_CERT_ED,
-  K_SR_FLAG,
-  K_COMMIT,
-  K_PREVIOUS_SRV,
-  K_CURRENT_SRV,
-  K_PACKAGE,
-
-  A_PURPOSE,
-  A_LAST_LISTED,
-  A_UNKNOWN_,
-
-  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
-  R_VERSION,
-  R_PERMANENT_KEY,
-  R_SECRET_ID_PART,
-  R_PUBLICATION_TIME,
-  R_PROTOCOL_VERSIONS,
-  R_INTRODUCTION_POINTS,
-  R_SIGNATURE,
-
-  R_IPO_IDENTIFIER,
-  R_IPO_IP_ADDRESS,
-  R_IPO_ONION_PORT,
-  R_IPO_ONION_KEY,
-  R_IPO_SERVICE_KEY,
-
-  C_CLIENT_NAME,
-  C_DESCRIPTOR_COOKIE,
-  C_CLIENT_KEY,
-
-  ERR_,
-  EOF_,
-  NIL_
-} directory_keyword;
-
-#define MIN_ANNOTATION A_PURPOSE
-#define MAX_ANNOTATION A_UNKNOWN_
-
-/** Structure to hold a single directory token.
- *
- * We parse a directory by breaking it into "tokens", each consisting
- * of a keyword, a line full of arguments, and a binary object.  The
- * arguments and object are both optional, depending on the keyword
- * type.
- *
- * This structure is only allocated in memareas; do not allocate it on
- * the heap, or token_clear() won't work.
- */
-typedef struct directory_token_t {
-  directory_keyword tp;        /**< Type of the token. */
-  int n_args:30;               /**< Number of elements in args */
-  char **args;                 /**< Array of arguments from keyword line. */
-
-  char *object_type;           /**< -----BEGIN [object_type]-----*/
-  size_t object_size;          /**< Bytes in object_body */
-  char *object_body;           /**< Contents of object, base64-decoded. */
-
-  crypto_pk_t *key;        /**< For public keys only.  Heap-allocated. */
-
-  char *error;                 /**< For ERR_ tokens only. */
-} directory_token_t;
-
-/* ********************************************************************** */
-
-/** We use a table of rules to decide how to parse each token type. */
-
-/** Rules for whether the keyword needs an object. */
-typedef enum {
-  NO_OBJ,        /**< No object, ever. */
-  NEED_OBJ,      /**< Object is required. */
-  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
-  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
-  NEED_KEY,      /**< Object is required, and must be a public key. */
-  OBJ_OK,        /**< Object is optional. */
-} obj_syntax;
-
-#define AT_START 1
-#define AT_END 2
-
-/** Determines the parsing rules for a single token type. */
-typedef struct token_rule_t {
-  /** The string value of the keyword identifying the type of item. */
-  const char *t;
-  /** The corresponding directory_keyword enum. */
-  directory_keyword v;
-  /** Minimum number of arguments for this item */
-  int min_args;
-  /** Maximum number of arguments for this item */
-  int max_args;
-  /** If true, we concatenate all arguments for this item into a single
-   * string. */
-  int concat_args;
-  /** Requirements on object syntax for this item. */
-  obj_syntax os;
-  /** Lowest number of times this item may appear in a document. */
-  int min_cnt;
-  /** Highest number of times this item may appear in a document. */
-  int max_cnt;
-  /** One or more of AT_START/AT_END to limit where the item may appear in a
-   * document. */
-  int pos;
-  /** True iff this token is an annotation. */
-  int is_annotation;
-} token_rule_t;
+#include "dirauth/dirvote.h"
 
-/**
- * @name macros for defining token rules
- *
- * Helper macros to define token tables.  's' is a string, 't' is a
- * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
- * object syntax.
- */
-/**@{*/
-
-/** Appears to indicate the end of a table. */
-#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
-/** An item with no restrictions: used for obsolete document types */
-#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
-/** An item with no restrictions on multiplicity or location. */
-#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
-/** An item that must appear exactly once */
-#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
-/** An item that must appear exactly once, at the start of the document */
-#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
-/** An item that must appear exactly once, at the end of the document */
-#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
-/** An item that must appear one or more times */
-#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
-/** An item that must appear no more than once */
-#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
-/** An annotation that must appear no more than once */
-#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
-
-/** Argument multiplicity: any number of arguments. */
-#define ARGS        0,INT_MAX,0
-/** Argument multiplicity: no arguments. */
-#define NO_ARGS     0,0,0
-/** Argument multiplicity: concatenate all arguments. */
-#define CONCAT_ARGS 1,1,1
-/** Argument multiplicity: at least <b>n</b> arguments. */
-#define GE(n)       n,INT_MAX,0
-/** Argument multiplicity: exactly <b>n</b> arguments. */
-#define EQ(n)       n,n,0
-/**@}*/
+/****************************************************************************/
 
 /** List of tokens recognized in router descriptors */
 static token_rule_t routerdesc_token_table[] = {
@@ -619,6 +363,7 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
 static int router_get_hash_impl_helper(const char *s, size_t s_len,
                             const char *start_str,
                             const char *end_str, char end_c,
+                            int log_severity,
                             const char **start_out, const char **end_out);
 static int router_get_hash_impl(const char *s, size_t s_len, char *digest,
                                 const char *start_str, const char *end_str,
@@ -628,30 +373,9 @@ static int router_get_hashes_impl(const char *s, size_t s_len,
                                   common_digests_t *digests,
                                   const char *start_str, const char *end_str,
                                   char end_char);
-static void token_clear(directory_token_t *tok);
-static smartlist_t *find_all_by_keyword(smartlist_t *s, directory_keyword k);
 static smartlist_t *find_all_exitpolicy(smartlist_t *s);
-static directory_token_t *find_by_keyword_(smartlist_t *s,
-                                           directory_keyword keyword,
-                                           const char *keyword_str);
-#define find_by_keyword(s, keyword) find_by_keyword_((s), (keyword), #keyword)
-static directory_token_t *find_opt_by_keyword(smartlist_t *s,
-                                              directory_keyword keyword);
-
-#define TS_ANNOTATIONS_OK 1
-#define TS_NOCHECK 2
-#define TS_NO_NEW_ANNOTATIONS 4
-static int tokenize_string(memarea_t *area,
-                           const char *start, const char *end,
-                           smartlist_t *out,
-                           token_rule_t *table,
-                           int flags);
-static directory_token_t *get_next_token(memarea_t *area,
-                                         const char **s,
-                                         const char *eos,
-                                         token_rule_t *table);
-#define CST_CHECK_AUTHORITY   (1<<0)
-#define CST_NO_CHECK_OBJTYPE  (1<<1)
+
+#define CST_NO_CHECK_OBJTYPE  (1<<0)
 static int check_signature_token(const char *digest,
                                  ssize_t digest_len,
                                  directory_token_t *tok,
@@ -668,9 +392,9 @@ static int check_signature_token(const char *digest,
   log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
             name, (unsigned long)alloc, (unsigned long)used);     \
   STMT_END
-#else
+#else /* !(defined(DEBUG_AREA_ALLOC)) */
 #define DUMP_AREA(a,name) STMT_NIL
-#endif
+#endif /* defined(DEBUG_AREA_ALLOC) */
 
 /* Dump mechanism for unparseable descriptors */
 
@@ -981,7 +705,7 @@ dump_desc_populate_one_file, (const char *dirname, const char *f))
     goto done;
     /* LCOV_EXCL_STOP */
   }
-#endif
+#endif /* SIZE_MAX > UINT64_MAX */
   if (BUG(st.st_size < 0)) {
     /* LCOV_EXCL_START
      * Should be impossible, since the OS isn't supposed to be b0rken. */
@@ -995,7 +719,7 @@ dump_desc_populate_one_file, (const char *dirname, const char *f))
    * filename.
    */
   if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
-                       DIGEST_SHA256) != 0) {
+                       DIGEST_SHA256) < 0) {
     /* Weird, but okay */
     log_info(LD_DIR,
              "Unable to hash content of %s from unparseable descriptors "
@@ -1144,8 +868,8 @@ dump_desc_populate_fifo_from_directory(const char *dirname)
  * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
  * than one descriptor to disk per minute. If there is already such a
  * file in the data directory, overwrite it. */
-STATIC void
-dump_desc(const char *desc, const char *type)
+MOCK_IMPL(STATIC void,
+dump_desc,(const char *desc, const char *type))
 {
   tor_assert(desc);
   tor_assert(type);
@@ -1159,7 +883,7 @@ dump_desc(const char *desc, const char *type)
   /* Get the hash for logging purposes anyway */
   len = strlen(desc);
   if (crypto_digest256((char *)digest_sha256, desc, len,
-                       DIGEST_SHA256) != 0) {
+                       DIGEST_SHA256) < 0) {
     log_info(LD_DIR,
              "Unable to parse descriptor of type %s, and unable to even hash"
              " it!", type);
@@ -1269,6 +993,41 @@ router_get_router_hash(const char *s, size_t s_len, char *digest)
                               DIGEST_SHA1);
 }
 
+/** Try to find the start and end of the signed portion of a networkstatus
+ * document in <b>s</b>. On success, set <b>start_out</b> to the first
+ * character of the document, and <b>end_out</b> to a position one after the
+ * final character of the signed document, and return 0.  On failure, return
+ * -1. */
+int
+router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              const char **start_out,
+                                              const char **end_out)
+{
+  return router_get_hash_impl_helper(s, strlen(s),
+                                     "network-status-version",
+                                     "\ndirectory-signature",
+                                     ' ', LOG_INFO,
+                                     start_out, end_out);
+}
+
+/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the
+ * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no
+ * signed portion can be identified.  Return 0 on success, -1 on failure. */
+int
+router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                           const char *s)
+{
+  const char *start, *end;
+  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+    start = s;
+    end = s + strlen(s);
+  }
+  tor_assert(start);
+  tor_assert(end);
+  return crypto_digest256((char*)digest_out, start, end-start,
+                          DIGEST_SHA3_256);
+}
+
 /** Set <b>digests</b> to all the digests of the consensus document in
  * <b>s</b> */
 int
@@ -1453,28 +1212,15 @@ tor_version_is_obsolete(const char *myversion, const char *versionlist)
   return ret;
 }
 
-/** Return true iff <b>key</b> is allowed to sign directories.
- */
-static int
-dir_signing_key_is_trusted(crypto_pk_t *key)
+MOCK_IMPL(STATIC int,
+signed_digest_equals, (const uint8_t *d1, const uint8_t *d2, size_t len))
 {
-  char digest[DIGEST_LEN];
-  if (!key) return 0;
-  if (crypto_pk_get_digest(key, digest) < 0) {
-    log_warn(LD_DIR, "Error computing dir-signing-key digest");
-    return 0;
-  }
-  if (!router_digest_is_trusted_dir(digest)) {
-    log_warn(LD_DIR, "Listed dir-signing-key is not trusted");
-    return 0;
-  }
-  return 1;
+  return tor_memeq(d1, d2, len);
 }
 
 /** Check whether the object body of the token in <b>tok</b> has a good
- * signature for <b>digest</b> using key <b>pkey</b>.  If
- * <b>CST_CHECK_AUTHORITY</b> is set, make sure that <b>pkey</b> is the key of
- * a directory authority.  If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
+ * signature for <b>digest</b> using key <b>pkey</b>.
+ * If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
  * the object type of the signature object. Use <b>doctype</b> as the type of
  * the document when generating log messages.  Return 0 on success, negative
  * on failure.
@@ -1489,7 +1235,6 @@ check_signature_token(const char *digest,
 {
   char *signed_digest;
   size_t keysize;
-  const int check_authority = (flags & CST_CHECK_AUTHORITY);
   const int check_objtype = ! (flags & CST_NO_CHECK_OBJTYPE);
 
   tor_assert(pkey);
@@ -1497,12 +1242,6 @@ check_signature_token(const char *digest,
   tor_assert(digest);
   tor_assert(doctype);
 
-  if (check_authority && !dir_signing_key_is_trusted(pkey)) {
-    log_warn(LD_DIR, "Key on %s did not come from an authority; rejecting",
-             doctype);
-    return -1;
-  }
-
   if (check_objtype) {
     if (strcmp(tok->object_type, "SIGNATURE")) {
       log_warn(LD_DIR, "Bad object type on %s signature", doctype);
@@ -1521,7 +1260,8 @@ check_signature_token(const char *digest,
   }
   //  log_debug(LD_DIR,"Signed %s hash starts %s", doctype,
   //            hex_str(signed_digest,4));
-  if (tor_memneq(digest, signed_digest, digest_len)) {
+  if (! signed_digest_equals((const uint8_t *)digest,
+                             (const uint8_t *)signed_digest, digest_len)) {
     log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype);
     tor_free(signed_digest);
     return -1;
@@ -1689,9 +1429,9 @@ dump_distinct_digest_count(int severity)
     verified_digests = digestmap_new();
   tor_log(severity, LD_GENERAL, "%d *distinct* router digests verified",
       digestmap_size(verified_digests));
-#else
+#else /* !(defined(COUNT_DISTINCT_DIGESTS)) */
   (void)severity; /* suppress "unused parameter" warning */
-#endif
+#endif /* defined(COUNT_DISTINCT_DIGESTS) */
 }
 
 /** Try to find an IPv6 OR port in <b>list</b> of directory_token_t's
@@ -2087,7 +1827,8 @@ router_parse_entry_from_string(const char *s, const char *end,
 
       if (router_get_hash_impl_helper(s, end-s, "router ",
                                       "\nrouter-sig-ed25519",
-                                      ' ', &signed_start, &signed_end) < 0) {
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
         log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor");
         goto err;
       }
@@ -2100,12 +1841,13 @@ router_parse_entry_from_string(const char *s, const char *end,
 
       ed25519_checkable_t check[3];
       int check_ok[3];
-      if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) {
+      time_t expires = TIME_MAX;
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, &expires) < 0) {
         log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
         goto err;
       }
       if (tor_cert_get_checkable_sig(&check[1],
-                                     ntor_cc_cert, &ntor_cc_pk) < 0) {
+                               ntor_cc_cert, &ntor_cc_pk, &expires) < 0) {
         log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert.");
         goto err;
       }
@@ -2135,10 +1877,7 @@ router_parse_entry_from_string(const char *s, const char *end,
       }
 
       /* We check this before adding it to the routerlist. */
-      if (cert->valid_until < ntor_cc_cert->valid_until)
-        router->cert_expiration_time = cert->valid_until;
-      else
-        router->cert_expiration_time = ntor_cc_cert->valid_until;
+      router->cert_expiration_time = expires;
     }
   }
 
@@ -2160,12 +1899,19 @@ router_parse_entry_from_string(const char *s, const char *end,
     }
   }
 
-  if ((tok = find_opt_by_keyword(tokens, K_PLATFORM))) {
-    router->platform = tor_strdup(tok->args[0]);
-  }
+  {
+    const char *version = NULL, *protocols = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PLATFORM))) {
+      router->platform = tor_strdup(tok->args[0]);
+      version = tok->args[0];
+    }
 
-  if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
-    router->protocol_list = tor_strdup(tok->args[0]);
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      router->protocol_list = tor_strdup(tok->args[0]);
+      protocols = tok->args[0];
+    }
+
+    summarize_protover_flags(&router->pv, protocols, version);
   }
 
   if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) {
@@ -2220,7 +1966,7 @@ router_parse_entry_from_string(const char *s, const char *end,
                  escaped(tok->args[i]));
         goto err;
       }
-      smartlist_add(router->declared_family, tor_strdup(tok->args[i]));
+      smartlist_add_strdup(router->declared_family, tok->args[i]);
     }
   }
 
@@ -2261,7 +2007,6 @@ router_parse_entry_from_string(const char *s, const char *end,
   }
 
   tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE);
-  note_crypto_pk_op(VERIFY_RTR);
 #ifdef COUNT_DISTINCT_DIGESTS
   if (!verified_digests)
     verified_digests = digestmap_new();
@@ -2332,6 +2077,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
    * parse that's covered by the hash. */
   int can_dl_again = 0;
 
+  if (BUG(s == NULL))
+    return NULL;
+
   if (!end) {
     end = s + strlen(s);
   }
@@ -2439,7 +2187,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
 
       if (router_get_hash_impl_helper(s, end-s, "extra-info ",
                                       "\nrouter-sig-ed25519",
-                                      ' ', &signed_start, &signed_end) < 0) {
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
         log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo");
         goto err;
       }
@@ -2452,7 +2201,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
 
       ed25519_checkable_t check[2];
       int check_ok[2];
-      if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) {
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, NULL) < 0) {
         log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
         goto err;
       }
@@ -2492,7 +2241,6 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
   }
 
   if (key) {
-    note_crypto_pk_op(VERIFY_RTR);
     if (check_signature_token(digest, DIGEST_LEN, tok, key, 0,
                               "extra-info") < 0)
       goto err;
@@ -2793,6 +2541,50 @@ routerstatus_parse_guardfraction(const char *guardfraction_str,
   return 0;
 }
 
+/** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
+ * falling back or correcting them based on <b>version</b> as appropriate.
+ */
+STATIC void
+summarize_protover_flags(protover_summary_flags_t *out,
+                         const char *protocols,
+                         const char *version)
+{
+  tor_assert(out);
+  memset(out, 0, sizeof(*out));
+  if (protocols) {
+    out->protocols_known = 1;
+    out->supports_extend2_cells =
+      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+    out->supports_ed25519_link_handshake_compat =
+      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_link_handshake_any =
+      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+    out->supports_ed25519_hs_intro =
+      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+    out->supports_v3_hsdir =
+      protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                      PROTOVER_HSDIR_V3);
+    out->supports_v3_rendezvous_point =
+      protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  }
+  if (version && !strcmpstart(version, "Tor ")) {
+    if (!out->protocols_known) {
+      /* The version is a "Tor" version, and where there is no
+       * list of protocol versions that we should be looking at instead. */
+
+      out->supports_extend2_cells =
+        tor_version_as_new_as(version, "0.2.4.8-alpha");
+      out->protocols_known = 1;
+    } else {
+      /* Bug #22447 forces us to filter on this version. */
+      if (!tor_version_as_new_as(version, "0.3.0.8")) {
+        out->supports_v3_hsdir = 0;
+      }
+    }
+  }
+}
+
 /** Given a string at *<b>s</b>, containing a routerstatus object, and an
  * empty smartlist at <b>tokens</b>, parse and return the first router status
  * object in the string, and advance *<b>s</b> to just after the end of the
@@ -2846,7 +2638,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
       goto err;
     }
   } else if (flav == FLAV_MICRODESC) {
-    offset = -1; /* There is no identity digest */
+    offset = -1; /* There is no descriptor digest in an md consensus r line */
   }
 
   if (vote_rs) {
@@ -2955,29 +2747,23 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     /* These are implied true by having been included in a consensus made
      * with a given method */
     rs->is_flagged_running = 1; /* Starting with consensus method 4. */
-    if (consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES)
-      rs->is_valid = 1;
-  }
-  int found_protocol_list = 0;
-  if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
-    found_protocol_list = 1;
-    rs->protocols_known = 1;
-    rs->supports_extend2_cells =
-      protocol_list_supports_protocol(tok->args[0], PRT_RELAY, 2);
+    rs->is_valid = 1; /* Starting with consensus method 24. */
   }
-  if ((tok = find_opt_by_keyword(tokens, K_V))) {
-    tor_assert(tok->n_args == 1);
-    if (!strcmpstart(tok->args[0], "Tor ") && !found_protocol_list) {
-      /* We only do version checks like this in the case where
-       * the version is a "Tor" version, and where there is no
-       * list of protocol versions that we should be looking at instead. */
-      rs->supports_extend2_cells =
-        tor_version_as_new_as(tok->args[0], "0.2.4.8-alpha");
-      rs->protocols_known = 1;
-    }
-    if (vote_rs) {
-      vote_rs->version = tor_strdup(tok->args[0]);
+  {
+    const char *protocols = NULL, *version = NULL;
+    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
+      tor_assert(tok->n_args == 1);
+      protocols = tok->args[0];
+    }
+    if ((tok = find_opt_by_keyword(tokens, K_V))) {
+      tor_assert(tok->n_args == 1);
+      version = tok->args[0];
+      if (vote_rs) {
+        vote_rs->version = tor_strdup(tok->args[0]);
+      }
     }
+
+    summarize_protover_flags(&rs->pv, protocols, version);
   }
 
   /* handle weighting/bandwidth info */
@@ -3111,7 +2897,6 @@ compare_vote_routerstatus_entries(const void **_a, const void **_b)
 int
 networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
 {
-  int64_t weight_scale;
   int64_t G=0, M=0, E=0, D=0, T=0;
   double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed;
   double Gtotal=0, Mtotal=0, Etotal=0;
@@ -3119,7 +2904,8 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
   int valid = 1;
   (void) consensus_method;
 
-  weight_scale = networkstatus_get_weight_scale_param(ns);
+  const int64_t weight_scale = networkstatus_get_weight_scale_param(ns);
+  tor_assert(weight_scale >= 1);
   Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1);
   Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1);
   Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1);
@@ -3181,7 +2967,7 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
   }
 
   Wgg /= weight_scale;
-  Wgm /= weight_scale;
+  Wgm /= weight_scale; (void) Wgm; // unused from here on.
   Wgd /= weight_scale;
 
   Wmg /= weight_scale;
@@ -3189,8 +2975,8 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
   Wme /= weight_scale;
   Wmd /= weight_scale;
 
-  Weg /= weight_scale;
-  Wem /= weight_scale;
+  Weg /= weight_scale; (void) Weg; // unused from here on.
+  Wem /= weight_scale; (void) Wem; // unused from here on.
   Wee /= weight_scale;
   Wed /= weight_scale;
 
@@ -3500,60 +3286,6 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
   return valid;
 }
 
-/** Parse and extract all SR commits from <b>tokens</b> and place them in
- *  <b>ns</b>. */
-static void
-extract_shared_random_commits(networkstatus_t *ns, smartlist_t *tokens)
-{
-  smartlist_t *chunks = NULL;
-
-  tor_assert(ns);
-  tor_assert(tokens);
-  /* Commits are only present in a vote. */
-  tor_assert(ns->type == NS_TYPE_VOTE);
-
-  ns->sr_info.commits = smartlist_new();
-
-  smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT);
-  /* It's normal that a vote might contain no commits even if it participates
-   * in the SR protocol. Don't treat it as an error. */
-  if (commits == NULL) {
-    goto end;
-  }
-
-  /* Parse the commit. We do NO validation of number of arguments or ordering
-   * for forward compatibility, it's the parse commit job to inform us if it's
-   * supported or not. */
-  chunks = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) {
-    /* Extract all arguments and put them in the chunks list. */
-    for (int i = 0; i < tok->n_args; i++) {
-      smartlist_add(chunks, tok->args[i]);
-    }
-    sr_commit_t *commit = sr_parse_commit(chunks);
-    smartlist_clear(chunks);
-    if (commit == NULL) {
-      /* Get voter identity so we can warn that this dirauth vote contains
-       * commit we can't parse. */
-      networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0);
-      tor_assert(voter);
-      log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.",
-               escaped(tok->object_body),
-               hex_str(voter->identity_digest,
-                       sizeof(voter->identity_digest)));
-      /* Commitment couldn't be parsed. Continue onto the next commit because
-       * this one could be unsupported for instance. */
-      continue;
-    }
-    /* Add newly created commit object to the vote. */
-    smartlist_add(ns->sr_info.commits, commit);
-  } SMARTLIST_FOREACH_END(tok);
-
- end:
-  smartlist_free(chunks);
-  smartlist_free(commits);
-}
-
 /** Check if a shared random value of type <b>srv_type</b> is in
  *  <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return
  *  -1 on failure, 0 on success. The resulting srv is allocated on the heap and
@@ -3615,8 +3347,8 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
     voter_identity = "consensus";
   }
 
-  /* We extract both and on error, everything is stopped because it means
-   * the votes is malformed for the shared random value(s). */
+  /* We extract both, and on error everything is stopped because it means
+   * the vote is malformed for the shared random value(s). */
   if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) {
     log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s",
              voter_identity);
@@ -3639,6 +3371,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   networkstatus_voter_info_t *voter = NULL;
   networkstatus_t *ns = NULL;
   common_digests_t ns_digests;
+  uint8_t sha3_as_signed[DIGEST256_LEN];
   const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
   directory_token_t *tok;
   struct in_addr in;
@@ -3652,7 +3385,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests)) {
+  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
@@ -3669,6 +3403,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   ns = tor_malloc_zero(sizeof(networkstatus_t));
   memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
+  memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed));
 
   tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
   tor_assert(tok);
@@ -3723,9 +3458,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
     if (tok) {
       for (i=0; i < tok->n_args; ++i)
-        smartlist_add(ns->supported_methods, tor_strdup(tok->args[i]));
+        smartlist_add_strdup(ns->supported_methods, tok->args[i]);
     } else {
-      smartlist_add(ns->supported_methods, tor_strdup("1"));
+      smartlist_add_strdup(ns->supported_methods, "1");
     }
   } else {
     tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
@@ -3807,7 +3542,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     ns->package_lines = smartlist_new();
     if (package_lst) {
       SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
-                    smartlist_add(ns->package_lines, tor_strdup(t->args[0])));
+                    smartlist_add_strdup(ns->package_lines, t->args[0]));
     }
     smartlist_free(package_lst);
   }
@@ -3816,7 +3551,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   ns->known_flags = smartlist_new();
   inorder = 1;
   for (i = 0; i < tok->n_args; ++i) {
-    smartlist_add(ns->known_flags, tor_strdup(tok->args[i]));
+    smartlist_add_strdup(ns->known_flags, tok->args[i]);
     if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
       log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
       inorder = 0;
@@ -3868,7 +3603,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
       tor_free(last_kwd);
       last_kwd = tor_strndup(tok->args[i], eq_pos);
-      smartlist_add(ns->net_params, tor_strdup(tok->args[i]));
+      smartlist_add_strdup(ns->net_params, tok->args[i]);
     }
     if (!inorder) {
       log_warn(LD_DIR, "params not in order");
@@ -3988,13 +3723,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   /* If this is a vote document, check if information about the shared
      randomness protocol is included, and extract it. */
   if (ns->type == NS_TYPE_VOTE) {
-    /* Does this authority participates in the SR protocol? */
-    tok = find_opt_by_keyword(tokens, K_SR_FLAG);
-    if (tok) {
-      ns->sr_info.participate = 1;
-      /* Get the SR commitments and reveals from the vote. */
-      extract_shared_random_commits(ns, tokens);
-    }
+    dirvote_parse_sr_commits(ns, tokens);
   }
   /* For both a vote and consensus, extract the shared random values. */
   if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) {
@@ -4011,11 +3740,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     if (ns->type != NS_TYPE_CONSENSUS) {
       vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
-                                               rs, 0, 0))
+                                               rs, 0, 0)) {
         smartlist_add(ns->routerstatus_list, rs);
-      else {
-        tor_free(rs->version);
-        tor_free(rs);
+      } else {
+        vote_routerstatus_free(rs);
       }
     } else {
       routerstatus_t *rs;
@@ -4024,7 +3752,6 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
                                                      ns->consensus_method,
                                                      flav))) {
         /* Use exponential-backoff scheduling when downloading microdescs */
-        rs->dl_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL;
         smartlist_add(ns->routerstatus_list, rs);
       }
     }
@@ -4111,7 +3838,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
         log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
         goto err;
       }
-      smartlist_add(ns->weight_params, tor_strdup(tok->args[i]));
+      smartlist_add_strdup(ns->weight_params, tok->args[i]);
     }
   }
 
@@ -4186,7 +3913,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
     }
 
-    if (voter_get_sig_by_algorithm(v, sig->alg)) {
+    if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) {
       /* We already parsed a vote with this algorithm from this voter. Use the
          first one. */
       log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus "
@@ -4272,7 +3999,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 /** Return the common_digests_t that holds the digests of the
  * <b>flavor_name</b>-flavored networkstatus according to the detached
  * signatures document <b>sigs</b>, allocating a new common_digests_t as
- * neeeded. */
+ * needed. */
 static common_digests_t *
 detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name)
 {
@@ -4286,7 +4013,7 @@ detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name)
 
 /** Return the list of signatures of the <b>flavor_name</b>-flavored
  * networkstatus according to the detached signatures document <b>sigs</b>,
- * allocating a new common_digests_t as neeeded. */
+ * allocating a new common_digests_t as needed. */
 static smartlist_t *
 detached_get_signatures(ns_detached_signatures_t *sigs,
                         const char *flavor_name)
@@ -4740,445 +4467,6 @@ assert_addr_policy_ok(smartlist_t *lst)
   });
 }
 
-/*
- * Low-level tokenizer for router descriptors and directories.
- */
-
-/** Free all resources allocated for <b>tok</b> */
-static void
-token_clear(directory_token_t *tok)
-{
-  if (tok->key)
-    crypto_pk_free(tok->key);
-}
-
-#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
-#define ALLOC(sz) memarea_alloc(area,sz)
-#define STRDUP(str) memarea_strdup(area,str)
-#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
-
-#define RET_ERR(msg)                                               \
-  STMT_BEGIN                                                       \
-    if (tok) token_clear(tok);                                      \
-    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
-    tok->tp = ERR_;                                                \
-    tok->error = STRDUP(msg);                                      \
-    goto done_tokenizing;                                          \
-  STMT_END
-
-/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
- * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
- * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
- * conform to the syntax we wanted.
- **/
-static inline directory_token_t *
-token_check_object(memarea_t *area, const char *kwd,
-                   directory_token_t *tok, obj_syntax o_syn)
-{
-  char ebuf[128];
-  switch (o_syn) {
-    case NO_OBJ:
-      /* No object is allowed for this token. */
-      if (tok->object_body) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      if (tok->key) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    case NEED_OBJ:
-      /* There must be a (non-key) object. */
-      if (!tok->object_body) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
-    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
-      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
-                     kwd, crypto_pk_num_bits(tok->key));
-        RET_ERR(ebuf);
-      }
-      /* fall through */
-    case NEED_KEY: /* There must be some kind of key. */
-      if (!tok->key) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      if (o_syn != NEED_SKEY_1024) {
-        if (crypto_pk_key_is_private(tok->key)) {
-          tor_snprintf(ebuf, sizeof(ebuf),
-               "Private key given for %s, which wants a public key", kwd);
-          RET_ERR(ebuf);
-        }
-      } else { /* o_syn == NEED_SKEY_1024 */
-        if (!crypto_pk_key_is_private(tok->key)) {
-          tor_snprintf(ebuf, sizeof(ebuf),
-               "Public key given for %s, which wants a private key", kwd);
-          RET_ERR(ebuf);
-        }
-      }
-      break;
-    case OBJ_OK:
-      /* Anything goes with this token. */
-      break;
-  }
-
- done_tokenizing:
-  return tok;
-}
-
-/** Helper: parse space-separated arguments from the string <b>s</b> ending at
- * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
- * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
- * all storage in <b>area</b>.  Return the number of arguments parsed, or
- * return -1 if there was an insanely high number of arguments. */
-static inline int
-get_token_arguments(memarea_t *area, directory_token_t *tok,
-                    const char *s, const char *eol)
-{
-/** Largest number of arguments we'll accept to any token, ever. */
-#define MAX_ARGS 512
-  char *mem = memarea_strndup(area, s, eol-s);
-  char *cp = mem;
-  int j = 0;
-  char *args[MAX_ARGS];
-  while (*cp) {
-    if (j == MAX_ARGS)
-      return -1;
-    args[j++] = cp;
-    cp = (char*)find_whitespace(cp);
-    if (!cp || !*cp)
-      break; /* End of the line. */
-    *cp++ = '\0';
-    cp = (char*)eat_whitespace(cp);
-  }
-  tok->n_args = j;
-  tok->args = memarea_memdup(area, args, j*sizeof(char*));
-  return j;
-#undef MAX_ARGS
-}
-
-/** Helper function: read the next token from *s, advance *s to the end of the
- * token, and return the parsed token.  Parse *<b>s</b> according to the list
- * of tokens in <b>table</b>.
- */
-static directory_token_t *
-get_next_token(memarea_t *area,
-               const char **s, const char *eos, token_rule_t *table)
-{
-  /** Reject any object at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
-  /** Reject any line at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_LINE_LENGTH (128*1024)
-
-  const char *next, *eol, *obstart;
-  size_t obname_len;
-  int i;
-  directory_token_t *tok;
-  obj_syntax o_syn = NO_OBJ;
-  char ebuf[128];
-  const char *kwd = "";
-
-  tor_assert(area);
-  tok = ALLOC_ZERO(sizeof(directory_token_t));
-  tok->tp = ERR_;
-
-  /* Set *s to first token, eol to end-of-line, next to after first token */
-  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
-  tor_assert(eos >= *s);
-  eol = memchr(*s, '\n', eos-*s);
-  if (!eol)
-    eol = eos;
-  if (eol - *s > MAX_LINE_LENGTH) {
-    RET_ERR("Line far too long");
-  }
-
-  next = find_whitespace_eos(*s, eol);
-
-  if (!strcmp_len(*s, "opt", next-*s)) {
-    /* Skip past an "opt" at the start of the line. */
-    *s = eat_whitespace_eos_no_nl(next, eol);
-    next = find_whitespace_eos(*s, eol);
-  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
-    RET_ERR("Unexpected EOF");
-  }
-
-  /* Search the table for the appropriate entry.  (I tried a binary search
-   * instead, but it wasn't any faster.) */
-  for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
-      /* We've found the keyword. */
-      kwd = table[i].t;
-      tok->tp = table[i].v;
-      o_syn = table[i].os;
-      *s = eat_whitespace_eos_no_nl(next, eol);
-      /* We go ahead whether there are arguments or not, so that tok->args is
-       * always set if we want arguments. */
-      if (table[i].concat_args) {
-        /* The keyword takes the line as a single argument */
-        tok->args = ALLOC(sizeof(char*));
-        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
-        tok->n_args = 1;
-      } else {
-        /* This keyword takes multiple arguments. */
-        if (get_token_arguments(area, tok, *s, eol)<0) {
-          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
-          RET_ERR(ebuf);
-        }
-        *s = eol;
-      }
-      if (tok->n_args < table[i].min_args) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
-        RET_ERR(ebuf);
-      } else if (tok->n_args > table[i].max_args) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    }
-  }
-
-  if (tok->tp == ERR_) {
-    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
-    if (*s < eol && **s == '@')
-      tok->tp = A_UNKNOWN_;
-    else
-      tok->tp = K_OPT;
-    tok->args = ALLOC(sizeof(char*));
-    tok->args[0] = STRNDUP(*s, eol-*s);
-    tok->n_args = 1;
-    o_syn = OBJ_OK;
-  }
-
-  /* Check whether there's an object present */
-  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
-  tor_assert(eos >= *s);
-  eol = memchr(*s, '\n', eos-*s);
-  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
-    goto check_object;
-
-  obstart = *s; /* Set obstart to start of object spec */
-  if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
-      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
-    RET_ERR("Malformed object: bad begin line");
-  }
-  tok->object_type = STRNDUP(*s+11, eol-*s-16);
-  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
-  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
-
-  /* Go to the end of the object */
-  next = tor_memstr(*s, eos-*s, "-----END ");
-  if (!next) {
-    RET_ERR("Malformed object: missing object end line");
-  }
-  tor_assert(eos >= next);
-  eol = memchr(next, '\n', eos-next);
-  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
-    eol = eos;
-  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
-  if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
-    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
-             tok->object_type);
-    ebuf[sizeof(ebuf)-1] = '\0';
-    RET_ERR(ebuf);
-  }
-  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
-    RET_ERR("Couldn't parse object: missing footer or object much too big.");
-
-  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
-      RET_ERR("Couldn't parse public key.");
-  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
-      RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
-  }
-  *s = eol;
-
- check_object:
-  tok = token_check_object(area, kwd, tok, o_syn);
-
- done_tokenizing:
-  return tok;
-
-#undef RET_ERR
-#undef ALLOC
-#undef ALLOC_ZERO
-#undef STRDUP
-#undef STRNDUP
-}
-
-/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
- * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
- * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
- * entire string.
- */
-static int
-tokenize_string(memarea_t *area,
-                const char *start, const char *end, smartlist_t *out,
-                token_rule_t *table, int flags)
-{
-  const char **s;
-  directory_token_t *tok = NULL;
-  int counts[NIL_];
-  int i;
-  int first_nonannotation;
-  int prev_len = smartlist_len(out);
-  tor_assert(area);
-
-  s = &start;
-  if (!end) {
-    end = start+strlen(start);
-  } else {
-    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
-    if (memchr(start, '\0', end-start)) {
-      log_warn(LD_DIR, "parse error: internal NUL character.");
-      return -1;
-    }
-  }
-  for (i = 0; i < NIL_; ++i)
-    counts[i] = 0;
-
-  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
-
-  while (*s < end && (!tok || tok->tp != EOF_)) {
-    tok = get_next_token(area, s, end, table);
-    if (tok->tp == ERR_) {
-      log_warn(LD_DIR, "parse error: %s", tok->error);
-      token_clear(tok);
-      return -1;
-    }
-    ++counts[tok->tp];
-    smartlist_add(out, tok);
-    *s = eat_whitespace_eos(*s, end);
-  }
-
-  if (flags & TS_NOCHECK)
-    return 0;
-
-  if ((flags & TS_ANNOTATIONS_OK)) {
-    first_nonannotation = -1;
-    for (i = 0; i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
-        first_nonannotation = i;
-        break;
-      }
-    }
-    if (first_nonannotation < 0) {
-      log_warn(LD_DIR, "parse error: item contains only annotations");
-      return -1;
-    }
-    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
-        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
-        return -1;
-      }
-    }
-    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
-      if (first_nonannotation != prev_len) {
-        log_warn(LD_DIR, "parse error: Unexpected annotations.");
-        return -1;
-      }
-    }
-  } else {
-    for (i=0;  i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
-        log_warn(LD_DIR, "parse error: no annotations allowed.");
-        return -1;
-      }
-    }
-    first_nonannotation = 0;
-  }
-  for (i = 0; table[i].t; ++i) {
-    if (counts[table[i].v] < table[i].min_cnt) {
-      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
-      return -1;
-    }
-    if (counts[table[i].v] > table[i].max_cnt) {
-      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
-      return -1;
-    }
-    if (table[i].pos & AT_START) {
-      if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
-        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
-        return -1;
-      }
-    }
-    if (table[i].pos & AT_END) {
-      if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
-        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
-        return -1;
-      }
-    }
-  }
-  return 0;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
- * NULL if no such keyword is found.
- */
-static directory_token_t *
-find_opt_by_keyword(smartlist_t *s, directory_keyword keyword)
-{
-  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
-  return NULL;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
- * with an assert if no such keyword is found.
- */
-static directory_token_t *
-find_by_keyword_(smartlist_t *s, directory_keyword keyword,
-                 const char *keyword_as_string)
-{
-  directory_token_t *tok = find_opt_by_keyword(s, keyword);
-  if (PREDICT_UNLIKELY(!tok)) {
-    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
-         "been validated. Internal error.", keyword_as_string, (int)keyword);
-    tor_assert(tok);
-  }
-  return tok;
-}
-
-/** If there are any directory_token_t entries in <b>s</b> whose keyword is
- * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
- * in the same order in which they occur in <b>s</b>.  Otherwise return
- * NULL. */
-static smartlist_t *
-find_all_by_keyword(smartlist_t *s, directory_keyword k)
-{
-  smartlist_t *out = NULL;
-  SMARTLIST_FOREACH(s, directory_token_t *, t,
-                    if (t->tp == k) {
-                      if (!out)
-                        out = smartlist_new();
-                      smartlist_add(out, t);
-                    });
-  return out;
-}
-
 /** Return a newly allocated smartlist of all accept or reject tokens in
  * <b>s</b>.
  */
@@ -5204,16 +4492,18 @@ static int
 router_get_hash_impl_helper(const char *s, size_t s_len,
                             const char *start_str,
                             const char *end_str, char end_c,
+                            int log_severity,
                             const char **start_out, const char **end_out)
 {
   const char *start, *end;
   start = tor_memstr(s, s_len, start_str);
   if (!start) {
-    log_warn(LD_DIR,"couldn't find start of hashed material \"%s\"",start_str);
+    log_fn(log_severity,LD_DIR,
+           "couldn't find start of hashed material \"%s\"",start_str);
     return -1;
   }
   if (start != s && *(start-1) != '\n') {
-    log_warn(LD_DIR,
+    log_fn(log_severity,LD_DIR,
              "first occurrence of \"%s\" is not at the start of a line",
              start_str);
     return -1;
@@ -5221,12 +4511,14 @@ router_get_hash_impl_helper(const char *s, size_t s_len,
   end = tor_memstr(start+strlen(start_str),
                    s_len - (start-s) - strlen(start_str), end_str);
   if (!end) {
-    log_warn(LD_DIR,"couldn't find end of hashed material \"%s\"",end_str);
+    log_fn(log_severity,LD_DIR,
+           "couldn't find end of hashed material \"%s\"",end_str);
     return -1;
   }
   end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str));
   if (!end) {
-    log_warn(LD_DIR,"couldn't find EOL");
+    log_fn(log_severity,LD_DIR,
+           "couldn't find EOL");
     return -1;
   }
   ++end;
@@ -5250,17 +4542,28 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest,
                      digest_algorithm_t alg)
 {
   const char *start=NULL, *end=NULL;
-  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
                                   &start,&end)<0)
     return -1;
 
+  return router_compute_hash_final(digest, start, end-start, alg);
+}
+
+/** Compute the digest of the <b>len</b>-byte directory object at
+ * <b>start</b>, using <b>alg</b>. Store the result in <b>digest</b>, which
+ * must be long enough to hold it. */
+MOCK_IMPL(STATIC int,
+router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg))
+{
   if (alg == DIGEST_SHA1) {
-    if (crypto_digest(digest, start, end-start)) {
+    if (crypto_digest(digest, start, len) < 0) {
       log_warn(LD_BUG,"couldn't compute digest");
       return -1;
     }
   } else {
-    if (crypto_digest256(digest, start, end-start, alg)) {
+    if (crypto_digest256(digest, start, len, alg) < 0) {
       log_warn(LD_BUG,"couldn't compute digest");
       return -1;
     }
@@ -5276,7 +4579,7 @@ router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests,
                        const char *end_str, char end_c)
 {
   const char *start=NULL, *end=NULL;
-  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
                                   &start,&end)<0)
     return -1;
 
@@ -5446,11 +4749,13 @@ microdescs_parse_from_string(const char *s, const char *eos,
         if (!strcmp(t->args[0], "ed25519")) {
           if (md->ed25519_identity_pkey) {
             log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+            smartlist_free(id_lines);
             goto next;
           }
           ed25519_public_key_t k;
           if (ed25519_public_from_base64(&k, t->args[1])<0) {
             log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+            smartlist_free(id_lines);
             goto next;
           }
           md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
@@ -5476,7 +4781,7 @@ microdescs_parse_from_string(const char *s, const char *eos,
                    escaped(tok->args[i]));
           goto next;
         }
-        smartlist_add(md->family, tor_strdup(tok->args[i]));
+        smartlist_add_strdup(md->family, tok->args[i]);
       }
     }
 
@@ -5923,7 +5228,8 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
    * descriptor. */
   tok = find_by_keyword(tokens, R_PUBLICATION_TIME);
   tor_assert(tok->n_args == 1);
-  if (parse_iso_time_(tok->args[0], &result->timestamp, strict_time_fmt) < 0) {
+  if (parse_iso_time_(tok->args[0], &result->timestamp,
+                      strict_time_fmt, 0) < 0) {
     log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]);
     goto err;
   }
@@ -5962,12 +5268,14 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
   }
   /* Parse and verify signature. */
   tok = find_by_keyword(tokens, R_SIGNATURE);
-  note_crypto_pk_op(VERIFY_RTR);
   if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0,
                             "v2 rendezvous service descriptor") < 0)
     goto err;
   /* Verify that descriptor ID belongs to public key and secret ID part. */
-  crypto_pk_get_digest(result->pk, public_key_hash);
+  if (crypto_pk_get_digest(result->pk, public_key_hash) < 0) {
+    log_warn(LD_REND, "Unable to compute rend descriptor public key digest");
+    goto err;
+  }
   rend_get_descriptor_id_bytes(test_desc_id, public_key_hash,
                                secret_id_part);
   if (tor_memneq(desc_id_out, test_desc_id, DIGEST_LEN)) {
diff --git a/src/or/routerparse.h b/src/or/routerparse.h
index 01a5de88e8..418fd3acdb 100644
--- a/src/or/routerparse.h
+++ b/src/or/routerparse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,11 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest);
 int router_get_dir_hash(const char *s, char *digest);
 int router_get_networkstatus_v3_hashes(const char *s,
                                        common_digests_t *digests);
+int router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                                  const char **start_out,
+                                                  const char **end_out);
+int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                               const char *s);
 int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest);
 #define DIROBJ_MAX_SIG_LEN 256
 char *router_get_dirobj_signature(const char *digest,
@@ -113,7 +118,6 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
     (const char *dirname, const char *f));
 STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-STATIC void dump_desc(const char *desc, const char *type);
 STATIC void dump_desc_fifo_cleanup(void);
 struct memarea_t;
 STATIC routerstatus_t *routerstatus_parse_entry_from_string(
@@ -123,9 +127,19 @@ STATIC routerstatus_t *routerstatus_parse_entry_from_string(
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
                                      consensus_flavor_t flav);
-#endif
+MOCK_DECL(STATIC void,dump_desc,(const char *desc, const char *type));
+MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg));
+MOCK_DECL(STATIC int, signed_digest_equals,
+          (const uint8_t *d1, const uint8_t *d2, size_t len));
+
+STATIC void summarize_protover_flags(protover_summary_flags_t *out,
+                                     const char *protocols,
+                                     const char *version);
+#endif /* defined(ROUTERPARSE_PRIVATE) */
 
 #define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1"
 
-#endif
+#endif /* !defined(TOR_ROUTERPARSE_H) */
 
diff --git a/src/or/routerset.c b/src/or/routerset.c
index 58b66ea777..a2599b316c 100644
--- a/src/or/routerset.c
+++ b/src/or/routerset.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,6 +28,7 @@
 #define ROUTERSET_PRIVATE
 
 #include "or.h"
+#include "bridges.h"
 #include "geoip.h"
 #include "nodelist.h"
 #include "policies.h"
@@ -262,12 +263,12 @@ routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set)
     geoip_get_country("A1") >= 0;
 
   if (add_unknown) {
-    smartlist_add(set->country_names, tor_strdup("??"));
-    smartlist_add(set->list, tor_strdup("{??}"));
+    smartlist_add_strdup(set->country_names, "??");
+    smartlist_add_strdup(set->list, "{??}");
   }
   if (add_a1) {
-    smartlist_add(set->country_names, tor_strdup("a1"));
-    smartlist_add(set->list, tor_strdup("{a1}"));
+    smartlist_add_strdup(set->country_names, "a1");
+    smartlist_add_strdup(set->list, "{a1}");
   }
 
   if (add_unknown || add_a1) {
@@ -334,6 +335,18 @@ routerset_contains_node(const routerset_t *set, const node_t *node)
     return 0;
 }
 
+/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+int
+routerset_contains_bridge(const routerset_t *set, const bridge_info_t *bridge)
+{
+  const char *id = (const char*)bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+  return routerset_contains(set, &addrport->addr, addrport->port,
+                            NULL, id, -1);
+}
+
 /** Add every known node_t that is a member of <b>routerset</b> to
  * <b>out</b>, but never add any that are part of <b>excludeset</b>.
  * If <b>running_only</b>, only add the running ones. */
@@ -349,7 +362,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
     /* No routers are specified by type; all are given by name or digest.
      * we can do a lookup in O(len(routerset)). */
     SMARTLIST_FOREACH(routerset->list, const char *, name, {
-        const node_t *node = node_get_by_nickname(name, 1);
+        const node_t *node = node_get_by_nickname(name, 0);
         if (node) {
           if (!running_only || node->is_running)
             if (!routerset_contains_node(excludeset, node))
@@ -424,7 +437,7 @@ routerset_equal(const routerset_t *old, const routerset_t *new)
 
 /** Free all storage held in <b>routerset</b>. */
 void
-routerset_free(routerset_t *routerset)
+routerset_free_(routerset_t *routerset)
 {
   if (!routerset)
     return;
diff --git a/src/or/routerset.h b/src/or/routerset.h
index c2f7205c3e..53e8c66c5e 100644
--- a/src/or/routerset.h
+++ b/src/or/routerset.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,8 +26,11 @@ int routerset_contains_routerstatus(const routerset_t *set,
                                     country_t country);
 int routerset_contains_extendinfo(const routerset_t *set,
                                   const extend_info_t *ei);
-
+struct bridge_info_t;
+int routerset_contains_bridge(const routerset_t *set,
+                              const struct bridge_info_t *bridge);
 int routerset_contains_node(const routerset_t *set, const node_t *node);
+
 void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
                              const routerset_t *excludeset,
                              int running_only);
@@ -37,7 +40,8 @@ void routerset_subtract_nodes(smartlist_t *out,
 
 char *routerset_to_string(const routerset_t *routerset);
 int routerset_equal(const routerset_t *old, const routerset_t *new);
-void routerset_free(routerset_t *routerset);
+void routerset_free_(routerset_t *routerset);
+#define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
 int routerset_len(const routerset_t *set);
 
 #ifdef ROUTERSET_PRIVATE
@@ -79,6 +83,6 @@ struct routerset_t {
    * reloaded. */
   bitarray_t *countries;
 };
-#endif
-#endif
+#endif /* defined(ROUTERSET_PRIVATE) */
+#endif /* !defined(TOR_ROUTERSET_H) */
 
diff --git a/src/or/scheduler.c b/src/or/scheduler.c
index 49ac1b939a..da894294bf 100644
--- a/src/or/scheduler.c
+++ b/src/or/scheduler.c
@@ -1,178 +1,406 @@
-/* * Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-/**
- * \file scheduler.c
- * \brief Relay scheduling system
- **/
-
 #include "or.h"
-
-#define TOR_CHANNEL_INTERNAL_ /* For channel_flush_some_cells() */
-#include "channel.h"
+#include "config.h"
 
 #include "compat_libevent.h"
 #define SCHEDULER_PRIVATE_
+#define SCHEDULER_KIST_PRIVATE
 #include "scheduler.h"
+#include "main.h"
+#include "buffers.h"
+#define TOR_CHANNEL_INTERNAL_
+#include "channeltls.h"
 
-#include <event2/event.h>
-
-/*
- * Scheduler high/low watermarks
- */
-
-static uint32_t sched_q_low_water = 16384;
-static uint32_t sched_q_high_water = 32768;
-
-/*
- * Maximum cells to flush in a single call to channel_flush_some_cells();
- * setting this low means more calls, but too high and we could overshoot
- * sched_q_high_water.
- */
-
-static uint32_t sched_max_flush_cells = 16;
-
-/*
- * Write scheduling works by keeping track of which channels can
- * accept cells, and have cells to write.  From the scheduler's perspective,
- * a channel can be in four possible states:
+/**
+ * \file scheduler.c
+ * \brief Channel scheduling system: decides which channels should send and
+ * receive when.
  *
- * 1.) Not open for writes, no cells to send
- *     - Not much to do here, and the channel will have scheduler_state ==
- *       SCHED_CHAN_IDLE
- *     - Transitions from:
- *       - Open for writes/has cells by simultaneously draining all circuit
+ * This module is the global/common parts of the scheduling system. This system
+ * is what decides what channels get to send cells on their circuits and when.
+ *
+ * Terms:
+ * - "Scheduling system": the collection of scheduler*.{h,c} files and their
+ *   aggregate behavior.
+ * - "Scheduler implementation": a scheduler_t. The scheduling system has one
+ *   active scheduling implementation at a time.
+ *
+ * In this file you will find state that any scheduler implementation can have
+ * access to as well as the functions the rest of Tor uses to interact with the
+ * scheduling system.
+ *
+ * The earliest versions of Tor approximated a kind of round-robin system
+ * among active connections, but only approximated it. It would only consider
+ * one connection (roughly equal to a channel in today's terms) at a time, and
+ * thus could only prioritize circuits against others on the same connection.
+ *
+ * Then in response to the KIST paper[0], Tor implemented a global
+ * circuit scheduler. It was supposed to prioritize circuits across many
+ * channels, but wasn't effective. It is preserved in scheduler_vanilla.c.
+ *
+ * [0]: http://www.robgjansen.com/publications/kist-sec2014.pdf
+ *
+ * Then we actually got around to implementing KIST for real. We decided to
+ * modularize the scheduler so new ones can be implemented. You can find KIST
+ * in scheduler_kist.c.
+ *
+ * Channels have one of four scheduling states based on whether or not they
+ * have cells to send and whether or not they are able to send.
+ *
+ * <ol>
+ * <li>
+ *   Not open for writes, no cells to send.
+ *     <ul><li> Not much to do here, and the channel will have scheduler_state
+ *       == SCHED_CHAN_IDLE
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Open for writes/has cells by simultaneously draining all circuit
  *         queues and filling the output buffer.
- *     - Transitions to:
- *       - Not open for writes/has cells by arrival of cells on an attached
+ *       </ul>
+ *     <li> Transitions to:
+ *      <ul>
+ *       <li> Not open for writes/has cells by arrival of cells on an attached
  *         circuit (this would be driven from append_cell_to_circuit_queue())
- *       - Open for writes/no cells by a channel type specific path;
+ *       <li> Open for writes/no cells by a channel type specific path;
  *         driven from connection_or_flushed_some() for channel_tls_t.
+ *      </ul>
+ *    </ul>
  *
- * 2.) Open for writes, no cells to send
- *     - Not much here either; this will be the state an idle but open channel
- *       can be expected to settle in.  It will have scheduler_state ==
- *       SCHED_CHAN_WAITING_FOR_CELLS
- *     - Transitions from:
- *       - Not open for writes/no cells by flushing some of the output
+ * <li> Open for writes, no cells to send
+ *   <ul>
+ *     <li>Not much here either; this will be the state an idle but open
+ *       channel can be expected to settle in.  It will have scheduler_state
+ *       == SCHED_CHAN_WAITING_FOR_CELLS
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/no cells by flushing some of the output
  *         buffer.
- *       - Open for writes/has cells by the scheduler moving cells from
+ *       <li>Open for writes/has cells by the scheduler moving cells from
  *         circuit queues to channel output queue, but not having enough
  *         to fill the output queue.
- *     - Transitions to:
- *       - Open for writes/has cells by arrival of new cells on an attached
+ *       </ul>
+ *     <li> Transitions to:
+ *       <ul>
+ *        <li>Open for writes/has cells by arrival of new cells on an attached
  *         circuit, in append_cell_to_circuit_queue()
+ *       </ul>
+ *     </ul>
  *
- * 3.) Not open for writes, cells to send
- *     - This is the state of a busy circuit limited by output bandwidth;
+ * <li>Not open for writes, cells to send
+ *     <ul>
+ *     <li>This is the state of a busy circuit limited by output bandwidth;
  *       cells have piled up in the circuit queues waiting to be relayed.
  *       The channel will have scheduler_state == SCHED_CHAN_WAITING_TO_WRITE.
- *     - Transitions from:
- *       - Not open for writes/no cells by arrival of cells on an attached
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/no cells by arrival of cells on an attached
  *         circuit
- *       - Open for writes/has cells by filling an output buffer without
+ *       <li>Open for writes/has cells by filling an output buffer without
  *         draining all cells from attached circuits
- *    - Transitions to:
- *       - Opens for writes/has cells by draining some of the output buffer
+ *       </ul>
+ *    <li> Transitions to:
+ *       <ul>
+ *       <li>Opens for writes/has cells by draining some of the output buffer
  *         via the connection_or_flushed_some() path (for channel_tls_t).
+ *       </ul>
+ *    </ul>
  *
- * 4.) Open for writes, cells to send
- *     - This connection is ready to relay some cells and waiting for
+ * <li>Open for writes, cells to send
+ *     <ul>
+ *     <li>This connection is ready to relay some cells and waiting for
  *       the scheduler to choose it.  The channel will have scheduler_state ==
  *       SCHED_CHAN_PENDING.
- *     - Transitions from:
- *       - Not open for writes/has cells by the connection_or_flushed_some()
+ *     <li>Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/has cells by the connection_or_flushed_some()
  *         path
- *       - Open for writes/no cells by the append_cell_to_circuit_queue()
+ *       <li>Open for writes/no cells by the append_cell_to_circuit_queue()
  *         path
- *     - Transitions to:
- *       - Not open for writes/no cells by draining all circuit queues and
- *         simultaneously filling the output buffer.
- *       - Not open for writes/has cells by writing enough cells to fill the
+ *       </ul>
+ *     <li> Transitions to:
+ *       <ul>
+ *        <li>Not open for writes/no cells by draining all circuit queues and
+ *          simultaneously filling the output buffer.
+ *        <li>Not open for writes/has cells by writing enough cells to fill the
  *         output buffer
- *       - Open for writes/no cells by draining all attached circuit queues
+ *        <li>Open for writes/no cells by draining all attached circuit queues
  *         without also filling the output buffer
+ *       </ul>
+ *    </ul>
+ * </ol>
  *
  * Other event-driven parts of the code move channels between these scheduling
- * states by calling scheduler functions; the scheduler only runs on open-for-
- * writes/has-cells channels and is the only path for those to transition to
- * other states.  The scheduler_run() function gives us the opportunity to do
- * scheduling work, and is called from other scheduler functions whenever a
- * state transition occurs, and periodically from the main event loop.
+ * states by calling scheduler functions. The scheduling system builds up a
+ * list of channels in the SCHED_CHAN_PENDING state that the scheduler
+ * implementation should then use when it runs. Scheduling implementations need
+ * to properly update channel states during their scheduler_t->run() function
+ * as that is the only opportunity for channels to move from SCHED_CHAN_PENDING
+ * to any other state.
+ *
+ * The remainder of this file is a small amount of state that any scheduler
+ * implementation should have access to, and the functions the rest of Tor uses
+ * to interact with the scheduling system.
  */
 
-/* Scheduler global data structures */
+/*****************************************************************************
+ * Scheduling system state
+ *
+ * State that can be accessed from any scheduler implementation (but not
+ * outside the scheduling system)
+ *****************************************************************************/
+
+/** DOCDOC */
+STATIC const scheduler_t *the_scheduler;
 
-/*
+/**
  * We keep a list of channels that are pending - i.e, have cells to write
- * and can accept them to send.  The enum scheduler_state in channel_t
+ * and can accept them to send. The enum scheduler_state in channel_t
  * is reserved for our use.
+ *
+ * Priority queue of channels that can write and have cells (pending work)
  */
-
-/* Pqueue of channels that can write and have cells (pending work) */
 STATIC smartlist_t *channels_pending = NULL;
 
-/*
+/**
  * This event runs the scheduler from its callback, and is manually
  * activated whenever a channel enters open for writes/cells to send.
  */
+STATIC struct mainloop_event_t *run_sched_ev = NULL;
 
-STATIC struct event *run_sched_ev = NULL;
+static int have_logged_kist_suddenly_disabled = 0;
 
-/*
- * Queue heuristic; this is not the queue size, but an 'effective queuesize'
- * that ages out contributions from stalled channels.
- */
+/*****************************************************************************
+ * Scheduling system static function definitions
+ *
+ * Functions that can only be accessed from this file.
+ *****************************************************************************/
 
-STATIC uint64_t queue_heuristic = 0;
+/** Return a human readable string for the given scheduler type. */
+static const char *
+get_scheduler_type_string(scheduler_types_t type)
+{
+  switch (type) {
+  case SCHEDULER_VANILLA:
+    return "Vanilla";
+  case SCHEDULER_KIST:
+    return "KIST";
+  case SCHEDULER_KIST_LITE:
+    return "KISTLite";
+  case SCHEDULER_NONE:
+    /* fallthrough */
+  default:
+    tor_assert_unreached();
+    return "(N/A)";
+  }
+}
 
-/*
- * Timestamp for last queue heuristic update
+/**
+ * Scheduler event callback; this should get triggered once per event loop
+ * if any scheduling work was created during the event loop.
  */
+static void
+scheduler_evt_callback(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
 
-STATIC time_t queue_heuristic_timestamp = 0;
+  log_debug(LD_SCHED, "Scheduler event callback called");
 
-/* Scheduler static function declarations */
+  /* Run the scheduler. This is a mandatory function. */
 
-static void scheduler_evt_callback(evutil_socket_t fd,
-                                   short events, void *arg);
-static int scheduler_more_work(void);
-static void scheduler_retrigger(void);
-#if 0
-static void scheduler_trigger(void);
-#endif
+  /* We might as well assert on this. If this function doesn't exist, no cells
+   * are getting scheduled. Things are very broken. scheduler_t says the run()
+   * function is mandatory. */
+  tor_assert(the_scheduler->run);
+  the_scheduler->run();
 
-/* Scheduler function implementations */
+  /* Schedule itself back in if it has more work. */
 
-/** Free everything and shut down the scheduling system */
+  /* Again, might as well assert on this mandatory scheduler_t function. If it
+   * doesn't exist, there's no way to tell libevent to run the scheduler again
+   * in the future. */
+  tor_assert(the_scheduler->schedule);
+  the_scheduler->schedule();
+}
 
-void
-scheduler_free_all(void)
+/** Using the global options, select the scheduler we should be using. */
+static void
+select_scheduler(void)
 {
-  log_debug(LD_SCHED, "Shutting down scheduler");
-
-  if (run_sched_ev) {
-    if (event_del(run_sched_ev) < 0) {
-      log_warn(LD_BUG, "Problem deleting run_sched_ev");
+  scheduler_t *new_scheduler = NULL;
+
+#ifdef TOR_UNIT_TESTS
+  /* This is hella annoying to set in the options for every test that passes
+   * through the scheduler and there are many so if we don't explicitly have
+   * a list of types set, just put the vanilla one. */
+  if (get_options()->SchedulerTypes_ == NULL) {
+    the_scheduler = get_vanilla_scheduler();
+    return;
+  }
+#endif /* defined(TOR_UNIT_TESTS) */
+
+  /* This list is ordered that is first entry has the first priority. Thus, as
+   * soon as we find a scheduler type that we can use, we use it and stop. */
+  SMARTLIST_FOREACH_BEGIN(get_options()->SchedulerTypes_, int *, type) {
+    switch (*type) {
+    case SCHEDULER_VANILLA:
+      new_scheduler = get_vanilla_scheduler();
+      goto end;
+    case SCHEDULER_KIST:
+      if (!scheduler_can_use_kist()) {
+#ifdef HAVE_KIST_SUPPORT
+        if (!have_logged_kist_suddenly_disabled) {
+          /* We should only log this once in most cases. If it was the kernel
+           * losing support for kist that caused scheduler_can_use_kist() to
+           * return false, then this flag makes sure we only log this message
+           * once. If it was the consensus that switched from "yes use kist"
+           * to "no don't use kist", then we still set the flag so we log
+           * once, but we unset the flag elsewhere if we ever can_use_kist()
+           * again.
+           */
+          have_logged_kist_suddenly_disabled = 1;
+          log_notice(LD_SCHED, "Scheduler type KIST has been disabled by "
+                               "the consensus or no kernel support.");
+        }
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+        log_info(LD_SCHED, "Scheduler type KIST not built in");
+#endif /* defined(HAVE_KIST_SUPPORT) */
+        continue;
+      }
+      /* This flag will only get set in one of two cases:
+       * 1 - the kernel lost support for kist. In that case, we don't expect to
+       *     ever end up here
+       * 2 - the consensus went from "yes use kist" to "no don't use kist".
+       * We might end up here if the consensus changes back to "yes", in which
+       * case we might want to warn the user again if it goes back to "no"
+       * yet again. Thus we unset the flag */
+      have_logged_kist_suddenly_disabled = 0;
+      new_scheduler = get_kist_scheduler();
+      scheduler_kist_set_full_mode();
+      goto end;
+    case SCHEDULER_KIST_LITE:
+      new_scheduler = get_kist_scheduler();
+      scheduler_kist_set_lite_mode();
+      goto end;
+    case SCHEDULER_NONE:
+      /* fallthrough */
+    default:
+      /* Our option validation should have caught this. */
+      tor_assert_unreached();
     }
-    tor_event_free(run_sched_ev);
-    run_sched_ev = NULL;
+  } SMARTLIST_FOREACH_END(type);
+
+ end:
+  if (new_scheduler == NULL) {
+    log_err(LD_SCHED, "Tor was unable to select a scheduler type. Please "
+                      "make sure Schedulers is correctly configured with "
+                      "what Tor does support.");
+    /* We weren't able to choose a scheduler which means that none of the ones
+     * set in Schedulers are supported or usable. We will respect the user
+     * wishes of using what it has been configured and don't do a sneaky
+     * fallback. Because this can be changed at runtime, we have to stop tor
+     * right now. */
+    exit(1); // XXXX bad exit
   }
 
-  if (channels_pending) {
-    smartlist_free(channels_pending);
-    channels_pending = NULL;
-  }
+  /* Set the chosen scheduler. */
+  the_scheduler = new_scheduler;
 }
 
 /**
- * Comparison function to use when sorting pending channels
+ * Helper function called from a few different places. It changes the
+ * scheduler implementation, if necessary. And if it did, it then tells the
+ * old one to free its state and the new one to initialize.
  */
+static void
+set_scheduler(void)
+{
+  const scheduler_t *old_scheduler = the_scheduler;
+  scheduler_types_t old_scheduler_type = SCHEDULER_NONE;
+
+  /* We keep track of the type in order to log only if the type switched. We
+   * can't just use the scheduler pointers because KIST and KISTLite share the
+   * same object. */
+  if (the_scheduler) {
+    old_scheduler_type = the_scheduler->type;
+  }
+
+  /* From the options, select the scheduler type to set. */
+  select_scheduler();
+  tor_assert(the_scheduler);
+
+  /* We look at the pointer difference in case the old sched and new sched
+   * share the same scheduler object, as is the case with KIST and KISTLite. */
+  if (old_scheduler != the_scheduler) {
+    /* Allow the old scheduler to clean up, if needed. */
+    if (old_scheduler && old_scheduler->free_all) {
+      old_scheduler->free_all();
+    }
+
+    /* Initialize the new scheduler. */
+    if (the_scheduler->init) {
+      the_scheduler->init();
+    }
+  }
+
+  /* Finally we notice log if we switched schedulers. We use the type in case
+   * two schedulers share a scheduler object. */
+  if (old_scheduler_type != the_scheduler->type) {
+    log_notice(LD_CONFIG, "Scheduler type %s has been enabled.",
+               get_scheduler_type_string(the_scheduler->type));
+  }
+}
 
-MOCK_IMPL(STATIC int,
+/*****************************************************************************
+ * Scheduling system private function definitions
+ *
+ * Functions that can only be accessed from scheduler*.c
+ *****************************************************************************/
+
+/** Returns human readable string for the given channel scheduler state. */
+const char *
+get_scheduler_state_string(int scheduler_state)
+{
+  switch (scheduler_state) {
+  case SCHED_CHAN_IDLE:
+    return "IDLE";
+  case SCHED_CHAN_WAITING_FOR_CELLS:
+    return "WAITING_FOR_CELLS";
+  case SCHED_CHAN_WAITING_TO_WRITE:
+    return "WAITING_TO_WRITE";
+  case SCHED_CHAN_PENDING:
+    return "PENDING";
+  default:
+    return "(invalid)";
+  }
+}
+
+/** Helper that logs channel scheduler_state changes. Use this instead of
+ * setting scheduler_state directly. */
+void
+scheduler_set_channel_state(channel_t *chan, int new_state)
+{
+  log_debug(LD_SCHED, "chan %" PRIu64 " changed from scheduler state %s to %s",
+      chan->global_identifier,
+      get_scheduler_state_string(chan->scheduler_state),
+      get_scheduler_state_string(new_state));
+  chan->scheduler_state = new_state;
+}
+
+/** Return the pending channel list. */
+smartlist_t *
+get_channels_pending(void)
+{
+  return channels_pending;
+}
+
+/** Comparison function to use when sorting pending channels. */
+MOCK_IMPL(int,
 scheduler_compare_channels, (const void *c1_v, const void *c2_v))
 {
-  channel_t *c1 = NULL, *c2 = NULL;
+  const channel_t *c1 = NULL, *c2 = NULL;
   /* These are a workaround for -Wbad-function-cast throwing a fit */
   const circuitmux_policy_t *p1, *p2;
   uintptr_t p1_i, p2_i;
@@ -180,11 +408,8 @@ scheduler_compare_channels, (const void *c1_v, const void *c2_v))
   tor_assert(c1_v);
   tor_assert(c2_v);
 
-  c1 = (channel_t *)(c1_v);
-  c2 = (channel_t *)(c2_v);
-
-  tor_assert(c1);
-  tor_assert(c2);
+  c1 = (const channel_t *)(c1_v);
+  c2 = (const channel_t *)(c2_v);
 
   if (c1 != c2) {
     if (circuitmux_get_policy(c1->cmux) ==
@@ -211,36 +436,80 @@ scheduler_compare_channels, (const void *c1_v, const void *c2_v))
   }
 }
 
-/*
- * Scheduler event callback; this should get triggered once per event loop
- * if any scheduling work was created during the event loop.
- */
+/*****************************************************************************
+ * Scheduling system global functions
+ *
+ * Functions that can be accessed from anywhere in Tor.
+ *****************************************************************************/
 
-static void
-scheduler_evt_callback(evutil_socket_t fd, short events, void *arg)
+/**
+ * This is how the scheduling system is notified of Tor's configuration
+ * changing. For example: a SIGHUP was issued.
+ */
+void
+scheduler_conf_changed(void)
 {
-  (void)fd;
-  (void)events;
-  (void)arg;
-  log_debug(LD_SCHED, "Scheduler event callback called");
+  /* Let the scheduler decide what it should do. */
+  set_scheduler();
 
-  tor_assert(run_sched_ev);
+  /* Then tell the (possibly new) scheduler that we have new options. */
+  if (the_scheduler->on_new_options) {
+    the_scheduler->on_new_options();
+  }
+}
 
-  /* Run the scheduler */
-  scheduler_run();
+/**
+ * Whenever we get a new consensus, this function is called.
+ */
+void
+scheduler_notify_networkstatus_changed(void)
+{
+  /* Maybe the consensus param made us change the scheduler. */
+  set_scheduler();
 
-  /* Do we have more work to do? */
-  if (scheduler_more_work()) scheduler_retrigger();
+  /* Then tell the (possibly new) scheduler that we have a new consensus */
+  if (the_scheduler->on_new_consensus) {
+    the_scheduler->on_new_consensus();
+  }
 }
 
-/** Mark a channel as no longer ready to accept writes */
+/**
+ * Free everything scheduling-related from main.c. Note this is only called
+ * when Tor is shutting down, while scheduler_t->free_all() is called both when
+ * Tor is shutting down and when we are switching schedulers.
+ */
+void
+scheduler_free_all(void)
+{
+  log_debug(LD_SCHED, "Shutting down scheduler");
+
+  if (run_sched_ev) {
+    mainloop_event_free(run_sched_ev);
+    run_sched_ev = NULL;
+  }
 
+  if (channels_pending) {
+    /* We don't have ownership of the objects in this list. */
+    smartlist_free(channels_pending);
+    channels_pending = NULL;
+  }
+
+  if (the_scheduler && the_scheduler->free_all) {
+    the_scheduler->free_all();
+  }
+  the_scheduler = NULL;
+}
+
+/** Mark a channel as no longer ready to accept writes. */
 MOCK_IMPL(void,
 scheduler_channel_doesnt_want_writes,(channel_t *chan))
 {
-  tor_assert(chan);
-
-  tor_assert(channels_pending);
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
 
   /* If it's already in pending, we can put it in waiting_to_write */
   if (chan->scheduler_state == SCHED_CHAN_PENDING) {
@@ -251,13 +520,9 @@ scheduler_channel_doesnt_want_writes,(channel_t *chan))
      */
     smartlist_pqueue_remove(channels_pending,
                             scheduler_compare_channels,
-                            STRUCT_OFFSET(channel_t, sched_heap_idx),
+                            offsetof(channel_t, sched_heap_idx),
                             chan);
-    chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-    log_debug(LD_SCHED,
-              "Channel " U64_FORMAT " at %p went from pending "
-              "to waiting_to_write",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
+    scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
   } else {
     /*
      * It's not in pending, so it can't become waiting_to_write; it's
@@ -265,41 +530,39 @@ scheduler_channel_doesnt_want_writes,(channel_t *chan))
      * waiting_for_cells (remove it, can't write any more).
      */
     if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
-      chan->scheduler_state = SCHED_CHAN_IDLE;
-      log_debug(LD_SCHED,
-                "Channel " U64_FORMAT " at %p left waiting_for_cells",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
+      scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
     }
   }
 }
 
-/** Mark a channel as having waiting cells */
-
+/** Mark a channel as having waiting cells. */
 MOCK_IMPL(void,
 scheduler_channel_has_waiting_cells,(channel_t *chan))
 {
-  int became_pending = 0;
-
-  tor_assert(chan);
-  tor_assert(channels_pending);
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
 
-  /* First, check if this one also writeable */
+  /* First, check if it's also writeable */
   if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
     /*
      * It's in channels_waiting_for_cells, so it shouldn't be in any of
      * the other lists.  It has waiting cells now, so it goes to
      * channels_pending.
      */
-    chan->scheduler_state = SCHED_CHAN_PENDING;
-    smartlist_pqueue_add(channels_pending,
-                         scheduler_compare_channels,
-                         STRUCT_OFFSET(channel_t, sched_heap_idx),
-                         chan);
-    log_debug(LD_SCHED,
-              "Channel " U64_FORMAT " at %p went from waiting_for_cells "
-              "to pending",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
-    became_pending = 1;
+    scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
+    if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+      smartlist_pqueue_add(channels_pending,
+                           scheduler_compare_channels,
+                           offsetof(channel_t, sched_heap_idx),
+                           chan);
+    }
+    /* If we made a channel pending, we potentially have scheduling work to
+     * do. */
+    the_scheduler->schedule();
   } else {
     /*
      * It's not in waiting_for_cells, so it can't become pending; it's
@@ -308,256 +571,121 @@ scheduler_channel_has_waiting_cells,(channel_t *chan))
      */
     if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE ||
           chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-      log_debug(LD_SCHED,
-                "Channel " U64_FORMAT " at %p entered waiting_to_write",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
     }
   }
+}
 
-  /*
-   * If we made a channel pending, we potentially have scheduling work
-   * to do.
-   */
-  if (became_pending) scheduler_retrigger();
+/** Add the scheduler event to the set of pending events with next_run being
+ * the longest time libevent should wait before triggering the event. */
+void
+scheduler_ev_add(const struct timeval *next_run)
+{
+  tor_assert(run_sched_ev);
+  tor_assert(next_run);
+  if (BUG(mainloop_event_schedule(run_sched_ev, next_run) < 0)) {
+    log_warn(LD_SCHED, "Adding to libevent failed. Next run time was set to: "
+                       "%ld.%06ld", next_run->tv_sec, (long)next_run->tv_usec);
+    return;
+  }
 }
 
-/** Set up the scheduling system */
+/** Make the scheduler event active with the given flags. */
+void
+scheduler_ev_active(void)
+{
+  tor_assert(run_sched_ev);
+  mainloop_event_activate(run_sched_ev);
+}
 
+/*
+ * Initialize everything scheduling-related from config.c. Note this is only
+ * called when Tor is starting up, while scheduler_t->init() is called both
+ * when Tor is starting up and when we are switching schedulers.
+ */
 void
 scheduler_init(void)
 {
   log_debug(LD_SCHED, "Initting scheduler");
 
-  tor_assert(!run_sched_ev);
-  run_sched_ev = tor_event_new(tor_libevent_get_base(), -1,
-                               0, scheduler_evt_callback, NULL);
-
+  // Two '!' because we really do want to check if the pointer is non-NULL
+  IF_BUG_ONCE(!!run_sched_ev) {
+    log_warn(LD_SCHED, "We should not already have a libevent scheduler event."
+             "I'll clean the old one up, but this is odd.");
+    mainloop_event_free(run_sched_ev);
+    run_sched_ev = NULL;
+  }
+  run_sched_ev = mainloop_event_new(scheduler_evt_callback, NULL);
   channels_pending = smartlist_new();
-  queue_heuristic = 0;
-  queue_heuristic_timestamp = approx_time();
-}
-
-/** Check if there's more scheduling work */
 
-static int
-scheduler_more_work(void)
-{
-  tor_assert(channels_pending);
-
-  return ((scheduler_get_queue_heuristic() < sched_q_low_water) &&
-          ((smartlist_len(channels_pending) > 0))) ? 1 : 0;
+  set_scheduler();
 }
 
-/** Retrigger the scheduler in a way safe to use from the callback */
-
-static void
-scheduler_retrigger(void)
-{
-  tor_assert(run_sched_ev);
-  event_active(run_sched_ev, EV_TIMEOUT, 1);
-}
-
-/** Notify the scheduler of a channel being closed */
-
+/*
+ * If a channel is going away, this is how the scheduling system is informed
+ * so it can do any freeing necessary. This ultimately calls
+ * scheduler_t->on_channel_free() so the current scheduler can release any
+ * state specific to this channel.
+ */
 MOCK_IMPL(void,
 scheduler_release_channel,(channel_t *chan))
 {
-  tor_assert(chan);
-  tor_assert(channels_pending);
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
 
-  if (chan->scheduler_state == SCHED_CHAN_PENDING) {
+  /* Try to remove the channel from the pending list regardless of its
+   * scheduler state. We can release a channel in many places in the tor code
+   * so we can't rely on the channel state (PENDING) to remove it from the
+   * list.
+   *
+   * For instance, the channel can change state from OPEN to CLOSING while
+   * being handled in the scheduler loop leading to the channel being in
+   * PENDING state but not in the pending list. Furthermore, we release the
+   * channel when it changes state to close and a second time when we free it.
+   * Not ideal at all but for now that is the way it is. */
+  if (chan->sched_heap_idx != -1) {
     smartlist_pqueue_remove(channels_pending,
                             scheduler_compare_channels,
-                            STRUCT_OFFSET(channel_t, sched_heap_idx),
+                            offsetof(channel_t, sched_heap_idx),
                             chan);
   }
 
-  chan->scheduler_state = SCHED_CHAN_IDLE;
-}
-
-/** Run the scheduling algorithm if necessary */
-
-MOCK_IMPL(void,
-scheduler_run, (void))
-{
-  int n_cells, n_chans_before, n_chans_after;
-  uint64_t q_len_before, q_heur_before, q_len_after, q_heur_after;
-  ssize_t flushed, flushed_this_time;
-  smartlist_t *to_readd = NULL;
-  channel_t *chan = NULL;
-
-  log_debug(LD_SCHED, "We have a chance to run the scheduler");
-
-  if (scheduler_get_queue_heuristic() < sched_q_low_water) {
-    n_chans_before = smartlist_len(channels_pending);
-    q_len_before = channel_get_global_queue_estimate();
-    q_heur_before = scheduler_get_queue_heuristic();
-
-    while (scheduler_get_queue_heuristic() <= sched_q_high_water &&
-           smartlist_len(channels_pending) > 0) {
-      /* Pop off a channel */
-      chan = smartlist_pqueue_pop(channels_pending,
-                                  scheduler_compare_channels,
-                                  STRUCT_OFFSET(channel_t, sched_heap_idx));
-      tor_assert(chan);
-
-      /* Figure out how many cells we can write */
-      n_cells = channel_num_cells_writeable(chan);
-      if (n_cells > 0) {
-        log_debug(LD_SCHED,
-                  "Scheduler saw pending channel " U64_FORMAT " at %p with "
-                  "%d cells writeable",
-                  U64_PRINTF_ARG(chan->global_identifier), chan, n_cells);
-
-        flushed = 0;
-        while (flushed < n_cells &&
-               scheduler_get_queue_heuristic() <= sched_q_high_water) {
-          flushed_this_time =
-            channel_flush_some_cells(chan,
-                                     MIN(sched_max_flush_cells,
-                                         (size_t) n_cells - flushed));
-          if (flushed_this_time <= 0) break;
-          flushed += flushed_this_time;
-        }
-
-        if (flushed < n_cells) {
-          /* We ran out of cells to flush */
-          chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
-          log_debug(LD_SCHED,
-                    "Channel " U64_FORMAT " at %p "
-                    "entered waiting_for_cells from pending",
-                    U64_PRINTF_ARG(chan->global_identifier),
-                    chan);
-        } else {
-          /* The channel may still have some cells */
-          if (channel_more_to_flush(chan)) {
-          /* The channel goes to either pending or waiting_to_write */
-            if (channel_num_cells_writeable(chan) > 0) {
-              /* Add it back to pending later */
-              if (!to_readd) to_readd = smartlist_new();
-              smartlist_add(to_readd, chan);
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "is still pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            } else {
-              /* It's waiting to be able to write more */
-              chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "entered waiting_to_write from pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            }
-          } else {
-            /* No cells left; it can go to idle or waiting_for_cells */
-            if (channel_num_cells_writeable(chan) > 0) {
-              /*
-               * It can still accept writes, so it goes to
-               * waiting_for_cells
-               */
-              chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "entered waiting_for_cells from pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            } else {
-              /*
-               * We exactly filled up the output queue with all available
-               * cells; go to idle.
-               */
-              chan->scheduler_state = SCHED_CHAN_IDLE;
-              log_debug(LD_SCHED,
-                        "Channel " U64_FORMAT " at %p "
-                        "become idle from pending",
-                        U64_PRINTF_ARG(chan->global_identifier),
-                        chan);
-            }
-          }
-        }
-
-        log_debug(LD_SCHED,
-                  "Scheduler flushed %d cells onto pending channel "
-                  U64_FORMAT " at %p",
-                  (int)flushed, U64_PRINTF_ARG(chan->global_identifier),
-                  chan);
-      } else {
-        log_info(LD_SCHED,
-                 "Scheduler saw pending channel " U64_FORMAT " at %p with "
-                 "no cells writeable",
-                 U64_PRINTF_ARG(chan->global_identifier), chan);
-        /* Put it back to WAITING_TO_WRITE */
-        chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE;
-      }
-    }
-
-    /* Readd any channels we need to */
-    if (to_readd) {
-      SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
-        readd_chan->scheduler_state = SCHED_CHAN_PENDING;
-        smartlist_pqueue_add(channels_pending,
-                             scheduler_compare_channels,
-                             STRUCT_OFFSET(channel_t, sched_heap_idx),
-                             readd_chan);
-      } SMARTLIST_FOREACH_END(readd_chan);
-      smartlist_free(to_readd);
-    }
-
-    n_chans_after = smartlist_len(channels_pending);
-    q_len_after = channel_get_global_queue_estimate();
-    q_heur_after = scheduler_get_queue_heuristic();
-    log_debug(LD_SCHED,
-              "Scheduler handled %d of %d pending channels, queue size from "
-              U64_FORMAT " to " U64_FORMAT ", queue heuristic from "
-              U64_FORMAT " to " U64_FORMAT,
-              n_chans_before - n_chans_after, n_chans_before,
-              U64_PRINTF_ARG(q_len_before), U64_PRINTF_ARG(q_len_after),
-              U64_PRINTF_ARG(q_heur_before), U64_PRINTF_ARG(q_heur_after));
+  if (the_scheduler->on_channel_free) {
+    the_scheduler->on_channel_free(chan);
   }
+  scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
 }
 
-/** Trigger the scheduling event so we run the scheduler later */
-
-#if 0
-static void
-scheduler_trigger(void)
-{
-  log_debug(LD_SCHED, "Triggering scheduler event");
-
-  tor_assert(run_sched_ev);
-
-  event_add(run_sched_ev, EV_TIMEOUT, 1);
-}
-#endif
-
 /** Mark a channel as ready to accept writes */
 
 void
 scheduler_channel_wants_writes(channel_t *chan)
 {
-  int became_pending = 0;
-
-  tor_assert(chan);
-  tor_assert(channels_pending);
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
+  IF_BUG_ONCE(!channels_pending) {
+    return;
+  }
 
   /* If it's already in waiting_to_write, we can put it in pending */
   if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) {
     /*
      * It can write now, so it goes to channels_pending.
      */
-    smartlist_pqueue_add(channels_pending,
-                         scheduler_compare_channels,
-                         STRUCT_OFFSET(channel_t, sched_heap_idx),
-                         chan);
-    chan->scheduler_state = SCHED_CHAN_PENDING;
-    log_debug(LD_SCHED,
-              "Channel " U64_FORMAT " at %p went from waiting_to_write "
-              "to pending",
-              U64_PRINTF_ARG(chan->global_identifier), chan);
-    became_pending = 1;
+    scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
+    if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+      smartlist_pqueue_add(channels_pending,
+                           scheduler_compare_channels,
+                           offsetof(channel_t, sched_heap_idx),
+                           chan);
+    }
+    /* We just made a channel pending, we have scheduling work to do. */
+    the_scheduler->schedule();
   } else {
     /*
      * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending;
@@ -565,143 +693,74 @@ scheduler_channel_wants_writes(channel_t *chan)
      */
     if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS ||
           chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
-      log_debug(LD_SCHED,
-                "Channel " U64_FORMAT " at %p entered waiting_for_cells",
-                U64_PRINTF_ARG(chan->global_identifier), chan);
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
     }
   }
+}
+
+/* Log warn the given channel and extra scheduler context as well. This is
+ * used by SCHED_BUG() in order to be able to extract as much information as
+ * we can when we hit a bug. Channel chan can be NULL. */
+void
+scheduler_bug_occurred(const channel_t *chan)
+{
+  char buf[128];
+
+  if (chan != NULL) {
+    const size_t outbuf_len =
+      buf_datalen(TO_CONN(BASE_CHAN_TO_TLS((channel_t *) chan)->conn)->outbuf);
+    tor_snprintf(buf, sizeof(buf),
+                 "Channel %" PRIu64 " in state %s and scheduler state %s."
+                 " Num cells on cmux: %d. Connection outbuf len: %lu.",
+                 chan->global_identifier,
+                 channel_state_to_string(chan->state),
+                 get_scheduler_state_string(chan->scheduler_state),
+                 circuitmux_num_cells(chan->cmux),
+                 (unsigned long)outbuf_len);
+  }
 
-  /*
-   * If we made a channel pending, we potentially have scheduling work
-   * to do.
-   */
-  if (became_pending) scheduler_retrigger();
+  {
+    char *msg;
+    /* Rate limit every 60 seconds. If we start seeing this every 60 sec, we
+     * know something is stuck/wrong. It *should* be loud but not too much. */
+    static ratelim_t rlimit = RATELIM_INIT(60);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_warn(LD_BUG, "%s Num pending channels: %d. "
+                       "Channel in pending list: %s.%s",
+               (chan != NULL) ? buf : "No channel in bug context.",
+               smartlist_len(channels_pending),
+               (smartlist_pos(channels_pending, chan) == -1) ? "no" : "yes",
+               msg);
+      tor_free(msg);
+    }
+  }
 }
 
-/**
- * Notify the scheduler that a channel's position in the pqueue may have
- * changed
- */
+#ifdef TOR_UNIT_TESTS
 
+/*
+ * Notify scheduler that a channel's queue position may have changed.
+ */
 void
 scheduler_touch_channel(channel_t *chan)
 {
-  tor_assert(chan);
+  IF_BUG_ONCE(!chan) {
+    return;
+  }
 
   if (chan->scheduler_state == SCHED_CHAN_PENDING) {
     /* Remove and re-add it */
     smartlist_pqueue_remove(channels_pending,
                             scheduler_compare_channels,
-                            STRUCT_OFFSET(channel_t, sched_heap_idx),
+                            offsetof(channel_t, sched_heap_idx),
                             chan);
     smartlist_pqueue_add(channels_pending,
                          scheduler_compare_channels,
-                         STRUCT_OFFSET(channel_t, sched_heap_idx),
+                         offsetof(channel_t, sched_heap_idx),
                          chan);
   }
   /* else no-op, since it isn't in the queue */
 }
 
-/**
- * Notify the scheduler of a queue size adjustment, to recalculate the
- * queue heuristic.
- */
-
-void
-scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj)
-{
-  time_t now = approx_time();
-
-  log_debug(LD_SCHED,
-            "Queue size adjustment by %s" U64_FORMAT " for channel "
-            U64_FORMAT,
-            (dir >= 0) ? "+" : "-",
-            U64_PRINTF_ARG(adj),
-            U64_PRINTF_ARG(chan->global_identifier));
-
-  /* Get the queue heuristic up to date */
-  scheduler_update_queue_heuristic(now);
-
-  /* Adjust as appropriate */
-  if (dir >= 0) {
-    /* Increasing it */
-    queue_heuristic += adj;
-  } else {
-    /* Decreasing it */
-    if (queue_heuristic > adj) queue_heuristic -= adj;
-    else queue_heuristic = 0;
-  }
-
-  log_debug(LD_SCHED,
-            "Queue heuristic is now " U64_FORMAT,
-            U64_PRINTF_ARG(queue_heuristic));
-}
-
-/**
- * Query the current value of the queue heuristic
- */
-
-STATIC uint64_t
-scheduler_get_queue_heuristic(void)
-{
-  time_t now = approx_time();
-
-  scheduler_update_queue_heuristic(now);
-
-  return queue_heuristic;
-}
-
-/**
- * Adjust the queue heuristic value to the present time
- */
-
-STATIC void
-scheduler_update_queue_heuristic(time_t now)
-{
-  time_t diff;
-
-  if (queue_heuristic_timestamp == 0) {
-    /*
-     * Nothing we can sensibly do; must not have been initted properly.
-     * Oh well.
-     */
-    queue_heuristic_timestamp = now;
-  } else if (queue_heuristic_timestamp < now) {
-    diff = now - queue_heuristic_timestamp;
-    /*
-     * This is a simple exponential age-out; the other proposed alternative
-     * was a linear age-out using the bandwidth history in rephist.c; I'm
-     * going with this out of concern that if an adversary can jam the
-     * scheduler long enough, it would cause the bandwidth to drop to
-     * zero and render the aging mechanism ineffective thereafter.
-     */
-    if (0 <= diff && diff < 64) queue_heuristic >>= diff;
-    else queue_heuristic = 0;
-
-    queue_heuristic_timestamp = now;
-
-    log_debug(LD_SCHED,
-              "Queue heuristic is now " U64_FORMAT,
-              U64_PRINTF_ARG(queue_heuristic));
-  }
-  /* else no update needed, or time went backward */
-}
-
-/**
- * Set scheduler watermarks and flush size
- */
-
-void
-scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush)
-{
-  /* Sanity assertions - caller should ensure these are true */
-  tor_assert(lo > 0);
-  tor_assert(hi > lo);
-  tor_assert(max_flush > 0);
-
-  sched_q_low_water = lo;
-  sched_q_high_water = hi;
-  sched_max_flush_cells = max_flush;
-}
+#endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/or/scheduler.h b/src/or/scheduler.h
index 3dcfd2faca..08b02e286f 100644
--- a/src/or/scheduler.h
+++ b/src/or/scheduler.h
@@ -1,9 +1,9 @@
-/* * Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file scheduler.h
- * \brief Header file for scheduler.c
+ * \brief Header file for scheduler*.c
  **/
 
 #ifndef TOR_SCHEDULER_H
@@ -13,45 +13,206 @@
 #include "channel.h"
 #include "testsupport.h"
 
-/* Global-visibility scheduler functions */
+/** Scheduler type, we build an ordered list with those values from the
+ * parsed strings in Schedulers. The reason to do such a thing is so we can
+ * quickly and without parsing strings select the scheduler at anytime. */
+typedef enum {
+  SCHEDULER_NONE =     -1,
+  SCHEDULER_VANILLA =   1,
+  SCHEDULER_KIST =      2,
+  SCHEDULER_KIST_LITE = 3,
+} scheduler_types_t;
+
+/**
+ * A scheduler implementation is a collection of function pointers. If you
+ * would like to add a new scheduler called foo, create scheduler_foo.c,
+ * implement at least the mandatory ones, and implement get_foo_scheduler()
+ * that returns a complete scheduler_t for your foo scheduler. See
+ * scheduler_kist.c for an example.
+ *
+ * These function pointers SHOULD NOT be used anywhere outside of the
+ * scheduling source files. The rest of Tor should communicate with the
+ * scheduling system through the functions near the bottom of this file, and
+ * those functions will call into the current scheduler implementation as
+ * necessary.
+ *
+ * If your scheduler doesn't need to implement something (for example: it
+ * doesn't create any state for itself, thus it has nothing to free when Tor
+ * is shutting down), then set that function pointer to NULL.
+ */
+typedef struct scheduler_s {
+  /* Scheduler type. This is used for logging when the scheduler is switched
+   * during runtime. */
+  scheduler_types_t type;
+
+  /* (Optional) To be called when we want to prepare a scheduler for use.
+   * Perhaps Tor just started and we are the lucky chosen scheduler, or
+   * perhaps Tor is switching to this scheduler. No matter the case, this is
+   * where we would prepare any state and initialize parameters. You might
+   * think of this as the opposite of free_all(). */
+  void (*init)(void);
+
+  /* (Optional) To be called when we want to tell the scheduler to delete all
+   * of its state (if any). Perhaps Tor is shutting down or perhaps we are
+   * switching schedulers. */
+  void (*free_all)(void);
+
+  /* (Mandatory) Libevent controls the main event loop in Tor, and this is
+   * where we register with libevent the next execution of run_sched_ev [which
+   * ultimately calls run()]. */
+  void (*schedule)(void);
+
+  /* (Mandatory) This is the heart of a scheduler! This is where the
+   * excitement happens! Here libevent has given us the chance to execute, and
+   * we should do whatever we need to do in order to move some cells from
+   * their circuit queues to output buffers in an intelligent manner. We
+   * should do this quickly. When we are done, we'll try to schedule() ourself
+   * if more work needs to be done to setup the next scheduling run. */
+  void (*run)(void);
+
+  /*
+   * External event not related to the scheduler but that can influence it.
+   */
+
+  /* (Optional) To be called whenever Tor finds out about a new consensus.
+   * First the scheduling system as a whole will react to the new consensus
+   * and change the scheduler if needed. After that, the current scheduler
+   * (which might be new) will call this so it has the chance to react to the
+   * new consensus too. If there's a consensus parameter that your scheduler
+   * wants to keep an eye on, this is where you should check for it.  */
+  void (*on_new_consensus)(void);
+
+  /* (Optional) To be called when a channel is being freed. Sometimes channels
+   * go away (for example: the relay on the other end is shutting down). If
+   * the scheduler keeps any channel-specific state and has memory to free
+   * when channels go away, implement this and free it here. */
+  void (*on_channel_free)(const channel_t *);
+
+  /* (Optional) To be called whenever Tor is reloading configuration options.
+   * For example: SIGHUP was issued and Tor is rereading its torrc. A
+   * scheduler should use this as an opportunity to parse and cache torrc
+   * options so that it doesn't have to call get_options() all the time. */
+  void (*on_new_options)(void);
+} scheduler_t;
+
+/*****************************************************************************
+ * Globally visible scheduler variables/values
+ *
+ * These are variables/constants that all of Tor should be able to see.
+ *****************************************************************************/
+
+/* Default interval that KIST runs (in ms). */
+#define KIST_SCHED_RUN_INTERVAL_DEFAULT 10
+/* Minimum interval that KIST runs. This value disables KIST. */
+#define KIST_SCHED_RUN_INTERVAL_MIN 0
+/* Maximum interval that KIST runs (in ms). */
+#define KIST_SCHED_RUN_INTERVAL_MAX 100
+
+/*****************************************************************************
+ * Globally visible scheduler functions
+ *
+ * These functions are how the rest of Tor communicates with the scheduling
+ * system.
+ *****************************************************************************/
 
-/* Set up and shut down the scheduler from main.c */
-void scheduler_free_all(void);
 void scheduler_init(void);
-MOCK_DECL(void, scheduler_run, (void));
+void scheduler_free_all(void);
+void scheduler_conf_changed(void);
+void scheduler_notify_networkstatus_changed(void);
+MOCK_DECL(void, scheduler_release_channel, (channel_t *chan));
 
-/* Mark channels as having cells or wanting/not wanting writes */
-MOCK_DECL(void,scheduler_channel_doesnt_want_writes,(channel_t *chan));
-MOCK_DECL(void,scheduler_channel_has_waiting_cells,(channel_t *chan));
+/*
+ * Ways for a channel to interact with the scheduling system. A channel only
+ * really knows (i) whether or not it has cells it wants to send, and
+ * (ii) whether or not it would like to write.
+ */
 void scheduler_channel_wants_writes(channel_t *chan);
+MOCK_DECL(void, scheduler_channel_doesnt_want_writes, (channel_t *chan));
+MOCK_DECL(void, scheduler_channel_has_waiting_cells, (channel_t *chan));
 
-/* Notify the scheduler of a channel being closed */
-MOCK_DECL(void,scheduler_release_channel,(channel_t *chan));
+/*****************************************************************************
+ * Private scheduler functions
+ *
+ * These functions are only visible to the scheduling system, the current
+ * scheduler implementation, and tests.
+ *****************************************************************************/
+#ifdef SCHEDULER_PRIVATE_
 
-/* Notify scheduler of queue size adjustments */
-void scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj);
+/*********************************
+ * Defined in scheduler.c
+ *********************************/
 
-/* Notify scheduler that a channel's queue position may have changed */
-void scheduler_touch_channel(channel_t *chan);
+void scheduler_set_channel_state(channel_t *chan, int new_state);
+const char *get_scheduler_state_string(int scheduler_state);
 
-/* Adjust the watermarks from config file*/
-void scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush);
+/* Triggers a BUG() and extra information with chan if available. */
+#define SCHED_BUG(cond, chan) \
+  (PREDICT_UNLIKELY(cond) ? \
+   ((BUG(cond)) ? (scheduler_bug_occurred(chan), 1) : 0) : 0)
 
-/* Things only scheduler.c and its test suite should see */
+void scheduler_bug_occurred(const channel_t *chan);
 
-#ifdef SCHEDULER_PRIVATE_
-MOCK_DECL(STATIC int, scheduler_compare_channels,
+smartlist_t *get_channels_pending(void);
+MOCK_DECL(int, scheduler_compare_channels,
           (const void *c1_v, const void *c2_v));
-STATIC uint64_t scheduler_get_queue_heuristic(void);
-STATIC void scheduler_update_queue_heuristic(time_t now);
+void scheduler_ev_active(void);
+void scheduler_ev_add(const struct timeval *next_run);
 
 #ifdef TOR_UNIT_TESTS
 extern smartlist_t *channels_pending;
-extern struct event *run_sched_ev;
-extern uint64_t queue_heuristic;
-extern time_t queue_heuristic_timestamp;
-#endif
-#endif
+extern struct mainloop_event_t *run_sched_ev;
+extern const scheduler_t *the_scheduler;
+void scheduler_touch_channel(channel_t *chan);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/*********************************
+ * Defined in scheduler_kist.c
+ *********************************/
+
+#ifdef SCHEDULER_KIST_PRIVATE
+
+/* Socket table entry which holds information of a channel's socket and kernel
+ * TCP information. Only used by KIST. */
+typedef struct socket_table_ent_s {
+  HT_ENTRY(socket_table_ent_s) node;
+  const channel_t *chan;
+  /* Amount written this scheduling run */
+  uint64_t written;
+  /* Amount that can be written this scheduling run */
+  uint64_t limit;
+  /* TCP info from the kernel */
+  uint32_t cwnd;
+  uint32_t unacked;
+  uint32_t mss;
+  uint32_t notsent;
+} socket_table_ent_t;
+
+typedef HT_HEAD(outbuf_table_s, outbuf_table_ent_s) outbuf_table_t;
+
+MOCK_DECL(int, channel_should_write_to_kernel,
+          (outbuf_table_t *table, channel_t *chan));
+MOCK_DECL(void, channel_write_to_kernel, (channel_t *chan));
+MOCK_DECL(void, update_socket_info_impl, (socket_table_ent_t *ent));
+
+int scheduler_can_use_kist(void);
+void scheduler_kist_set_full_mode(void);
+void scheduler_kist_set_lite_mode(void);
+scheduler_t *get_kist_scheduler(void);
+int kist_scheduler_run_interval(void);
+
+#ifdef TOR_UNIT_TESTS
+extern int32_t sched_run_interval;
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* defined(SCHEDULER_KIST_PRIVATE) */
+
+/*********************************
+ * Defined in scheduler_vanilla.c
+ *********************************/
+
+scheduler_t *get_vanilla_scheduler(void);
+
+#endif /* defined(SCHEDULER_PRIVATE_) */
 
 #endif /* !defined(TOR_SCHEDULER_H) */
 
diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c
new file mode 100644
index 0000000000..e941626371
--- /dev/null
+++ b/src/or/scheduler_kist.c
@@ -0,0 +1,834 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define SCHEDULER_KIST_PRIVATE
+
+#include "or.h"
+#include "buffers.h"
+#include "config.h"
+#include "connection.h"
+#include "networkstatus.h"
+#define TOR_CHANNEL_INTERNAL_
+#include "channel.h"
+#include "channeltls.h"
+#define SCHEDULER_PRIVATE_
+#include "scheduler.h"
+
+#ifdef HAVE_KIST_SUPPORT
+/* Kernel interface needed for KIST. */
+#include <netinet/tcp.h>
+#include <linux/sockios.h>
+#endif /* HAVE_KIST_SUPPORT */
+
+/*****************************************************************************
+ * Data structures and supporting functions
+ *****************************************************************************/
+
+/* Socket_table hash table stuff. The socket_table keeps track of per-socket
+ * limit information imposed by kist and used by kist. */
+
+static uint32_t
+socket_table_ent_hash(const socket_table_ent_t *ent)
+{
+  return (uint32_t)ent->chan->global_identifier;
+}
+
+static unsigned
+socket_table_ent_eq(const socket_table_ent_t *a, const socket_table_ent_t *b)
+{
+  return a->chan == b->chan;
+}
+
+typedef HT_HEAD(socket_table_s, socket_table_ent_s) socket_table_t;
+
+static socket_table_t socket_table = HT_INITIALIZER();
+
+HT_PROTOTYPE(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
+             socket_table_ent_eq)
+HT_GENERATE2(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
+             socket_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+
+/* outbuf_table hash table stuff. The outbuf_table keeps track of which
+ * channels have data sitting in their outbuf so the kist scheduler can force
+ * a write from outbuf to kernel periodically during a run and at the end of a
+ * run. */
+
+typedef struct outbuf_table_ent_s {
+  HT_ENTRY(outbuf_table_ent_s) node;
+  channel_t *chan;
+} outbuf_table_ent_t;
+
+static uint32_t
+outbuf_table_ent_hash(const outbuf_table_ent_t *ent)
+{
+  return (uint32_t)ent->chan->global_identifier;
+}
+
+static unsigned
+outbuf_table_ent_eq(const outbuf_table_ent_t *a, const outbuf_table_ent_t *b)
+{
+  return a->chan->global_identifier == b->chan->global_identifier;
+}
+
+HT_PROTOTYPE(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq)
+HT_GENERATE2(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+
+/*****************************************************************************
+ * Other internal data
+ *****************************************************************************/
+
+/* Store the last time the scheduler was run so we can decide when to next run
+ * the scheduler based on it. */
+static monotime_t scheduler_last_run;
+/* This is a factor for the extra_space calculation in kist per-socket limits.
+ * It is the number of extra congestion windows we want to write to the kernel.
+ */
+static double sock_buf_size_factor = 1.0;
+/* How often the scheduler runs. */
+STATIC int sched_run_interval = KIST_SCHED_RUN_INTERVAL_DEFAULT;
+
+#ifdef HAVE_KIST_SUPPORT
+/* Indicate if KIST lite mode is on or off. We can disable it at runtime.
+ * Important to have because of the KISTLite -> KIST possible transition. */
+static unsigned int kist_lite_mode = 0;
+/* Indicate if we don't have the kernel support. This can happen if the kernel
+ * changed and it doesn't recognized the values passed to the syscalls needed
+ * by KIST. In that case, fallback to the naive approach. */
+static unsigned int kist_no_kernel_support = 0;
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+static unsigned int kist_lite_mode = 1;
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
+/*****************************************************************************
+ * Internally called function implementations
+ *****************************************************************************/
+
+/* Little helper function to get the length of a channel's output buffer */
+static inline size_t
+channel_outbuf_length(channel_t *chan)
+{
+  /* In theory, this can not happen because we can not scheduler a channel
+   * without a connection that has its outbuf initialized. Just in case, bug
+   * on this so we can understand a bit more why it happened. */
+  if (SCHED_BUG(BASE_CHAN_TO_TLS(chan)->conn == NULL, chan)) {
+    return 0;
+  }
+  return buf_datalen(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->outbuf);
+}
+
+/* Little helper function for HT_FOREACH_FN. */
+static int
+each_channel_write_to_kernel(outbuf_table_ent_t *ent, void *data)
+{
+  (void) data; /* Make compiler happy. */
+  channel_write_to_kernel(ent->chan);
+  return 0; /* Returning non-zero removes the element from the table. */
+}
+
+/* Free the given outbuf table entry ent. */
+static int
+free_outbuf_info_by_ent(outbuf_table_ent_t *ent, void *data)
+{
+  (void) data; /* Make compiler happy. */
+  log_debug(LD_SCHED, "Freeing outbuf table entry from chan=%" PRIu64,
+            ent->chan->global_identifier);
+  tor_free(ent);
+  return 1; /* So HT_FOREACH_FN will remove the element */
+}
+
+/* Free the given socket table entry ent. */
+static int
+free_socket_info_by_ent(socket_table_ent_t *ent, void *data)
+{
+  (void) data; /* Make compiler happy. */
+  log_debug(LD_SCHED, "Freeing socket table entry from chan=%" PRIu64,
+            ent->chan->global_identifier);
+  tor_free(ent);
+  return 1; /* So HT_FOREACH_FN will remove the element */
+}
+
+/* Clean up socket_table. Probably because the KIST sched impl is going away */
+static void
+free_all_socket_info(void)
+{
+  HT_FOREACH_FN(socket_table_s, &socket_table, free_socket_info_by_ent, NULL);
+  HT_CLEAR(socket_table_s, &socket_table);
+}
+
+static socket_table_ent_t *
+socket_table_search(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t search, *ent = NULL;
+  search.chan = chan;
+  ent = HT_FIND(socket_table_s, table, &search);
+  return ent;
+}
+
+/* Free a socket entry in table for the given chan. */
+static void
+free_socket_info_by_chan(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (!ent)
+    return;
+  log_debug(LD_SCHED, "scheduler free socket info for chan=%" PRIu64,
+            chan->global_identifier);
+  HT_REMOVE(socket_table_s, table, ent);
+  free_socket_info_by_ent(ent, NULL);
+}
+
+/* Perform system calls for the given socket in order to calculate kist's
+ * per-socket limit as documented in the function body. */
+MOCK_IMPL(void,
+update_socket_info_impl, (socket_table_ent_t *ent))
+{
+#ifdef HAVE_KIST_SUPPORT
+  int64_t tcp_space, extra_space;
+  const tor_socket_t sock =
+    TO_CONN(BASE_CHAN_TO_TLS((channel_t *) ent->chan)->conn)->s;
+  struct tcp_info tcp;
+  socklen_t tcp_info_len = sizeof(tcp);
+
+  if (kist_no_kernel_support || kist_lite_mode) {
+    goto fallback;
+  }
+
+  /* Gather information */
+  if (getsockopt(sock, SOL_TCP, TCP_INFO, (void *)&(tcp), &tcp_info_len) < 0) {
+    if (errno == EINVAL) {
+      /* Oops, this option is not provided by the kernel, we'll have to
+       * disable KIST entirely. This can happen if tor was built on a machine
+       * with the support previously or if the kernel was updated and lost the
+       * support. */
+      log_notice(LD_SCHED, "Looks like our kernel doesn't have the support "
+                           "for KIST anymore. We will fallback to the naive "
+                           "approach. Remove KIST from the Schedulers list "
+                           "to disable.");
+      kist_no_kernel_support = 1;
+    }
+    goto fallback;
+  }
+  if (ioctl(sock, SIOCOUTQNSD, &(ent->notsent)) < 0) {
+    if (errno == EINVAL) {
+      log_notice(LD_SCHED, "Looks like our kernel doesn't have the support "
+                           "for KIST anymore. We will fallback to the naive "
+                           "approach. Remove KIST from the Schedulers list "
+                           "to disable.");
+      /* Same reason as the above. */
+      kist_no_kernel_support = 1;
+    }
+    goto fallback;
+  }
+  ent->cwnd = tcp.tcpi_snd_cwnd;
+  ent->unacked = tcp.tcpi_unacked;
+  ent->mss = tcp.tcpi_snd_mss;
+
+  /* In order to reduce outbound kernel queuing delays and thus improve Tor's
+   * ability to prioritize circuits, KIST wants to set a socket write limit
+   * that is near the amount that the socket would be able to immediately send
+   * into the Internet.
+   *
+   * We first calculate how much the socket could send immediately (assuming
+   * completely full packets) according to the congestion window and the number
+   * of unacked packets.
+   *
+   * Then we add a little extra space in a controlled way. We do this so any
+   * when the kernel gets ACKs back for data currently sitting in the "TCP
+   * space", it will already have some more data to send immediately. It will
+   * not have to wait for the scheduler to run again. The amount of extra space
+   * is a factor of the current congestion window. With the suggested
+   * sock_buf_size_factor value of 1.0, we allow at most 2*cwnd bytes to sit in
+   * the kernel: 1 cwnd on the wire waiting for ACKs and 1 cwnd ready and
+   * waiting to be sent when those ACKs finally come.
+   *
+   * In the below diagram, we see some bytes in the TCP-space (denoted by '*')
+   * that have be sent onto the wire and are waiting for ACKs. We have a little
+   * more room in "TCP space" that we can fill with data that will be
+   * immediately sent. We also see the "extra space" KIST calculates. The sum
+   * of the empty "TCP space" and the "extra space" is the kist-imposed write
+   * limit for this socket.
+   *
+   * <----------------kernel-outbound-socket-queue----------------|
+   * <*********---------------------------------------------------|
+   * |----TCP-space-----|----extra-space-----|
+   * |------------------|
+   *                    ^ ((cwnd - unacked) * mss) bytes
+   *                    |--------------------|
+   *                                         ^ ((cwnd * mss) * factor) bytes
+   */
+
+  /* These values from the kernel are uint32_t, they will always fit into a
+   * int64_t tcp_space variable but if the congestion window cwnd is smaller
+   * than the unacked packets, the remaining TCP space is set to 0. */
+  if (ent->cwnd >= ent->unacked) {
+    tcp_space = (ent->cwnd - ent->unacked) * (int64_t)(ent->mss);
+  } else {
+    tcp_space = 0;
+  }
+
+  /* The clamp_double_to_int64 makes sure the first part fits into an int64_t.
+   * In fact, if sock_buf_size_factor is still forced to be >= 0 in config.c,
+   * then it will be positive for sure. Then we subtract a uint32_t. Getting a
+   * negative value is OK, see after how it is being handled. */
+  extra_space =
+    clamp_double_to_int64(
+                 (ent->cwnd * (int64_t)ent->mss) * sock_buf_size_factor) -
+    ent->notsent - (int64_t)channel_outbuf_length((channel_t *) ent->chan);
+  if ((tcp_space + extra_space) < 0) {
+    /* This means that the "notsent" queue is just too big so we shouldn't put
+     * more in the kernel for now. */
+    ent->limit = 0;
+  } else {
+    /* The positive sum of two int64_t will always fit into an uint64_t.
+     * And we know this will always be positive, since we checked above. */
+    ent->limit = (uint64_t)tcp_space + (uint64_t)extra_space;
+  }
+  return;
+
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+  goto fallback;
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
+ fallback:
+  /* If all of a sudden we don't have kist support, we just zero out all the
+   * variables for this socket since we don't know what they should be. We
+   * also allow the socket to write as much as it can from the estimated
+   * number of cells the lower layer can accept, effectively returning it to
+   * Vanilla scheduler behavior. */
+  ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0;
+  /* This function calls the specialized channel object (currently channeltls)
+   * and ask how many cells it can write on the outbuf which we then multiply
+   * by the size of the cells for this channel. The cast is because this
+   * function requires a non-const channel object, meh. */
+  ent->limit = channel_num_cells_writeable((channel_t *) ent->chan) *
+               (get_cell_network_size(ent->chan->wide_circ_ids) +
+                TLS_PER_CELL_OVERHEAD);
+}
+
+/* Given a socket that isn't in the table, add it.
+ * Given a socket that is in the table, re-init values that need init-ing
+ * every scheduling run
+ */
+static void
+init_socket_info(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (!ent) {
+    log_debug(LD_SCHED, "scheduler init socket info for chan=%" PRIu64,
+              chan->global_identifier);
+    ent = tor_malloc_zero(sizeof(*ent));
+    ent->chan = chan;
+    HT_INSERT(socket_table_s, table, ent);
+  }
+  ent->written = 0;
+}
+
+/* Add chan to the outbuf table if it isn't already in it. If it is, then don't
+ * do anything */
+static void
+outbuf_table_add(outbuf_table_t *table, channel_t *chan)
+{
+  outbuf_table_ent_t search, *ent;
+  search.chan = chan;
+  ent = HT_FIND(outbuf_table_s, table, &search);
+  if (!ent) {
+    log_debug(LD_SCHED, "scheduler init outbuf info for chan=%" PRIu64,
+              chan->global_identifier);
+    ent = tor_malloc_zero(sizeof(*ent));
+    ent->chan = chan;
+    HT_INSERT(outbuf_table_s, table, ent);
+  }
+}
+
+static void
+outbuf_table_remove(outbuf_table_t *table, channel_t *chan)
+{
+  outbuf_table_ent_t search, *ent;
+  search.chan = chan;
+  ent = HT_FIND(outbuf_table_s, table, &search);
+  if (ent) {
+    HT_REMOVE(outbuf_table_s, table, ent);
+    free_outbuf_info_by_ent(ent, NULL);
+  }
+}
+
+/* Set the scheduler running interval. */
+static void
+set_scheduler_run_interval(void)
+{
+  int old_sched_run_interval = sched_run_interval;
+  sched_run_interval = kist_scheduler_run_interval();
+  if (old_sched_run_interval != sched_run_interval) {
+    log_info(LD_SCHED, "Scheduler KIST changing its running interval "
+                       "from %" PRId32 " to %" PRId32,
+             old_sched_run_interval, sched_run_interval);
+  }
+}
+
+/* Return true iff the channel hasn't hit its kist-imposed write limit yet */
+static int
+socket_can_write(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (SCHED_BUG(!ent, chan)) {
+    return 1; // Just return true, saying that kist wouldn't limit the socket
+  }
+
+  /* We previously calculated a write limit for this socket. In the below
+   * calculation, first determine how much room is left in bytes. Then divide
+   * that by the amount of space a cell takes. If there's room for at least 1
+   * cell, then KIST will allow the socket to write. */
+  int64_t kist_limit_space =
+    (int64_t) (ent->limit - ent->written) /
+    (CELL_MAX_NETWORK_SIZE + TLS_PER_CELL_OVERHEAD);
+  return kist_limit_space > 0;
+}
+
+/* Update the channel's socket kernel information. */
+static void
+update_socket_info(socket_table_t *table, const channel_t *chan)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (SCHED_BUG(!ent, chan)) {
+    return; // Whelp. Entry didn't exist for some reason so nothing to do.
+  }
+  update_socket_info_impl(ent);
+  log_debug(LD_SCHED, "chan=%" PRIu64 " updated socket info, limit: %" PRIu64
+                      ", cwnd: %" PRIu32 ", unacked: %" PRIu32
+                      ", notsent: %" PRIu32 ", mss: %" PRIu32,
+            ent->chan->global_identifier, ent->limit, ent->cwnd, ent->unacked,
+            ent->notsent, ent->mss);
+}
+
+/* Increment the channel's socket written value by the number of bytes. */
+static void
+update_socket_written(socket_table_t *table, channel_t *chan, size_t bytes)
+{
+  socket_table_ent_t *ent = NULL;
+  ent = socket_table_search(table, chan);
+  if (SCHED_BUG(!ent, chan)) {
+    return; // Whelp. Entry didn't exist so nothing to do.
+  }
+
+  log_debug(LD_SCHED, "chan=%" PRIu64 " wrote %lu bytes, old was %" PRIi64,
+            chan->global_identifier, (unsigned long) bytes, ent->written);
+
+  ent->written += bytes;
+}
+
+/*
+ * A naive KIST impl would write every single cell all the way to the kernel.
+ * That would take a lot of system calls. A less bad KIST impl would write a
+ * channel's outbuf to the kernel only when we are switching to a different
+ * channel. But if we have two channels with equal priority, we end up writing
+ * one cell for each and bouncing back and forth. This KIST impl avoids that
+ * by only writing a channel's outbuf to the kernel if it has 8 cells or more
+ * in it.
+ */
+MOCK_IMPL(int, channel_should_write_to_kernel,
+          (outbuf_table_t *table, channel_t *chan))
+{
+  outbuf_table_add(table, chan);
+  /* CELL_MAX_NETWORK_SIZE * 8 because we only want to write the outbuf to the
+   * kernel if there's 8 or more cells waiting */
+  return channel_outbuf_length(chan) > (CELL_MAX_NETWORK_SIZE * 8);
+}
+
+/* Little helper function to write a channel's outbuf all the way to the
+ * kernel */
+MOCK_IMPL(void, channel_write_to_kernel, (channel_t *chan))
+{
+  log_debug(LD_SCHED, "Writing %lu bytes to kernel for chan %" PRIu64,
+            (unsigned long)channel_outbuf_length(chan),
+            chan->global_identifier);
+  connection_handle_write(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), 0);
+}
+
+/* Return true iff the scheduler has work to perform. */
+static int
+have_work(void)
+{
+  smartlist_t *cp = get_channels_pending();
+  IF_BUG_ONCE(!cp) {
+    return 0; // channels_pending doesn't exist so... no work?
+  }
+  return smartlist_len(cp) > 0;
+}
+
+/* Function of the scheduler interface: free_all() */
+static void
+kist_free_all(void)
+{
+  free_all_socket_info();
+}
+
+/* Function of the scheduler interface: on_channel_free() */
+static void
+kist_on_channel_free_fn(const channel_t *chan)
+{
+  free_socket_info_by_chan(&socket_table, chan);
+}
+
+/* Function of the scheduler interface: on_new_consensus() */
+static void
+kist_scheduler_on_new_consensus(void)
+{
+  set_scheduler_run_interval();
+}
+
+/* Function of the scheduler interface: on_new_options() */
+static void
+kist_scheduler_on_new_options(void)
+{
+  sock_buf_size_factor = get_options()->KISTSockBufSizeFactor;
+
+  /* Calls kist_scheduler_run_interval which calls get_options(). */
+  set_scheduler_run_interval();
+}
+
+/* Function of the scheduler interface: init() */
+static void
+kist_scheduler_init(void)
+{
+  /* When initializing the scheduler, the last run could be 0 because it is
+   * declared static or a value in the past that was set when it was last
+   * used. In both cases, we want to initialize it to now so we don't risk
+   * using the value 0 which doesn't play well with our monotonic time
+   * interface.
+   *
+   * One side effect is that the first scheduler run will be at the next tick
+   * that is in now + 10 msec (KIST_SCHED_RUN_INTERVAL_DEFAULT) by default. */
+  monotime_get(&scheduler_last_run);
+
+  kist_scheduler_on_new_options();
+  IF_BUG_ONCE(sched_run_interval == 0) {
+    log_warn(LD_SCHED, "We are initing the KIST scheduler and noticed the "
+             "KISTSchedRunInterval is telling us to not use KIST. That's "
+             "weird! We'll continue using KIST, but at %" PRId32 "ms.",
+             KIST_SCHED_RUN_INTERVAL_DEFAULT);
+    sched_run_interval = KIST_SCHED_RUN_INTERVAL_DEFAULT;
+  }
+}
+
+/* Function of the scheduler interface: schedule() */
+static void
+kist_scheduler_schedule(void)
+{
+  struct monotime_t now;
+  struct timeval next_run;
+  int64_t diff;
+
+  if (!have_work()) {
+    return;
+  }
+  monotime_get(&now);
+
+  /* If time is really monotonic, we can never have now being smaller than the
+   * last scheduler run. The scheduler_last_run at first is set to 0.
+   * Unfortunately, not all platforms guarantee monotonic time so we log at
+   * info level but don't make it more noisy. */
+  diff = monotime_diff_msec(&scheduler_last_run, &now);
+  if (diff < 0) {
+    log_info(LD_SCHED, "Monotonic time between now and last run of scheduler "
+                       "is negative: %" PRId64 ". Setting diff to 0.", diff);
+    diff = 0;
+  }
+  if (diff < sched_run_interval) {
+    next_run.tv_sec = 0;
+    /* Takes 1000 ms -> us. This will always be valid because diff can NOT be
+     * negative and can NOT be bigger than sched_run_interval so values can
+     * only go from 1000 usec (diff set to interval - 1) to 100000 usec (diff
+     * set to 0) for the maximum allowed run interval (100ms). */
+    next_run.tv_usec = (int) ((sched_run_interval - diff) * 1000);
+    /* Re-adding an event reschedules it. It does not duplicate it. */
+    scheduler_ev_add(&next_run);
+  } else {
+    scheduler_ev_active();
+  }
+}
+
+/* Function of the scheduler interface: run() */
+static void
+kist_scheduler_run(void)
+{
+  /* Define variables */
+  channel_t *chan = NULL; // current working channel
+  /* The last distinct chan served in a sched loop. */
+  channel_t *prev_chan = NULL;
+  int flush_result; // temporarily store results from flush calls
+  /* Channels to be re-adding to pending at the end */
+  smartlist_t *to_readd = NULL;
+  smartlist_t *cp = get_channels_pending();
+
+  outbuf_table_t outbuf_table = HT_INITIALIZER();
+
+  /* For each pending channel, collect new kernel information */
+  SMARTLIST_FOREACH_BEGIN(cp, const channel_t *, pchan) {
+      init_socket_info(&socket_table, pchan);
+      update_socket_info(&socket_table, pchan);
+  } SMARTLIST_FOREACH_END(pchan);
+
+  log_debug(LD_SCHED, "Running the scheduler. %d channels pending",
+            smartlist_len(cp));
+
+  /* The main scheduling loop. Loop until there are no more pending channels */
+  while (smartlist_len(cp) > 0) {
+    /* get best channel */
+    chan = smartlist_pqueue_pop(cp, scheduler_compare_channels,
+                                offsetof(channel_t, sched_heap_idx));
+    if (SCHED_BUG(!chan, NULL)) {
+      /* Some-freaking-how a NULL got into the channels_pending. That should
+       * never happen, but it should be harmless to ignore it and keep looping.
+       */
+      continue;
+    }
+    outbuf_table_add(&outbuf_table, chan);
+
+    /* if we have switched to a new channel, consider writing the previous
+     * channel's outbuf to the kernel. */
+    if (!prev_chan) {
+      prev_chan = chan;
+    }
+    if (prev_chan != chan) {
+      if (channel_should_write_to_kernel(&outbuf_table, prev_chan)) {
+        channel_write_to_kernel(prev_chan);
+        outbuf_table_remove(&outbuf_table, prev_chan);
+      }
+      prev_chan = chan;
+    }
+
+    /* Only flush and write if the per-socket limit hasn't been hit */
+    if (socket_can_write(&socket_table, chan)) {
+      /* flush to channel queue/outbuf */
+      flush_result = (int)channel_flush_some_cells(chan, 1); // 1 for num cells
+      /* XXX: While flushing cells, it is possible that the connection write
+       * fails leading to the channel to be closed which triggers a release
+       * and free its entry in the socket table. And because of a engineering
+       * design issue, the error is not propagated back so we don't get an
+       * error at this point. So before we continue, make sure the channel is
+       * open and if not just ignore it. See #23751. */
+      if (!CHANNEL_IS_OPEN(chan)) {
+        /* Channel isn't open so we put it back in IDLE mode. It is either
+         * renegotiating its TLS session or about to be released. */
+        scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
+        continue;
+      }
+      /* flush_result has the # cells flushed */
+      if (flush_result > 0) {
+        update_socket_written(&socket_table, chan, flush_result *
+                              (CELL_MAX_NETWORK_SIZE + TLS_PER_CELL_OVERHEAD));
+      } else {
+        /* XXX: This can happen because tor sometimes does flush in an
+         * opportunistic way cells from the circuit to the outbuf so the
+         * channel can end up here without having anything to flush nor needed
+         * to write to the kernel. Hopefully we'll fix that soon but for now
+         * we have to handle this case which happens kind of often. */
+        log_debug(LD_SCHED,
+                 "We didn't flush anything on a chan that we think "
+                 "can write and wants to write. The channel's state is '%s' "
+                 "and in scheduler state '%s'. We're going to mark it as "
+                 "waiting_for_cells (as that's most likely the issue) and "
+                 "stop scheduling it this round.",
+                 channel_state_to_string(chan->state),
+                 get_scheduler_state_string(chan->scheduler_state));
+        scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+        continue;
+      }
+    }
+
+    /* Decide what to do with the channel now */
+
+    if (!channel_more_to_flush(chan) &&
+        !socket_can_write(&socket_table, chan)) {
+
+      /* Case 1: no more cells to send, and cannot write */
+
+      /*
+       * You might think we should put the channel in SCHED_CHAN_IDLE. And
+       * you're probably correct. While implementing KIST, we found that the
+       * scheduling system would sometimes lose track of channels when we did
+       * that. We suspect it has to do with the difference between "can't
+       * write because socket/outbuf is full" and KIST's "can't write because
+       * we've arbitrarily decided that that's enough for now." Sometimes
+       * channels run out of cells at the same time they hit their
+       * kist-imposed write limit and maybe the rest of Tor doesn't put the
+       * channel back in pending when it is supposed to.
+       *
+       * This should be investigated again. It is as simple as changing
+       * SCHED_CHAN_WAITING_FOR_CELLS to SCHED_CHAN_IDLE and seeing if Tor
+       * starts having serious throughput issues. Best done in shadow/chutney.
+       */
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+    } else if (!channel_more_to_flush(chan)) {
+
+      /* Case 2: no more cells to send, but still open for writes */
+
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+    } else if (!socket_can_write(&socket_table, chan)) {
+
+      /* Case 3: cells to send, but cannot write */
+
+      /*
+       * We want to write, but can't. If we left the channel in
+       * channels_pending, we would never exit the scheduling loop. We need to
+       * add it to a temporary list of channels to be added to channels_pending
+       * after the scheduling loop is over. They can hopefully be taken care of
+       * in the next scheduling round.
+       */
+      if (!to_readd) {
+        to_readd = smartlist_new();
+      }
+      smartlist_add(to_readd, chan);
+    } else {
+
+      /* Case 4: cells to send, and still open for writes */
+
+      scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
+      if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+        smartlist_pqueue_add(cp, scheduler_compare_channels,
+                             offsetof(channel_t, sched_heap_idx), chan);
+      }
+    }
+  } /* End of main scheduling loop */
+
+  /* Write the outbuf of any channels that still have data */
+  HT_FOREACH_FN(outbuf_table_s, &outbuf_table, each_channel_write_to_kernel,
+                NULL);
+  /* We are done with it. */
+  HT_FOREACH_FN(outbuf_table_s, &outbuf_table, free_outbuf_info_by_ent, NULL);
+  HT_CLEAR(outbuf_table_s, &outbuf_table);
+
+  log_debug(LD_SCHED, "len pending=%d, len to_readd=%d",
+            smartlist_len(cp),
+            (to_readd ? smartlist_len(to_readd) : -1));
+
+  /* Re-add any channels we need to */
+  if (to_readd) {
+    SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
+      scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
+      if (!smartlist_contains(cp, readd_chan)) {
+        if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+          /* XXXX Note that the check above is in theory redundant with
+           * the smartlist_contains check.  But let's make sure we're
+           * not messing anything up, and leave them both for now. */
+          smartlist_pqueue_add(cp, scheduler_compare_channels,
+                             offsetof(channel_t, sched_heap_idx), readd_chan);
+        }
+      }
+    } SMARTLIST_FOREACH_END(readd_chan);
+    smartlist_free(to_readd);
+  }
+
+  monotime_get(&scheduler_last_run);
+}
+
+/*****************************************************************************
+ * Externally called function implementations not called through scheduler_t
+ *****************************************************************************/
+
+/* Stores the kist scheduler function pointers. */
+static scheduler_t kist_scheduler = {
+  .type = SCHEDULER_KIST,
+  .free_all = kist_free_all,
+  .on_channel_free = kist_on_channel_free_fn,
+  .init = kist_scheduler_init,
+  .on_new_consensus = kist_scheduler_on_new_consensus,
+  .schedule = kist_scheduler_schedule,
+  .run = kist_scheduler_run,
+  .on_new_options = kist_scheduler_on_new_options,
+};
+
+/* Return the KIST scheduler object. If it didn't exists, return a newly
+ * allocated one but init() is not called. */
+scheduler_t *
+get_kist_scheduler(void)
+{
+  return &kist_scheduler;
+}
+
+/* Check the torrc (and maybe consensus) for the configured KIST scheduler run
+ * interval.
+ * - If torrc > 0, then return the positive torrc value (should use KIST, and
+ *   should use the set value)
+ * - If torrc == 0, then look in the consensus for what the value should be.
+ *   - If == 0, then return 0 (don't use KIST)
+ *   - If > 0, then return the positive consensus value
+ *   - If consensus doesn't say anything, return 10 milliseconds, default.
+ */
+int
+kist_scheduler_run_interval(void)
+{
+  int run_interval = get_options()->KISTSchedRunInterval;
+
+  if (run_interval != 0) {
+    log_debug(LD_SCHED, "Found KISTSchedRunInterval=%" PRId32 " in torrc. "
+                        "Using that.", run_interval);
+    return run_interval;
+  }
+
+  log_debug(LD_SCHED, "KISTSchedRunInterval=0, turning to the consensus.");
+
+  /* Will either be the consensus value or the default. Note that 0 can be
+   * returned which means the consensus wants us to NOT use KIST. */
+  return networkstatus_get_param(NULL, "KISTSchedRunInterval",
+                                 KIST_SCHED_RUN_INTERVAL_DEFAULT,
+                                 KIST_SCHED_RUN_INTERVAL_MIN,
+                                 KIST_SCHED_RUN_INTERVAL_MAX);
+}
+
+/* Set KISTLite mode that is KIST without kernel support. */
+void
+scheduler_kist_set_lite_mode(void)
+{
+  kist_lite_mode = 1;
+  kist_scheduler.type = SCHEDULER_KIST_LITE;
+  log_info(LD_SCHED,
+           "Setting KIST scheduler without kernel support (KISTLite mode)");
+}
+
+/* Set KIST mode that is KIST with kernel support. */
+void
+scheduler_kist_set_full_mode(void)
+{
+  kist_lite_mode = 0;
+  kist_scheduler.type = SCHEDULER_KIST;
+  log_info(LD_SCHED,
+           "Setting KIST scheduler with kernel support (KIST mode)");
+}
+
+#ifdef HAVE_KIST_SUPPORT
+
+/* Return true iff the scheduler subsystem should use KIST. */
+int
+scheduler_can_use_kist(void)
+{
+  if (kist_no_kernel_support) {
+    /* We have no kernel support so we can't use KIST. */
+    return 0;
+  }
+
+  /* We do have the support, time to check if we can get the interval that the
+   * consensus can be disabling. */
+  int run_interval = kist_scheduler_run_interval();
+  log_debug(LD_SCHED, "Determined KIST sched_run_interval should be "
+                      "%" PRId32 ". Can%s use KIST.",
+           run_interval, (run_interval > 0 ? "" : " not"));
+  return run_interval > 0;
+}
+
+#else /* !(defined(HAVE_KIST_SUPPORT)) */
+
+int
+scheduler_can_use_kist(void)
+{
+  return 0;
+}
+
+#endif /* defined(HAVE_KIST_SUPPORT) */
+
diff --git a/src/or/scheduler_vanilla.c b/src/or/scheduler_vanilla.c
new file mode 100644
index 0000000000..b674d8256c
--- /dev/null
+++ b/src/or/scheduler_vanilla.c
@@ -0,0 +1,175 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "config.h"
+#define TOR_CHANNEL_INTERNAL_
+#include "channel.h"
+#define SCHEDULER_PRIVATE_
+#include "scheduler.h"
+
+/*****************************************************************************
+ * Other internal data
+ *****************************************************************************/
+
+/* Maximum cells to flush in a single call to channel_flush_some_cells(); */
+#define MAX_FLUSH_CELLS 1000
+
+/*****************************************************************************
+ * Externally called function implementations
+ *****************************************************************************/
+
+/* Return true iff the scheduler has work to perform. */
+static int
+have_work(void)
+{
+  smartlist_t *cp = get_channels_pending();
+  IF_BUG_ONCE(!cp) {
+    return 0; // channels_pending doesn't exist so... no work?
+  }
+  return smartlist_len(cp) > 0;
+}
+
+/** Re-trigger the scheduler in a way safe to use from the callback */
+
+static void
+vanilla_scheduler_schedule(void)
+{
+  if (!have_work()) {
+    return;
+  }
+
+  /* Activate our event so it can process channels. */
+  scheduler_ev_active();
+}
+
+static void
+vanilla_scheduler_run(void)
+{
+  int n_cells, n_chans_before, n_chans_after;
+  ssize_t flushed, flushed_this_time;
+  smartlist_t *cp = get_channels_pending();
+  smartlist_t *to_readd = NULL;
+  channel_t *chan = NULL;
+
+  log_debug(LD_SCHED, "We have a chance to run the scheduler");
+
+  n_chans_before = smartlist_len(cp);
+
+  while (smartlist_len(cp) > 0) {
+    /* Pop off a channel */
+    chan = smartlist_pqueue_pop(cp,
+                                scheduler_compare_channels,
+                                offsetof(channel_t, sched_heap_idx));
+    IF_BUG_ONCE(!chan) {
+      /* Some-freaking-how a NULL got into the channels_pending. That should
+       * never happen, but it should be harmless to ignore it and keep looping.
+       */
+      continue;
+    }
+
+    /* Figure out how many cells we can write */
+    n_cells = channel_num_cells_writeable(chan);
+    if (n_cells > 0) {
+      log_debug(LD_SCHED,
+                "Scheduler saw pending channel " U64_FORMAT " at %p with "
+                "%d cells writeable",
+                U64_PRINTF_ARG(chan->global_identifier), chan, n_cells);
+
+      flushed = 0;
+      while (flushed < n_cells) {
+        flushed_this_time =
+          channel_flush_some_cells(chan,
+                        MIN(MAX_FLUSH_CELLS, (size_t) n_cells - flushed));
+        if (flushed_this_time <= 0) break;
+        flushed += flushed_this_time;
+      }
+
+      if (flushed < n_cells) {
+        /* We ran out of cells to flush */
+        scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+      } else {
+        /* The channel may still have some cells */
+        if (channel_more_to_flush(chan)) {
+        /* The channel goes to either pending or waiting_to_write */
+          if (channel_num_cells_writeable(chan) > 0) {
+            /* Add it back to pending later */
+            if (!to_readd) to_readd = smartlist_new();
+            smartlist_add(to_readd, chan);
+            log_debug(LD_SCHED,
+                      "Channel " U64_FORMAT " at %p "
+                      "is still pending",
+                      U64_PRINTF_ARG(chan->global_identifier),
+                      chan);
+          } else {
+            /* It's waiting to be able to write more */
+            scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
+          }
+        } else {
+          /* No cells left; it can go to idle or waiting_for_cells */
+          if (channel_num_cells_writeable(chan) > 0) {
+            /*
+             * It can still accept writes, so it goes to
+             * waiting_for_cells
+             */
+            scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
+          } else {
+            /*
+             * We exactly filled up the output queue with all available
+             * cells; go to idle.
+             */
+            scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
+          }
+        }
+      }
+
+      log_debug(LD_SCHED,
+                "Scheduler flushed %d cells onto pending channel "
+                U64_FORMAT " at %p",
+                (int)flushed, U64_PRINTF_ARG(chan->global_identifier),
+                chan);
+    } else {
+      log_info(LD_SCHED,
+               "Scheduler saw pending channel " U64_FORMAT " at %p with "
+               "no cells writeable",
+               U64_PRINTF_ARG(chan->global_identifier), chan);
+      /* Put it back to WAITING_TO_WRITE */
+      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
+    }
+  }
+
+  /* Readd any channels we need to */
+  if (to_readd) {
+    SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
+      scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
+      smartlist_pqueue_add(cp,
+                           scheduler_compare_channels,
+                           offsetof(channel_t, sched_heap_idx),
+                           readd_chan);
+    } SMARTLIST_FOREACH_END(readd_chan);
+    smartlist_free(to_readd);
+  }
+
+  n_chans_after = smartlist_len(cp);
+  log_debug(LD_SCHED, "Scheduler handled %d of %d pending channels",
+            n_chans_before - n_chans_after, n_chans_before);
+}
+
+/* Stores the vanilla scheduler function pointers. */
+static scheduler_t vanilla_scheduler = {
+  .type = SCHEDULER_VANILLA,
+  .free_all = NULL,
+  .on_channel_free = NULL,
+  .init = NULL,
+  .on_new_consensus = NULL,
+  .schedule = vanilla_scheduler_schedule,
+  .run = vanilla_scheduler_run,
+  .on_new_options = NULL,
+};
+
+scheduler_t *
+get_vanilla_scheduler(void)
+{
+  return &vanilla_scheduler;
+}
+
diff --git a/src/or/shared_random_client.c b/src/or/shared_random_client.c
new file mode 100644
index 0000000000..3aef83cef4
--- /dev/null
+++ b/src/or/shared_random_client.c
@@ -0,0 +1,259 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_client.c
+ * \brief This file contains functions that are from the shared random
+ *        subsystem but used by many part of tor. The full feature is built
+ *        as part of the dirauth module.
+ **/
+
+#define SHARED_RANDOM_CLIENT_PRIVATE
+#include "shared_random_client.h"
+
+#include "config.h"
+#include "voting_schedule.h"
+#include "networkstatus.h"
+#include "util.h"
+#include "util_format.h"
+
+/* Convert a given srv object to a string for the control port. This doesn't
+ * fail and the srv object MUST be valid. */
+static char *
+srv_to_control_string(const sr_srv_t *srv)
+{
+  char *srv_str;
+  char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+  tor_assert(srv);
+
+  sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv);
+  tor_asprintf(&srv_str, "%s", srv_hash_encoded);
+  return srv_str;
+}
+
+/* Return the voting interval of the tor vote subsystem. */
+int
+get_voting_interval(void)
+{
+  int interval;
+  networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL));
+
+  if (consensus) {
+    interval = (int)(consensus->fresh_until - consensus->valid_after);
+  } else {
+    /* Same for both a testing and real network. We voluntarily ignore the
+     * InitialVotingInterval since it complexifies things and it doesn't
+     * affect the SR protocol. */
+    interval = get_options()->V3AuthVotingInterval;
+  }
+  tor_assert(interval > 0);
+  return interval;
+}
+
+/* Given the time <b>now</b>, return the start time of the current round of
+ * the SR protocol. For example, if it's 23:47:08, the current round thus
+ * started at 23:47:00 for a voting interval of 10 seconds. */
+time_t
+get_start_time_of_current_round(void)
+{
+  const or_options_t *options = get_options();
+  int voting_interval = get_voting_interval();
+  /* First, get the start time of the next round */
+  time_t next_start = voting_schedule_get_next_valid_after_time();
+  /* Now roll back next_start by a voting interval to find the start time of
+     the current round. */
+  time_t curr_start = voting_schedule_get_start_of_next_interval(
+                                     next_start - voting_interval - 1,
+                                     voting_interval,
+                                     options->TestingV3AuthVotingStartOffset);
+  return curr_start;
+}
+
+/*
+ * Public API
+ */
+
+/* Encode the given shared random value and put it in dst. Destination
+ * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
+void
+sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
+{
+  int ret;
+  /* Extra byte for the NULL terminated char. */
+  char buf[SR_SRV_VALUE_BASE64_LEN + 1];
+
+  tor_assert(dst);
+  tor_assert(srv);
+  tor_assert(dst_len >= sizeof(buf));
+
+  ret = base64_encode(buf, sizeof(buf), (const char *) srv->value,
+                      sizeof(srv->value), 0);
+  /* Always expect the full length without the NULL byte. */
+  tor_assert(ret == (sizeof(buf) - 1));
+  tor_assert(ret <= (int) dst_len);
+  strlcpy(dst, buf, dst_len);
+}
+
+/* Return the current SRV string representation for the control port. Return a
+ * newly allocated string on success containing the value else "" if not found
+ * or if we don't have a valid consensus yet. */
+char *
+sr_get_current_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.current_srv) {
+    srv_str = srv_to_control_string(c->sr_info.current_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return the previous SRV string representation for the control port. Return
+ * a newly allocated string on success containing the value else "" if not
+ * found or if we don't have a valid consensus yet. */
+char *
+sr_get_previous_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.previous_srv) {
+    srv_str = srv_to_control_string(c->sr_info.previous_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return current shared random value from the latest consensus. Caller can
+ * NOT keep a reference to the returned pointer. Return NULL if none. */
+const sr_srv_t *
+sr_get_current(const networkstatus_t *ns)
+{
+  const networkstatus_t *consensus;
+
+  /* Use provided ns else get a live one */
+  if (ns) {
+    consensus = ns;
+  } else {
+    consensus = networkstatus_get_live_consensus(approx_time());
+  }
+  /* Ideally we would never be asked for an SRV without a live consensus. Make
+   * sure this assumption is correct. */
+  tor_assert_nonfatal(consensus);
+
+  if (consensus) {
+    return consensus->sr_info.current_srv;
+  }
+  return NULL;
+}
+
+/* Return previous shared random value from the latest consensus. Caller can
+ * NOT keep a reference to the returned pointer. Return NULL if none. */
+const sr_srv_t *
+sr_get_previous(const networkstatus_t *ns)
+{
+  const networkstatus_t *consensus;
+
+  /* Use provided ns else get a live one */
+  if (ns) {
+    consensus = ns;
+  } else {
+    consensus = networkstatus_get_live_consensus(approx_time());
+  }
+  /* Ideally we would never be asked for an SRV without a live consensus. Make
+   * sure this assumption is correct. */
+  tor_assert_nonfatal(consensus);
+
+  if (consensus) {
+    return consensus->sr_info.previous_srv;
+  }
+  return NULL;
+}
+
+/* Parse a list of arguments from a SRV value either from a vote, consensus
+ * or from our disk state and return a newly allocated srv object. NULL is
+ * returned on error.
+ *
+ * The arguments' order:
+ *    num_reveals, value
+ */
+sr_srv_t *
+sr_parse_srv(const smartlist_t *args)
+{
+  char *value;
+  int ok, ret;
+  uint64_t num_reveals;
+  sr_srv_t *srv = NULL;
+
+  tor_assert(args);
+
+  if (smartlist_len(args) < 2) {
+    goto end;
+  }
+
+  /* First argument is the number of reveal values */
+  num_reveals = tor_parse_uint64(smartlist_get(args, 0),
+                                 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    goto end;
+  }
+  /* Second and last argument is the shared random value it self. */
+  value = smartlist_get(args, 1);
+  if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) {
+    goto end;
+  }
+
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = num_reveals;
+  /* We subtract one byte from the srclen because the function ignores the
+   * '=' character in the given buffer. This is broken but it's a documented
+   * behavior of the implementation. */
+  ret = base64_decode((char *) srv->value, sizeof(srv->value), value,
+                      SR_SRV_VALUE_BASE64_LEN - 1);
+  if (ret != sizeof(srv->value)) {
+    tor_free(srv);
+    srv = NULL;
+    goto end;
+  }
+ end:
+  return srv;
+}
+
+/** Return the start time of the current SR protocol run. For example, if the
+ *  time is 23/06/2017 23:47:08 and a full SR protocol run is 24 hours, this
+ *  function should return 23/06/2017 00:00:00. */
+time_t
+sr_state_get_start_time_of_current_protocol_run(time_t now)
+{
+  int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  int voting_interval = get_voting_interval();
+  /* Find the time the current round started. */
+  time_t beginning_of_current_round = get_start_time_of_current_round();
+
+  /* Get current SR protocol round */
+  int current_round = (now / voting_interval) % total_rounds;
+
+  /* Get start time by subtracting the time elapsed from the beginning of the
+     protocol run */
+  time_t time_elapsed_since_start_of_run = current_round * voting_interval;
+  return beginning_of_current_round - time_elapsed_since_start_of_run;
+}
+
+/** Return the time (in seconds) it takes to complete a full SR protocol phase
+ *  (e.g. the commit phase). */
+unsigned int
+sr_state_get_phase_duration(void)
+{
+  return SHARED_RANDOM_N_ROUNDS * get_voting_interval();
+}
+
+/** Return the time (in seconds) it takes to complete a full SR protocol run */
+unsigned int
+sr_state_get_protocol_run_duration(void)
+{
+  int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
+  return total_protocol_rounds * get_voting_interval();
+}
+
diff --git a/src/or/shared_random_client.h b/src/or/shared_random_client.h
new file mode 100644
index 0000000000..89c608d45f
--- /dev/null
+++ b/src/or/shared_random_client.h
@@ -0,0 +1,47 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shared_random_client.h
+ * \brief Header file for shared_random_client.c.
+ **/
+
+#ifndef TOR_SHARED_RANDOM_CLIENT_H
+#define TOR_SHARED_RANDOM_CLIENT_H
+
+/* Dirauth module. */
+#include "dirauth/shared_random.h"
+
+/* Helper functions. */
+void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv);
+int get_voting_interval(void);
+
+/* Control port functions. */
+char *sr_get_current_for_control(void);
+char *sr_get_previous_for_control(void);
+
+/* SRV functions. */
+const sr_srv_t *sr_get_current(const networkstatus_t *ns);
+const sr_srv_t *sr_get_previous(const networkstatus_t *ns);
+sr_srv_t *sr_parse_srv(const smartlist_t *args);
+
+/*
+ * Shared Random State API
+ */
+
+/* Each protocol phase has 12 rounds  */
+#define SHARED_RANDOM_N_ROUNDS 12
+/* Number of phase we have in a protocol. */
+#define SHARED_RANDOM_N_PHASES 2
+
+time_t sr_state_get_start_time_of_current_protocol_run(time_t now);
+unsigned int sr_state_get_phase_duration(void);
+unsigned int sr_state_get_protocol_run_duration(void);
+time_t get_start_time_of_current_round(void);
+
+#ifdef TOR_UNIT_TESTS
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* TOR_SHARED_RANDOM_CLIENT_H */
+
diff --git a/src/or/statefile.c b/src/or/statefile.c
index 8fa4324b25..c81ea44e06 100644
--- a/src/or/statefile.c
+++ b/src/or/statefile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,8 +34,10 @@
 #include "config.h"
 #include "confparse.h"
 #include "connection.h"
+#include "control.h"
 #include "entrynodes.h"
 #include "hibernate.h"
+#include "main.h"
 #include "rephist.h"
 #include "router.h"
 #include "sandbox.h"
@@ -53,10 +55,14 @@ static config_abbrev_t state_abbrevs_[] = {
   { NULL, NULL, 0, 0},
 };
 
+/** dummy instance of or_state_t, used for type-checking its
+ * members with CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(or_state_t);
+
 /*XXXX these next two are duplicates or near-duplicates from config.c */
 #define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(or_state_t, member),  \
-      initvalue }
+  { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member),       \
+      initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) }
 /** As VAR, but the option name and member name are the same. */
 #define V(member,conftype,initvalue)                                    \
   VAR(#member, conftype, member, initvalue)
@@ -85,6 +91,8 @@ static config_var_t state_vars_[] = {
   VAR("TransportProxy",               LINELIST_S, TransportProxies, NULL),
   V(TransportProxies,                 LINELIST_V, NULL),
 
+  V(HidServRevCounter,            LINELIST, NULL),
+
   V(BWHistoryReadEnds,                ISOTIME,  NULL),
   V(BWHistoryReadInterval,            UINT,     "900"),
   V(BWHistoryReadValues,              CSV,      ""),
@@ -102,6 +110,8 @@ static config_var_t state_vars_[] = {
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
+  V(Guard,                            LINELIST, NULL),
+
   V(TorVersion,                       STRING,   NULL),
 
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
@@ -111,7 +121,8 @@ static config_var_t state_vars_[] = {
   V(CircuitBuildAbandonedCount,       UINT,     "0"),
   VAR("CircuitBuildTimeBin",          LINELIST_S, BuildtimeHistogram, NULL),
   VAR("BuildtimeHistogram",           LINELIST_V, BuildtimeHistogram, NULL),
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
+
+  END_OF_CONFIG_VARS
 };
 
 #undef VAR
@@ -129,14 +140,15 @@ static int or_state_validate_cb(void *old_options, void *options,
 /** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
 static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST, STRUCT_OFFSET(or_state_t, ExtraLines), NULL
+  "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL
+  CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines)
 };
 
 /** Configuration format for or_state_t. */
 static const config_format_t state_format = {
   sizeof(or_state_t),
   OR_STATE_MAGIC,
-  STRUCT_OFFSET(or_state_t, magic_),
+  offsetof(or_state_t, magic_),
   state_abbrevs_,
   NULL,
   state_vars_,
@@ -400,10 +412,15 @@ or_state_load(void)
     log_info(LD_GENERAL, "Loaded state from \"%s\"", fname);
     /* Warn the user if their clock has been set backwards,
      * they could be tricked into using old consensuses */
-    time_t apparent_skew = new_state->LastWritten - time(NULL);
-    if (apparent_skew > 0)
+    time_t apparent_skew = time(NULL) - new_state->LastWritten;
+    if (apparent_skew < 0) {
+      /* Initialize bootstrap event reporting because we might call
+       * clock_skew_warning() before the bootstrap state is
+       * initialized, causing an assertion failure. */
+      control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
       clock_skew_warning(NULL, (long)apparent_skew, 1, LD_GENERAL,
                          "local state file", fname);
+    }
   } else {
     log_info(LD_GENERAL, "Initialized state");
   }
@@ -655,8 +672,6 @@ save_transport_to_state(const char *transport,
     *next = line = tor_malloc_zero(sizeof(config_line_t));
     line->key = tor_strdup("TransportProxy");
     tor_asprintf(&line->value, "%s %s", transport, fmt_addrport(addr, port));
-
-    next = &(line->next);
   }
 
   if (!get_options()->AvoidDiskWrites)
@@ -666,8 +681,20 @@ save_transport_to_state(const char *transport,
   tor_free(transport_addrport);
 }
 
+/** Change the next_write time of <b>state</b> to <b>when</b>, unless the
+ * state is already scheduled to be written to disk earlier than <b>when</b>.
+ */
+void
+or_state_mark_dirty(or_state_t *state, time_t when)
+{
+  if (state->next_write > when) {
+    state->next_write = when;
+    reschedule_or_state_save();
+  }
+}
+
 STATIC void
-or_state_free(or_state_t *state)
+or_state_free_(or_state_t *state)
 {
   if (!state)
     return;
diff --git a/src/or/statefile.h b/src/or/statefile.h
index b13743481d..5aa2ca9320 100644
--- a/src/or/statefile.h
+++ b/src/or/statefile.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_STATEFILE_H
@@ -17,12 +17,14 @@ char *get_stored_bindaddr_for_server_transport(const char *transport);
 int or_state_load(void);
 int or_state_loaded(void);
 void or_state_free_all(void);
+void or_state_mark_dirty(or_state_t *state, time_t when);
 
 #ifdef STATEFILE_PRIVATE
 STATIC config_line_t *get_transport_in_state_by_name(const char *transport);
-STATIC void or_state_free(or_state_t *state);
+STATIC void or_state_free_(or_state_t *state);
+#define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st))
 STATIC or_state_t *or_state_new(void);
 #endif
 
-#endif
+#endif /* !defined(TOR_STATEFILE_H) */
 
diff --git a/src/or/status.c b/src/or/status.c
index fa2238b9f9..4b8033d114 100644
--- a/src/or/status.c
+++ b/src/or/status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,8 +25,9 @@
 #include "main.h"
 #include "rephist.h"
 #include "hibernate.h"
-#include "rephist.h"
 #include "statefile.h"
+#include "hs_stats.h"
+#include "hs_service.h"
 #include "dos.h"
 
 static void log_accounting(const time_t now, const or_options_t *options);
@@ -40,7 +41,7 @@ count_circuits(void)
 }
 
 /** Take seconds <b>secs</b> and return a newly allocated human-readable
- * uptime string */
+ * uptime string. */
 STATIC char *
 secs_to_uptime(long secs)
 {
@@ -86,6 +87,26 @@ bytes_to_usage(uint64_t bytes)
   return bw_string;
 }
 
+/** Log some usage info about our onion service(s). */
+static void
+log_onion_service_stats(void)
+{
+  unsigned int num_services = hs_service_get_num_services();
+
+  /* If there are no active onion services, no need to print logs */
+  if (num_services == 0) {
+    return;
+  }
+
+  log_notice(LD_HEARTBEAT,
+             "Our onion service%s received %u v2 and %u v3 INTRODUCE2 cells "
+             "and attempted to launch %d rendezvous circuits.",
+             num_services == 1 ? "" : "s",
+             hs_stats_get_n_introduce2_v2_cells(),
+             hs_stats_get_n_introduce2_v3_cells(),
+             hs_stats_get_n_rendezvous_launches());
+}
+
 /** Log a "heartbeat" message describing Tor's status and history so that the
  * user can know that there is indeed a running Tor.  Return 0 on success and
  * -1 on failure. */
@@ -159,6 +180,23 @@ log_heartbeat(time_t now)
     tor_free(msg);
   }
 
+  if (options->MainloopStats) {
+    const uint64_t main_loop_success_count = get_main_loop_success_count();
+    const uint64_t main_loop_error_count = get_main_loop_error_count();
+    const uint64_t main_loop_idle_count = get_main_loop_idle_count();
+
+    log_fn(LOG_NOTICE, LD_HEARTBEAT, "Main event loop statistics: "
+         U64_FORMAT " successful returns, "
+         U64_FORMAT " erroneous returns, and "
+         U64_FORMAT " idle returns.",
+         U64_PRINTF_ARG(main_loop_success_count),
+         U64_PRINTF_ARG(main_loop_error_count),
+         U64_PRINTF_ARG(main_loop_idle_count));
+  }
+
+  /** Now, if we are an HS service, log some stats about our usage */
+  log_onion_service_stats();
+
   tor_free(uptime);
   tor_free(bw_sent);
   tor_free(bw_rcvd);
diff --git a/src/or/status.h b/src/or/status.h
index b97e835037..49da6abc0f 100644
--- a/src/or/status.h
+++ b/src/or/status.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_STATUS_H
@@ -14,5 +14,5 @@ STATIC char *secs_to_uptime(long secs);
 STATIC char *bytes_to_usage(uint64_t bytes);
 #endif
 
-#endif
+#endif /* !defined(TOR_STATUS_H) */
 
diff --git a/src/or/tor_api.c b/src/or/tor_api.c
new file mode 100644
index 0000000000..4260cc88f4
--- /dev/null
+++ b/src/or/tor_api.c
@@ -0,0 +1,88 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_api.c
+ **/
+
+#include "tor_api.h"
+#include "tor_api_internal.h"
+
+// Include this after the above headers, to insure that they don't
+// depend on anything else.
+#include "orconfig.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// We don't want to use tor_malloc and tor_free here, since this needs
+// to run before anything is initialized at all, and ought to run when
+// we're not linked to anything at all.
+
+#define raw_malloc malloc
+#define raw_free free
+
+tor_main_configuration_t *
+tor_main_configuration_new(void)
+{
+  static const char *fake_argv[] = { "tor" };
+  tor_main_configuration_t *cfg = raw_malloc(sizeof(*cfg));
+  if (cfg == NULL)
+    return NULL;
+
+  memset(cfg, 0, sizeof(*cfg));
+
+  cfg->argc = 1;
+  cfg->argv = (char **) fake_argv;
+
+  return cfg;
+}
+
+int
+tor_main_configuration_set_command_line(tor_main_configuration_t *cfg,
+                                        int argc, char *argv[])
+{
+  if (cfg == NULL)
+    return -1;
+  cfg->argc = argc;
+  cfg->argv = argv;
+  return 0;
+}
+
+void
+tor_main_configuration_free(tor_main_configuration_t *cfg)
+{
+  if (cfg == NULL)
+    return;
+  raw_free(cfg);
+}
+
+/* Main entry point for the Tor process.  Called from main().
+ *
+ * This function is distinct from main() only so we can link main.c into
+ * the unittest binary without conflicting with the unittests' main.
+ *
+ * Some embedders have historically called this function; but that usage is
+ * deprecated: they should use tor_run_main() instead.
+ */
+int
+tor_main(int argc, char *argv[])
+{
+  tor_main_configuration_t *cfg = tor_main_configuration_new();
+  if (!cfg) {
+    puts("INTERNAL ERROR: Allocation failure. Cannot proceed");
+    return 1;
+  }
+  if (tor_main_configuration_set_command_line(cfg, argc, argv) < 0) {
+    puts("INTERNAL ERROR: Can't set command line. Cannot proceed.");
+    return 1;
+  }
+  int rv = tor_run_main(cfg);
+  tor_main_configuration_free(cfg);
+  return rv;
+}
+
diff --git a/src/or/tor_api.h b/src/or/tor_api.h
new file mode 100644
index 0000000000..6d4a9518e0
--- /dev/null
+++ b/src/or/tor_api.h
@@ -0,0 +1,101 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_api.h
+ * \brief Public C API for the Tor network service.
+ *
+ * This interface is intended for use by programs that need to link Tor as
+ * a library, and launch it in a separate thread.  If you have the ability
+ * to run Tor as a separate executable, you should probably do that instead
+ * of embedding it as a library.
+ *
+ * To use this API, first construct a tor_main_configuration_t object using
+ * tor_main_configuration_new().  Then, you use one or more other function
+ * calls (such as tor_main_configuration_set_command_line() to configure how
+ * Tor should be run.  Finally, you pass the configuration object to
+ * tor_run_main().
+ *
+ * At this point, tor_run_main() will block its thread to run a Tor daemon;
+ * when the Tor daemon exits, it will return.  See notes on bugs and
+ * limitations below.
+ *
+ * There is no other public C API to Tor: calling any C Tor function not
+ * documented in this file is not guaranteed to be stable.
+ **/
+
+#ifndef TOR_API_H
+#define TOR_API_H
+
+typedef struct tor_main_configuration_t tor_main_configuration_t;
+
+/**
+ * Create and return a new tor_main_configuration().
+ */
+tor_main_configuration_t *tor_main_configuration_new(void);
+
+/**
+ * Set the command-line arguments in <b>cfg</b>.
+ *
+ * The <b>argc</b> and <b>argv</b> values here are as for main().  The
+ * contents of the argv pointer must remain unchanged until tor_run_main() has
+ * finished and you call tor_main_configuration_free().
+ *
+ * Return 0 on success, -1 on failure.
+ */
+int tor_main_configuration_set_command_line(tor_main_configuration_t *cfg,
+                                            int argc, char *argv[]);
+
+/**
+ * Release all storage held in <b>cfg</b>.
+ *
+ * Once you have passed a tor_main_configuration_t to tor_run_main(), you
+ * must not free it until tor_run_main() has finished.
+ */
+void tor_main_configuration_free(tor_main_configuration_t *cfg);
+
+/**
+ * Run the tor process, as if from the command line.
+ *
+ * The command line arguments from tor_main_configuration_set_command_line()
+ * are taken as if they had been passed to main().
+ *
+ * This function will not return until Tor is done running.  It returns zero
+ * on success, and nonzero on failure.
+ *
+ * If you want to control when Tor exits, make sure to configure a control
+ * socket. The OwningControllerFD option may be helpful there.
+ *
+ * BUG 23847: Sometimes, if you call tor_main a second time (after it has
+ * returned), Tor may crash or behave strangely.  We have fixed all issues of
+ * this type that we could find, but more may remain.
+ *
+ * LIMITATION: You cannot run more than one instance of Tor in the same
+ * process at the same time. Concurrent calls will cause undefined behavior.
+ * We do not currently have plans to change this.
+ *
+ * LIMITATION: While we will try to fix any problems found here, you
+ * should be aware that Tor was originally written to run as its own
+ * process, and that the functionality of this file was added later.  If
+ * you find any bugs or strange behavior, please report them, and we'll
+ * try to straighten them out.
+ */
+int tor_run_main(const tor_main_configuration_t *);
+
+/**
+ * Run the tor process, as if from the command line.
+ *
+ * @deprecated Using this function from outside Tor is deprecated; you should
+ * use tor_run_main() instead.
+ *
+ * BUGS: This function has all the same bugs as tor_run_main().
+ *
+ * LIMITATIONS: This function has all the limitations of tor_run_main().
+ */
+int tor_main(int argc, char **argv);
+
+#endif /* !defined(TOR_API_H) */
+
diff --git a/src/or/tor_api_internal.h b/src/or/tor_api_internal.h
new file mode 100644
index 0000000000..10b6278b7b
--- /dev/null
+++ b/src/or/tor_api_internal.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_API_INTERNAL_H
+#define TOR_API_INTERNAL_H
+
+/* The contents of this type are private; don't mess with them from outside
+ * Tor. */
+struct tor_main_configuration_t {
+  /** As in main() */
+  int argc;
+  /** As in main(). This pointer is owned by the caller */
+  char **argv;
+};
+
+#endif /* !defined(TOR_API_INTERNAL_H) */
+
diff --git a/src/or/tor_main.c b/src/or/tor_main.c
index d67eda2ac9..703669ac99 100644
--- a/src/or/tor_main.c
+++ b/src/or/tor_main.c
@@ -1,19 +1,12 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-extern const char tor_git_revision[];
-
-/** String describing which Tor Git repository version the source was
- * built from.  This string is generated by a bit of shell kludging in
- * src/or/include.am, and is usually right.
- */
-const char tor_git_revision[] =
-#ifndef _MSC_VER
-#include "micro-revision.i"
+#include "orconfig.h"
+#ifdef ENABLE_RESTART_DEBUGGING
+#include <stdlib.h>
 #endif
-  "";
 
 /**
  * \file tor_main.c
@@ -26,14 +19,23 @@ const char tor_git_revision[] =
 int tor_main(int argc, char *argv[]);
 
 /** We keep main() in a separate file so that our unit tests can use
- * functions from main.c)
+ * functions from main.c.
  */
 int
 main(int argc, char *argv[])
 {
-  int r = tor_main(argc, argv);
+  int r;
+#ifdef ENABLE_RESTART_DEBUGGING
+  int restart_count = getenv("TOR_DEBUG_RESTART") ? 1 : 0;
+ again:
+#endif
+  r = tor_main(argc, argv);
   if (r < 0 || r > 255)
     return 1;
+#ifdef ENABLE_RESTART_DEBUGGING
+  else if (r == 0 && restart_count--)
+    goto again;
+#endif
   else
     return r;
 }
diff --git a/src/or/torcert.c b/src/or/torcert.c
index a6a33c675a..1c5afd965a 100644
--- a/src/or/torcert.c
+++ b/src/or/torcert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,9 +6,28 @@
  *
  * \brief Implementation for ed25519-signed certificates as used in the Tor
  * protocol.
+ *
+ * This certificate format is designed to be simple and compact; it's
+ * documented in tor-spec.txt in the torspec.git repository.  All of the
+ * certificates in this format are signed with an Ed25519 key; the
+ * contents themselves may be another Ed25519 key, a digest of a
+ * RSA key, or some other material.
+ *
+ * In this module there is also support for a crooss-certification of
+ * Ed25519 identities using (older) RSA1024 identities.
+ *
+ * Tor uses other types of certificate too, beyond those described in this
+ * module. Notably, our use of TLS requires us to touch X.509 certificates,
+ * even though sensible people would stay away from those. Our X.509
+ * certificates are represented with tor_x509_cert_t, and implemented in
+ * tortls.c.  We also have a separate certificate type that authorities
+ * use to authenticate their RSA signing keys with their RSA identity keys:
+ * that one is authority_cert_t, and it's mostly handled in routerlist.c.
  */
 
-#include "crypto.h"
+#include "or.h"
+#include "config.h"
+#include "crypto_util.h"
 #include "torcert.h"
 #include "ed25519_cert.h"
 #include "torlog.h"
@@ -57,29 +76,39 @@ tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
   ed25519_signature_t signature;
   if (ed25519_sign(&signature, encoded,
                    real_len-ED25519_SIG_LEN, signing_key)<0) {
+    /* LCOV_EXCL_START */
     log_warn(LD_BUG, "Can't sign certificate");
     goto err;
+    /* LCOV_EXCL_STOP */
   }
   memcpy(sig, signature.sig, ED25519_SIG_LEN);
 
   torcert = tor_cert_parse(encoded, real_len);
   if (! torcert) {
+    /* LCOV_EXCL_START */
     log_warn(LD_BUG, "Generated a certificate we cannot parse");
     goto err;
+    /* LCOV_EXCL_STOP */
   }
 
   if (tor_cert_checksig(torcert, &signing_key->pubkey, now) < 0) {
-    log_warn(LD_BUG, "Generated a certificate whose signature we can't check");
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Generated a certificate whose signature we can't "
+             "check: %s", tor_cert_describe_signature_status(torcert));
     goto err;
+    /* LCOV_EXCL_STOP */
   }
 
   tor_free(encoded);
 
   goto done;
 
+ /* LCOV_EXCL_START */
  err:
   tor_cert_free(torcert);
   torcert = NULL;
+ /* LCOV_EXCL_STOP */
+
  done:
   ed25519_cert_free(cert);
   tor_free(encoded);
@@ -109,7 +138,7 @@ tor_cert_create(const ed25519_keypair_t *signing_key,
 
 /** Release all storage held for <b>cert</b>. */
 void
-tor_cert_free(tor_cert_t *cert)
+tor_cert_free_(tor_cert_t *cert)
 {
   if (! cert)
     return;
@@ -137,7 +166,12 @@ tor_cert_parse(const uint8_t *encoded, const size_t len)
   cert->encoded_len = len;
 
   memcpy(cert->signed_key.pubkey, parsed->certified_key, 32);
-  cert->valid_until = parsed->exp_field * 3600;
+  int64_t valid_until_64 = ((int64_t)parsed->exp_field) * 3600;
+#if SIZEOF_TIME_T < SIZEOF_INT64_T
+  if (valid_until_64 > TIME_MAX)
+    valid_until_64 = TIME_MAX - 1;
+#endif
+  cert->valid_until = (time_t) valid_until_64;
   cert->cert_type = parsed->cert_type;
 
   for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) {
@@ -164,11 +198,17 @@ tor_cert_parse(const uint8_t *encoded, const size_t len)
 }
 
 /** Fill in <b>checkable_out</b> with the information needed to check
- * the signature on <b>cert</b> with <b>pubkey</b>. */
+ * the signature on <b>cert</b> with <b>pubkey</b>.
+ *
+ * On success, if <b>expiration_out</b> is provided, and it is some time
+ * _after_ the expiration time of this certificate, set it to the
+ * expiration time of this certificate.
+ */
 int
 tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
                            const tor_cert_t *cert,
-                           const ed25519_public_key_t *pubkey)
+                           const ed25519_public_key_t *pubkey,
+                           time_t *expiration_out)
 {
   if (! pubkey) {
     if (cert->signing_key_included)
@@ -185,6 +225,10 @@ tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
   memcpy(checkable_out->signature.sig,
          cert->encoded + signed_len, ED25519_SIG_LEN);
 
+  if (expiration_out) {
+    *expiration_out = MIN(*expiration_out, cert->valid_until);
+  }
+
   return 0;
 }
 
@@ -199,14 +243,15 @@ tor_cert_checksig(tor_cert_t *cert,
 {
   ed25519_checkable_t checkable;
   int okay;
+  time_t expires = TIME_MAX;
 
-  if (now && now > cert->valid_until) {
-    cert->cert_expired = 1;
+  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey, &expires) < 0)
     return -1;
-  }
 
-  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey) < 0)
+  if (now && now > expires) {
+    cert->cert_expired = 1;
     return -1;
+  }
 
   if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) {
     cert->sig_bad = 1;
@@ -223,6 +268,24 @@ tor_cert_checksig(tor_cert_t *cert,
   }
 }
 
+/** Return a string describing the status of the signature on <b>cert</b>
+ *
+ * Will always be "unchecked" unless tor_cert_checksig has been called.
+ */
+const char *
+tor_cert_describe_signature_status(const tor_cert_t *cert)
+{
+  if (cert->cert_expired) {
+    return "expired";
+  } else if (cert->sig_bad) {
+    return "mis-signed";
+  } else if (cert->sig_ok) {
+    return "okay";
+  } else {
+    return "unchecked";
+  }
+}
+
 /** Return a new copy of <b>cert</b> */
 tor_cert_t *
 tor_cert_dup(const tor_cert_t *cert)
@@ -255,6 +318,8 @@ tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
   return tor_cert_eq(cert1, cert2);
 }
 
+#define RSA_ED_CROSSCERT_PREFIX "Tor TLS RSA/Ed25519 cross-certificate"
+
 /** Create new cross-certification object to certify <b>ed_key</b> as the
  * master ed25519 identity key for the RSA identity key <b>rsa_key</b>.
  * Allocates and stores the encoded certificate in *<b>cert</b>, and returns
@@ -265,6 +330,10 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
                                time_t expires,
                                uint8_t **cert)
 {
+  // It is later than 1985, since otherwise there would be no C89
+  // compilers. (Try to diagnose #22466.)
+  tor_assert_nonfatal(expires >= 15 * 365 * 86400);
+
   uint8_t *res;
 
   rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new();
@@ -279,11 +348,21 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
   ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
   tor_assert(sz > 0 && sz <= alloc_sz);
 
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+
   const int signed_part_len = 32 + 4;
+  crypto_digest_add_bytes(d, (char*)res, signed_part_len);
+
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
   int siglen = crypto_pk_private_sign(rsa_key,
                                       (char*)rsa_ed_crosscert_getarray_sig(cc),
                                       rsa_ed_crosscert_getlen_sig(cc),
-                                      (char*)res, signed_part_len);
+                                      (char*)digest, sizeof(digest));
   tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key));
   tor_assert(siglen <= UINT8_MAX);
   cc->sig_len = siglen;
@@ -295,3 +374,352 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
   return sz;
 }
 
+/**
+ * Check whether the <b>crosscert_len</b> byte certificate in <b>crosscert</b>
+ * is in fact a correct cross-certification of <b>master_key</b> using
+ * the RSA key <b>rsa_id_key</b>.
+ *
+ * Also reject the certificate if it expired before
+ * <b>reject_if_expired_before</b>.
+ *
+ * Return 0 on success, negative on failure.
+ */
+MOCK_IMPL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before))
+{
+  rsa_ed_crosscert_t *cc = NULL;
+  int rv;
+
+#define ERR(code, s)                                            \
+  do {                                                          \
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,                      \
+           "Received a bad RSA->Ed25519 crosscert: %s",         \
+           (s));                                                \
+    rv = (code);                                                \
+    goto err;                                                   \
+  } while (0)
+
+  if (BUG(crypto_pk_keysize(rsa_id_key) > PK_BYTES))
+    return -1;
+
+  if (BUG(!crosscert))
+    return -1;
+
+  ssize_t parsed_len = rsa_ed_crosscert_parse(&cc, crosscert, crosscert_len);
+  if (parsed_len < 0 || crosscert_len != (size_t)parsed_len) {
+    ERR(-2, "Unparseable or overlong crosscert");
+  }
+
+  if (tor_memneq(rsa_ed_crosscert_getarray_ed_key(cc),
+                 master_key->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    ERR(-3, "Crosscert did not match Ed25519 key");
+  }
+
+  const uint32_t expiration_date = rsa_ed_crosscert_get_expiration(cc);
+  const uint64_t expiration_time = ((uint64_t)expiration_date) * 3600;
+
+  if (reject_if_expired_before < 0 ||
+      expiration_time < (uint64_t)reject_if_expired_before) {
+    ERR(-4, "Crosscert is expired");
+  }
+
+  const uint8_t *eos = rsa_ed_crosscert_get_end_of_signed(cc);
+  const uint8_t *sig = rsa_ed_crosscert_getarray_sig(cc);
+  const uint8_t siglen = rsa_ed_crosscert_get_sig_len(cc);
+  tor_assert(eos >= crosscert);
+  tor_assert((size_t)(eos - crosscert) <= crosscert_len);
+  tor_assert(siglen == rsa_ed_crosscert_getlen_sig(cc));
+
+  /* Compute the digest */
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+  crypto_digest_add_bytes(d, (char*)crosscert, eos-crosscert);
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  /* Now check the signature */
+  uint8_t signed_[PK_BYTES];
+  int signed_len = crypto_pk_public_checksig(rsa_id_key,
+                                          (char*)signed_, sizeof(signed_),
+                                          (char*)sig, siglen);
+  if (signed_len < DIGEST256_LEN) {
+    ERR(-5, "Bad signature, or length of signed data not as expected");
+  }
+
+  if (tor_memneq(digest, signed_, DIGEST256_LEN)) {
+    ERR(-6, "The signature was good, but it didn't match the data");
+  }
+
+  rv = 0;
+ err:
+  rsa_ed_crosscert_free(cc);
+  return rv;
+}
+
+/** Construct and return a new empty or_handshake_certs object */
+or_handshake_certs_t *
+or_handshake_certs_new(void)
+{
+  return tor_malloc_zero(sizeof(or_handshake_certs_t));
+}
+
+/** Release all storage held in <b>certs</b> */
+void
+or_handshake_certs_free_(or_handshake_certs_t *certs)
+{
+  if (!certs)
+    return;
+
+  tor_x509_cert_free(certs->auth_cert);
+  tor_x509_cert_free(certs->link_cert);
+  tor_x509_cert_free(certs->id_cert);
+
+  tor_cert_free(certs->ed_id_sign);
+  tor_cert_free(certs->ed_sign_link);
+  tor_cert_free(certs->ed_sign_auth);
+  tor_free(certs->ed_rsa_crosscert);
+
+  memwipe(certs, 0xBD, sizeof(*certs));
+  tor_free(certs);
+}
+
+#undef ERR
+#define ERR(s)                                                  \
+  do {                                                          \
+    log_fn(severity, LD_PROTOCOL,                               \
+           "Received a bad CERTS cell: %s",                     \
+           (s));                                                \
+    return 0;                                                   \
+  } while (0)
+
+int
+or_handshake_certs_rsa_ok(int severity,
+                          or_handshake_certs_t *certs,
+                          tor_tls_t *tls,
+                          time_t now)
+{
+  tor_x509_cert_t *link_cert = certs->link_cert;
+  tor_x509_cert_t *auth_cert = certs->auth_cert;
+  tor_x509_cert_t *id_cert = certs->id_cert;
+
+  if (certs->started_here) {
+    if (! (id_cert && link_cert))
+      ERR("The certs we wanted (ID, Link) were missing");
+    if (! tor_tls_cert_matches_key(tls, link_cert))
+      ERR("The link certificate didn't match the TLS public key");
+    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, now, 0))
+      ERR("The link certificate was not valid");
+    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  } else {
+    if (! (id_cert && auth_cert))
+      ERR("The certs we wanted (ID, Auth) were missing");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, now, 1))
+      ERR("The authentication certificate was not valid");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  }
+
+  return 1;
+}
+
+/** Check all the ed25519 certificates in <b>certs</b> against each other, and
+ * against the peer certificate in <b>tls</b> if appropriate.  On success,
+ * return 0; on failure, return a negative value and warn at level
+ * <b>severity</b> */
+int
+or_handshake_certs_ed25519_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now)
+{
+  ed25519_checkable_t check[10];
+  unsigned n_checkable = 0;
+  time_t expiration = TIME_MAX;
+
+#define ADDCERT(cert, pk)                                               \
+  do {                                                                  \
+    tor_assert(n_checkable < ARRAY_LENGTH(check));                      \
+    if (tor_cert_get_checkable_sig(&check[n_checkable++], cert, pk,     \
+                                   &expiration) < 0)                    \
+      ERR("Could not get checkable cert.");                             \
+  } while (0)
+
+  if (! certs->ed_id_sign || !certs->ed_id_sign->signing_key_included) {
+    ERR("No Ed25519 signing key");
+  }
+  ADDCERT(certs->ed_id_sign, NULL);
+
+  if (certs->started_here) {
+    if (! certs->ed_sign_link)
+      ERR("No Ed25519 link key");
+    {
+      /* check for a match with the TLS cert. */
+      tor_x509_cert_t *peer_cert = tor_tls_get_peer_cert(tls);
+      if (BUG(!peer_cert)) {
+        /* This is a bug, because if we got to this point, we are a connection
+         * that was initiated here, and we completed a TLS handshake. The
+         * other side *must* have given us a certificate! */
+        ERR("No x509 peer cert"); // LCOV_EXCL_LINE
+      }
+      const common_digests_t *peer_cert_digests =
+        tor_x509_cert_get_cert_digests(peer_cert);
+      int okay = tor_memeq(peer_cert_digests->d[DIGEST_SHA256],
+                           certs->ed_sign_link->signed_key.pubkey,
+                           DIGEST256_LEN);
+      tor_x509_cert_free(peer_cert);
+      if (!okay)
+        ERR("Link certificate does not match TLS certificate");
+    }
+
+    ADDCERT(certs->ed_sign_link, &certs->ed_id_sign->signed_key);
+
+  } else {
+    if (! certs->ed_sign_auth)
+      ERR("No Ed25519 link authentication key");
+    ADDCERT(certs->ed_sign_auth, &certs->ed_id_sign->signed_key);
+  }
+
+  if (expiration < now) {
+    ERR("At least one certificate expired.");
+  }
+
+  /* Okay, we've gotten ready to check all the Ed25519 certificates.
+   * Now, we are going to check the RSA certificate's cross-certification
+   * with the ED certificates.
+   *
+   * FFFF In the future, we might want to make this optional.
+   */
+
+  tor_x509_cert_t *rsa_id_cert = certs->id_cert;
+  if (!rsa_id_cert) {
+    ERR("Missing legacy RSA ID certificate");
+  }
+  if (! tor_tls_cert_is_valid(severity, rsa_id_cert, rsa_id_cert, now, 1)) {
+    ERR("The legacy RSA ID certificate was not valid");
+  }
+  if (! certs->ed_rsa_crosscert) {
+    ERR("Missing RSA->Ed25519 crosscert");
+  }
+  crypto_pk_t *rsa_id_key = tor_tls_cert_get_key(rsa_id_cert);
+  if (!rsa_id_key) {
+    ERR("RSA ID cert had no RSA key");
+  }
+
+  if (rsa_ed25519_crosscert_check(certs->ed_rsa_crosscert,
+                                  certs->ed_rsa_crosscert_len,
+                                  rsa_id_key,
+                                  &certs->ed_id_sign->signing_key,
+                                  now) < 0) {
+    crypto_pk_free(rsa_id_key);
+    ERR("Invalid RSA->Ed25519 crosscert");
+  }
+  crypto_pk_free(rsa_id_key);
+  rsa_id_key = NULL;
+
+  /* FFFF We could save a little time in the client case by queueing
+   * this batch to check it later, along with the signature from the
+   * AUTHENTICATE cell. That will change our data flow a bit, though,
+   * so I say "postpone". */
+
+  if (ed25519_checksig_batch(NULL, check, n_checkable) < 0) {
+    ERR("At least one Ed25519 certificate was badly signed");
+  }
+
+  return 1;
+}
+
+/**
+ * Check the Ed certificates and/or the RSA certificates, as appropriate.  If
+ * we obtained an Ed25519 identity, set *ed_id_out. If we obtained an RSA
+ * identity, set *rs_id_out. Otherwise, set them both to NULL.
+ */
+void
+or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out)
+{
+  tor_assert(ed_id_out);
+  tor_assert(rsa_id_out);
+
+  *ed_id_out = NULL;
+  *rsa_id_out = NULL;
+
+  if (certs->ed_id_sign) {
+    if (or_handshake_certs_ed25519_ok(severity, certs, tls, now)) {
+      tor_assert(certs->ed_id_sign);
+      tor_assert(certs->id_cert);
+
+      *ed_id_out = &certs->ed_id_sign->signing_key;
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+
+      /* If we reached this point, we did not look at any of the
+       * subsidiary RSA certificates, so we'd better just remove them.
+       */
+      tor_x509_cert_free(certs->link_cert);
+      tor_x509_cert_free(certs->auth_cert);
+      certs->link_cert = certs->auth_cert = NULL;
+    }
+    /* We do _not_ fall through here.  If you provided us Ed25519
+     * certificates, we expect to verify them! */
+  } else {
+    /* No ed25519 keys given in the CERTS cell */
+    if (or_handshake_certs_rsa_ok(severity, certs, tls, now)) {
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+    }
+  }
+}
+
+/* === ENCODING === */
+
+/* Encode the ed25519 certificate <b>cert</b> and put the newly allocated
+ * string in <b>cert_str_out</b>. Return 0 on success else a negative value. */
+int
+tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out)
+{
+  int ret = -1;
+  char *ed_cert_b64 = NULL;
+  size_t ed_cert_b64_len;
+
+  tor_assert(cert);
+  tor_assert(cert_str_out);
+
+  /* Get the encoded size and add the NUL byte. */
+  ed_cert_b64_len = base64_encode_size(cert->encoded_len,
+                                       BASE64_ENCODE_MULTILINE) + 1;
+  ed_cert_b64 = tor_malloc_zero(ed_cert_b64_len);
+
+  /* Base64 encode the encoded certificate. */
+  if (base64_encode(ed_cert_b64, ed_cert_b64_len,
+                    (const char *) cert->encoded, cert->encoded_len,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    /* LCOV_EXCL_START */
+    log_err(LD_BUG, "Couldn't base64-encode ed22519 cert!");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Put everything together in a NUL terminated string. */
+  tor_asprintf(cert_str_out,
+               "-----BEGIN ED25519 CERT-----\n"
+               "%s"
+               "-----END ED25519 CERT-----",
+               ed_cert_b64);
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(ed_cert_b64);
+  return ret;
+}
+
diff --git a/src/or/torcert.h b/src/or/torcert.h
index 9c819c0abb..18ca60b5a8 100644
--- a/src/or/torcert.h
+++ b/src/or/torcert.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TORCERT_H_INCLUDED
@@ -6,12 +6,15 @@
 
 #include "crypto_ed25519.h"
 
-#define SIGNED_KEY_TYPE_ED25519 0x01
+#define SIGNED_KEY_TYPE_ED25519     0x01
 
-#define CERT_TYPE_ID_SIGNING    0x04
-#define CERT_TYPE_SIGNING_LINK  0x05
-#define CERT_TYPE_SIGNING_AUTH  0x06
-#define CERT_TYPE_ONION_ID      0x0A
+#define CERT_TYPE_ID_SIGNING        0x04
+#define CERT_TYPE_SIGNING_LINK      0x05
+#define CERT_TYPE_SIGNING_AUTH      0x06
+#define CERT_TYPE_SIGNING_HS_DESC   0x08
+#define CERT_TYPE_AUTH_HS_IP_KEY    0x09
+#define CERT_TYPE_ONION_ID          0x0A
+#define CERT_TYPE_CROSS_HS_IP_KEYS  0x0B
 
 #define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1
 
@@ -54,14 +57,17 @@ tor_cert_t *tor_cert_create(const ed25519_keypair_t *signing_key,
 
 tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen);
 
-void tor_cert_free(tor_cert_t *cert);
+void tor_cert_free_(tor_cert_t *cert);
+#define tor_cert_free(cert) FREE_AND_NULL(tor_cert_t, tor_cert_free_, (cert))
 
 int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
-                                 const tor_cert_t *out,
-                                 const ed25519_public_key_t *pubkey);
+                               const tor_cert_t *out,
+                               const ed25519_public_key_t *pubkey,
+                               time_t *expiration_out);
 
 int tor_cert_checksig(tor_cert_t *cert,
                       const ed25519_public_key_t *pubkey, time_t now);
+const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
 
 tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
 int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
@@ -71,6 +77,33 @@ ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
                                        const crypto_pk_t *rsa_key,
                                        time_t expires,
                                        uint8_t **cert);
-
-#endif
+MOCK_DECL(int,
+rsa_ed25519_crosscert_check, (const uint8_t *crosscert,
+                              const size_t crosscert_len,
+                              const crypto_pk_t *rsa_id_key,
+                              const ed25519_public_key_t *master_key,
+                              const time_t reject_if_expired_before));
+
+or_handshake_certs_t *or_handshake_certs_new(void);
+void or_handshake_certs_free_(or_handshake_certs_t *certs);
+#define or_handshake_certs_free(certs) \
+  FREE_AND_NULL(or_handshake_certs_t, or_handshake_certs_free_, (certs))
+int or_handshake_certs_rsa_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now);
+int or_handshake_certs_ed25519_ok(int severity,
+                                  or_handshake_certs_t *certs,
+                                  tor_tls_t *tls,
+                                  time_t now);
+void or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out);
+
+int tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out);
+
+#endif /* !defined(TORCERT_H_INCLUDED) */
 
diff --git a/src/or/transports.c b/src/or/transports.c
index 7a52b737e4..614fc81da8 100644
--- a/src/or/transports.c
+++ b/src/or/transports.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -91,13 +91,13 @@
 
 #define PT_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "config.h"
 #include "circuitbuild.h"
 #include "transports.h"
 #include "util.h"
 #include "router.h"
 #include "statefile.h"
-#include "entrynodes.h"
 #include "connection_or.h"
 #include "ext_orport.h"
 #include "control.h"
@@ -135,7 +135,7 @@ static smartlist_t *transport_list = NULL;
 /** Returns a transport_t struct for a transport proxy supporting the
     protocol <b>name</b> listening at <b>addr</b>:<b>port</b> using
     SOCKS version <b>socks_ver</b>. */
-static transport_t *
+STATIC transport_t *
 transport_new(const tor_addr_t *addr, uint16_t port,
               const char *name, int socks_ver,
               const char *extra_info_args)
@@ -154,7 +154,7 @@ transport_new(const tor_addr_t *addr, uint16_t port,
 
 /** Free the pluggable transport struct <b>transport</b>. */
 void
-transport_free(transport_t *transport)
+transport_free_(transport_t *transport)
 {
   if (!transport)
     return;
@@ -222,8 +222,8 @@ transport_copy(const transport_t *transport)
 
 /** Returns the transport in our transport list that has the name <b>name</b>.
  *  Else returns NULL. */
-transport_t *
-transport_get_by_name(const char *name)
+MOCK_IMPL(transport_t *,
+transport_get_by_name,(const char *name))
 {
   tor_assert(name);
 
@@ -430,7 +430,7 @@ add_transport_to_proxy(const char *transport, managed_proxy_t *mp)
 {
   tor_assert(mp->transports_to_launch);
   if (!smartlist_contains_string(mp->transports_to_launch, transport))
-    smartlist_add(mp->transports_to_launch, tor_strdup(transport));
+    smartlist_add_strdup(mp->transports_to_launch, transport);
 }
 
 /** Called when a SIGHUP occurs. Returns true if managed proxy
@@ -480,7 +480,6 @@ proxy_needs_restart(const managed_proxy_t *mp)
  *  preparations and then flag its state so that it will be relaunched
  *  in the next tick. */
 static void
-
 proxy_prepare_for_restart(managed_proxy_t *mp)
 {
   transport_t *t_tmp = NULL;
@@ -528,12 +527,12 @@ launch_managed_proxy(managed_proxy_t *mp)
                                 (const char **)mp->argv,
                                 env,
                                 &mp->process_handle);
-#else
+#else /* !(defined(_WIN32)) */
   retval = tor_spawn_background(mp->argv[0],
                                 (const char **)mp->argv,
                                 env,
                                 &mp->process_handle);
-#endif
+#endif /* defined(_WIN32) */
 
   process_environment_free(env);
 
@@ -591,7 +590,7 @@ pt_configure_remaining_proxies(void)
     }
 
     /* If the proxy is not fully configured, try to configure it
-       futher. */
+       further. */
     if (!proxy_configuration_finished(mp))
       if (configure_proxy(mp) == 1)
         at_least_a_proxy_config_finished = 1;
@@ -1026,48 +1025,71 @@ parse_method_error(const char *line, int is_server)
            line+strlen(error)+1);
 }
 
-/** Parses an SMETHOD <b>line</b> and if well-formed it registers the
- *  new transport in <b>mp</b>. */
-STATIC int
-parse_smethod_line(const char *line, managed_proxy_t *mp)
+/** A helper for parse_{c,s}method_line(), bootstraps its
+ *  functionalities. If <b>is_smethod</b> is true then the
+ *  the line to parse is a SMETHOD line otherwise it is a
+ *  CMETHOD line*/
+static int
+parse_method_line_helper(const char *line,
+                         managed_proxy_t *mp,
+                         int is_smethod)
 {
+  int item_index = 0;
   int r;
-  smartlist_t *items = NULL;
 
-  char *method_name=NULL;
+  char *transport_name=NULL;
   char *args_string=NULL;
   char *addrport=NULL;
-  tor_addr_t tor_addr;
+  int socks_ver=PROXY_NONE;
   char *address=NULL;
   uint16_t port = 0;
 
+  const char *method_str = is_smethod ? PROTO_SMETHOD : PROTO_CMETHOD;
+  const int min_args_count = is_smethod ? 3 : 4;
+
+  tor_addr_t tor_addr;
   transport_t *transport=NULL;
+  smartlist_t *items= smartlist_new();
 
-  items = smartlist_new();
   smartlist_split_string(items, line, NULL,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-  if (smartlist_len(items) < 3) {
-    log_warn(LD_CONFIG, "Server managed proxy sent us a SMETHOD line "
-             "with too few arguments.");
+  if (smartlist_len(items) < min_args_count) {
+    log_warn(LD_CONFIG, "Managed proxy sent us a %s line "
+             "with too few arguments.", method_str);
     goto err;
   }
 
-  /* Example of legit SMETHOD line:
-     SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */
-
-  tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD));
+  tor_assert(!strcmp(smartlist_get(items, item_index),method_str));
+  ++item_index;
 
-  method_name = smartlist_get(items,1);
-  if (!string_is_C_identifier(method_name)) {
+  transport_name = smartlist_get(items,item_index);
+  ++item_index;
+  if (!string_is_C_identifier(transport_name)) {
     log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).",
-             method_name);
+             transport_name);
     goto err;
   }
 
-  addrport = smartlist_get(items, 2);
+  /** Check for the proxy method sent to us in CMETHOD line. */
+  if (!is_smethod) {
+    const char *socks_ver_str = smartlist_get(items,item_index);
+    ++item_index;
+
+    if (!strcmp(socks_ver_str,"socks4")) {
+      socks_ver = PROXY_SOCKS4;
+    } else if (!strcmp(socks_ver_str,"socks5")) {
+      socks_ver = PROXY_SOCKS5;
+    } else {
+      log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol "
+               "we don't recognize. (%s)", socks_ver_str);
+      goto err;
+    }
+  }
+
+  addrport = smartlist_get(items, item_index);
+  ++item_index;
   if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) {
-    log_warn(LD_CONFIG, "Error parsing transport "
-             "address '%s'", addrport);
+    log_warn(LD_CONFIG, "Error parsing transport address '%s'", addrport);
     goto err;
   }
 
@@ -1082,10 +1104,11 @@ parse_smethod_line(const char *line, managed_proxy_t *mp)
     goto err;
   }
 
-  if (smartlist_len(items) > 3) {
+  /** Check for options in the SMETHOD line. */
+  if (is_smethod && smartlist_len(items) > min_args_count) {
     /* Seems like there are also some [options] in the SMETHOD line.
        Let's see if we can parse them. */
-    char *options_string = smartlist_get(items, 3);
+    char *options_string = smartlist_get(items, item_index);
     log_debug(LD_CONFIG, "Got options_string: %s", options_string);
     if (!strcmpstart(options_string, "ARGS:")) {
       args_string = options_string+strlen("ARGS:");
@@ -1093,17 +1116,20 @@ parse_smethod_line(const char *line, managed_proxy_t *mp)
     }
   }
 
-  transport = transport_new(&tor_addr, port, method_name,
-                            PROXY_NONE, args_string);
-  if (!transport)
-    goto err;
+  transport = transport_new(&tor_addr, port, transport_name,
+                            socks_ver, args_string);
 
   smartlist_add(mp->transports, transport);
 
-  /* For now, notify the user so that they know where the server
-     transport is listening. */
-  log_info(LD_CONFIG, "Server transport %s at %s:%d.",
-           method_name, address, (int)port);
+  /** Logs info about line parsing success for client or server */
+  if (is_smethod) {
+    log_info(LD_CONFIG, "Server transport %s at %s:%d.",
+             transport_name, address, (int)port);
+  } else {
+    log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. "
+             "Attached to managed proxy.",
+             transport_name, address, (int)port, socks_ver);
+  }
 
   r=0;
   goto done;
@@ -1118,95 +1144,24 @@ parse_smethod_line(const char *line, managed_proxy_t *mp)
   return r;
 }
 
+/** Parses an SMETHOD <b>line</b> and if well-formed it registers the
+ *  new transport in <b>mp</b>. */
+STATIC int
+parse_smethod_line(const char *line, managed_proxy_t *mp)
+{
+  /* Example of legit SMETHOD line:
+     SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */
+  return parse_method_line_helper(line, mp, 1);
+}
+
 /** Parses a CMETHOD <b>line</b>, and if well-formed it registers
  *  the new transport in <b>mp</b>. */
 STATIC int
 parse_cmethod_line(const char *line, managed_proxy_t *mp)
 {
-  int r;
-  smartlist_t *items = NULL;
-
-  char *method_name=NULL;
-
-  char *socks_ver_str=NULL;
-  int socks_ver=PROXY_NONE;
-
-  char *addrport=NULL;
-  tor_addr_t tor_addr;
-  char *address=NULL;
-  uint16_t port = 0;
-
-  transport_t *transport=NULL;
-
-  items = smartlist_new();
-  smartlist_split_string(items, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-  if (smartlist_len(items) < 4) {
-    log_warn(LD_CONFIG, "Client managed proxy sent us a CMETHOD line "
-             "with too few arguments.");
-    goto err;
-  }
-
-  tor_assert(!strcmp(smartlist_get(items,0),PROTO_CMETHOD));
-
-  method_name = smartlist_get(items,1);
-  if (!string_is_C_identifier(method_name)) {
-    log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).",
-             method_name);
-    goto err;
-  }
-
-  socks_ver_str = smartlist_get(items,2);
-
-  if (!strcmp(socks_ver_str,"socks4")) {
-    socks_ver = PROXY_SOCKS4;
-  } else if (!strcmp(socks_ver_str,"socks5")) {
-    socks_ver = PROXY_SOCKS5;
-  } else {
-    log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol "
-             "we don't recognize. (%s)", socks_ver_str);
-    goto err;
-  }
-
-  addrport = smartlist_get(items, 3);
-  if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) {
-    log_warn(LD_CONFIG, "Error parsing transport "
-             "address '%s'", addrport);
-    goto err;
-  }
-
-  if (!port) {
-    log_warn(LD_CONFIG,
-             "Transport address '%s' has no port.", addrport);
-    goto err;
-  }
-
-  if (tor_addr_parse(&tor_addr, address) < 0) {
-    log_warn(LD_CONFIG, "Error parsing transport address '%s'", address);
-    goto err;
-  }
-
-  transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL);
-  if (!transport)
-    goto err;
-
-  smartlist_add(mp->transports, transport);
-
-  log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. "
-           "Attached to managed proxy.",
-           method_name, address, (int)port, socks_ver);
-
-  r=0;
-  goto done;
-
- err:
-  r = -1;
-
- done:
-  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
-  smartlist_free(items);
-  tor_free(address);
-  return r;
+  /* Example of legit CMETHOD line:
+     CMETHOD obfs2 socks5 127.0.0.1:35713 */
+  return parse_method_line_helper(line, mp, 0);
 }
 
 /** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */
@@ -1322,7 +1277,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     tor_free(state_tmp);
   }
 
-  smartlist_add(envs, tor_strdup("TOR_PT_MANAGED_TRANSPORT_VER=1"));
+  smartlist_add_strdup(envs, "TOR_PT_MANAGED_TRANSPORT_VER=1");
 
   {
     char *transports_to_launch =
diff --git a/src/or/transports.h b/src/or/transports.h
index 7de90dcbec..022b926a03 100644
--- a/src/or/transports.h
+++ b/src/or/transports.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,9 +35,10 @@ void sweep_transport_list(void);
 MOCK_DECL(int, transport_add_from_config,
           (const tor_addr_t *addr, uint16_t port,
            const char *name, int socks_ver));
-void transport_free(transport_t *transport);
+void transport_free_(transport_t *transport);
+#define transport_free(tr) FREE_AND_NULL(transport_t, transport_free_, (tr))
 
-transport_t *transport_get_by_name(const char *name);
+MOCK_DECL(transport_t*, transport_get_by_name, (const char *name));
 
 MOCK_DECL(void, pt_kickstart_proxy,
           (const smartlist_t *transport_list, char **proxy_argv,
@@ -112,6 +113,9 @@ typedef struct {
   smartlist_t *transports;
 } managed_proxy_t;
 
+STATIC transport_t *transport_new(const tor_addr_t *addr, uint16_t port,
+                                  const char *name, int socks_ver,
+                                  const char *extra_info_args);
 STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp);
 STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp);
 
@@ -133,7 +137,7 @@ STATIC char* get_pt_proxy_uri(void);
 
 STATIC void free_execve_args(char **arg);
 
-#endif
+#endif /* defined(PT_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_TRANSPORTS_H) */
 
diff --git a/src/or/voting_schedule.c b/src/or/voting_schedule.c
new file mode 100644
index 0000000000..d230a6dbcd
--- /dev/null
+++ b/src/or/voting_schedule.c
@@ -0,0 +1,192 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voting_schedule.c
+ * \brief This file contains functions that are from the directory authority
+ *        subsystem related to voting specifically but used by many part of
+ *        tor. The full feature is built as part of the dirauth module.
+ **/
+
+#define VOTING_SCHEDULE_PRIVATE
+#include "voting_schedule.h"
+
+#include "or.h"
+#include "config.h"
+#include "networkstatus.h"
+
+/* =====
+ * Vote scheduling
+ * ===== */
+
+/** Return the start of the next interval of size <b>interval</b> (in
+ * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
+ * starts a fresh interval, and if the last interval of a day would be
+ * truncated to less than half its size, it is rolled into the
+ * previous interval. */
+time_t
+voting_schedule_get_start_of_next_interval(time_t now, int interval,
+                                           int offset)
+{
+  struct tm tm;
+  time_t midnight_today=0;
+  time_t midnight_tomorrow;
+  time_t next;
+
+  tor_gmtime_r(&now, &tm);
+  tm.tm_hour = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
+
+  if (tor_timegm(&tm, &midnight_today) < 0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
+    // LCOV_EXCL_STOP
+  }
+  midnight_tomorrow = midnight_today + (24*60*60);
+
+  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
+
+  /* Intervals never cross midnight. */
+  if (next > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  /* If the interval would only last half as long as it's supposed to, then
+   * skip over to the next day. */
+  if (next + interval/2 > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  next += offset;
+  if (next - interval > now)
+    next -= interval;
+
+  return next;
+}
+
+/* Populate and return a new voting_schedule_t that can be used to schedule
+ * voting. The object is allocated on the heap and it's the responsibility of
+ * the caller to free it. Can't fail. */
+static voting_schedule_t *
+get_voting_schedule(const or_options_t *options, time_t now, int severity)
+{
+  int interval, vote_delay, dist_delay;
+  time_t start;
+  time_t end;
+  networkstatus_t *consensus;
+  voting_schedule_t *new_voting_schedule;
+
+  new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t));
+
+  consensus = networkstatus_get_live_consensus(now);
+
+  if (consensus) {
+    interval = (int)( consensus->fresh_until - consensus->valid_after );
+    vote_delay = consensus->vote_seconds;
+    dist_delay = consensus->dist_seconds;
+
+    /* Note down the consensus valid after, so that we detect outdated voting
+     * schedules in case of skewed clocks etc. */
+    new_voting_schedule->live_consensus_valid_after = consensus->valid_after;
+  } else {
+    interval = options->TestingV3AuthInitialVotingInterval;
+    vote_delay = options->TestingV3AuthInitialVoteDelay;
+    dist_delay = options->TestingV3AuthInitialDistDelay;
+  }
+
+  tor_assert(interval > 0);
+
+  if (vote_delay + dist_delay > interval/2)
+    vote_delay = dist_delay = interval / 4;
+
+  start = new_voting_schedule->interval_starts =
+    voting_schedule_get_start_of_next_interval(now,interval,
+                                      options->TestingV3AuthVotingStartOffset);
+  end = voting_schedule_get_start_of_next_interval(start+1, interval,
+                                      options->TestingV3AuthVotingStartOffset);
+
+  tor_assert(end > start);
+
+  new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2);
+  new_voting_schedule->voting_ends = start - dist_delay;
+  new_voting_schedule->fetch_missing_votes =
+    start - dist_delay - (vote_delay/2);
+  new_voting_schedule->voting_starts = start - dist_delay - vote_delay;
+
+  {
+    char tbuf[ISO_TIME_LEN+1];
+    format_iso_time(tbuf, new_voting_schedule->interval_starts);
+    tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: "
+            "consensus_set=%d, interval=%d",
+            tbuf, consensus?1:0, interval);
+  }
+
+  return new_voting_schedule;
+}
+
+#define voting_schedule_free(s) \
+  FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s))
+
+/** Frees a voting_schedule_t. This should be used instead of the generic
+ * tor_free. */
+static void
+voting_schedule_free_(voting_schedule_t *voting_schedule_to_free)
+{
+  if (!voting_schedule_to_free)
+    return;
+  tor_free(voting_schedule_to_free);
+}
+
+voting_schedule_t voting_schedule;
+
+/* Using the time <b>now</b>, return the next voting valid-after time. */
+time_t
+voting_schedule_get_next_valid_after_time(void)
+{
+  time_t now = approx_time();
+  bool need_to_recalculate_voting_schedule = false;
+
+  /* This is a safe guard in order to make sure that the voting schedule
+   * static object is at least initialized. Using this function with a zeroed
+   * voting schedule can lead to bugs. */
+  if (tor_mem_is_zero((const char *) &voting_schedule,
+                      sizeof(voting_schedule))) {
+    need_to_recalculate_voting_schedule = true;
+    goto done; /* no need for next check if we have to recalculate anyway */
+  }
+
+  /* Also make sure we are not using an outdated voting schedule. If we have a
+   * newer consensus, make sure we recalculate the voting schedule. */
+  const networkstatus_t *ns = networkstatus_get_live_consensus(now);
+  if (ns && ns->valid_after != voting_schedule.live_consensus_valid_after) {
+    log_info(LD_DIR, "Voting schedule is outdated: recalculating (%d/%d)",
+             (int) ns->valid_after,
+             (int) voting_schedule.live_consensus_valid_after);
+    need_to_recalculate_voting_schedule = true;
+  }
+
+ done:
+  if (need_to_recalculate_voting_schedule) {
+    voting_schedule_recalculate_timing(get_options(), now);
+    voting_schedule.created_on_demand = 1;
+  }
+
+  return voting_schedule.interval_starts;
+}
+
+/** Set voting_schedule to hold the timing for the next vote we should be
+ * doing. All type of tor do that because HS subsystem needs the timing as
+ * well to function properly. */
+void
+voting_schedule_recalculate_timing(const or_options_t *options, time_t now)
+{
+  voting_schedule_t *new_voting_schedule;
+
+  /* get the new voting schedule */
+  new_voting_schedule = get_voting_schedule(options, now, LOG_INFO);
+  tor_assert(new_voting_schedule);
+
+  /* Fill in the global static struct now */
+  memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
+  voting_schedule_free(new_voting_schedule);
+}
+
diff --git a/src/or/voting_schedule.h b/src/or/voting_schedule.h
new file mode 100644
index 0000000000..087701408e
--- /dev/null
+++ b/src/or/voting_schedule.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file voting_schedule.h
+ * \brief Header file for voting_schedule.c.
+ **/
+
+#ifndef TOR_VOTING_SCHEDULE_H
+#define TOR_VOTING_SCHEDULE_H
+
+#include "or.h"
+
+/** Scheduling information for a voting interval. */
+typedef struct {
+  /** When do we generate and distribute our vote for this interval? */
+  time_t voting_starts;
+  /** When do we send an HTTP request for any votes that we haven't
+   * been posted yet?*/
+  time_t fetch_missing_votes;
+  /** When do we give up on getting more votes and generate a consensus? */
+  time_t voting_ends;
+  /** When do we send an HTTP request for any signatures we're expecting to
+   * see on the consensus? */
+  time_t fetch_missing_signatures;
+  /** When do we publish the consensus? */
+  time_t interval_starts;
+
+  /* True iff we have generated and distributed our vote. */
+  int have_voted;
+  /* True iff we've requested missing votes. */
+  int have_fetched_missing_votes;
+  /* True iff we have built a consensus and sent the signatures around. */
+  int have_built_consensus;
+  /* True iff we've fetched missing signatures. */
+  int have_fetched_missing_signatures;
+  /* True iff we have published our consensus. */
+  int have_published_consensus;
+
+  /* True iff this voting schedule was set on demand meaning not through the
+   * normal vote operation of a dirauth or when a consensus is set. This only
+   * applies to a directory authority that needs to recalculate the voting
+   * timings only for the first vote even though this object was initilized
+   * prior to voting. */
+  int created_on_demand;
+
+  /** The valid-after time of the last live consensus that filled this voting
+   *  schedule.  It's used to detect outdated voting schedules. */
+  time_t live_consensus_valid_after;
+} voting_schedule_t;
+
+/* Public API. */
+
+extern voting_schedule_t voting_schedule;
+
+void voting_schedule_recalculate_timing(const or_options_t *options,
+                                        time_t now);
+
+time_t voting_schedule_get_start_of_next_interval(time_t now,
+                                                  int interval,
+                                                  int offset);
+time_t voting_schedule_get_next_valid_after_time(void);
+
+#endif /* TOR_VOTING_SCHEDULE_H */
+
diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in
new file mode 100644
index 0000000000..6eddc75459
--- /dev/null
+++ b/src/rust/.cargo/config.in
@@ -0,0 +1,12 @@
+[source]
+
+@RUST_DL@ [source.crates-io]
+@RUST_DL@ registry = 'https://github.com/rust-lang/crates.io-index'
+@RUST_DL@ replace-with = 'vendored-sources'
+
+@RUST_DL@ [source.vendored-sources]
+@RUST_DL@ directory = '@TOR_RUST_DEPENDENCIES@'
+
+[build]
+@RUST_WARN@ rustflags = [ "-D", "warnings" ]
+@RUST_TARGET_PROP@
diff --git a/src/rust/.rustfmt.toml b/src/rust/.rustfmt.toml
new file mode 100644
index 0000000000..f25bd51883
--- /dev/null
+++ b/src/rust/.rustfmt.toml
@@ -0,0 +1,2 @@
+max_width = 80
+comment_width = 80
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
new file mode 100644
index 0000000000..3a067a39dc
--- /dev/null
+++ b/src/rust/Cargo.lock
@@ -0,0 +1,121 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "crypto"
+version = "0.0.1"
+dependencies = [
+ "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "external 0.0.1",
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smartlist 0.0.1",
+ "tor_allocate 0.0.1",
+ "tor_log 0.1.0",
+]
+
+[[package]]
+name = "digest"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "external"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smartlist 0.0.1",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "protover"
+version = "0.0.1"
+dependencies = [
+ "external 0.0.1",
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smartlist 0.0.1",
+ "tor_allocate 0.0.1",
+ "tor_log 0.1.0",
+ "tor_util 0.0.1",
+]
+
+[[package]]
+name = "rand"
+version = "0.5.0-pre.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.2.0-pre.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "smartlist"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tor_allocate"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tor_log"
+version = "0.1.0"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tor_allocate 0.0.1",
+]
+
+[[package]]
+name = "tor_rust"
+version = "0.1.0"
+dependencies = [
+ "protover 0.0.1",
+ "tor_util 0.0.1",
+]
+
+[[package]]
+name = "tor_util"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tor_allocate 0.0.1",
+ "tor_log 0.1.0",
+]
+
+[[package]]
+name = "typenum"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603"
+"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d"
+"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff"
+"checksum rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3795e4701d9628a63a84d0289e66279883b40df165fca7caed7b87122447032a"
+"checksum rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7255ffbdb188d5be1a69b6f9f3cf187de4207430b9e79ed5b76458a6b20de9a"
+"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd"
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
new file mode 100644
index 0000000000..de8693ea33
--- /dev/null
+++ b/src/rust/Cargo.toml
@@ -0,0 +1,26 @@
+[workspace]
+members = [
+    "crypto",
+    "external",
+    "protover",
+    "smartlist",
+    "tor_allocate",
+    "tor_log",
+    "tor_rust",
+    "tor_util",
+]
+
+# Can remove panic="abort" when this issue is fixed:
+# https://github.com/rust-lang/rust/issues/52652
+[profile.dev]
+panic = "abort"
+
+[profile.release]
+debug = true
+panic = "abort"
+
+[profile.test]
+panic = "abort"
+
+[profile.bench]
+panic = "abort"
diff --git a/src/rust/build.rs b/src/rust/build.rs
new file mode 100644
index 0000000000..b943aa5535
--- /dev/null
+++ b/src/rust/build.rs
@@ -0,0 +1,179 @@
+//! Build script for Rust modules in Tor.
+//!
+//! We need to use this because some of our Rust tests need to use some
+//! of our C modules, which need to link some external libraries.
+//!
+//! This script works by looking at a "config.rust" file generated by our
+//! configure script, and then building a set of options for cargo to pass to
+//! the compiler.
+
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+use std::io::prelude::*;
+use std::io;
+use std::path::PathBuf;
+
+/// Wrapper around a key-value map.
+struct Config(
+    HashMap<String,String>
+);
+
+/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR
+/// location provided by cargo and recursing up the directory tree.  Note that
+/// we need to look in the OUT_DIR, since autoconf will place generated files
+/// in the build directory.
+fn find_cfg() -> io::Result<String> {
+    let mut path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    loop {
+        path.push("config.rust");
+        if path.exists() {
+            return Ok(path.to_str().unwrap().to_owned());
+        }
+        path.pop(); // remove config.rust
+        if ! path.pop() { // can't remove last part of directory
+            return Err(io::Error::new(io::ErrorKind::NotFound,
+                                      "No config.rust"));
+        }
+    }
+}
+
+impl Config {
+    /// Find the config.rust file and try to parse it.
+    ///
+    /// The file format is a series of lines of the form KEY=VAL, with
+    /// any blank lines and lines starting with # ignored.
+    fn load() -> io::Result<Config> {
+        let path = find_cfg()?;
+        let f = File::open(&path)?;
+        let reader = io::BufReader::new(f);
+        let mut map = HashMap::new();
+        for line in reader.lines() {
+            let s = line?;
+            if s.trim().starts_with("#") || s.trim() == "" {
+                continue;
+            }
+            let idx = match s.find("=") {
+                None => {
+                    return Err(io::Error::new(io::ErrorKind::InvalidData,
+                                              "missing ="));
+                },
+                Some(x) => x
+            };
+            let (var,eq_val) = s.split_at(idx);
+            let val = &eq_val[1..];
+            map.insert(var.to_owned(), val.to_owned());
+        }
+        Ok(Config(map))
+    }
+
+    /// Return a reference to the value whose key is 'key'.
+    ///
+    /// Panics if 'key' is not found in the configuration.
+    fn get(&self, key : &str) -> &str {
+        self.0.get(key).unwrap()
+    }
+
+    /// Add a dependency on a static C library that is part of Tor, by name.
+    fn component(&self, s : &str) {
+        println!("cargo:rustc-link-lib=static={}", s);
+    }
+
+    /// Add a dependency on a native library that is not part of Tor, by name.
+    fn dependency(&self, s : &str) {
+        println!("cargo:rustc-link-lib={}", s);
+    }
+
+    /// Add a link path, relative to Tor's build directory.
+    fn link_relpath(&self, s : &str) {
+        let builddir = self.get("BUILDDIR");
+        println!("cargo:rustc-link-search=native={}/{}", builddir, s);
+    }
+
+    /// Add an absolute link path.
+    fn link_path(&self, s : &str) {
+        println!("cargo:rustc-link-search=native={}", s);
+    }
+
+    /// Parse the CFLAGS in s, looking for -l and -L items, and adding
+    /// rust configuration as appropriate.
+    fn from_cflags(&self, s : &str) {
+        let mut next_is_lib = false;
+        let mut next_is_path = false;
+        for ent in self.get(s).split_whitespace() {
+            if next_is_lib {
+                self.dependency(ent);
+                next_is_lib = false;
+            } else if next_is_path {
+                self.link_path(ent);
+                next_is_path = false;
+            } else if ent == "-l" {
+                next_is_lib = true;
+            } else if ent == "-L" {
+                next_is_path = true;
+            } else if ent.starts_with("-L") {
+                self.link_path(&ent[2..]);
+            } else if ent.starts_with("-l") {
+                self.dependency(&ent[2..]);
+            }
+        }
+    }
+}
+
+pub fn main() {
+    let cfg = Config::load().unwrap();
+    let package = env::var("CARGO_PKG_NAME").unwrap();
+
+    match package.as_ref() {
+        "crypto" => {
+            // Right now, I'm having a separate configuration for each Rust
+            // package, since I'm hoping we can trim them down.  Once we have a
+            // second Rust package that needs to use this build script, let's
+            // extract some of this stuff into a module.
+            //
+            // This is a ridiculous amount of code to be pulling in just
+            // to test our crypto library: modularity would be our
+            // friend here.
+            cfg.from_cflags("TOR_LDFLAGS_zlib");
+            cfg.from_cflags("TOR_LDFLAGS_openssl");
+            cfg.from_cflags("TOR_LDFLAGS_libevent");
+
+            cfg.link_relpath("src/common");
+            cfg.link_relpath("src/ext/keccak-tiny");
+            cfg.link_relpath("src/ext/keccak-tiny");
+            cfg.link_relpath("src/ext/ed25519/ref10");
+            cfg.link_relpath("src/ext/ed25519/donna");
+            cfg.link_relpath("src/trunnel");
+
+            // Note that we can't pull in "libtor-testing", or else we
+            // will have dependencies on all the other rust packages that
+            // tor uses.  We must be careful with factoring and dependencies
+            // moving forward!
+            cfg.component("or-crypto-testing");
+            cfg.component("or-ctime-testing");
+            cfg.component("or-testing");
+            cfg.component("or-event-testing");
+            cfg.component("or-ctime-testing");
+            cfg.component("curve25519_donna");
+            cfg.component("keccak-tiny");
+            cfg.component("ed25519_ref10");
+            cfg.component("ed25519_donna");
+            cfg.component("or-trunnel-testing");
+
+            cfg.from_cflags("TOR_ZLIB_LIBS");
+            cfg.from_cflags("TOR_LIB_MATH");
+            cfg.from_cflags("TOR_OPENSSL_LIBS");
+            cfg.from_cflags("TOR_LIBEVENT_LIBS");
+            cfg.from_cflags("TOR_LIB_WS32");
+            cfg.from_cflags("TOR_LIB_GDI");
+            cfg.from_cflags("TOR_LIB_USERENV");
+            cfg.from_cflags("CURVE25519_LIBS");
+            cfg.from_cflags("TOR_LZMA_LIBS");
+            cfg.from_cflags("TOR_ZSTD_LIBS");
+            cfg.from_cflags("LIBS");
+        },
+        _ => {
+            panic!("No configuration in build.rs for package {}", package);
+        }
+    }
+}
diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml
new file mode 100644
index 0000000000..869e0d6256
--- /dev/null
+++ b/src/rust/crypto/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+authors = ["The Tor Project",
+           "Isis Lovecruft <isis@torproject.org>"]
+name = "crypto"
+version = "0.0.1"
+publish = false
+build = "../build.rs"
+
+[lib]
+name = "crypto"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
+[dependencies]
+libc = "=0.2.39"
+digest = "=0.7.2"
+rand_core = { version = "=0.2.0-pre.0", default-features = false }
+
+external = { path = "../external" }
+smartlist = { path = "../smartlist" }
+tor_allocate = { path = "../tor_allocate" }
+tor_log = { path = "../tor_log" }
+
+[dev-dependencies]
+rand = { version = "=0.5.0-pre.2", default-features = false }
+rand_core = { version = "=0.2.0-pre.0", default-features = false }
+
+[features]
diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs
new file mode 100644
index 0000000000..a2463b89eb
--- /dev/null
+++ b/src/rust/crypto/digests/mod.rs
@@ -0,0 +1,7 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Hash Digests and eXtendible Output Functions (XOFs)
+
+pub mod sha2;
diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs
new file mode 100644
index 0000000000..03e0843dc0
--- /dev/null
+++ b/src/rust/crypto/digests/sha2.rs
@@ -0,0 +1,222 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Hash Digests and eXtendible Output Functions (XOFs)
+
+pub use digest::Digest;
+
+use digest::BlockInput;
+use digest::FixedOutput;
+use digest::Input;
+use digest::generic_array::GenericArray;
+use digest::generic_array::typenum::U32;
+use digest::generic_array::typenum::U64;
+
+use external::crypto_digest::CryptoDigest;
+use external::crypto_digest::DigestAlgorithm;
+use external::crypto_digest::get_256_bit_digest;
+use external::crypto_digest::get_512_bit_digest;
+
+pub use external::crypto_digest::DIGEST256_LEN;
+pub use external::crypto_digest::DIGEST512_LEN;
+
+/// The block size for both SHA-256 and SHA-512 digests is 512 bits/64 bytes.
+///
+/// Unfortunately, we have to use the generic_array crate currently to express
+/// this at compile time.  Later, in the future, when Rust implements const
+/// generics, we'll be able to remove this dependency (actually, it will get
+/// removed from the digest crate, which is currently `pub use`ing it).
+type BlockSize = U64;
+
+/// A SHA2-256 digest.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_dup`
+#[derive(Clone)]
+pub struct Sha256 {
+    engine: CryptoDigest,
+}
+
+/// Construct a new, default instance of a `Sha256` hash digest function.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha256, Digest};
+///
+/// let mut hasher: Sha256 = Sha256::default();
+/// ```
+///
+/// # Returns
+///
+/// A new `Sha256` digest.
+impl Default for Sha256 {
+    fn default() -> Sha256 {
+        Sha256{ engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_256)) }
+    }
+}
+
+impl BlockInput for Sha256 {
+    type BlockSize = BlockSize;
+}
+
+/// Input `msg` into the digest.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha256, Digest};
+///
+/// let mut hasher: Sha256 = Sha256::default();
+///
+/// hasher.input(b"foo");
+/// hasher.input(b"bar");
+/// ```
+impl Input for Sha256 {
+    fn process(&mut self, msg: &[u8]) {
+        self.engine.add_bytes(&msg);
+    }
+}
+
+/// Retrieve the output hash from everything which has been fed into this
+/// `Sha256` digest thus far.
+///
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest in external::crypto_digest.
+impl FixedOutput for Sha256 {
+    type OutputSize = U32;
+
+    fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> {
+        let buffer: [u8; DIGEST256_LEN] = get_256_bit_digest(self.engine);
+
+        GenericArray::from(buffer)
+    }
+}
+
+/// A SHA2-512 digest.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_dup`
+#[derive(Clone)]
+pub struct Sha512 {
+    engine: CryptoDigest,
+}
+
+/// Construct a new, default instance of a `Sha512` hash digest function.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha512, Digest};
+///
+/// let mut hasher: Sha512 = Sha512::default();
+/// ```
+///
+/// # Returns
+///
+/// A new `Sha512` digest.
+impl Default for Sha512 {
+    fn default() -> Sha512 {
+        Sha512{ engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_512)) }
+    }
+}
+
+impl BlockInput for Sha512 {
+    type BlockSize = BlockSize;
+}
+
+/// Input `msg` into the digest.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use crypto::digests::sha2::{Sha512, Digest};
+///
+/// let mut hasher: Sha512 = Sha512::default();
+///
+/// hasher.input(b"foo");
+/// hasher.input(b"bar");
+/// ```
+impl Input for Sha512 {
+    fn process(&mut self, msg: &[u8]) {
+        self.engine.add_bytes(&msg);
+    }
+}
+
+/// Retrieve the output hash from everything which has been fed into this
+/// `Sha512` digest thus far.
+///
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest in external::crypto_digest.
+impl FixedOutput for Sha512 {
+    type OutputSize = U64;
+
+    fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> {
+        let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine);
+
+        GenericArray::clone_from_slice(&buffer)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use digest::Digest;
+
+    use super::*;
+
+    #[test]
+    fn sha256_default() {
+        let _: Sha256 = Sha256::default();
+    }
+
+    #[test]
+    fn sha256_digest() {
+        let mut h: Sha256 = Sha256::new();
+        let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN];
+        let expected = [151, 223, 53, 136, 181, 163, 242, 75, 171, 195,
+                        133, 27, 55, 47, 11, 167, 26, 157, 205, 222, 212,
+                        59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157];
+
+        h.input(b"foo");
+        h.input(b"bar");
+        h.input(b"baz");
+
+        result.copy_from_slice(h.fixed_result().as_slice());
+
+        println!("{:?}", &result[..]);
+
+        assert_eq!(result, expected);
+    }
+
+    #[test]
+    fn sha512_default() {
+        let _: Sha512 = Sha512::default();
+    }
+
+    #[test]
+    fn sha512_digest() {
+        let mut h: Sha512 = Sha512::new();
+        let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN];
+
+        let expected = [203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167,
+                153, 217, 233, 8, 190, 69, 231, 103, 245, 209, 71, 212, 116,
+                73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20, 122,
+                221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38,
+                179, 237, 61, 126, 246, 240, 103, 202, 153, 24, 90];
+
+        h.input(b"foo");
+        h.input(b"bar");
+        h.input(b"baz");
+
+        result.copy_from_slice(h.fixed_result().as_slice());
+
+        println!("{:?}", &result[..]);
+
+        assert_eq!(&result[..], &expected[..]);
+    }
+}
diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs
new file mode 100644
index 0000000000..f72a859dd7
--- /dev/null
+++ b/src/rust/crypto/lib.rs
@@ -0,0 +1,45 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Common cryptographic functions and utilities.
+//!
+//! # Hash Digests and eXtendable Output Functions (XOFs)
+//!
+//! The `digests` module contains submodules for specific hash digests
+//! and extendable output functions.
+//!
+//! ```rust,no_run
+//! use crypto::digests::sha2::*;
+//!
+//! let mut hasher: Sha256 = Sha256::default();
+//! let mut result: [u8; 32] = [0u8; 32];
+//!
+//! hasher.input(b"foo");
+//! hasher.input(b"bar");
+//! hasher.input(b"baz");
+//!
+//! result.copy_from_slice(hasher.result().as_slice());
+//!
+//! assert!(result == [b'X'; DIGEST256_LEN]);
+//! ```
+
+#[deny(missing_docs)]
+
+// External crates from cargo or TOR_RUST_DEPENDENCIES.
+extern crate digest;
+extern crate libc;
+extern crate rand_core;
+
+// External dependencies for tests.
+#[cfg(test)]
+extern crate rand as rand_crate;
+
+// Our local crates.
+extern crate external;
+#[cfg(not(test))]
+#[macro_use]
+extern crate tor_log;
+
+pub mod digests;  // Unfortunately named "digests" plural to avoid name conflict with the digest crate
+pub mod rand;
diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs
new file mode 100644
index 0000000000..82d02a70bb
--- /dev/null
+++ b/src/rust/crypto/rand/mod.rs
@@ -0,0 +1,6 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+// Internal dependencies
+pub mod rng;
diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs
new file mode 100644
index 0000000000..07a0a7bdc7
--- /dev/null
+++ b/src/rust/crypto/rand/rng.rs
@@ -0,0 +1,140 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Wrappers for Tor's random number generators to provide implementations of
+//! `rand_core` traits.
+
+// This is the real implementation, in use in production, which calls into our C
+// wrappers in /src/common/crypto_rand.c, which call into OpenSSL, system
+// libraries, and make syscalls.
+#[cfg(not(test))]
+mod internal {
+    use std::u64;
+
+    use rand_core::CryptoRng;
+    use rand_core::Error;
+    use rand_core::RngCore;
+    use rand_core::impls::next_u32_via_fill;
+    use rand_core::impls::next_u64_via_fill;
+
+    use external::c_tor_crypto_rand;
+    use external::c_tor_crypto_strongest_rand;
+    use external::c_tor_crypto_seed_rng;
+
+    use tor_log::LogDomain;
+    use tor_log::LogSeverity;
+
+    /// Largest strong entropy request permitted.
+    //
+    // C_RUST_COUPLED: `MAX_STRONGEST_RAND_SIZE` /src/common/crypto_rand.c
+    const MAX_STRONGEST_RAND_SIZE: usize = 256;
+
+    /// A wrapper around OpenSSL's RNG.
+    pub struct TorRng {
+        // This private, zero-length field forces the struct to be treated the
+        // same as its opaque C couterpart.
+        _unused: [u8; 0],
+    }
+
+    /// Mark `TorRng` as being suitable for cryptographic purposes.
+    impl CryptoRng for TorRng {}
+
+    impl TorRng {
+        // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c
+        #[allow(dead_code)]
+        pub fn new() -> Self {
+            if !c_tor_crypto_seed_rng() {
+                tor_log_msg!(LogSeverity::Warn, LogDomain::General,
+                             "TorRng::from_seed()",
+                             "The RNG could not be seeded!");
+            }
+            // XXX also log success at info level —isis
+            TorRng{ _unused: [0u8; 0] }
+        }
+    }
+
+    impl RngCore for TorRng {
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u32(&mut self) -> u32 {
+            next_u32_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u64(&mut self) -> u64 {
+            next_u64_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            c_tor_crypto_rand(dest);
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
+            Ok(self.fill_bytes(dest))
+        }
+    }
+
+    /// A CSPRNG which hashes together randomness from OpenSSL's RNG and entropy
+    /// obtained from the operating system.
+    pub struct TorStrongestRng {
+        // This private, zero-length field forces the struct to be treated the
+        // same as its opaque C couterpart.
+        _unused: [u8; 0],
+    }
+
+    /// Mark `TorRng` as being suitable for cryptographic purposes.
+    impl CryptoRng for TorStrongestRng {}
+
+    impl TorStrongestRng {
+        // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c
+        #[allow(dead_code)]
+        pub fn new() -> Self {
+            if !c_tor_crypto_seed_rng() {
+                tor_log_msg!(LogSeverity::Warn, LogDomain::General,
+                             "TorStrongestRng::from_seed()",
+                             "The RNG could not be seeded!");
+            }
+            // XXX also log success at info level —isis
+            TorStrongestRng{ _unused: [0u8; 0] }
+        }
+    }
+
+    impl RngCore for TorStrongestRng {
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u32(&mut self) -> u32 {
+            next_u32_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn next_u64(&mut self) -> u64 {
+            next_u64_via_fill(self)
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn fill_bytes(&mut self, dest: &mut [u8]) {
+            debug_assert!(dest.len() <= MAX_STRONGEST_RAND_SIZE);
+
+            c_tor_crypto_strongest_rand(dest);
+        }
+
+        // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c
+        fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> {
+            Ok(self.fill_bytes(dest))
+        }
+    }
+}
+
+// For testing, we expose a pure-Rust implementation.
+#[cfg(test)]
+mod internal {
+    // It doesn't matter if we pretend ChaCha is a CSPRNG in tests.
+    pub use rand_crate::ChaChaRng as TorRng;
+    pub use rand_crate::ChaChaRng as TorStrongestRng;
+}
+
+// Finally, expose the public functionality of whichever appropriate internal
+// module.
+pub use self::internal::*;
+
diff --git a/src/rust/external/Cargo.toml b/src/rust/external/Cargo.toml
new file mode 100644
index 0000000000..60ec03be40
--- /dev/null
+++ b/src/rust/external/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "external"
+
+[dependencies]
+libc = "=0.2.39"
+
+[dependencies.smartlist]
+path = "../smartlist"
+
+[lib]
+name = "external"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs
new file mode 100644
index 0000000000..3e8801f203
--- /dev/null
+++ b/src/rust/external/crypto_digest.rs
@@ -0,0 +1,406 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Bindings to external digest and XOF functions which live within
+//! src/common/crypto_digest.[ch].
+//!
+//! We wrap our C implementations in src/common/crypto_digest.[ch] with more
+//! Rusty types and interfaces in src/rust/crypto/digest/.
+
+use std::process::abort;
+
+use libc::c_char;
+use libc::c_int;
+use libc::size_t;
+use libc::uint8_t;
+
+use smartlist::Stringlist;
+
+/// Length of the output of our message digest.
+pub const DIGEST_LEN: usize = 20;
+
+/// Length of the output of our second (improved) message digests.  (For now
+/// this is just sha256, but it could be any other 256-bit digest.)
+pub const DIGEST256_LEN: usize = 32;
+
+/// Length of the output of our 64-bit optimized message digests (SHA512).
+pub const DIGEST512_LEN: usize = 64;
+
+/// Length of a sha1 message digest when encoded in base32 with trailing = signs
+/// removed.
+pub const BASE32_DIGEST_LEN: usize = 32;
+
+/// Length of a sha1 message digest when encoded in base64 with trailing = signs
+/// removed.
+pub const BASE64_DIGEST_LEN: usize = 27;
+
+/// Length of a sha256 message digest when encoded in base64 with trailing =
+/// signs removed.
+pub const BASE64_DIGEST256_LEN: usize = 43;
+
+/// Length of a sha512 message digest when encoded in base64 with trailing =
+/// signs removed.
+pub const BASE64_DIGEST512_LEN: usize = 86;
+
+/// Length of hex encoding of SHA1 digest, not including final NUL.
+pub const HEX_DIGEST_LEN: usize = 40;
+
+/// Length of hex encoding of SHA256 digest, not including final NUL.
+pub const HEX_DIGEST256_LEN: usize = 64;
+
+/// Length of hex encoding of SHA512 digest, not including final NUL.
+pub const HEX_DIGEST512_LEN: usize = 128;
+
+/// Our C code uses an enum to declare the digest algorithm types which we know
+/// about.  However, because enums are implementation-defined in C, we can
+/// neither work with them directly nor translate them into Rust enums.
+/// Instead, we represent them as a u8 (under the assumption that we'll never
+/// support more than 256 hash functions).
+#[allow(non_camel_case_types)]
+type digest_algorithm_t = u8;
+
+const DIGEST_SHA1: digest_algorithm_t = 0;
+const DIGEST_SHA256: digest_algorithm_t = 1;
+const DIGEST_SHA512: digest_algorithm_t = 2;
+const DIGEST_SHA3_256: digest_algorithm_t = 3;
+const DIGEST_SHA3_512: digest_algorithm_t = 4;
+
+/// The number of hash digests we produce for a `common_digests_t`.
+///
+/// We can't access these from Rust, because their definitions in C require
+/// introspecting the `digest_algorithm_t` typedef, which is an enum, so we have
+/// to redefine them here.
+const N_COMMON_DIGEST_ALGORITHMS: usize = DIGEST_SHA256 as usize + 1;
+
+/// A digest function.
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct crypto_digest_t {
+    // This private, zero-length field forces the struct to be treated the same
+    // as its opaque C couterpart.
+    _unused: [u8; 0],
+}
+
+/// An eXtendible Output Function (XOF).
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct crypto_xof_t {
+    // This private, zero-length field forces the struct to be treated the same
+    // as its opaque C couterpart.
+    _unused: [u8; 0],
+}
+
+/// A set of all the digests we commonly compute, taken on a single
+/// string.  Any digests that are shorter than 512 bits are right-padded
+/// with 0 bits.
+///
+/// Note that this representation wastes 44 bytes for the SHA1 case, so
+/// don't use it for anything where we need to allocate a whole bunch at
+/// once.
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct common_digests_t {
+    pub d: [[c_char; N_COMMON_DIGEST_ALGORITHMS]; DIGEST256_LEN],
+}
+
+/// A `smartlist_t` is just an alias for the `#[repr(C)]` type `Stringlist`, to
+/// make it more clear that we're working with a smartlist which is owned by C.
+#[allow(non_camel_case_types)]
+// BINDGEN_GENERATED: This type isn't actually bindgen generated, but the code
+// below it which uses it is.  As such, this comes up as "dead code" as well.
+#[allow(dead_code)]
+type smartlist_t = Stringlist;
+
+/// All of the external functions from `src/common/crypto_digest.h`.
+///
+/// These are kept private because they should be wrapped with Rust to make their usage safer.
+//
+// BINDGEN_GENERATED: These definitions were generated with bindgen and cleaned
+// up manually.  As such, there are more bindings than are likely necessary or
+// which are in use.
+#[allow(dead_code)]
+extern "C" {
+    fn crypto_digest(digest: *mut c_char, m: *const c_char, len: size_t) -> c_int;
+    fn crypto_digest256(digest: *mut c_char, m: *const c_char, len: size_t,
+                        algorithm: digest_algorithm_t) -> c_int;
+    fn crypto_digest512(digest: *mut c_char, m: *const c_char, len: size_t,
+                        algorithm: digest_algorithm_t) -> c_int;
+    fn crypto_common_digests(ds_out: *mut common_digests_t, m: *const c_char, len: size_t) -> c_int;
+    fn crypto_digest_smartlist_prefix(digest_out: *mut c_char, len_out: size_t, prepend: *const c_char,
+                                      lst: *const smartlist_t, append: *const c_char, alg: digest_algorithm_t);
+    fn crypto_digest_smartlist(digest_out: *mut c_char, len_out: size_t,
+                               lst: *const smartlist_t, append: *const c_char, alg: digest_algorithm_t);
+    fn crypto_digest_algorithm_get_name(alg: digest_algorithm_t) -> *const c_char;
+    fn crypto_digest_algorithm_get_length(alg: digest_algorithm_t) -> size_t;
+    fn crypto_digest_algorithm_parse_name(name: *const c_char) -> c_int;
+    fn crypto_digest_new() -> *mut crypto_digest_t;
+    fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t;
+    fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t;
+    fn crypto_digest_free_(digest: *mut crypto_digest_t);
+    fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t);
+    fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t);
+    fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t;
+    fn crypto_digest_assign(into: *mut crypto_digest_t, from: *const crypto_digest_t);
+    fn crypto_hmac_sha256(hmac_out: *mut c_char, key: *const c_char, key_len: size_t,
+                          msg: *const c_char, msg_len: size_t);
+    fn crypto_mac_sha3_256(mac_out: *mut uint8_t, len_out: size_t,
+                           key: *const uint8_t, key_len: size_t,
+                           msg: *const uint8_t, msg_len: size_t);
+    fn crypto_xof_new() -> *mut crypto_xof_t;
+    fn crypto_xof_add_bytes(xof: *mut crypto_xof_t, data: *const uint8_t, len: size_t);
+    fn crypto_xof_squeeze_bytes(xof: *mut crypto_xof_t, out: *mut uint8_t, len: size_t);
+    fn crypto_xof_free(xof: *mut crypto_xof_t);
+}
+
+/// A wrapper around a `digest_algorithm_t`.
+pub enum DigestAlgorithm {
+    SHA2_256,
+    SHA2_512,
+    SHA3_256,
+    SHA3_512,
+}
+
+impl From<DigestAlgorithm> for digest_algorithm_t {
+    fn from(digest: DigestAlgorithm) -> digest_algorithm_t {
+        match digest {
+            DigestAlgorithm::SHA2_256 => DIGEST_SHA256,
+            DigestAlgorithm::SHA2_512 => DIGEST_SHA512,
+            DigestAlgorithm::SHA3_256 => DIGEST_SHA3_256,
+            DigestAlgorithm::SHA3_512 => DIGEST_SHA3_512,
+        }
+    }
+}
+
+/// A wrapper around a mutable pointer to a `crypto_digest_t`.
+pub struct CryptoDigest(*mut crypto_digest_t);
+
+/// Explicitly copy the state of a `CryptoDigest` hash digest context.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_dup`
+impl Clone for CryptoDigest {
+    fn clone(&self) -> CryptoDigest {
+        let digest: *mut crypto_digest_t;
+
+        unsafe {
+            digest = crypto_digest_dup(self.0 as *const crypto_digest_t);
+        }
+
+        // See the note in the implementation of CryptoDigest for the
+        // reasoning for `abort()` here.
+        if digest.is_null() {
+            abort();
+        }
+
+        CryptoDigest(digest)
+    }
+}
+
+impl CryptoDigest {
+    /// A wrapper to call one of the C functions `crypto_digest_new`,
+    /// `crypto_digest256_new`, or `crypto_digest512_new`.
+    ///
+    /// # Warnings
+    ///
+    /// This function will `abort()` the entire process in an "abnormal" fashion,
+    /// i.e. not unwinding this or any other thread's stack, running any
+    /// destructors, or calling any panic/exit hooks) if `tor_malloc()` (called in
+    /// `crypto_digest256_new()`) is unable to allocate memory.
+    ///
+    /// # Returns
+    ///
+    /// A new `CryptoDigest`, which is a wrapper around a opaque representation
+    /// of a `crypto_digest_t`.  The underlying `crypto_digest_t` _MUST_ only
+    /// ever be handled via a raw pointer, and never introspected.
+    ///
+    /// # C_RUST_COUPLED
+    ///
+    /// * `crypto_digest_new`
+    /// * `crypto_digest256_new`
+    /// * `crypto_digest512_new`
+    /// * `tor_malloc` (called by `crypto_digest256_new`, but we make
+    ///    assumptions about its behvaiour and return values here)
+    pub fn new(algorithm: Option<DigestAlgorithm>) -> CryptoDigest {
+        let digest: *mut crypto_digest_t;
+
+        if algorithm.is_none() {
+            unsafe {
+                digest = crypto_digest_new();
+            }
+        } else {
+            let algo: digest_algorithm_t = algorithm.unwrap().into(); // can't fail because it's Some
+
+            unsafe {
+                // XXX This is a pretty awkward API to use from Rust...
+                digest = match algo {
+                    DIGEST_SHA1     => crypto_digest_new(),
+                    DIGEST_SHA256   => crypto_digest256_new(DIGEST_SHA256),
+                    DIGEST_SHA3_256 => crypto_digest256_new(DIGEST_SHA3_256),
+                    DIGEST_SHA512   => crypto_digest512_new(DIGEST_SHA512),
+                    DIGEST_SHA3_512 => crypto_digest512_new(DIGEST_SHA3_512),
+                    _               => abort(),
+                }
+            }
+        }
+
+        // In our C code, `crypto_digest*_new()` allocates memory with
+        // `tor_malloc()`.  In `tor_malloc()`, if the underlying malloc
+        // implementation fails to allocate the requested memory and returns a
+        // NULL pointer, we call `exit(1)`.  In the case that this `exit(1)` is
+        // called within a worker, be that a process or a thread, the inline
+        // comments within `tor_malloc()` mention "that's ok, since the parent
+        // will run out of memory soon anyway".  However, if it takes long
+        // enough for the worker to die, and it manages to return a NULL pointer
+        // to our Rust code, our Rust is now in an irreparably broken state and
+        // may exhibit undefined behaviour.  An even worse scenario, if/when we
+        // have parent/child processes/threads controlled by Rust, would be that
+        // the UB contagion in Rust manages to spread to other children before
+        // the entire process (hopefully terminates).
+        //
+        // However, following the assumptions made in `tor_malloc()` that
+        // calling `exit(1)` in a child is okay because the parent will
+        // eventually run into the same errors, and also to stymie any UB
+        // contagion in the meantime, we call abort!() here to terminate the
+        // entire program immediately.
+        if digest.is_null() {
+            abort();
+        }
+
+        CryptoDigest(digest)
+    }
+
+    /// A wrapper to call the C function `crypto_digest_add_bytes`.
+    ///
+    /// # Inputs
+    ///
+    /// * `bytes`: a byte slice of bytes to be added into this digest.
+    ///
+    /// # C_RUST_COUPLED
+    ///
+    /// * `crypto_digest_add_bytes`
+    pub fn add_bytes(&self, bytes: &[u8]) {
+        unsafe {
+            crypto_digest_add_bytes(self.0 as *mut crypto_digest_t,
+                                    bytes.as_ptr() as *const c_char,
+                                    bytes.len() as size_t)
+        }
+    }
+}
+
+impl Drop for CryptoDigest {
+    fn drop(&mut self) {
+        unsafe {
+            crypto_digest_free_(self.0 as *mut crypto_digest_t);
+        }
+    }
+}
+
+/// Get the 256-bit digest output of a `crypto_digest_t`.
+///
+/// # Inputs
+///
+/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA256` or a
+///   `DIGEST_SHA3_256`.
+///
+/// # Warning
+///
+/// Calling this function with a `CryptoDigest` which is neither SHA2-256 or
+/// SHA3-256 is a programming error.  Since we cannot introspect the opaque
+/// struct from Rust, however, there is no way for us to check that the correct
+/// one is being passed in.  That is up to you, dear programmer.  If you mess
+/// up, you will get a incorrectly-sized hash digest in return, and it will be
+/// your fault.  Don't do that.
+///
+/// # Returns
+///
+/// A 256-bit hash digest, as a `[u8; 32]`.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_get_digest`
+/// * `DIGEST256_LEN`
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest w.r.t. output array size.
+pub fn get_256_bit_digest(digest: CryptoDigest) -> [u8; DIGEST256_LEN] {
+    let mut buffer: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN];
+
+    unsafe {
+        crypto_digest_get_digest(digest.0,
+                                 buffer.as_mut_ptr() as *mut c_char,
+                                 DIGEST256_LEN as size_t);
+
+        if buffer.as_ptr().is_null() {
+            abort();
+        }
+    }
+    buffer
+}
+
+/// Get the 512-bit digest output of a `crypto_digest_t`.
+///
+/// # Inputs
+///
+/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA512` or a
+///   `DIGEST_SHA3_512`.
+///
+/// # Warning
+///
+/// Calling this function with a `CryptoDigest` which is neither SHA2-512 or
+/// SHA3-512 is a programming error.  Since we cannot introspect the opaque
+/// struct from Rust, however, there is no way for us to check that the correct
+/// one is being passed in.  That is up to you, dear programmer.  If you mess
+/// up, you will get a incorrectly-sized hash digest in return, and it will be
+/// your fault.  Don't do that.
+///
+/// # Returns
+///
+/// A 512-bit hash digest, as a `[u8; 64]`.
+///
+/// # C_RUST_COUPLED
+///
+/// * `crypto_digest_get_digest`
+/// * `DIGEST512_LEN`
+//
+// FIXME: Once const generics land in Rust, we should genericise calling
+// crypto_digest_get_digest w.r.t. output array size.
+pub fn get_512_bit_digest(digest: CryptoDigest) -> [u8; DIGEST512_LEN] {
+    let mut buffer: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN];
+
+    unsafe {
+        crypto_digest_get_digest(digest.0,
+                                 buffer.as_mut_ptr() as *mut c_char,
+                                 DIGEST512_LEN as size_t);
+
+        if buffer.as_ptr().is_null() {
+            abort();
+        }
+    }
+    buffer
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_layout_common_digests_t() {
+        assert_eq!(::std::mem::size_of::<common_digests_t>(), 64usize,
+                   concat!("Size of: ", stringify!(common_digests_t)));
+        assert_eq!(::std::mem::align_of::<common_digests_t>(), 1usize,
+                   concat!("Alignment of ", stringify!(common_digests_t)));
+    }
+
+    #[test]
+    fn test_layout_crypto_digest_t() {
+        assert_eq!(::std::mem::size_of::<crypto_digest_t>(), 0usize,
+                   concat!("Size of: ", stringify!(crypto_digest_t)));
+        assert_eq!(::std::mem::align_of::<crypto_digest_t>(), 1usize,
+                   concat!("Alignment of ", stringify!(crypto_digest_t)));
+    }
+}
diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs
new file mode 100644
index 0000000000..af1ade0161
--- /dev/null
+++ b/src/rust/external/crypto_rand.rs
@@ -0,0 +1,87 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Bindings to external (P)RNG interfaces and utilities in 
+//! src/common/crypto_rand.[ch].
+//!
+//! We wrap our C implementations in src/common/crypto_rand.[ch] here in order
+//! to provide wrappers with native Rust types, and then provide more Rusty
+//! types and and trait implementations in src/rust/crypto/rand/.
+
+use std::time::Duration;
+
+use libc::c_double;
+use libc::c_int;
+use libc::size_t;
+use libc::time_t;
+use libc::uint8_t;
+
+extern "C" {
+    fn crypto_seed_rng() -> c_int;
+    fn crypto_rand(out: *mut uint8_t, out_len: size_t);
+    fn crypto_strongest_rand(out: *mut uint8_t, out_len: size_t);
+    fn crypto_rand_time_range(min: time_t, max: time_t) -> time_t;
+    fn crypto_rand_double() -> c_double;
+}
+
+/// Seed OpenSSL's random number generator with bytes from the operating
+/// system.
+///
+/// # Returns
+///
+/// `true` on success; `false` on failure.
+pub fn c_tor_crypto_seed_rng() -> bool {
+    let ret: c_int;
+
+    unsafe {
+        ret = crypto_seed_rng();
+    }
+    match ret {
+        0 => return true,
+        _ => return false,
+    }
+}
+
+/// Fill the bytes of `dest` with random data.
+pub fn c_tor_crypto_rand(dest: &mut [u8]) {
+    unsafe {
+        crypto_rand(dest.as_mut_ptr(), dest.len() as size_t);
+    }
+}
+
+/// Fill the bytes of `dest` with "strong" random data by hashing
+/// together randomness obtained from OpenSSL's RNG and the operating
+/// system.
+pub fn c_tor_crypto_strongest_rand(dest: &mut [u8]) {
+    // We'll let the C side panic if the len is larger than
+    // MAX_STRONGEST_RAND_SIZE, rather than potentially panicking here.  A
+    // paranoid caller should assert on the length of dest *before* calling this
+    // function.
+    unsafe {
+        crypto_strongest_rand(dest.as_mut_ptr(), dest.len() as size_t);
+    }
+}
+
+/// Get a random time, in seconds since the Unix Epoch.
+///
+/// # Returns
+///
+/// A `std::time::Duration` of seconds since the Unix Epoch.
+pub fn c_tor_crypto_rand_time_range(min: &Duration, max: &Duration) -> Duration {
+    let ret: time_t;
+
+    unsafe {
+        ret = crypto_rand_time_range(min.as_secs() as time_t, max.as_secs() as time_t);
+    }
+
+    Duration::from_secs(ret as u64)
+}
+
+/// Return a pseudorandom 64-bit float, chosen uniformly from the range [0.0, 1.0).
+pub fn c_tor_crypto_rand_double() -> f64 {
+    unsafe {
+        crypto_rand_double()
+    }
+}
+
diff --git a/src/rust/external/external.rs b/src/rust/external/external.rs
new file mode 100644
index 0000000000..b9e17f021d
--- /dev/null
+++ b/src/rust/external/external.rs
@@ -0,0 +1,33 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+use libc::{c_char, c_int};
+use std::ffi::CString;
+
+extern "C" {
+    fn tor_version_as_new_as(
+        platform: *const c_char,
+        cutoff: *const c_char,
+    ) -> c_int;
+}
+
+/// Wrap calls to tor_version_as_new_as, defined in src/or/routerparse.c
+pub fn c_tor_version_as_new_as(platform: &str, cutoff: &str) -> bool {
+    // CHK: These functions should log a warning if an error occurs. This
+    // can be added when integration with tor's logger is added to rust
+    let c_platform = match CString::new(platform) {
+        Ok(n) => n,
+        Err(_) => return false,
+    };
+
+    let c_cutoff = match CString::new(cutoff) {
+        Ok(n) => n,
+        Err(_) => return false,
+    };
+
+    let result: c_int = unsafe {
+        tor_version_as_new_as(c_platform.as_ptr(), c_cutoff.as_ptr())
+    };
+
+    result == 1
+}
diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs
new file mode 100644
index 0000000000..b72a4f6e4c
--- /dev/null
+++ b/src/rust/external/lib.rs
@@ -0,0 +1,19 @@
+//! Copyright (c) 2016-2018, The Tor Project, Inc. */
+//! See LICENSE for licensing information */
+
+//! Interface for external calls to tor C ABI
+//!
+//! The purpose of this module is to provide a clean interface for when Rust
+//! modules need to interact with functionality in tor C code rather than each
+//! module implementing this functionality repeatedly.
+
+extern crate libc;
+
+extern crate smartlist;
+
+pub mod crypto_digest;
+mod crypto_rand;
+mod external;
+
+pub use crypto_rand::*;
+pub use external::*;
diff --git a/src/rust/include.am b/src/rust/include.am
new file mode 100644
index 0000000000..5e5b0b3faf
--- /dev/null
+++ b/src/rust/include.am
@@ -0,0 +1,41 @@
+include src/rust/tor_rust/include.am
+
+EXTRA_DIST +=\
+	src/rust/build.rs \
+	src/rust/Cargo.toml \
+	src/rust/Cargo.lock \
+	src/rust/.cargo/config.in \
+	src/rust/crypto/Cargo.toml \
+	src/rust/crypto/lib.rs \
+	src/rust/crypto/digests/mod.rs \
+	src/rust/crypto/digests/sha2.rs \
+	src/rust/crypto/rand/mod.rs \
+	src/rust/crypto/rand/rng.rs \
+	src/rust/external/Cargo.toml \
+	src/rust/external/crypto_digest.rs \
+	src/rust/external/crypto_rand.rs \
+	src/rust/external/external.rs \
+	src/rust/external/lib.rs \
+	src/rust/protover/Cargo.toml \
+	src/rust/protover/errors.rs \
+	src/rust/protover/protoset.rs \
+	src/rust/protover/ffi.rs \
+	src/rust/protover/lib.rs \
+	src/rust/protover/protover.rs \
+	src/rust/protover/tests/protover.rs \
+	src/rust/smartlist/Cargo.toml \
+	src/rust/smartlist/lib.rs \
+	src/rust/smartlist/smartlist.rs \
+	src/rust/tor_allocate/Cargo.toml \
+	src/rust/tor_allocate/lib.rs \
+	src/rust/tor_allocate/tor_allocate.rs \
+	src/rust/tor_log/Cargo.toml \
+	src/rust/tor_log/lib.rs \
+	src/rust/tor_log/tor_log.rs \
+	src/rust/tor_rust/Cargo.toml \
+	src/rust/tor_rust/include.am \
+	src/rust/tor_rust/lib.rs \
+	src/rust/tor_util/Cargo.toml \
+	src/rust/tor_util/ffi.rs \
+	src/rust/tor_util/lib.rs \
+	src/rust/tor_util/strings.rs
diff --git a/src/rust/protover/Cargo.toml b/src/rust/protover/Cargo.toml
new file mode 100644
index 0000000000..a8480e142a
--- /dev/null
+++ b/src/rust/protover/Cargo.toml
@@ -0,0 +1,30 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "protover"
+
+[features]
+
+[dependencies]
+libc = "=0.2.39"
+
+[dependencies.smartlist]
+path = "../smartlist"
+
+[dependencies.external]
+path = "../external"
+
+[dependencies.tor_util]
+path = "../tor_util"
+
+[dependencies.tor_allocate]
+path = "../tor_allocate"
+
+[dependencies.tor_log]
+path = "../tor_log"
+
+[lib]
+name = "protover"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
diff --git a/src/rust/protover/errors.rs b/src/rust/protover/errors.rs
new file mode 100644
index 0000000000..d9dc73381f
--- /dev/null
+++ b/src/rust/protover/errors.rs
@@ -0,0 +1,46 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Various errors which may occur during protocol version parsing.
+
+use std::fmt;
+use std::fmt::Display;
+
+/// All errors which may occur during protover parsing routines.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
+#[allow(missing_docs)] // See Display impl for error descriptions
+pub enum ProtoverError {
+    Overlap,
+    LowGreaterThanHigh,
+    Unparseable,
+    ExceedsMax,
+    ExceedsExpansionLimit,
+    UnknownProtocol,
+    ExceedsNameLimit,
+    InvalidProtocol,
+}
+
+/// Descriptive error messages for `ProtoverError` variants.
+impl Display for ProtoverError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ProtoverError::Overlap
+                => write!(f, "Two or more (low, high) protover ranges would overlap once expanded."),
+            ProtoverError::LowGreaterThanHigh
+                => write!(f, "The low in a (low, high) protover range was greater than high."),
+            ProtoverError::Unparseable
+                => write!(f, "The protover string was unparseable."),
+            ProtoverError::ExceedsMax
+                => write!(f, "The high in a (low, high) protover range exceeds u32::MAX."),
+            ProtoverError::ExceedsExpansionLimit
+                => write!(f, "The protover string would exceed the maximum expansion limit."),
+            ProtoverError::UnknownProtocol
+                => write!(f, "A protocol in the protover string we attempted to parse is unknown."),
+            ProtoverError::ExceedsNameLimit
+                => write!(f, "An unrecognised protocol name was too long."),
+            ProtoverError::InvalidProtocol
+                => write!(f, "A protocol name includes invalid characters."),
+        }
+    }
+}
diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs
new file mode 100644
index 0000000000..7386e988c5
--- /dev/null
+++ b/src/rust/protover/ffi.rs
@@ -0,0 +1,260 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! FFI functions, only to be called from C.
+//!
+//! Equivalent C versions of this api are in `src/or/protover.c`
+
+use libc::{c_char, c_int, uint32_t};
+use std::ffi::CStr;
+
+use smartlist::*;
+use tor_allocate::allocate_and_copy_string;
+
+use errors::ProtoverError;
+use protover::*;
+
+/// Translate C enums to Rust Proto enums, using the integer value of the C
+/// enum to map to its associated Rust enum.
+///
+/// C_RUST_COUPLED: src/or/protover.h `protocol_type_t`
+fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> {
+    match c_proto {
+        0 => Ok(Protocol::Link),
+        1 => Ok(Protocol::LinkAuth),
+        2 => Ok(Protocol::Relay),
+        3 => Ok(Protocol::DirCache),
+        4 => Ok(Protocol::HSDir),
+        5 => Ok(Protocol::HSIntro),
+        6 => Ok(Protocol::HSRend),
+        7 => Ok(Protocol::Desc),
+        8 => Ok(Protocol::Microdesc),
+        9 => Ok(Protocol::Cons),
+        _ => Err(ProtoverError::UnknownProtocol),
+    }
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::all_supported
+#[no_mangle]
+pub extern "C" fn protover_all_supported(
+    c_relay_version: *const c_char,
+    missing_out: *mut *mut c_char,
+) -> c_int {
+
+    if c_relay_version.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_relay_version) };
+
+    let relay_version = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+
+    let relay_proto_entry: UnvalidatedProtoEntry =
+        match UnvalidatedProtoEntry::from_str_any_len(relay_version) {
+        Ok(n)  => n,
+        Err(_) => return 1,
+    };
+    let maybe_unsupported: Option<UnvalidatedProtoEntry> = relay_proto_entry.all_supported();
+
+    if maybe_unsupported.is_some() {
+        let unsupported: UnvalidatedProtoEntry = maybe_unsupported.unwrap();
+        if missing_out.is_null() {
+            return 0;
+        }
+        let ptr = allocate_and_copy_string(&unsupported.to_string());
+        unsafe { *missing_out = ptr };
+
+        return 0;
+    }
+
+    1
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::list_supports_protocol
+#[no_mangle]
+pub extern "C" fn protocol_list_supports_protocol(
+    c_protocol_list: *const c_char,
+    c_protocol: uint32_t,
+    version: uint32_t,
+) -> c_int {
+    if c_protocol_list.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) };
+
+    let protocol_list = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+    let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() {
+        Ok(n)  => n,
+        Err(_) => return 0,
+    };
+    let protocol: UnknownProtocol = match translate_to_rust(c_protocol) {
+        Ok(n) => n.into(),
+        Err(_) => return 0,
+    };
+    match proto_entry.supports_protocol(&protocol, &version) {
+        false => return 0,
+        true  => return 1,
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn protover_contains_long_protocol_names_(
+    c_protocol_list: *const c_char
+) -> c_int {
+    if c_protocol_list.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) };
+
+    let protocol_list = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1
+    };
+
+    let protocol_entry : Result<UnvalidatedProtoEntry,_> =
+        protocol_list.parse();
+
+    match protocol_entry {
+        Ok(_) => 0,
+        Err(_) => 1,
+    }
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::list_supports_protocol_or_later
+#[no_mangle]
+pub extern "C" fn protocol_list_supports_protocol_or_later(
+    c_protocol_list: *const c_char,
+    c_protocol: uint32_t,
+    version: uint32_t,
+) -> c_int {
+    if c_protocol_list.is_null() {
+        return 1;
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) };
+
+    let protocol_list = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return 1,
+    };
+
+    let protocol = match translate_to_rust(c_protocol) {
+        Ok(n) => n,
+        Err(_) => return 0,
+    };
+
+    let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() {
+        Ok(n)  => n,
+        Err(_) => return 1,
+    };
+
+    if proto_entry.supports_protocol_or_later(&protocol.into(), &version) {
+        return 1;
+    }
+    0
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::get_supported_protocols
+#[no_mangle]
+pub extern "C" fn protover_get_supported_protocols() -> *const c_char {
+    let supported: &'static CStr;
+
+    supported = get_supported_protocols_cstr();
+    supported.as_ptr()
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::compute_vote
+//
+// Why is the threshold a signed integer? —isis
+#[no_mangle]
+pub extern "C" fn protover_compute_vote(
+    list: *const Stringlist,
+    threshold: c_int
+) -> *mut c_char {
+
+    if list.is_null() {
+        let empty = String::new();
+        return allocate_and_copy_string(&empty);
+    }
+
+    // Dereference of raw pointer requires an unsafe block. The pointer is
+    // checked above to ensure it is not null.
+    let data: Vec<String> = unsafe { (*list).get_list() };
+    let hold: usize = threshold as usize;
+    let mut proto_entries: Vec<UnvalidatedProtoEntry> = Vec::new();
+
+    for datum in data {
+        let entry: UnvalidatedProtoEntry = match datum.parse() {
+            Ok(n)  => n,
+            Err(_) => continue
+        };
+        proto_entries.push(entry);
+    }
+    let vote: UnvalidatedProtoEntry = ProtoverVote::compute(&proto_entries, &hold);
+
+    allocate_and_copy_string(&vote.to_string())
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::is_supported_here
+#[no_mangle]
+pub extern "C" fn protover_is_supported_here(
+    c_protocol: uint32_t,
+    version: uint32_t,
+) -> c_int {
+    let protocol = match translate_to_rust(c_protocol) {
+        Ok(n) => n,
+        Err(_) => return 0,
+    };
+
+    let is_supported = is_supported_here(&protocol, &version);
+
+    return if is_supported { 1 } else { 0 };
+}
+
+/// Provide an interface for C to translate arguments and return types for
+/// protover::compute_for_old_tor
+#[no_mangle]
+pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const c_char {
+    let supported: &'static CStr;
+    let empty: &'static CStr;
+
+    empty = cstr!("");
+
+    if version.is_null() {
+        return empty.as_ptr();
+    }
+
+    // Require an unsafe block to read the version from a C string. The pointer
+    // is checked above to ensure it is not null.
+    let c_str: &CStr = unsafe { CStr::from_ptr(version) };
+
+    let version = match c_str.to_str() {
+        Ok(n) => n,
+        Err(_) => return empty.as_ptr(),
+    };
+
+    supported = compute_for_old_tor_cstr(&version);
+    supported.as_ptr()
+}
diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs
new file mode 100644
index 0000000000..ce964196fd
--- /dev/null
+++ b/src/rust/protover/lib.rs
@@ -0,0 +1,39 @@
+//! Copyright (c) 2016-2017, The Tor Project, Inc. */
+//! See LICENSE for licensing information */
+
+//! Versioning information for different pieces of the Tor protocol.
+//!
+//! The below description is taken from src/rust/protover.c, which is currently
+//! enabled by default. We are in the process of experimenting with Rust in
+//! tor, and this protover module is implemented to help achieve this goal.
+//!
+//! Starting in version 0.2.9.3-alpha, Tor places separate version numbers on
+//! each of the different components of its protocol. Relays use these numbers
+//! to advertise what versions of the protocols they can support, and clients
+//! use them to find what they can ask a given relay to do.  Authorities vote
+//! on the supported protocol versions for each relay, and also vote on the
+//! which protocols you should have to support in order to be on the Tor
+//! network. All Tor instances use these required/recommended protocol versions
+//! to tell what level of support for recent protocols each relay has, and
+//! to decide whether they should be running given their current protocols.
+//!
+//! The main advantage of these protocol versions numbers over using Tor
+//! version numbers is that they allow different implementations of the Tor
+//! protocols to develop independently, without having to claim compatibility
+//! with specific versions of Tor.
+
+#[deny(missing_docs)]
+
+extern crate libc;
+extern crate smartlist;
+extern crate external;
+extern crate tor_allocate;
+#[macro_use]
+extern crate tor_util;
+
+pub mod errors;
+pub mod protoset;
+mod protover;
+pub mod ffi;
+
+pub use protover::*;
diff --git a/src/rust/protover/protoset.rs b/src/rust/protover/protoset.rs
new file mode 100644
index 0000000000..465b8f2850
--- /dev/null
+++ b/src/rust/protover/protoset.rs
@@ -0,0 +1,673 @@
+// Copyright (c) 2018, The Tor Project, Inc.
+// Copyright (c) 2018, isis agora lovecruft
+// See LICENSE for licensing information
+
+//! Sets for lazily storing ordered, non-overlapping ranges of integers.
+
+use std::cmp;
+use std::iter;
+use std::slice;
+use std::str::FromStr;
+use std::u32;
+
+use errors::ProtoverError;
+
+/// A single version number.
+pub type Version = u32;
+
+/// A `ProtoSet` stores an ordered `Vec<T>` of `(low, high)` pairs of ranges of
+/// non-overlapping protocol versions.
+///
+/// # Examples
+///
+/// ```
+/// use std::str::FromStr;
+///
+/// use protover::errors::ProtoverError;
+/// use protover::protoset::ProtoSet;
+/// use protover::protoset::Version;
+///
+/// # fn do_test() -> Result<ProtoSet, ProtoverError> {
+/// let protoset: ProtoSet = ProtoSet::from_str("3-5,8")?;
+///
+/// // We could also equivalently call:
+/// let protoset: ProtoSet = "3-5,8".parse()?;
+///
+/// assert!(protoset.contains(&4));
+/// assert!(!protoset.contains(&7));
+///
+/// let expanded: Vec<Version> = protoset.clone().into();
+///
+/// assert_eq!(&expanded[..], &[3, 4, 5, 8]);
+///
+/// let contracted: String = protoset.clone().to_string();
+///
+/// assert_eq!(contracted, "3-5,8".to_string());
+/// # Ok(protoset)
+/// # }
+/// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct ProtoSet {
+    pub(crate) pairs: Vec<(Version, Version)>,
+}
+
+impl Default for ProtoSet {
+    fn default() -> Self {
+        let pairs: Vec<(Version, Version)> = Vec::new();
+
+        ProtoSet{ pairs }
+    }
+}
+
+impl<'a> ProtoSet {
+    /// Create a new `ProtoSet` from a slice of `(low, high)` pairs.
+    ///
+    /// # Inputs
+    ///
+    /// We do not assume the input pairs are deduplicated or ordered.
+    pub fn from_slice(low_high_pairs: &'a [(Version, Version)]) -> Result<Self, ProtoverError> {
+        let mut pairs: Vec<(Version, Version)> = Vec::with_capacity(low_high_pairs.len());
+
+        for &(low, high) in low_high_pairs {
+            pairs.push((low, high));
+        }
+        // Sort the pairs without reallocation and remove all duplicate pairs.
+        pairs.sort_unstable();
+        pairs.dedup();
+
+        ProtoSet{ pairs }.is_ok()
+    }
+}
+
+/// Expand this `ProtoSet` to a `Vec` of all its `Version`s.
+///
+/// # Examples
+///
+/// ```
+/// use std::str::FromStr;
+/// use protover::protoset::ProtoSet;
+/// use protover::protoset::Version;
+/// # use protover::errors::ProtoverError;
+///
+/// # fn do_test() -> Result<Vec<Version>, ProtoverError> {
+/// let protoset: ProtoSet = ProtoSet::from_str("3-5,21")?;
+/// let versions: Vec<Version> = protoset.into();
+///
+/// assert_eq!(&versions[..], &[3, 4, 5, 21]);
+/// #
+/// # Ok(versions)
+/// # }
+/// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+/// ```
+impl Into<Vec<Version>> for ProtoSet {
+    fn into(self) -> Vec<Version> {
+        let mut versions: Vec<Version> = Vec::new();
+
+        for &(low, high) in self.iter() {
+            versions.extend(low..high + 1);
+        }
+        versions
+    }
+}
+
+impl ProtoSet {
+    /// Get an iterator over the `(low, high)` `pairs` in this `ProtoSet`.
+    pub fn iter(&self) -> slice::Iter<(Version, Version)> {
+        self.pairs.iter()
+    }
+
+    /// Expand this `ProtoSet` into a `Vec` of all its `Version`s.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<bool, ProtoverError> {
+    /// let protoset: ProtoSet = "3-5,9".parse()?;
+    ///
+    /// assert_eq!(protoset.expand(), vec![3, 4, 5, 9]);
+    ///
+    /// let protoset: ProtoSet = "1,3,5-7".parse()?;
+    ///
+    /// assert_eq!(protoset.expand(), vec![1, 3, 5, 6, 7]);
+    /// #
+    /// # Ok(true)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    pub fn expand(self) -> Vec<Version> {
+        self.into()
+    }
+
+    pub fn len(&self) -> usize {
+        let mut length: usize = 0;
+
+        for &(low, high) in self.iter() {
+            length += (high as usize - low as usize) + 1;
+        }
+
+        length
+    }
+
+    /// Check that this `ProtoSet` is well-formed.
+    ///
+    /// This is automatically called in `ProtoSet::from_str()`.
+    ///
+    /// # Errors
+    ///
+    /// * `ProtoverError::LowGreaterThanHigh`: if its `pairs` were not
+    ///   well-formed, i.e. a `low` in a `(low, high)` was higher than the
+    ///   previous `high`,
+    /// * `ProtoverError::Overlap`: if one or more of the `pairs` are
+    ///   overlapping,
+    /// * `ProtoverError::ExceedsMax`: if the number of versions when expanded
+    ///   would exceed `MAX_PROTOCOLS_TO_EXPAND`, and
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` is this `Protoset`, and whose `Err` is one of the
+    /// errors enumerated in the Errors section above.
+    fn is_ok(self) -> Result<ProtoSet, ProtoverError> {
+        let mut last_high: Version = 0;
+
+        for &(low, high) in self.iter() {
+            if low == u32::MAX || high == u32::MAX {
+                return Err(ProtoverError::ExceedsMax);
+            }
+            if low <= last_high {
+                return Err(ProtoverError::Overlap);
+            } else if low > high {
+                return Err(ProtoverError::LowGreaterThanHigh);
+            }
+            last_high = high;
+        }
+
+        Ok(self)
+    }
+
+    /// Determine if this `ProtoSet` contains no `Version`s.
+    ///
+    /// # Returns
+    ///
+    /// * `true` if this `ProtoSet`'s length is zero, and
+    /// * `false` otherwise.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// let protoset: ProtoSet = ProtoSet::default();
+    ///
+    /// assert!(protoset.is_empty());
+    /// ```
+    pub fn is_empty(&self) -> bool {
+        self.pairs.len() == 0
+    }
+
+    /// Determine if `version` is included within this `ProtoSet`.
+    ///
+    /// # Inputs
+    ///
+    /// * `version`: a `Version`.
+    ///
+    /// # Returns
+    ///
+    /// `true` if the `version` is contained within this set; `false` otherwise.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<ProtoSet, ProtoverError> {
+    /// let protoset: ProtoSet = ProtoSet::from_slice(&[(0, 5), (7, 9), (13, 14)])?;
+    ///
+    /// assert!(protoset.contains(&5));
+    /// assert!(!protoset.contains(&10));
+    /// #
+    /// # Ok(protoset)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    pub fn contains(&self, version: &Version) -> bool {
+        for &(low, high) in self.iter() {
+            if low <= *version && *version <= high {
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Returns all the `Version`s in `self` which are not also in the `other`
+    /// `ProtoSet`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<bool, ProtoverError> {
+    /// let protoset: ProtoSet = "1,3-6,10-12,15-16".parse()?;
+    /// let other: ProtoSet = "2,5-7,9-11,14-20".parse()?;
+    ///
+    /// let subset: ProtoSet = protoset.and_not_in(&other);
+    ///
+    /// assert_eq!(subset.expand(), vec![1, 3, 4, 12]);
+    /// #
+    /// # Ok(true)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    pub fn and_not_in(&self, other: &Self) -> Self {
+        if self.is_empty() || other.is_empty() {
+            return self.clone();
+        }
+
+        let pairs = self.iter().flat_map(|&(lo, hi)| {
+            let the_end = (hi + 1, hi + 1); // special case to mark the end of the range.
+            let excluded_ranges = other
+                .iter()
+                .cloned() // have to be owned tuples, to match iter::once(the_end).
+                .skip_while(move|&(_, hi2)| hi2 < lo) // skip the non-overlapping ranges.
+                .take_while(move|&(lo2, _)| lo2 <= hi) // take all the overlapping ones.
+                .chain(iter::once(the_end));
+
+            let mut nextlo = lo;
+            excluded_ranges.filter_map(move |(excluded_lo, excluded_hi)| {
+                let pair = if nextlo < excluded_lo {
+                    Some((nextlo, excluded_lo - 1))
+                } else {
+                    None
+                };
+                nextlo = cmp::min(excluded_hi, u32::MAX - 1) + 1;
+                pair
+            })
+        });
+
+        let pairs = pairs.collect();
+        ProtoSet::is_ok(ProtoSet{ pairs }).expect("should be already sorted")
+    }
+}
+
+impl FromStr for ProtoSet {
+    type Err = ProtoverError;
+
+    /// Parse the unique version numbers supported by a subprotocol from a string.
+    ///
+    /// # Inputs
+    ///
+    /// * `version_string`, a string comprised of "[0-9,-]"
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` value is a `ProtoSet` holding all of the unique
+    /// version numbers.
+    ///
+    /// The returned `Result`'s `Err` value is an `ProtoverError` appropriate to
+    /// the error.
+    ///
+    /// # Errors
+    ///
+    /// This function will error if:
+    ///
+    /// * the `version_string` is an equals (`"="`) sign,
+    /// * the expansion of a version range produces an error (see
+    ///   `expand_version_range`),
+    /// * any single version number is not parseable as an `u32` in radix 10, or
+    /// * there are greater than 2^16 version numbers to expand.
+    ///
+    /// # Examples
+    /// 
+    /// ```
+    /// use std::str::FromStr;
+    ///
+    /// use protover::errors::ProtoverError;
+    /// use protover::protoset::ProtoSet;
+    ///
+    /// # fn do_test() -> Result<ProtoSet, ProtoverError> {
+    /// let protoset: ProtoSet = ProtoSet::from_str("2-5,8")?;
+    ///
+    /// assert!(protoset.contains(&5));
+    /// assert!(!protoset.contains(&10));
+    ///
+    /// // We can also equivalently call `ProtoSet::from_str` by doing (all
+    /// // implementations of `FromStr` can be called this way, this one isn't
+    /// // special):
+    /// let protoset: ProtoSet = "4-6,12".parse()?;
+    ///
+    /// // Calling it (either way) can take really large ranges (up to `u32::MAX`):
+    /// let protoset: ProtoSet = "1-70000".parse()?;
+    /// let protoset: ProtoSet = "1-4294967296".parse()?;
+    ///
+    /// // There are lots of ways to get an `Err` from this function.  Here are
+    /// // a few:
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("="));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-"));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("not_an_int"));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("3-"));
+    /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-,4"));
+    ///
+    /// // Things which would get parsed into an _empty_ `ProtoSet` are,
+    /// // however, legal, and result in an empty `ProtoSet`:
+    /// assert_eq!(Ok(ProtoSet::default()), ProtoSet::from_str(""));
+    /// assert_eq!(Ok(ProtoSet::default()), ProtoSet::from_str(",,,"));
+    /// #
+    /// # Ok(protoset)
+    /// # }
+    /// # fn main() { do_test(); }  // wrap the test so we can use the ? operator
+    /// ```
+    fn from_str(version_string: &str) -> Result<Self, Self::Err> {
+        let mut pairs: Vec<(Version, Version)> = Vec::new();
+        let pieces: ::std::str::Split<char> = version_string.split(',');
+
+        for p in pieces {
+            if p.is_empty() {
+                continue;
+            } else if p.contains('-') {
+                let mut pair = p.splitn(2, '-');
+
+                let low  = pair.next().ok_or(ProtoverError::Unparseable)?;
+                let high = pair.next().ok_or(ProtoverError::Unparseable)?;
+
+                let lo: Version =  low.parse().or(Err(ProtoverError::Unparseable))?;
+                let hi: Version = high.parse().or(Err(ProtoverError::Unparseable))?;
+
+                if lo == u32::MAX || hi == u32::MAX {
+                    return Err(ProtoverError::ExceedsMax);
+                }
+                pairs.push((lo, hi));
+            } else {
+                let v: u32 = p.parse().or(Err(ProtoverError::Unparseable))?;
+
+                if v == u32::MAX {
+                    return Err(ProtoverError::ExceedsMax);
+                }
+                pairs.push((v, v));
+            }
+        }
+        // If we were passed in an empty string, or
+        // simply a comma, or a pile of commas, then return an empty ProtoSet.
+        if pairs.len() == 0 {
+            return Ok(ProtoSet::default());
+        }
+        ProtoSet::from_slice(&pairs[..])
+    }
+}
+
+impl ToString for ProtoSet {
+    /// Contracts a `ProtoSet` of versions into a string.
+    ///
+    /// # Returns
+    ///
+    /// A `String` representation of this `ProtoSet` in ascending order.
+    fn to_string(&self) -> String {
+        let mut final_output: Vec<String> = Vec::new();
+
+        for &(lo, hi) in self.iter() {
+            if lo != hi {
+                debug_assert!(lo < hi);
+                final_output.push(format!("{}-{}", lo, hi));
+            } else {
+                final_output.push(format!("{}", lo));
+            }
+        }
+        final_output.join(",")
+    }
+}
+
+/// Checks to see if there is a continuous range of integers, starting at the
+/// first in the list. Returns the last integer in the range if a range exists.
+///
+/// # Inputs
+///
+/// `list`, an ordered  vector of `u32` integers of "[0-9,-]" representing the
+/// supported versions for a single protocol.
+///
+/// # Returns
+///
+/// A `bool` indicating whether the list contains a range, starting at the first
+/// in the list, a`Version` of the last integer in the range, and a `usize` of
+/// the index of that version.
+///
+/// For example, if given vec![1, 2, 3, 5], find_range will return true,
+/// as there is a continuous range, and 3, which is the last number in the
+/// continuous range, and 2 which is the index of 3.
+fn find_range(list: &Vec<Version>) -> (bool, Version, usize) {
+    if list.len() == 0 {
+        return (false, 0, 0);
+    }
+
+    let mut index: usize = 0;
+    let mut iterable = list.iter().peekable();
+    let mut range_end = match iterable.next() {
+        Some(n) => *n,
+        None => return (false, 0, 0),
+    };
+
+    let mut has_range = false;
+
+    while iterable.peek().is_some() {
+        let n = *iterable.next().unwrap();
+        if n != range_end + 1 {
+            break;
+        }
+
+        has_range = true;
+        range_end = n;
+        index += 1;
+    }
+
+    (has_range, range_end, index)
+}
+
+impl From<Vec<Version>> for ProtoSet {
+    fn from(mut v: Vec<Version>) -> ProtoSet {
+        let mut version_pairs: Vec<(Version, Version)> = Vec::new();
+
+        v.sort_unstable();
+        v.dedup();
+
+        'vector: while !v.is_empty() {
+            let (has_range, end, index): (bool, Version, usize) = find_range(&v);
+
+            if has_range {
+                let first: Version = match v.first() {
+                    Some(x) => *x,
+                    None    => continue,
+                };
+                let last:  Version = match v.get(index) {
+                    Some(x) => *x,
+                    None    => continue,
+                };
+                debug_assert!(last == end, format!("last = {}, end = {}", last, end));
+
+                version_pairs.push((first, last));
+                v = v.split_off(index + 1);
+
+                if v.len() == 0 {
+                    break 'vector;
+                }
+            } else {
+                let last: Version = match v.get(index) {
+                    Some(x) => *x,
+                    None    => continue,
+                };
+                version_pairs.push((last, last));
+                v.remove(index);
+            }
+        }
+        ProtoSet::from_slice(&version_pairs[..]).unwrap_or(ProtoSet::default())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_find_range() {
+        assert_eq!((false, 0, 0), find_range(&vec![]));
+        assert_eq!((false, 1, 0), find_range(&vec![1]));
+        assert_eq!((true, 2, 1), find_range(&vec![1, 2]));
+        assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3]));
+        assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3, 5]));
+    }
+
+    macro_rules! assert_contains_each {
+        ($protoset:expr, $versions:expr) => (
+            for version in $versions {
+                assert!($protoset.contains(version));
+            }
+        )
+    }
+
+    macro_rules! test_protoset_contains_versions {
+        ($list:expr, $str:expr) => (
+            let versions: &[Version] = $list;
+            let protoset: Result<ProtoSet, ProtoverError> = ProtoSet::from_str($str);
+
+            assert!(protoset.is_ok());
+            let p = protoset.unwrap();
+            assert_contains_each!(p, versions);
+        )
+    }
+
+    #[test]
+    fn test_versions_from_str() {
+        test_protoset_contains_versions!(&[], "");
+        test_protoset_contains_versions!(&[1], "1");
+        test_protoset_contains_versions!(&[1, 2], "1,2");
+        test_protoset_contains_versions!(&[1, 2, 3], "1-3");
+        test_protoset_contains_versions!(&[1, 2, 5], "1-2,5");
+        test_protoset_contains_versions!(&[1, 3, 4, 5], "1,3-5");
+        test_protoset_contains_versions!(&[42, 55, 56, 57, 58], "42,55-58");
+    }
+
+    #[test]
+    fn test_versions_from_str_ab() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("a,b"));
+    }
+
+    #[test]
+    fn test_versions_from_str_negative_1() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1"));
+    }
+
+    #[test]
+    fn test_versions_from_str_hyphens() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("--1"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1-2"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1--2"));
+    }
+
+    #[test]
+    fn test_versions_from_str_triple() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-2-3"));
+    }
+
+    #[test]
+    fn test_versions_from_str_1exclam() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,!"));
+    }
+
+    #[test]
+    fn test_versions_from_str_percent_equal() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("%="));
+    }
+
+    #[test]
+    fn test_versions_from_str_whitespace() {
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2\n"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1\r,2"));
+        assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,\t2"));
+    }
+
+    #[test]
+    fn test_versions_from_str_overlap() {
+        assert_eq!(Err(ProtoverError::Overlap), ProtoSet::from_str("1-3,2-4"));
+    }
+
+    #[test]
+    fn test_versions_from_slice_overlap() {
+        assert_eq!(Err(ProtoverError::Overlap), ProtoSet::from_slice(&[(1, 3), (2, 4)]));
+    }
+
+    #[test]
+    fn test_versions_from_str_max() {
+        assert_eq!(Err(ProtoverError::ExceedsMax), ProtoSet::from_str("4294967295"));
+    }
+
+    #[test]
+    fn test_versions_from_slice_max() {
+        assert_eq!(Err(ProtoverError::ExceedsMax), ProtoSet::from_slice(&[(4294967295, 4294967295)]));
+    }
+
+    #[test]
+    fn test_protoset_contains() {
+        let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 5), (7, 9), (13, 14)]).unwrap();
+
+        for x in 1..6   { assert!(protoset.contains(&x), format!("should contain {}", x)); }
+        for x in 7..10  { assert!(protoset.contains(&x), format!("should contain {}", x)); }
+        for x in 13..15 { assert!(protoset.contains(&x), format!("should contain {}", x)); }
+
+        for x in [6, 10, 11, 12, 15, 42, 43, 44, 45, 1234584].iter() {
+            assert!(!protoset.contains(&x), format!("should not contain {}", x));
+        }
+    }
+
+    #[test]
+    fn test_protoset_contains_1_3() {
+        let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 3)]).unwrap();
+
+        for x in 1..4 { assert!(protoset.contains(&x), format!("should contain {}", x)); }
+    }
+
+    macro_rules! assert_protoset_from_vec_contains_all {
+        ($($x:expr),*) => (
+            let vec: Vec<Version> = vec!($($x),*);
+            let protoset: ProtoSet = vec.clone().into();
+
+            for x in vec.iter() {
+                assert!(protoset.contains(&x));
+            }
+        )
+    }
+
+    #[test]
+    fn test_protoset_from_vec_123() {
+        assert_protoset_from_vec_contains_all!(1, 2, 3);
+    }
+
+    #[test]
+    fn test_protoset_from_vec_1_315() {
+        assert_protoset_from_vec_contains_all!(1, 2, 3, 15);
+    }
+
+    #[test]
+    fn test_protoset_from_vec_unordered() {
+        let v: Vec<Version> = vec!(2, 3, 8, 4, 3, 9, 7, 2);
+        let ps: ProtoSet = v.into();
+
+        assert_eq!(ps.to_string(), "2-4,7-9");
+    }
+
+    #[test]
+    fn test_protoset_into_vec() {
+        let ps: ProtoSet = "1-13,42,9001,4294967294".parse().unwrap();
+        let v: Vec<Version> = ps.into();
+
+        assert!(v.contains(&7));
+        assert!(v.contains(&9001));
+        assert!(v.contains(&4294967294));
+    }
+}
+
+#[cfg(all(test, feature = "bench"))]
+mod bench {
+    use super::*;
+}
diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs
new file mode 100644
index 0000000000..68027056c4
--- /dev/null
+++ b/src/rust/protover/protover.rs
@@ -0,0 +1,914 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+use std::collections::HashMap;
+use std::collections::hash_map;
+use std::ffi::CStr;
+use std::fmt;
+use std::str;
+use std::str::FromStr;
+use std::string::String;
+
+use external::c_tor_version_as_new_as;
+
+use errors::ProtoverError;
+use protoset::Version;
+use protoset::ProtoSet;
+
+/// The first version of Tor that included "proto" entries in its descriptors.
+/// Authorities should use this to decide whether to guess proto lines.
+///
+/// C_RUST_COUPLED:
+///     src/or/protover.h `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`
+const FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS: &'static str = "0.2.9.3-alpha";
+
+/// The maximum number of subprotocol version numbers we will attempt to expand
+/// before concluding that someone is trying to DoS us
+///
+/// C_RUST_COUPLED: src/or/protover.c `MAX_PROTOCOLS_TO_EXPAND`
+const MAX_PROTOCOLS_TO_EXPAND: usize = (1<<16);
+
+/// The maximum size an `UnknownProtocol`'s name may be.
+pub(crate) const MAX_PROTOCOL_NAME_LENGTH: usize = 100;
+
+/// Known subprotocols in Tor. Indicates which subprotocol a relay supports.
+///
+/// C_RUST_COUPLED: src/or/protover.h `protocol_type_t`
+#[derive(Clone, Hash, Eq, PartialEq, Debug)]
+pub enum Protocol {
+    Cons,
+    Desc,
+    DirCache,
+    HSDir,
+    HSIntro,
+    HSRend,
+    Link,
+    LinkAuth,
+    Microdesc,
+    Relay,
+}
+
+impl fmt::Display for Protocol {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+/// Translates a string representation of a protocol into a Proto type.
+/// Error if the string is an unrecognized protocol name.
+///
+/// C_RUST_COUPLED: src/or/protover.c `PROTOCOL_NAMES`
+impl FromStr for Protocol {
+    type Err = ProtoverError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "Cons" => Ok(Protocol::Cons),
+            "Desc" => Ok(Protocol::Desc),
+            "DirCache" => Ok(Protocol::DirCache),
+            "HSDir" => Ok(Protocol::HSDir),
+            "HSIntro" => Ok(Protocol::HSIntro),
+            "HSRend" => Ok(Protocol::HSRend),
+            "Link" => Ok(Protocol::Link),
+            "LinkAuth" => Ok(Protocol::LinkAuth),
+            "Microdesc" => Ok(Protocol::Microdesc),
+            "Relay" => Ok(Protocol::Relay),
+            _ => Err(ProtoverError::UnknownProtocol),
+        }
+    }
+}
+
+/// A protocol string which is not one of the `Protocols` we currently know
+/// about.
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
+pub struct UnknownProtocol(String);
+
+impl fmt::Display for UnknownProtocol {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+fn is_valid_proto(s: &str) -> bool {
+    s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')
+}
+
+impl FromStr for UnknownProtocol {
+    type Err = ProtoverError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if !is_valid_proto(s) {
+            Err(ProtoverError::InvalidProtocol)
+        } else if s.len() <= MAX_PROTOCOL_NAME_LENGTH {
+            Ok(UnknownProtocol(s.to_string()))
+        } else {
+            Err(ProtoverError::ExceedsNameLimit)
+        }
+    }
+}
+
+impl UnknownProtocol {
+    /// Create an `UnknownProtocol`, ignoring whether or not it
+    /// exceeds MAX_PROTOCOL_NAME_LENGTH.
+    fn from_str_any_len(s: &str) -> Result<Self, ProtoverError> {
+        if !is_valid_proto(s) {
+            return Err(ProtoverError::InvalidProtocol);
+        }
+        Ok(UnknownProtocol(s.to_string()))
+    }
+}
+
+impl From<Protocol> for UnknownProtocol {
+    fn from(p: Protocol) -> UnknownProtocol {
+        UnknownProtocol(p.to_string())
+    }
+}
+
+/// Get a CStr representation of current supported protocols, for
+/// passing to C, or for converting to a `&str` for Rust.
+///
+/// # Returns
+///
+/// An `&'static CStr` whose value is the existing protocols supported by tor.
+/// Returned data is in the format as follows:
+///
+/// "HSDir=1-1 LinkAuth=1"
+///
+/// # Note
+///
+/// Rust code can use the `&'static CStr` as a normal `&'a str` by
+/// calling `protover::get_supported_protocols`.
+///
+//  C_RUST_COUPLED: src/or/protover.c `protover_get_supported_protocols`
+pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
+    cstr!("Cons=1-2 \
+           Desc=1-2 \
+           DirCache=1-2 \
+           HSDir=1-2 \
+           HSIntro=3-4 \
+           HSRend=1-2 \
+           Link=1-5 \
+           LinkAuth=1,3 \
+           Microdesc=1-2 \
+           Relay=1-2")
+}
+
+/// A map of protocol names to the versions of them which are supported.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ProtoEntry(HashMap<Protocol, ProtoSet>);
+
+impl Default for ProtoEntry {
+    fn default() -> ProtoEntry {
+        ProtoEntry( HashMap::new() )
+    }
+}
+
+impl ProtoEntry {
+    /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`.
+    pub fn iter(&self) -> hash_map::Iter<Protocol, ProtoSet> {
+        self.0.iter()
+    }
+
+    /// Translate the supported tor versions from a string into a
+    /// ProtoEntry, which is useful when looking up a specific
+    /// subprotocol.
+    pub fn supported() -> Result<Self, ProtoverError> {
+        let supported_cstr: &'static CStr = get_supported_protocols_cstr();
+        let supported: &str = supported_cstr.to_str().unwrap_or("");
+
+        supported.parse()
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    pub fn get(&self, protocol: &Protocol) -> Option<&ProtoSet> {
+        self.0.get(protocol)
+    }
+
+    pub fn insert(&mut self, key: Protocol, value: ProtoSet) {
+        self.0.insert(key, value);
+    }
+
+    pub fn remove(&mut self, key: &Protocol) -> Option<ProtoSet> {
+        self.0.remove(key)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
+impl FromStr for ProtoEntry {
+    type Err = ProtoverError;
+
+    /// Parse a string of subprotocol types and their version numbers.
+    ///
+    /// # Inputs
+    ///
+    /// * A `protocol_entry` string, comprised of a keywords, an "=" sign, and
+    /// one or more version numbers, each separated by a space.  For example,
+    /// `"Cons=3-4 HSDir=1"`.
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` value is a `ProtoEntry`, where the
+    /// first element is the subprotocol type (see `protover::Protocol`) and the last
+    /// element is an ordered set of `(low, high)` unique version numbers which are supported.
+    /// Otherwise, the `Err` value of this `Result` is a `ProtoverError`.
+    fn from_str(protocol_entry: &str) -> Result<ProtoEntry, ProtoverError> {
+        let mut proto_entry: ProtoEntry = ProtoEntry::default();
+        let entries = protocol_entry.split(' ');
+
+        for entry in entries {
+            let mut parts = entry.splitn(2, '=');
+
+            let proto = match parts.next() {
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+
+            let vers = match parts.next() {
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+            let versions: ProtoSet = vers.parse()?;
+            let proto_name: Protocol = proto.parse()?;
+
+            proto_entry.insert(proto_name, versions);
+
+            if proto_entry.len() > MAX_PROTOCOLS_TO_EXPAND {
+                return Err(ProtoverError::ExceedsMax);
+            }
+        }
+        Ok(proto_entry)
+    }
+}
+
+/// Generate an implementation of `ToString` for either a `ProtoEntry` or an
+/// `UnvalidatedProtoEntry`.
+macro_rules! impl_to_string_for_proto_entry {
+    ($t:ty) => (
+        impl ToString for $t {
+            fn to_string(&self) -> String {
+                let mut parts: Vec<String> = Vec::new();
+
+                for (protocol, versions) in self.iter() {
+                    parts.push(format!("{}={}", protocol.to_string(), versions.to_string()));
+                }
+                parts.sort_unstable();
+                parts.join(" ")
+            }
+        }
+    )
+}
+
+impl_to_string_for_proto_entry!(ProtoEntry);
+impl_to_string_for_proto_entry!(UnvalidatedProtoEntry);
+
+/// A `ProtoEntry`, but whose `Protocols` can be any `UnknownProtocol`, not just
+/// the supported ones enumerated in `Protocols`.  The protocol versions are
+/// validated, however.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct UnvalidatedProtoEntry(HashMap<UnknownProtocol, ProtoSet>);
+
+impl Default for UnvalidatedProtoEntry {
+    fn default() -> UnvalidatedProtoEntry {
+        UnvalidatedProtoEntry( HashMap::new() )
+    }
+}
+
+impl UnvalidatedProtoEntry {
+    /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`.
+    pub fn iter(&self) -> hash_map::Iter<UnknownProtocol, ProtoSet> {
+        self.0.iter()
+    }
+
+    pub fn get(&self, protocol: &UnknownProtocol) -> Option<&ProtoSet> {
+        self.0.get(protocol)
+    }
+
+    pub fn insert(&mut self, key: UnknownProtocol, value: ProtoSet) {
+        self.0.insert(key, value);
+    }
+
+    pub fn remove(&mut self, key: &UnknownProtocol) -> Option<ProtoSet> {
+        self.0.remove(key)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        let mut total: usize = 0;
+
+        for (_, versions) in self.iter() {
+            total += versions.len();
+        }
+        total
+    }
+
+    /// Determine if we support every protocol a client supports, and if not,
+    /// determine which protocols we do not have support for.
+    ///
+    /// # Returns
+    ///
+    /// Optionally, return parameters which the client supports but which we do not.
+    ///
+    /// # Examples
+    /// ```
+    /// use protover::UnvalidatedProtoEntry;
+    ///
+    /// let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap();
+    /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    /// assert_eq!(true, unsupported.is_none());
+    ///
+    /// let protocols: UnvalidatedProtoEntry = "Link=1-2 Wombat=9".parse().unwrap();
+    /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    /// assert_eq!(true, unsupported.is_some());
+    /// assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+    /// ```
+    pub fn all_supported(&self) -> Option<UnvalidatedProtoEntry> {
+        let mut unsupported: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+        let supported: ProtoEntry = match ProtoEntry::supported() {
+            Ok(x)  => x,
+            Err(_) => return None,
+        };
+
+        for (protocol, versions) in self.iter() {
+            let is_supported: Result<Protocol, ProtoverError> = protocol.0.parse();
+            let supported_protocol: Protocol;
+
+            // If the protocol wasn't even in the enum, then we definitely don't
+            // know about it and don't support any of its versions.
+            if is_supported.is_err() {
+                if !versions.is_empty() {
+                    unsupported.insert(protocol.clone(), versions.clone());
+                }
+                continue;
+            } else {
+                supported_protocol = is_supported.unwrap();
+            }
+
+            let maybe_supported_versions: Option<&ProtoSet> = supported.get(&supported_protocol);
+            let supported_versions: &ProtoSet;
+
+            // If the protocol wasn't in the map, then we don't know about it
+            // and don't support any of its versions.  Add its versions to the
+            // map (if it has versions).
+            if maybe_supported_versions.is_none() {
+                if !versions.is_empty() {
+                    unsupported.insert(protocol.clone(), versions.clone());
+                }
+                continue;
+            } else {
+                supported_versions = maybe_supported_versions.unwrap();
+            }
+            let unsupported_versions = versions.and_not_in(supported_versions);
+
+            if !unsupported_versions.is_empty() {
+                unsupported.insert(protocol.clone(), unsupported_versions);
+            }
+        }
+
+        if unsupported.is_empty() {
+            return None;
+        }
+        Some(unsupported)
+    }
+
+    /// Determine if we have support for some protocol and version.
+    ///
+    /// # Inputs
+    ///
+    /// * `proto`, an `UnknownProtocol` to test support for
+    /// * `vers`, a `Version` which we will go on to determine whether the
+    /// specified protocol supports.
+    ///
+    /// # Return
+    ///
+    /// Returns `true` iff this `UnvalidatedProtoEntry` includes support for the
+    /// indicated protocol and version, and `false` otherwise.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// use protover::*;
+    /// # use protover::errors::ProtoverError;
+    ///
+    /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> {
+    /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=1 Doggo=3-5".parse()?;
+    /// assert_eq!(true, proto.supports_protocol(&Protocol::Cons.into(), &1));
+    /// assert_eq!(false, proto.supports_protocol(&Protocol::Cons.into(), &5));
+    /// assert_eq!(true, proto.supports_protocol(&UnknownProtocol::from_str("Doggo")?, &4));
+    /// # Ok(proto)
+    /// # } fn main () { do_test(); }
+    /// ```
+    pub fn supports_protocol(&self, proto: &UnknownProtocol, vers: &Version) -> bool {
+        let supported_versions: &ProtoSet = match self.get(proto) {
+            Some(n) => n,
+            None => return false,
+        };
+        supported_versions.contains(&vers)
+    }
+
+    /// As `UnvalidatedProtoEntry::supports_protocol()`, but also returns `true`
+    /// if any later version of the protocol is supported.
+    ///
+    /// # Examples
+    /// ```
+    /// use protover::*;
+    /// # use protover::errors::ProtoverError;
+    ///
+    /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> {
+    /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=5".parse()?;
+    ///
+    /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &5));
+    /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &4));
+    /// assert_eq!(false, proto.supports_protocol_or_later(&Protocol::Cons.into(), &6));
+    /// # Ok(proto)
+    /// # } fn main () { do_test(); }
+    /// ```
+    pub fn supports_protocol_or_later(&self, proto: &UnknownProtocol, vers: &Version) -> bool {
+        let supported_versions: &ProtoSet = match self.get(&proto) {
+            Some(n) => n,
+            None => return false,
+        };
+        supported_versions.iter().any(|v| v.1 >= *vers)
+    }
+
+    /// Split a string containing (potentially) several protocols and their
+    /// versions into a `Vec` of tuples of string in `(protocol, versions)`
+    /// form.
+    ///
+    /// # Inputs
+    ///
+    /// A &str in the form `"Link=3-4 Cons=5"`.
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` variant is a `Vec<(&str, &str)>` of `(protocol,
+    /// versions)`, or whose `Err` variant is a `ProtoverError`.
+    ///
+    /// # Errors
+    ///
+    /// This will error with a `ProtoverError::Unparseable` if any of the
+    /// following are true:
+    ///
+    /// * If a protocol name is an empty string, e.g. `"Cons=1,3 =3-5"`.
+    /// * If a protocol name cannot be parsed as utf-8.
+    /// * If the version numbers are an empty string, e.g. `"Cons="`.
+    fn parse_protocol_and_version_str<'a>(protocol_string: &'a str)
+        -> Result<Vec<(&'a str, &'a str)>, ProtoverError>
+    {
+        let mut protovers: Vec<(&str, &str)> = Vec::new();
+
+        for subproto in protocol_string.split(' ') {
+            let mut parts = subproto.splitn(2, '=');
+
+            let name = match parts.next() {
+                Some("") => return Err(ProtoverError::Unparseable),
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+            let vers = match parts.next() {
+                Some(n) => n,
+                None => return Err(ProtoverError::Unparseable),
+            };
+            protovers.push((name, vers));
+        }
+        Ok(protovers)
+    }
+}
+
+impl FromStr for UnvalidatedProtoEntry {
+    type Err = ProtoverError;
+
+    /// Parses a protocol list without validating the protocol names.
+    ///
+    /// # Inputs
+    ///
+    /// * `protocol_string`, a string comprised of keys and values, both which are
+    /// strings. The keys are the protocol names while values are a string
+    /// representation of the supported versions.
+    ///
+    /// The input is _not_ expected to be a subset of the Protocol types
+    ///
+    /// # Returns
+    ///
+    /// A `Result` whose `Ok` value is a `ProtoSet` holding all of the
+    /// unique version numbers.
+    ///
+    /// The returned `Result`'s `Err` value is an `ProtoverError` whose `Display`
+    /// impl has a description of the error.
+    ///
+    /// # Errors
+    ///
+    /// This function will error if:
+    ///
+    /// * The protocol string does not follow the "protocol_name=version_list"
+    ///   expected format, or
+    /// * If the version string is malformed. See `impl FromStr for ProtoSet`.
+    fn from_str(protocol_string: &str) -> Result<UnvalidatedProtoEntry, ProtoverError> {
+        let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+        let parts: Vec<(&str, &str)> =
+            UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?;
+
+        for &(name, vers) in parts.iter() {
+            let versions = ProtoSet::from_str(vers)?;
+            let protocol = UnknownProtocol::from_str(name)?;
+
+            parsed.insert(protocol, versions);
+        }
+        Ok(parsed)
+    }
+}
+
+impl UnvalidatedProtoEntry {
+    /// Create an `UnknownProtocol`, ignoring whether or not it
+    /// exceeds MAX_PROTOCOL_NAME_LENGTH.
+    pub(crate) fn from_str_any_len(protocol_string: &str)
+        -> Result<UnvalidatedProtoEntry, ProtoverError>
+    {
+        let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+        let parts: Vec<(&str, &str)> =
+            UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?;
+
+        for &(name, vers) in parts.iter() {
+            let versions = ProtoSet::from_str(vers)?;
+            let protocol = UnknownProtocol::from_str_any_len(name)?;
+
+            parsed.insert(protocol, versions);
+        }
+        Ok(parsed)
+    }
+}
+
+/// Pretend a `ProtoEntry` is actually an `UnvalidatedProtoEntry`.
+impl From<ProtoEntry> for UnvalidatedProtoEntry {
+    fn from(proto_entry: ProtoEntry) -> UnvalidatedProtoEntry {
+        let mut unvalidated: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+
+        for (protocol, versions) in proto_entry.iter() {
+            unvalidated.insert(UnknownProtocol::from(protocol.clone()), versions.clone());
+        }
+        unvalidated
+    }
+}
+
+/// A mapping of protocols to a count of how many times each of their `Version`s
+/// were voted for or supported.
+///
+/// # Warning
+///
+/// The "protocols" are *not* guaranteed to be known/supported `Protocol`s, in
+/// order to allow new subprotocols to be introduced even if Directory
+/// Authorities don't yet know of them.
+pub struct ProtoverVote( HashMap<UnknownProtocol, HashMap<Version, usize>> );
+
+impl Default for ProtoverVote {
+    fn default() -> ProtoverVote {
+        ProtoverVote( HashMap::new() )
+    }
+}
+
+impl IntoIterator for ProtoverVote {
+    type Item = (UnknownProtocol, HashMap<Version, usize>);
+    type IntoIter = hash_map::IntoIter<UnknownProtocol, HashMap<Version, usize>>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}
+
+impl ProtoverVote {
+    pub fn entry(&mut self, key: UnknownProtocol)
+        -> hash_map::Entry<UnknownProtocol, HashMap<Version, usize>>
+    {
+        self.0.entry(key)
+    }
+
+    /// Protocol voting implementation.
+    ///
+    /// Given a slice of `UnvalidatedProtoEntry`s and a vote `threshold`, return
+    /// a new `UnvalidatedProtoEntry` encoding all of the protocols that are
+    /// listed by at least `threshold` of the inputs.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use protover::ProtoverVote;
+    /// use protover::UnvalidatedProtoEntry;
+    ///
+    /// let protos: &[UnvalidatedProtoEntry] = &["Link=3-4".parse().unwrap(),
+    ///                                          "Link=3".parse().unwrap()];
+    /// let vote = ProtoverVote::compute(protos, &2);
+    /// assert_eq!("Link=3", vote.to_string());
+    /// ```
+    // C_RUST_COUPLED: /src/or/protover.c protover_compute_vote
+    pub fn compute(proto_entries: &[UnvalidatedProtoEntry], threshold: &usize) -> UnvalidatedProtoEntry {
+        let mut all_count: ProtoverVote = ProtoverVote::default();
+        let mut final_output: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default();
+
+        if proto_entries.is_empty() {
+            return final_output;
+        }
+
+        // parse and collect all of the protos and their versions and collect them
+        for vote in proto_entries {
+            // C_RUST_DIFFERS: This doesn't actually differ, bu this check on
+            // the total is here to make it match.  Because the C version calls
+            // expand_protocol_list() which checks if there would be too many
+            // subprotocols *or* individual version numbers, i.e. more than
+            // MAX_PROTOCOLS_TO_EXPAND, and does this *per vote*, we need to
+            // match it's behaviour and ensure we're not allowing more than it
+            // would.
+            if vote.len() > MAX_PROTOCOLS_TO_EXPAND {
+                continue;
+            }
+
+            for (protocol, versions) in vote.iter() {
+                let supported_vers: &mut HashMap<Version, usize> =
+                    all_count.entry(protocol.clone()).or_insert(HashMap::new());
+
+                for version in versions.clone().expand() {
+                    let counter: &mut usize =
+                        supported_vers.entry(version).or_insert(0);
+                    *counter += 1;
+                }
+            }
+        }
+
+        for (protocol, mut versions) in all_count {
+            // Go through and remove versions that are less than the threshold
+            versions.retain(|_, count| *count as usize >= *threshold);
+
+            if versions.len() > 0 {
+                let voted_versions: Vec<Version> = versions.keys().cloned().collect();
+                let voted_protoset: ProtoSet = ProtoSet::from(voted_versions);
+
+                final_output.insert(protocol, voted_protoset);
+            }
+        }
+        final_output
+    }
+}
+
+/// Returns a boolean indicating whether the given protocol and version is
+/// supported in any of the existing Tor protocols
+///
+/// # Examples
+/// ```
+/// use protover::is_supported_here;
+/// use protover::Protocol;
+///
+/// let is_supported = is_supported_here(&Protocol::Link, &10);
+/// assert_eq!(false, is_supported);
+///
+/// let is_supported = is_supported_here(&Protocol::Link, &1);
+/// assert_eq!(true, is_supported);
+/// ```
+pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool {
+    let currently_supported: ProtoEntry = match ProtoEntry::supported() {
+        Ok(result) => result,
+        Err(_) => return false,
+    };
+    let supported_versions = match currently_supported.get(proto) {
+        Some(n) => n,
+        None => return false,
+    };
+    supported_versions.contains(vers)
+}
+
+/// Since older versions of Tor cannot infer their own subprotocols,
+/// determine which subprotocols are supported by older Tor versions.
+///
+/// # Inputs
+///
+/// * `version`, a string comprised of "[0-9a-z.-]"
+///
+/// # Returns
+///
+/// A `&'static CStr` encoding a list of protocol names and supported
+/// versions. The string takes the following format:
+///
+/// "HSDir=1-1 LinkAuth=1"
+///
+/// This function returns the protocols that are supported by the version input,
+/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`
+/// (but not older than 0.2.4.19).  For newer tors (or older than 0.2.4.19), it
+/// returns an empty string.
+///
+/// # Note
+///
+/// This function is meant to be called for/within FFI code.  If you'd
+/// like to use this code in Rust, please see `compute_for_old_tor()`.
+//
+// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor`
+pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static CStr {
+    let empty: &'static CStr = cstr!("");
+
+    if c_tor_version_as_new_as(version, FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS) {
+        return empty;
+    }
+    if c_tor_version_as_new_as(version, "0.2.9.1-alpha") {
+        return cstr!("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \
+                      Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2");
+    }
+    if c_tor_version_as_new_as(version, "0.2.7.5") {
+        return cstr!("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \
+                      Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2");
+    }
+    if c_tor_version_as_new_as(version, "0.2.4.19") {
+        return cstr!("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \
+                      Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2");
+    }
+    empty
+}
+
+/// Since older versions of Tor cannot infer their own subprotocols,
+/// determine which subprotocols are supported by older Tor versions.
+///
+/// # Inputs
+///
+/// * `version`, a string comprised of "[0-9a-z.-]"
+///
+/// # Returns
+///
+/// A `Result` whose `Ok` value is an `&'static str` encoding a list of protocol
+/// names and supported versions. The string takes the following format:
+///
+/// "HSDir=1-1 LinkAuth=1"
+///
+/// This function returns the protocols that are supported by the version input,
+/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`.
+/// (but not older than 0.2.4.19).  For newer tors (or older than 0.2.4.19), its
+/// `Ok` `Result` contains an empty string.
+///
+/// Otherwise, its `Err` contains a `ProtoverError::Unparseable` if the
+/// `version` string was invalid utf-8.
+///
+/// # Note
+///
+/// This function is meant to be called for/within non-FFI Rust code.
+//
+// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor`
+pub fn compute_for_old_tor(version: &str) -> Result<&'static str, ProtoverError> {
+    // .to_str() fails with a Utf8Error if it couldn't validate the
+    // utf-8, so convert that here into an Unparseable ProtoverError.
+    compute_for_old_tor_cstr(version).to_str().or(Err(ProtoverError::Unparseable))
+}
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+    use std::string::ToString;
+
+    use super::*;
+
+    macro_rules! parse_proto {
+        ($e:expr) => {{
+            let proto: Result<UnknownProtocol, _> = $e.parse();
+            let proto2 = UnknownProtocol::from_str_any_len($e);
+            assert_eq!(proto, proto2);
+            proto
+        }};
+    }
+
+    #[test]
+    fn test_protocol_from_str() {
+        assert!(parse_proto!("Cons").is_ok());
+        assert!(parse_proto!("123").is_ok());
+        assert!(parse_proto!("1-2-3").is_ok());
+
+        let err = Err(ProtoverError::InvalidProtocol);
+        assert_eq!(err, parse_proto!("a_b_c"));
+        assert_eq!(err, parse_proto!("a b"));
+        assert_eq!(err, parse_proto!("a,"));
+        assert_eq!(err, parse_proto!("b."));
+        assert_eq!(err, parse_proto!("é"));
+    }
+
+    macro_rules! assert_protoentry_is_parseable {
+        ($e:expr) => (
+            let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse();
+
+            assert!(protoentry.is_ok(), format!("{:?}", protoentry.err()));
+        )
+    }
+
+    macro_rules! assert_protoentry_is_unparseable {
+        ($e:expr) => (
+            let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse();
+
+            assert!(protoentry.is_err());
+        )
+    }
+
+    #[test]
+    fn test_protoentry_from_str_multiple_protocols_multiple_versions() {
+        assert_protoentry_is_parseable!("Cons=3-4 Link=1,3-5");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_empty() {
+        assert_protoentry_is_unparseable!("");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_single_protocol_single_version() {
+        assert_protoentry_is_parseable!("HSDir=1");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_unknown_protocol() {
+        assert_protoentry_is_unparseable!("Ducks=5-7,8");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_allowed_number_of_versions() {
+        assert_protoentry_is_parseable!("Desc=1-4294967294");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_too_many_versions() {
+        assert_protoentry_is_unparseable!("Desc=1-4294967295");
+    }
+
+    #[test]
+    fn test_protoentry_from_str_() {
+        assert_protoentry_is_unparseable!("");
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_single_protocol_single_version() {
+        let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+        assert_eq!(true, unsupported.is_none());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_multiple_protocol_multiple_versions() {
+        let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_none());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_three_values() {
+        let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_none());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_unknown_protocol() {
+        let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_some());
+        assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_unsupported_high_version() {
+        let protocols: UnvalidatedProtoEntry = "HSDir=12-100".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_some());
+        assert_eq!("HSDir=12-100", &unsupported.unwrap().to_string());
+    }
+
+    #[test]
+    fn test_protoentry_all_supported_unsupported_low_version() {
+        let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap();
+        let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+        assert_eq!(true, unsupported.is_some());
+        assert_eq!("HSIntro=2", &unsupported.unwrap().to_string());
+    }
+
+    #[test]
+    fn test_contract_protocol_list() {
+        let mut versions = "";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+
+        versions = "1";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+
+        versions = "1-2";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+
+        versions = "1,3";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+
+        versions = "1-4";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+
+        versions = "1,3,5-7";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+
+        versions = "1-3,500";
+        assert_eq!(String::from(versions), ProtoSet::from_str(&versions).unwrap().to_string());
+    }
+}
diff --git a/src/rust/protover/tests/protover.rs b/src/rust/protover/tests/protover.rs
new file mode 100644
index 0000000000..9258d869d7
--- /dev/null
+++ b/src/rust/protover/tests/protover.rs
@@ -0,0 +1,394 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+extern crate protover;
+
+use protover::ProtoEntry;
+use protover::ProtoverVote;
+use protover::UnvalidatedProtoEntry;
+use protover::errors::ProtoverError;
+
+#[test]
+fn parse_protocol_with_single_proto_and_single_version() {
+    let _: ProtoEntry = "Cons=1".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_multiple_versions() {
+    let _: ProtoEntry = "Cons=1-2".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_different_single_protocol_and_single_version() {
+    let _: ProtoEntry = "HSDir=1".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_supported_version() {
+    let _: ProtoEntry = "Desc=2".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_two_protocols_and_single_version() {
+    let _: ProtoEntry = "Cons=1 HSDir=1".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_two_sequential_versions() {
+    let _: ProtoEntry = "Desc=1-2".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_protocol_range() {
+    let _: ProtoEntry = "Link=1-4".parse().unwrap();
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_protocol_set() {
+    let _: ProtoEntry = "Link=3-4 Desc=2".parse().unwrap();
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_protocol_set() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_two_values() {
+    let protocols: UnvalidatedProtoEntry = "Microdesc=1-2 Relay=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_one_value() {
+    let protocols: UnvalidatedProtoEntry = "Microdesc=1-2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+#[should_panic]
+fn parse_protocol_unvalidated_with_empty() {
+    let _: UnvalidatedProtoEntry = "".parse().unwrap();
+}
+
+#[test]
+#[should_panic]
+fn parse_protocol_validated_with_empty() {
+    let _: UnvalidatedProtoEntry = "".parse().unwrap();
+}
+
+#[test]
+fn protocol_all_supported_with_three_values() {
+    let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_protocol() {
+    let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_versions() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-999".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Link=6-999", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_low_version() {
+    let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("HSIntro=2", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_unsupported_high_version() {
+    let protocols: UnvalidatedProtoEntry = "Cons=1-2,999".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Cons=999", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protocol_all_supported_with_mix_of_supported_and_unsupproted() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Wombat=9".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_some());
+    assert_eq!("Wombat=9", &unsupported.unwrap().to_string());
+}
+
+#[test]
+fn protover_string_supports_protocol_returns_true_for_single_supported() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap();
+    let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &1);
+    assert_eq!(true, is_supported);
+}
+
+#[test]
+fn protover_string_supports_protocol_returns_false_for_single_unsupported() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap();
+    let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2);
+    assert_eq!(false, is_supported);
+}
+
+#[test]
+fn protover_string_supports_protocol_returns_false_for_unsupported() {
+    let protocols: UnvalidatedProtoEntry = "Link=3-4".parse().unwrap();
+    let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2);
+    assert_eq!(false, is_supported);
+}
+
+#[test]
+#[should_panic]
+fn parse_protocol_with_unexpected_characters() {
+    let _: UnvalidatedProtoEntry = "Cons=*-%".parse().unwrap();
+}
+
+#[test]
+#[should_panic]
+fn protover_compute_vote_returns_empty_for_empty_string() {
+    let protocols: &[UnvalidatedProtoEntry] = &["".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_single_protocol_for_matching() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Cons=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_two_protocols_for_two_matching() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Link=1 Cons=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1 Link=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_one_protocol_when_one_out_of_two_matches() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Cons=1 Link=2".parse().unwrap(), "Cons=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &2);
+    assert_eq!("Cons=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_protocols_that_it_doesnt_currently_support() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Foo=1 Cons=2".parse().unwrap(), "Bar=1".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Bar=1 Cons=2 Foo=1", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_matching_for_mix() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Link=1-10,500 Cons=1,3-7,8".parse().unwrap()];
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1,3-8 Link=1-10,500", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_matching_for_longer_mix() {
+    let protocols: &[UnvalidatedProtoEntry] = &[
+        "Desc=1-10,500 Cons=1,3-7,8".parse().unwrap(),
+        "Link=123-456,78 Cons=2-6,8 Desc=9".parse().unwrap(),
+    ];
+
+    let listed = ProtoverVote::compute(protocols, &1);
+    assert_eq!("Cons=1-8 Desc=1-10,500 Link=78,123-456", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_returns_matching_for_longer_mix_with_threshold_two() {
+    let protocols: &[UnvalidatedProtoEntry] = &[
+        "Desc=1-10,500 Cons=1,3-7,8".parse().unwrap(),
+        "Link=123-456,78 Cons=2-6,8 Desc=9".parse().unwrap(),
+    ];
+
+    let listed = ProtoverVote::compute(protocols, &2);
+    assert_eq!("Cons=3-6,8 Desc=9", listed.to_string());
+}
+
+#[test]
+fn protover_compute_vote_handles_duplicated_versions() {
+    let protocols: &[UnvalidatedProtoEntry] = &["Cons=1".parse().unwrap(), "Cons=1".parse().unwrap()];
+    assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string());
+
+    let protocols: &[UnvalidatedProtoEntry] = &["Cons=1-2".parse().unwrap(), "Cons=1-2".parse().unwrap()];
+    assert_eq!("Cons=1-2", ProtoverVote::compute(protocols, &2).to_string());
+}
+
+#[test]
+fn protover_compute_vote_handles_invalid_proto_entries() {
+    let protocols: &[UnvalidatedProtoEntry] = &[
+        "Cons=1".parse().unwrap(),
+        "Cons=1".parse().unwrap(),
+        "Dinosaur=1".parse().unwrap(),
+    ];
+    assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string());
+}
+
+#[test]
+fn parse_protocol_with_single_protocol_and_two_nonsequential_versions() {
+    let _: ProtoEntry = "Desc=1,2".parse().unwrap();
+}
+
+#[test]
+fn protover_is_supported_here_returns_true_for_supported_protocol() {
+    assert_eq!(true, protover::is_supported_here(&protover::Protocol::Cons, &1));
+}
+
+#[test]
+fn protover_is_supported_here_returns_false_for_unsupported_protocol() {
+    assert_eq!(false, protover::is_supported_here(&protover::Protocol::Cons, &5));
+}
+
+#[test]
+fn protocol_all_supported_with_single_proto_and_single_version() {
+    let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_multiple_versions() {
+    let protocol: UnvalidatedProtoEntry = "Cons=1-2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_different_single_protocol_and_single_version() {
+    let protocol: UnvalidatedProtoEntry = "HSDir=1".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_supported_version() {
+    let protocol: UnvalidatedProtoEntry = "Desc=2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_two_protocols_and_single_version() {
+    let protocols: UnvalidatedProtoEntry = "Cons=1 HSDir=1".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_two_nonsequential_versions() {
+    let protocol: UnvalidatedProtoEntry = "Desc=1,2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_two_sequential_versions() {
+    let protocol: UnvalidatedProtoEntry = "Desc=1-2".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+#[test]
+fn protocol_all_supported_with_single_protocol_and_protocol_range() {
+    let protocol: UnvalidatedProtoEntry = "Link=1-4".parse().unwrap();
+    let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported();
+    assert_eq!(true, unsupported.is_none());
+}
+
+// By allowing us to add to votes, the C implementation allows us to
+// exceed the limit.
+#[test]
+fn protover_compute_vote_may_exceed_limit() {
+    let proto1: UnvalidatedProtoEntry = "Sleen=1-65535".parse().unwrap();
+    let proto2: UnvalidatedProtoEntry = "Sleen=100000".parse().unwrap();
+
+    let _result: UnvalidatedProtoEntry = ProtoverVote::compute(&[proto1, proto2], &1);
+}
+
+#[test]
+fn protover_all_supported_should_exclude_versions_we_actually_do_support() {
+    let proto: UnvalidatedProtoEntry = "Link=3-999".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-999".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex1() {
+    let proto: UnvalidatedProtoEntry = "Link=1-3,345-666".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=345-666".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex2() {
+    let proto: UnvalidatedProtoEntry = "Link=1-3,5-12".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-12".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_exclude_some_versions_and_entire_protocols() {
+    let proto: UnvalidatedProtoEntry = "Link=1-3,5-12 Quokka=9000-9001".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-12 Quokka=9000-9001".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_not_dos_anyones_computer() {
+    let proto: UnvalidatedProtoEntry = "Link=1-2147483648".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-2147483648".to_string());
+}
+
+#[test]
+fn protover_all_supported_should_not_dos_anyones_computer_max_versions() {
+    let proto: UnvalidatedProtoEntry = "Link=1-4294967294".parse().unwrap();
+    let result: String = proto.all_supported().unwrap().to_string();
+
+    assert_eq!(result, "Link=6-4294967294".to_string());
+}
+
+#[test]
+// C_RUST_DIFFERS: The C will return true (e.g. saying "yes, that's supported")
+// but set the msg to NULL (??? seems maybe potentially bad).  The Rust will
+// simply return a None.
+fn protover_all_supported_should_return_empty_string_for_weird_thing() {
+    let proto: UnvalidatedProtoEntry = "Fribble=".parse().unwrap();
+    let result: Option<UnvalidatedProtoEntry> = proto.all_supported();
+
+    assert!(result.is_none());
+}
+
+#[test]
+fn protover_unvalidatedprotoentry_should_err_entirely_unparseable_things() {
+    let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Fribble".parse();
+
+    assert_eq!(Err(ProtoverError::Unparseable), proto);
+}
+
+#[test]
+fn protover_all_supported_over_maximum_limit() {
+    let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Sleen=1-4294967295".parse();
+
+    assert_eq!(Err(ProtoverError::ExceedsMax), proto);
+}
diff --git a/src/rust/smartlist/Cargo.toml b/src/rust/smartlist/Cargo.toml
new file mode 100644
index 0000000000..6ddcbee8e9
--- /dev/null
+++ b/src/rust/smartlist/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "smartlist"
+
+[dependencies]
+libc = "0.2.39"
+
+[lib]
+name = "smartlist"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
diff --git a/src/rust/smartlist/lib.rs b/src/rust/smartlist/lib.rs
new file mode 100644
index 0000000000..14a8148315
--- /dev/null
+++ b/src/rust/smartlist/lib.rs
@@ -0,0 +1,8 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+extern crate libc;
+
+mod smartlist;
+
+pub use smartlist::*;
diff --git a/src/rust/smartlist/smartlist.rs b/src/rust/smartlist/smartlist.rs
new file mode 100644
index 0000000000..2a822d89f4
--- /dev/null
+++ b/src/rust/smartlist/smartlist.rs
@@ -0,0 +1,115 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+use std::slice;
+use libc::{c_char, c_int};
+use std::ffi::CStr;
+
+/// Smartlists are a type used in C code in tor to define a collection of a
+/// generic type, which has a capacity and a number used. Each Smartlist
+/// defines how to extract the list of values from the underlying C structure
+///
+/// Implementations are required to have a C representation, as this module
+/// serves purely to translate smartlists as defined in tor to vectors in Rust.
+pub trait Smartlist<T> {
+    fn get_list(&self) -> Vec<T>;
+}
+
+#[repr(C)]
+pub struct Stringlist {
+    pub list: *const *const c_char,
+    pub num_used: c_int,
+    pub capacity: c_int,
+}
+
+impl Smartlist<String> for Stringlist {
+    fn get_list(&self) -> Vec<String> {
+        let empty: Vec<String> = Vec::new();
+        let mut rust_list: Vec<String> = Vec::new();
+
+        if self.list.is_null() || self.num_used == 0 {
+            return empty;
+        }
+
+        // unsafe, as we need to extract the smartlist list into a vector of
+        // pointers, and then transform each element into a Rust string.
+        let elems: &[*const c_char] =
+            unsafe { slice::from_raw_parts(self.list, self.num_used as usize) };
+
+        for elem in elems.iter() {
+            if elem.is_null() {
+                continue;
+            }
+
+            // unsafe, as we need to create a cstring from the referenced
+            // element
+            let c_string = unsafe { CStr::from_ptr(*elem) };
+
+            let r_string = match c_string.to_str() {
+                Ok(n) => n,
+                Err(_) => return empty,
+            };
+
+            rust_list.push(String::from(r_string));
+        }
+
+        rust_list
+    }
+}
+
+// TODO: CHK: this module maybe should be tested from a test in C with a
+// smartlist as defined in tor.
+#[cfg(test)]
+mod test {
+    #[test]
+    fn test_get_list_of_strings() {
+        extern crate libc;
+
+        use std::ffi::CString;
+        use libc::c_char;
+
+        use super::Smartlist;
+        use super::Stringlist;
+
+        {
+            // test to verify that null pointers are gracefully handled
+            use std::ptr;
+
+            let sl = Stringlist {
+                list: ptr::null(),
+                num_used: 0,
+                capacity: 0,
+            };
+
+            let data = sl.get_list();
+            assert_eq!(0, data.len());
+        }
+
+        {
+            let args = vec![String::from("a"), String::from("b")];
+
+            // for each string, transform  it into a CString
+            let c_strings: Vec<_> = args.iter()
+                .map(|arg| CString::new(arg.as_str()).unwrap())
+                .collect();
+
+            // then, collect a pointer for each CString
+            let p_args: Vec<_> =
+                c_strings.iter().map(|arg| arg.as_ptr()).collect();
+
+            let p: *const *const c_char = p_args.as_ptr();
+
+            // This is the representation that we expect when receiving a
+            // smartlist at the Rust/C FFI layer.
+            let sl = Stringlist {
+                list: p,
+                num_used: 2,
+                capacity: 2,
+            };
+
+            let data = sl.get_list();
+            assert_eq!("a", &data[0]);
+            assert_eq!("b", &data[1]);
+        }
+    }
+}
diff --git a/src/rust/tor_allocate/Cargo.toml b/src/rust/tor_allocate/Cargo.toml
new file mode 100644
index 0000000000..468425f115
--- /dev/null
+++ b/src/rust/tor_allocate/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+authors = ["The Tor Project"]
+version = "0.0.1"
+name = "tor_allocate"
+
+[dependencies]
+libc = "=0.2.39"
+
+[lib]
+name = "tor_allocate"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
diff --git a/src/rust/tor_allocate/lib.rs b/src/rust/tor_allocate/lib.rs
new file mode 100644
index 0000000000..937a5dcf63
--- /dev/null
+++ b/src/rust/tor_allocate/lib.rs
@@ -0,0 +1,15 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! Allocation helper functions that allow data to be allocated in Rust
+//! using tor's specified allocator. In doing so, this can be later freed
+//! from C.
+//!
+//! This is currently a temporary solution, we will later use tor's allocator
+//! by default for any allocation that occurs in Rust. However, as this will
+//! stabalize in 2018, we can use this as a temporary measure.
+
+extern crate libc;
+
+mod tor_allocate;
+pub use tor_allocate::*;
diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs
new file mode 100644
index 0000000000..3c0037f139
--- /dev/null
+++ b/src/rust/tor_allocate/tor_allocate.rs
@@ -0,0 +1,108 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+// No-op defined purely for testing at the module level
+use libc::c_char;
+
+#[cfg(not(feature = "testing"))]
+use std::{ptr, slice, mem};
+use libc::c_void;
+
+// Define a no-op implementation for testing Rust modules without linking to C
+#[cfg(feature = "testing")]
+pub fn allocate_and_copy_string(s: &String) -> *mut c_char {
+    use std::ffi::CString;
+    CString::new(s.as_str()).unwrap().into_raw()
+}
+
+// Defined only for tests, used for testing purposes, so that we don't need
+// to link to tor C files. Uses the system allocator
+#[cfg(test)]
+unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void {
+    use libc::malloc;
+    malloc(size)
+}
+
+#[cfg(all(not(test), not(feature = "testing")))]
+extern "C" {
+    fn tor_malloc_(size: usize) -> *mut c_void;
+}
+
+/// Allocate memory using tor_malloc_ and copy an existing string into the
+/// allocated buffer, returning a pointer that can later be called in C.
+///
+/// # Inputs
+///
+/// * `src`, a reference to a String.
+///
+/// # Returns
+///
+/// A `*mut c_char` that should be freed by tor_free in C
+///
+#[cfg(not(feature = "testing"))]
+pub fn allocate_and_copy_string(src: &String) -> *mut c_char {
+    let bytes: &[u8] = src.as_bytes();
+
+    let size = mem::size_of_val::<[u8]>(bytes);
+    let size_one_byte = mem::size_of::<u8>();
+
+    // handle integer overflow when adding one to the calculated length
+    let size_with_null_byte = match size.checked_add(size_one_byte) {
+        Some(n) => n,
+        None => return ptr::null_mut(),
+    };
+
+    let dest = unsafe { tor_malloc_(size_with_null_byte) as *mut u8 };
+
+    if dest.is_null() {
+        return ptr::null_mut();
+    }
+
+    unsafe { ptr::copy_nonoverlapping(bytes.as_ptr(), dest, size) };
+
+    // set the last byte as null, using the ability to index into a slice
+    // rather than doing pointer arithmatic
+    let slice = unsafe { slice::from_raw_parts_mut(dest, size_with_null_byte) };
+    slice[size] = 0; // add a null terminator
+
+    dest as *mut c_char
+}
+
+#[cfg(test)]
+mod test {
+
+    #[test]
+    fn test_allocate_and_copy_string_with_empty() {
+        use std::ffi::CStr;
+        use libc::{free, c_void};
+
+        use tor_allocate::allocate_and_copy_string;
+
+        let empty = String::new();
+        let allocated_empty = allocate_and_copy_string(&empty);
+
+        let allocated_empty_rust =
+            unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() };
+
+        assert_eq!("", allocated_empty_rust);
+
+        unsafe { free(allocated_empty as *mut c_void) };
+    }
+
+    #[test]
+    fn test_allocate_and_copy_string_with_not_empty_string() {
+        use std::ffi::CStr;
+        use libc::{free, c_void};
+
+        use tor_allocate::allocate_and_copy_string;
+
+        let empty = String::from("foo bar biz");
+        let allocated_empty = allocate_and_copy_string(&empty);
+
+        let allocated_empty_rust =
+            unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() };
+
+        assert_eq!("foo bar biz", allocated_empty_rust);
+
+        unsafe { free(allocated_empty as *mut c_void) };
+    }
+}
diff --git a/src/rust/tor_log/Cargo.toml b/src/rust/tor_log/Cargo.toml
new file mode 100644
index 0000000000..9d06299c05
--- /dev/null
+++ b/src/rust/tor_log/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "tor_log"
+version = "0.1.0"
+authors = ["The Tor Project"]
+
+[lib]
+name = "tor_log"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
+[features]
+
+[dependencies]
+libc = "0.2.39"
+
+[dependencies.tor_allocate]
+path = "../tor_allocate"
diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs
new file mode 100644
index 0000000000..72f9e38339
--- /dev/null
+++ b/src/rust/tor_log/lib.rs
@@ -0,0 +1,16 @@
+//! Copyright (c) 2016-2017, The Tor Project, Inc. */
+//! See LICENSE for licensing information */
+
+//! Logging wrapper for Rust to utilize Tor's logger, found at
+//! src/common/log.c and src/common/torlog.h
+//!
+//! Exposes different interfaces depending on whether we are running in test
+//! or non-test mode. When testing, we use a no-op implementation,
+//! otherwise we link directly to C.
+
+extern crate libc;
+extern crate tor_allocate;
+
+mod tor_log;
+
+pub use tor_log::*;
diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs
new file mode 100644
index 0000000000..ad6725f0f2
--- /dev/null
+++ b/src/rust/tor_log/tor_log.rs
@@ -0,0 +1,270 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+// Note that these functions are untested due to the fact that there are no
+// return variables to test and they are calling into a C API.
+
+/// The related domain which the logging message is relevant. For example,
+/// log messages relevant to networking would use LogDomain::LdNet, whereas
+/// general messages can use LdGeneral.
+#[derive(Eq, PartialEq)]
+pub enum LogDomain {
+    Net,
+    General,
+}
+
+/// The severity level at which to log messages.
+#[derive(Eq, PartialEq)]
+pub enum LogSeverity {
+    Notice,
+    Warn,
+}
+
+/// Main entry point for Rust modules to log messages.
+///
+/// # Inputs
+///
+/// * A `severity` of type LogSeverity, which defines the level of severity the
+/// message will be logged.
+/// * A `domain` of type LogDomain, which defines the domain the log message
+/// will be associated with.
+/// * A `function` of type &str, which defines the name of the function where
+/// the message is being logged. There is a current RFC for a macro that
+/// defines function names. When it is, we should use it. See
+/// https://github.com/rust-lang/rfcs/pull/1719
+/// * A `message` of type &str, which is the log message itself.
+#[macro_export]
+macro_rules! tor_log_msg {
+    ($severity: path,
+     $domain: path,
+     $function: expr,
+     $($message:tt)*) =>
+    {
+        {
+            let msg = format!($($message)*);
+            $crate::tor_log_msg_impl($severity, $domain, $function, msg)
+        }
+    };
+}
+
+#[inline]
+pub fn tor_log_msg_impl(
+    severity: LogSeverity,
+    domain: LogDomain,
+    function: &str,
+    message: String,
+) {
+    use std::ffi::CString;
+
+    /// Default function name to log in case of errors when converting
+    /// a function name to a CString
+    const ERR_LOG_FUNCTION: &str = "tor_log_msg";
+
+    /// Default message to log in case of errors when converting a log
+    /// message to a CString
+    const ERR_LOG_MSG: &str = "Unable to log message from Rust \
+            module due to error when converting to CString";
+
+    let func = match CString::new(function) {
+        Ok(n) => n,
+        Err(_) => CString::new(ERR_LOG_FUNCTION).unwrap(),
+    };
+
+    let msg = match CString::new(message) {
+        Ok(n) => n,
+        Err(_) => CString::new(ERR_LOG_MSG).unwrap(),
+    };
+
+    // Bind to a local variable to preserve ownership. This is essential so
+    // that ownership is guaranteed until these local variables go out of scope
+    let func_ptr = func.as_ptr();
+    let msg_ptr = msg.as_ptr();
+
+    let c_severity = unsafe { log::translate_severity(severity) };
+    let c_domain = unsafe { log::translate_domain(domain) };
+
+    unsafe { log::tor_log_string(c_severity, c_domain, func_ptr, msg_ptr) }
+}
+
+/// This implementation is used when compiling for actual use, as opposed to
+/// testing.
+#[cfg(not(test))]
+pub mod log {
+    use libc::{c_char, c_int};
+    use super::LogDomain;
+    use super::LogSeverity;
+
+    /// Severity log types. These mirror definitions in /src/common/torlog.h
+    /// C_RUST_COUPLED: src/common/log.c, log domain types
+    extern "C" {
+        static LOG_WARN_: c_int;
+        static LOG_NOTICE_: c_int;
+    }
+
+    /// Domain log types. These mirror definitions in /src/common/torlog.h
+    /// C_RUST_COUPLED: src/common/log.c, log severity types
+    extern "C" {
+        static LD_NET_: u32;
+        static LD_GENERAL_: u32;
+    }
+
+    /// Translate Rust defintions of log domain levels to C. This exposes a 1:1
+    /// mapping between types.
+    #[inline]
+    pub unsafe fn translate_domain(domain: LogDomain) -> u32 {
+        match domain {
+            LogDomain::Net => LD_NET_,
+            LogDomain::General => LD_GENERAL_,
+        }
+    }
+
+    /// Translate Rust defintions of log severity levels to C. This exposes a
+    /// 1:1 mapping between types.
+    #[inline]
+    pub unsafe fn translate_severity(severity: LogSeverity) -> c_int {
+        match severity {
+            LogSeverity::Warn => LOG_WARN_,
+            LogSeverity::Notice => LOG_NOTICE_,
+        }
+    }
+
+    /// The main entry point into Tor's logger. When in non-test mode, this
+    /// will link directly with `tor_log_string` in /src/or/log.c
+    extern "C" {
+        pub fn tor_log_string(
+            severity: c_int,
+            domain: u32,
+            function: *const c_char,
+            string: *const c_char,
+        );
+    }
+}
+
+/// This module exposes no-op functionality for testing other Rust modules
+/// without linking to C.
+#[cfg(test)]
+pub mod log {
+    use libc::{c_char, c_int};
+    use super::LogDomain;
+    use super::LogSeverity;
+
+    pub static mut LAST_LOGGED_FUNCTION: *mut String = 0 as *mut String;
+    pub static mut LAST_LOGGED_MESSAGE: *mut String = 0 as *mut String;
+
+    pub unsafe fn tor_log_string(
+        _severity: c_int,
+        _domain: u32,
+        function: *const c_char,
+        message: *const c_char,
+    ) {
+        use std::ffi::CStr;
+
+        let f = CStr::from_ptr(function);
+        let fct = match f.to_str() {
+            Ok(n) => n,
+            Err(_) => "",
+        };
+        LAST_LOGGED_FUNCTION = Box::into_raw(Box::new(String::from(fct)));
+
+        let m = CStr::from_ptr(message);
+        let msg = match m.to_str() {
+            Ok(n) => n,
+            Err(_) => "",
+        };
+        LAST_LOGGED_MESSAGE = Box::into_raw(Box::new(String::from(msg)));
+    }
+
+    pub unsafe fn translate_domain(_domain: LogDomain) -> u32 {
+        1
+    }
+
+    pub unsafe fn translate_severity(_severity: LogSeverity) -> c_int {
+        1
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use tor_log::*;
+    use tor_log::log::{LAST_LOGGED_FUNCTION, LAST_LOGGED_MESSAGE};
+
+    #[test]
+    fn test_get_log_message() {
+        {
+            fn test_macro() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::Net,
+                    "test_macro",
+                    "test log message {}",
+                    "a",
+                    );
+            }
+
+            test_macro();
+
+            let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) };
+            assert_eq!("test_macro", *function);
+
+            let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) };
+            assert_eq!("test log message a", *message);
+        }
+
+        // test multiple inputs into the log message
+        {
+            fn test_macro() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::Net,
+                    "next_test_macro",
+                    "test log message {} {} {} {} {}",
+                    1,
+                    2,
+                    3,
+                    4,
+                    5
+                );
+            }
+
+            test_macro();
+
+            let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) };
+            assert_eq!("next_test_macro", *function);
+
+            let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) };
+            assert_eq!("test log message 1 2 3 4 5", *message);
+        }
+
+        // test how a long log message will be formatted
+        {
+            fn test_macro() {
+                tor_log_msg!(
+                    LogSeverity::Warn,
+                    LogDomain::Net,
+                    "test_macro",
+                    "{}",
+                    "All the world's a stage, and all the men and women \
+                    merely players: they have their exits and their \
+                    entrances; and one man in his time plays many parts, his \
+                    acts being seven ages."
+                );
+            }
+
+            test_macro();
+
+            let expected_string = "All the world's a \
+                stage, and all the men \
+                and women merely players: \
+                they have their exits and \
+                their entrances; and one man \
+                in his time plays many parts, \
+                his acts being seven ages.";
+
+            let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) };
+            assert_eq!("test_macro", *function);
+
+            let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) };
+            assert_eq!(expected_string, *message);
+        }
+    }
+}
diff --git a/src/rust/tor_rust/Cargo.toml b/src/rust/tor_rust/Cargo.toml
new file mode 100644
index 0000000000..86fad3ee76
--- /dev/null
+++ b/src/rust/tor_rust/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+authors = ["The Tor Project"]
+name = "tor_rust"
+version = "0.1.0"
+
+[lib]
+name = "tor_rust"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
+[dependencies.tor_util]
+path = "../tor_util"
+
+[dependencies.protover]
+path = "../protover"
+
diff --git a/src/rust/tor_rust/include.am b/src/rust/tor_rust/include.am
new file mode 100644
index 0000000000..99f3ede653
--- /dev/null
+++ b/src/rust/tor_rust/include.am
@@ -0,0 +1,28 @@
+EXTRA_DIST +=\
+	src/rust/tor_rust/Cargo.toml \
+	src/rust/tor_rust/lib.rs
+
+EXTRA_CARGO_OPTIONS=
+
+@TOR_RUST_LIB_PATH@: FORCE
+	( cd "$(abs_top_builddir)/src/rust" ; \
+		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
+		$(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \
+	        $(CARGO_ONLINE) \
+                --manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" )
+
+distclean-rust:
+	( cd "$(abs_top_builddir)/src/rust" ; \
+		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
+		$(CARGO) clean $(EXTRA_CARGO_OPTIONS) \
+	        $(CARGO_ONLINE) \
+                --manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" )
+	rm -rf "$(abs_top_builddir)/src/rust/registry"
+
+if USE_RUST
+build-rust: @TOR_RUST_LIB_PATH@
+else
+build-rust:
+endif
+
+FORCE:
diff --git a/src/rust/tor_rust/lib.rs b/src/rust/tor_rust/lib.rs
new file mode 100644
index 0000000000..c1585c0480
--- /dev/null
+++ b/src/rust/tor_rust/lib.rs
@@ -0,0 +1,5 @@
+extern crate tor_util;
+extern crate protover;
+
+pub use tor_util::*;
+pub use protover::*;
diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml
new file mode 100644
index 0000000000..a606a280b2
--- /dev/null
+++ b/src/rust/tor_util/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+authors = ["The Tor Project"]
+name = "tor_util"
+version = "0.0.1"
+
+[lib]
+name = "tor_util"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
+[dependencies.tor_allocate]
+path = "../tor_allocate"
+
+[dependencies.tor_log]
+path = "../tor_log"
+
+[dependencies]
+libc = "=0.2.39"
+
diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs
new file mode 100644
index 0000000000..32779ed476
--- /dev/null
+++ b/src/rust/tor_util/ffi.rs
@@ -0,0 +1,27 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! FFI functions to announce Rust support during tor startup, only to be
+//! called from C.
+//!
+
+use tor_log::{LogSeverity, LogDomain};
+
+/// Returns a short string to announce Rust support during startup.
+///
+/// # Examples
+/// ```c
+/// char *rust_str = rust_welcome_string();
+/// printf("%s", rust_str);
+/// tor_free(rust_str);
+/// ```
+#[no_mangle]
+pub extern "C" fn rust_log_welcome_string() {
+    tor_log_msg!(
+        LogSeverity::Notice,
+        LogDomain::General,
+        "rust_log_welcome_string",
+        "Tor is running with Rust integration. Please report \
+        any bugs you encounter."
+    );
+}
diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs
new file mode 100644
index 0000000000..94697b6069
--- /dev/null
+++ b/src/rust/tor_util/lib.rs
@@ -0,0 +1,14 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! Small module to announce Rust support during startup for demonstration
+//! purposes.
+
+extern crate libc;
+extern crate tor_allocate;
+
+#[macro_use]
+extern crate tor_log;
+
+pub mod ffi;
+pub mod strings;
diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs
new file mode 100644
index 0000000000..505191d913
--- /dev/null
+++ b/src/rust/tor_util/strings.rs
@@ -0,0 +1,140 @@
+// Copyright (c) 2016-2017, The Tor Project, Inc. */
+// See LICENSE for licensing information */
+
+//! Utilities for working with static strings.
+
+/// Create a `CStr` from a literal byte slice, appending a NUL byte to it first.
+///
+/// # Warning
+///
+/// The literal byte slice which is taken as an argument *MUST NOT* have any NUL
+/// bytes (`b"\0"`) in it, anywhere, or else an empty string will be returned
+/// (`CStr::from_bytes_with_nul_unchecked(b"\0")`) so as to avoid `panic!()`ing.
+///
+/// # Examples
+///
+/// ```
+/// #[macro_use]
+/// extern crate tor_util;
+///
+/// use std::ffi::CStr;
+///
+/// # fn do_test() -> Result<&'static CStr, &'static str> {
+/// let message: &'static str = "This is a test of the tsunami warning system.";
+/// let tuesday: &'static CStr;
+/// let original: &str;
+///
+/// tuesday = cstr!("This is a test of the tsunami warning system.");
+/// original = tuesday.to_str().or(Err("Couldn't unwrap CStr!"))?;
+///
+/// assert!(original == message);
+/// #
+/// # Ok(tuesday)
+/// # }
+/// # fn main() {
+/// #     do_test();  // so that we can use the ? operator in the test
+/// # }
+/// ```
+/// It is also possible to pass several string literals to this macro.  They
+/// will be concatenated together in the order of the arguments, unmodified,
+/// before finally being suffixed with a NUL byte:
+///
+/// ```
+/// #[macro_use]
+/// extern crate tor_util;
+/// #
+/// # use std::ffi::CStr;
+/// #
+/// # fn do_test() -> Result<&'static CStr, &'static str> {
+///
+/// let quux: &'static CStr = cstr!("foo", "bar", "baz");
+/// let orig: &'static str = quux.to_str().or(Err("Couldn't unwrap CStr!"))?;
+///
+/// assert!(orig == "foobarbaz");
+/// # Ok(quux)
+/// # }
+/// # fn main() {
+/// #     do_test();  // so that we can use the ? operator in the test
+/// # }
+/// ```
+/// This is useful for passing static strings to C from Rust FFI code.  To do so
+/// so, use the `.as_ptr()` method on the resulting `&'static CStr` to convert
+/// it to the Rust equivalent of a C `const char*`:
+///
+/// ```
+/// #[macro_use]
+/// extern crate tor_util;
+///
+/// use std::ffi::CStr;
+/// use std::os::raw::c_char;
+///
+/// pub extern "C" fn give_static_borrowed_string_to_c() -> *const c_char {
+///     let hello: &'static CStr = cstr!("Hello, language my parents wrote.");
+///
+///     hello.as_ptr()
+/// }
+/// # fn main() {
+/// #     let greetings = give_static_borrowed_string_to_c();
+/// # }
+/// ```
+/// Note that the C code this static borrowed string is passed to *MUST NOT*
+/// attempt to free the memory for the string.
+///
+/// # Note
+///
+/// An unfortunate limitation of the rustc compiler (as of 1.25.0-nightly), is
+/// that the first example above compiles, but if we were to change the
+/// assignment of `tuesday` as follows, it will fail to compile, because Rust
+/// macros are expanded at parse time, and at parse time there is no symbol
+/// table available.
+///
+/// ```ignore
+/// tuesday = cstr!(message);
+/// ```
+/// with the error message `error: expected a literal`.
+///
+/// # Returns
+///
+/// If the string literals passed as arguments contain no NUL bytes anywhere,
+/// then an `&'static CStr` containing the (concatenated) bytes of the string
+/// literal(s) passed as arguments, with a NUL byte appended, is returned.
+/// Otherwise, an `&'static CStr` containing a single NUL byte is returned (an
+/// "empty" string in C).
+#[macro_export]
+macro_rules! cstr {
+    ($($bytes:expr),*) => (
+        ::std::ffi::CStr::from_bytes_with_nul(
+            concat!($($bytes),*, "\0").as_bytes()
+        ).unwrap_or(
+            unsafe{
+                ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0")
+            }
+        )
+    )
+}
+
+#[cfg(test)]
+mod test {
+    use std::ffi::CStr;
+
+    #[test]
+    fn cstr_macro() {
+        let _: &'static CStr = cstr!("boo");
+    }
+
+    #[test]
+    fn cstr_macro_multi_input() {
+        let quux: &'static CStr = cstr!("foo", "bar", "baz");
+
+        assert!(quux.to_str().unwrap() == "foobarbaz");
+    }
+
+    #[test]
+    fn cstr_macro_bad_input() {
+        let waving:   &'static CStr = cstr!("waving not drowning o/");
+        let drowning: &'static CStr = cstr!("\0 drowning not waving");
+
+        assert!(waving.to_str().unwrap()   == "waving not drowning o/");
+        assert!(drowning.to_str().unwrap() == "")
+    }
+}
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index 0ba56d7036..cfbe281b94 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -12,11 +12,13 @@ LIBS = ..\..\..\build-alpha\lib\libevent.lib \
  crypt32.lib gdi32.lib user32.lib
 
 TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
-        test_containers.obj \
+        test_consdiff.obj test_containers.obj \
 	test_controller_events.obj test_crypto.obj test_data.obj test_dir.obj \
 	test_checkdir.obj test_microdesc.obj test_pt.obj test_util.obj \
         test_config.obj test_connection.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
+	test_channelpadding.obj \
+	test_circuitstats.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
diff --git a/src/test/bench.c b/src/test/bench.c
index 30984fda70..9ab23c9921 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -1,13 +1,8 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-extern const char tor_git_revision[];
-/* Ordinarily defined in tor_main.c; this bit is just here to provide one
- * since we're not linking to tor_main.c */
-const char tor_git_revision[] = "";
-
 /**
  * \file bench.c
  * \brief Benchmarks for lower level Tor modules.
@@ -17,7 +12,7 @@ const char tor_git_revision[] = "";
 
 #include "or.h"
 #include "onion_tap.h"
-#include "relay.h"
+#include "relay_crypto.h"
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
@@ -28,6 +23,8 @@ const char tor_git_revision[] = "";
 #include "crypto_curve25519.h"
 #include "onion_ntor.h"
 #include "crypto_ed25519.h"
+#include "crypto_rand.h"
+#include "consdiff.h"
 
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
@@ -57,7 +54,7 @@ perftime(void)
   return timespec_to_nsec(&ts) - nanostart;
 }
 
-#else
+#else /* !(defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)) */
 static struct timeval tv_start = { 0, 0 };
 static void
 reset_perftime(void)
@@ -72,7 +69,7 @@ perftime(void)
   timersub(&now, &tv_start, &out);
   return ((uint64_t)out.tv_sec)*1000000000 + out.tv_usec*1000;
 }
-#endif
+#endif /* defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) */
 
 #define NANOCOUNT(start,end,iters) \
   ( ((double)((end)-(start))) / (iters) )
@@ -120,7 +117,7 @@ bench_onion_TAP(void)
   uint64_t start, end;
   char os[TAP_ONIONSKIN_CHALLENGE_LEN];
   char or[TAP_ONIONSKIN_REPLY_LEN];
-  crypto_dh_t *dh_out;
+  crypto_dh_t *dh_out = NULL;
 
   key = crypto_pk_new();
   key2 = crypto_pk_new();
@@ -175,6 +172,7 @@ bench_onion_TAP(void)
          NANOCOUNT(start, end, iters)/1e3);
 
  done:
+  crypto_dh_free(dh_out);
   crypto_pk_free(key);
   crypto_pk_free(key2);
 }
@@ -198,6 +196,7 @@ bench_onion_ntor_impl(void)
   curve25519_public_key_generate(&keypair2.pubkey, &keypair2.seckey);
   dimap_add_entry(&keymap, keypair1.pubkey.public_key, &keypair1);
   dimap_add_entry(&keymap, keypair2.pubkey.public_key, &keypair2);
+  crypto_rand((char *)nodeid, sizeof(nodeid));
 
   reset_perftime();
   start = perftime();
@@ -507,10 +506,10 @@ bench_cell_ops(void)
   char key1[CIPHER_KEY_LEN], key2[CIPHER_KEY_LEN];
   crypto_rand(key1, sizeof(key1));
   crypto_rand(key2, sizeof(key2));
-  or_circ->p_crypto = crypto_cipher_new(key1);
-  or_circ->n_crypto = crypto_cipher_new(key2);
-  or_circ->p_digest = crypto_digest_new();
-  or_circ->n_digest = crypto_digest_new();
+  or_circ->crypto.f_crypto = crypto_cipher_new(key1);
+  or_circ->crypto.b_crypto = crypto_cipher_new(key2);
+  or_circ->crypto.f_digest = crypto_digest_new();
+  or_circ->crypto.b_digest = crypto_digest_new();
 
   reset_perftime();
 
@@ -520,7 +519,8 @@ bench_cell_ops(void)
     for (i = 0; i < iters; ++i) {
       char recognized = 0;
       crypt_path_t *layer_hint = NULL;
-      relay_crypt(TO_CIRCUIT(or_circ), cell, d, &layer_hint, &recognized);
+      relay_decrypt_cell(TO_CIRCUIT(or_circ), cell, d,
+                         &layer_hint, &recognized);
     }
     end = perftime();
     printf("%sbound cells: %.2f ns per cell. (%.2f ns per byte of payload)\n",
@@ -529,10 +529,7 @@ bench_cell_ops(void)
            NANOCOUNT(start,end,iters*CELL_PAYLOAD_SIZE));
   }
 
-  crypto_digest_free(or_circ->p_digest);
-  crypto_digest_free(or_circ->n_digest);
-  crypto_cipher_free(or_circ->p_crypto);
-  crypto_cipher_free(or_circ->n_crypto);
+  relay_crypto_clear(&or_circ->crypto);
   tor_free(or_circ);
   tor_free(cell);
 }
@@ -672,6 +669,28 @@ main(int argc, const char **argv)
   or_options_t *options;
 
   tor_threads_init();
+  tor_compress_init();
+
+  if (argc == 4 && !strcmp(argv[1], "diff")) {
+    init_logging(1);
+    const int N = 200;
+    char *f1 = read_file_to_str(argv[2], RFTS_BIN, NULL);
+    char *f2 = read_file_to_str(argv[3], RFTS_BIN, NULL);
+    if (! f1 || ! f2) {
+      perror("X");
+      return 1;
+    }
+    for (i = 0; i < N; ++i) {
+      char *diff = consensus_diff_generate(f1, f2);
+      tor_free(diff);
+    }
+    char *diff = consensus_diff_generate(f1, f2);
+    printf("%s", diff);
+    tor_free(f1);
+    tor_free(f2);
+    tor_free(diff);
+    return 0;
+  }
 
   for (i = 1; i < argc; ++i) {
     if (!strcmp(argv[i], "--list")) {
@@ -693,11 +712,15 @@ main(int argc, const char **argv)
     printf("Couldn't seed RNG; exiting.\n");
     return 1;
   }
+
+  init_protocol_warning_severity_level();
   crypto_init_siphash_key();
   options = options_new();
   init_logging(1);
   options->command = CMD_RUN_UNITTESTS;
   options->DataDirectory = tor_strdup("");
+  options->KeyDirectory = tor_strdup("");
+  options->CacheDirectory = tor_strdup("");
   options_init(options);
   if (set_options(options, &errmsg) < 0) {
     printf("Failed to set initial options: %s\n", errmsg);
diff --git a/src/test/bt_test.py b/src/test/bt_test.py
index a1efca00fb..20d5c93346 100755
--- a/src/test/bt_test.py
+++ b/src/test/bt_test.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2015, The Tor Project, Inc
+# Copyright 2013-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py
index d5a3a79910..f84d3002d3 100644
--- a/src/test/ed25519_exts_ref.py
+++ b/src/test/ed25519_exts_ref.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc
+# Copyright 2014-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
@@ -32,8 +32,7 @@ def curve25519ToEd25519(c, sign):
     return encodepoint([x,y])
 
 def blindESK(esk, param):
-    h = H("Derive temporary signing key" + param)
-    mult = 2**(b-2) + sum(2**i * bit(h,i) for i in range(3,b-2))
+    mult = 2**(b-2) + sum(2**i * bit(param,i) for i in range(3,b-2))
     s = decodeint(esk[:32])
     s_prime = (s * mult) % ell
     k = esk[32:]
@@ -42,8 +41,7 @@ def blindESK(esk, param):
     return encodeint(s_prime) + k_prime
 
 def blindPK(pk, param):
-    h = H("Derive temporary signing key" + param)
-    mult = 2**(b-2) + sum(2**i * bit(h,i) for i in range(3,b-2))
+    mult = 2**(b-2) + sum(2**i * bit(param,i) for i in range(3,b-2))
     P = decodepoint(pk)
     return encodepoint(scalarmult(P, mult))
 
@@ -69,6 +67,11 @@ def signatureWithESK(m,h,pk):
 def newSK():
     return os.urandom(32)
 
+def random_scalar(entropy_f): # 0..L-1 inclusive
+    # reduce the bias to a safe level by generating 256 extra bits
+    oversized = int(binascii.hexlify(entropy_f(32+32)), 16)
+    return oversized % ell
+
 # ------------------------------------------------------------
 
 MSG = "This is extremely silly. But it is also incredibly serious business!"
@@ -126,6 +129,31 @@ class SelfTest(unittest.TestCase):
 
         self._testSignatures(besk, bpk)
 
+    def testIdentity(self):
+        # Base point:
+        # B is the unique point (x, 4/5) \in E for which x is positive
+        By = 4 * inv(5)
+        Bx = xrecover(By)
+        B = [Bx % q,By % q]
+
+        # Get identity E by doing: E = l*B, where l is the group order
+        identity = scalarmult(B, ell)
+
+        # Get identity E by doing: E = l*A, where A is a random point
+        sk = newSK()
+        pk = decodepoint(publickey(sk))
+        identity2 = scalarmult(pk, ell)
+
+        # Check that identities match
+        assert(identity == identity2)
+        # Check that identity is the point (0,1)
+        assert(identity == [0L,1L])
+
+        # Check identity element: a*E = E, where a is a random scalar
+        scalar = random_scalar(os.urandom)
+        result = scalarmult(identity, scalar)
+        assert(result == identity == identity2)
+
 # ------------------------------------------------------------
 
 # From pprint.pprint([ binascii.b2a_hex(os.urandom(32)) for _ in xrange(8) ])
diff --git a/src/test/ed25519_vectors.inc b/src/test/ed25519_vectors.inc
index 760bafb971..60c863beba 100644
--- a/src/test/ed25519_vectors.inc
+++ b/src/test/ed25519_vectors.inc
@@ -91,21 +91,21 @@ static const char *ED25519_BLINDING_PARAMS[] = {
  * blinding parameter.
  */
 static const char *ED25519_BLINDED_SECRET_KEYS[] = {
-  "014e83abadb2ca9a27e0ffe23920333d817729f48700e97656ec2823d694050e171d43"
+  "293c3acff4e902f6f63ddc5d5caa2a57e771db4f24de65d4c28df3232f47fa01171d43"
       "f24e3f53e70ec7ac280044ac77d4942dee5d6807118a59bdf3ee647e89",
-  "fad8cca0b4335847795288b1452508752b253e64e6c7c78d4a02dbbd7d46aa0eb8ceff"
+  "38b88f9f9440358da544504ee152fb475528f7c51c285bd1c68b14ade8e29a07b8ceff"
       "20dfcf53eb52b891fc078c934efbf0353af7242e7dc51bb32a093afa29",
-  "116eb0ae0a4a91763365bdf86db427b00862db448487808788cc339ac10e5e089217f5"
+  "4d03ce16a3f3249846aac9de0a0075061495c3b027248eeee47da4ddbaf9e0049217f5"
       "2e92797462bd890fc274672e05c98f2c82970d640084781334aae0f940",
-  "bd1fbb0ee5acddc4adbcf5f33e95d9445f40326ce579fdd764a24483a9ccb20f509ece"
+  "51d7db01aaa0d937a9fd7c8c7381445a14d8fa61f43347af5460d7cd8fda9904509ece"
       "e77082ce088f7c19d5a00e955eeef8df6fa41686abc1030c2d76807733",
-  "237f5345cefe8573ce9fa7e216381a1172796c9e3f70668ab503b1352952530fb57b95"
+  "1f76cab834e222bd2546efa7e073425680ab88df186ff41327d3e40770129b00b57b95"
       "a440570659a440a3e4771465022a8e67af86bdf2d0990c54e7bb87ff9a",
-  "ba8ff23bc4ad2b739e1ccffc9fbc7837053ea81cdfdb15073f56411cfbae1d0ec492fc"
+  "c23588c23ee76093419d07b27c6df5922a03ac58f96c53671456a7d1bdbf560ec492fc"
       "87d5ec2a1b185ca5a40541fdef0b1e128fd5c2380c888bfa924711bcab",
-  "0fa68f969de038c7a90a4a74ee6167c77582006f2dedecc1956501ba6b6fb10391b476"
+  "3ed249c6932d076e1a2f6916975914b14e8c739da00992358b8f37d3e790650691b476"
       "8f8e556d78f4bdcb9a13b6f6066fe81d3134ae965dc48cd0785b3af2b8",
-  "deaa3456d1c21944d5dcd361a646858c6cf9336b0a6851d925717eb1ae186902053d9c"
+  "288cbfd923cb286d48c084555b5bdd06c05e92fb81acdb45271367f57515380e053d9c"
       "00c81e1331c06ab50087be8cfc7dc11691b132614474f1aa9c2503cccd",
 };
 
@@ -115,14 +115,14 @@ static const char *ED25519_BLINDED_SECRET_KEYS[] = {
  * blinding parameter.
  */
 static const char *ED25519_BLINDED_PUBLIC_KEYS[] = {
-  "722d6da6348e618967ef782e71061e27163a8b35f21856475d9d2023f65b6495",
-  "1dffa0586da6cbfcff2024eedf4fc6c818242d9a82dbbe635d6da1b975a1160d",
-  "5ed81f98fed5a6acda4ea6da2c34fab0ab359d950c510c256473f1f33ff438b4",
-  "6e6f92a54fb282120c46d9603df41135f025bc1f58f283809d04be96aeb04040",
-  "cda236f28edc4c7e02d18007b8dab49d669265b0f7aefb1824d7cc8e73a2cd63",
-  "367b03b17b67ca7329b89a520bdab91782402a41cd67264e34b5541a4b3f875b",
-  "8d486b03ac4e3b486b7a1d563706c7fdac75aee789a7cf6f22789eedeff61a31",
-  "9f297ff0aa2ceda91c5ab1b6446f12533d145940de6d850dc323417afde0cb78",
+  "1fc1fa4465bd9d4956fdbdc9d3acb3c7019bb8d5606b951c2e1dfe0b42eaeb41",
+  "1cbbd4a88ce8f165447f159d9f628ada18674158c4f7c5ead44ce8eb0fa6eb7e",
+  "c5419ad133ffde7e0ac882055d942f582054132b092de377d587435722deb028",
+  "3e08d0dc291066272e313014bfac4d39ad84aa93c038478a58011f431648105f",
+  "59381f06acb6bf1389ba305f70874eed3e0f2ab57cdb7bc69ed59a9b8899ff4d",
+  "2b946a484344eb1c17c89dd8b04196a84f3b7222c876a07a4cece85f676f87d9",
+  "c6b585129b135f8769df2eba987e76e089e80ba3a2a6729134d3b28008ac098e",
+  "0eefdc795b59cabbc194c6174e34ba9451e8355108520554ec285acabebb34ac",
 };
 
 /**
diff --git a/src/test/fakechans.h b/src/test/fakechans.h
index fa0e37dbe6..ab5d8461b6 100644
--- a/src/test/fakechans.h
+++ b/src/test/fakechans.h
@@ -1,4 +1,4 @@
- /* Copyright (c) 2014-2016, The Tor Project, Inc. */
+ /* Copyright (c) 2014-2017, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 #ifndef TOR_FAKECHANS_H
@@ -20,7 +20,6 @@ void scheduler_release_channel_mock(channel_t *ch);
 
 /* Query some counters used by the exposed mocks */
 int get_mock_scheduler_has_waiting_cells_count(void);
-int get_mock_scheduler_release_channel_count(void);
 
 #endif /* !defined(TOR_FAKECHANS_H) */
 
diff --git a/src/test/fuzz/dict/consensus b/src/test/fuzz/dict/consensus
new file mode 100644
index 0000000000..3fcd9ee7ff
--- /dev/null
+++ b/src/test/fuzz/dict/consensus
@@ -0,0 +1,52 @@
+"a"
+"additional-digest"
+"additional-signature"
+"bandwidth-weights"
+"client-versions"
+"consensus-digest"
+"consensus-method"
+"consensus-methods"
+"contact"
+"dir-address"
+"directory-footer"
+"directory-signature"
+"dir-identity-key"
+"dir-key-certificate-version"
+"dir-key-certification"
+"dir-key-crosscert"
+"dir-key-expires"
+"dir-key-published"
+"dir-signing-key"
+"dir-source"
+"fingerprint"
+"fresh-until"
+"id"
+"known-flags"
+"legacy-dir-key"
+"m"
+"network-status-version"
+"opt"
+"p"
+"package"
+"params"
+"pr"
+"published"
+"r"
+"recommended-client-protocols"
+"recommended-relay-protocols"
+"required-client-protocols"
+"required-relay-protocols"
+"s"
+"server-versions"
+"shared-rand-commit"
+"shared-rand-current-value"
+"shared-rand-participate"
+"shared-rand-previous-value"
+"signing-ed25519"
+"v"
+"valid-after"
+"valid-until"
+"vote-digest"
+"vote-status"
+"voting-delay"
+"w"
diff --git a/src/test/fuzz/dict/descriptor b/src/test/fuzz/dict/descriptor
new file mode 100644
index 0000000000..110ee3e820
--- /dev/null
+++ b/src/test/fuzz/dict/descriptor
@@ -0,0 +1,41 @@
+"reject"
+"accept"
+"reject6"
+"accept6"
+"router"
+"ipv6-policy"
+"signing-key"
+"onion-key"
+"ntor-onion-key"
+"router-signature"
+"published"
+"uptime"
+"fingerprint"
+"hibernating"
+"platform"
+"proto"
+"contact"
+"read-history"
+"write-history"
+"extra-info-digest"
+"hidden-service-dir"
+"identity-ed25519"
+"master-key-ed25519"
+"router-sig-ed25519"
+"onion-key-crosscert"
+"ntor-onion-key-crosscert"
+"allow-single-hop-exits"
+"family"
+"caches-extra-info"
+"or-address"
+"opt"
+ "bandwidth"
+"@purpose"
+"tunnelled-dir-server"
+"-----BEGIN"
+"-----END"
+"-----"
+"ED25519 CERT"
+"RSA PUBLIC KEY"
+"CROSSCERT"
+"SIGNATURE"
diff --git a/src/test/fuzz/dict/extrainfo b/src/test/fuzz/dict/extrainfo
new file mode 100644
index 0000000000..eba7a1e4ce
--- /dev/null
+++ b/src/test/fuzz/dict/extrainfo
@@ -0,0 +1,32 @@
+"cell-circuits-per-decile"
+"cell-processed-cells"
+"cell-queued-cells"
+"cell-stats-end"
+"cell-time-in-queue"
+"dirreq-stats-end"
+"dirreq-v2-direct-dl"
+"dirreq-v2-ips"
+"dirreq-v2-reqs"
+"dirreq-v2-resp"
+"dirreq-v2-share"
+"dirreq-v2-tunneled-dl"
+"dirreq-v3-direct-dl"
+"dirreq-v3-ips"
+"dirreq-v3-reqs"
+"dirreq-v3-resp"
+"dirreq-v3-share"
+"dirreq-v3-tunneled-dl"
+"entry-ips"
+"entry-stats-end"
+"exit-kibibytes-read"
+"exit-kibibytes-written"
+"exit-stats-end"
+"exit-streams-opened"
+"extra-info"
+"identity-ed25519"
+"opt"
+"published"
+"read-history"
+"router-sig-ed25519"
+"router-signature"
+"write-history"
diff --git a/src/test/fuzz/dict/hsdescv2 b/src/test/fuzz/dict/hsdescv2
new file mode 100644
index 0000000000..48788301dc
--- /dev/null
+++ b/src/test/fuzz/dict/hsdescv2
@@ -0,0 +1,8 @@
+"introduction-points"
+"permanent-key"
+"protocol-versions"
+"publication-time"
+"rendezvous-service-descriptor"
+"secret-id-part"
+"signature"
+"version"
diff --git a/src/test/fuzz/dict/hsdescv3 b/src/test/fuzz/dict/hsdescv3
new file mode 100644
index 0000000000..84e8db578a
--- /dev/null
+++ b/src/test/fuzz/dict/hsdescv3
@@ -0,0 +1,6 @@
+"hs-descriptor"
+"descriptor-lifetime"
+"descriptor-signing-key-cert"
+"revision-counter"
+"superencrypted"
+"signature"
diff --git a/src/test/fuzz/dict/http b/src/test/fuzz/dict/http
new file mode 100644
index 0000000000..3b0531579d
--- /dev/null
+++ b/src/test/fuzz/dict/http
@@ -0,0 +1,24 @@
+#
+# AFL dictionary for the Tor Directory protocol's HTTP headers
+# ------------------------------------------------------------
+#
+# Extracted from directory_handle_command() in the tor source code
+#
+# Copyright (c) 2016-2017, The Tor Project, Inc.
+# See LICENSE for licensing information
+#
+# Usage:
+# Select the dictionaries relevant to the part of the directory protocol you
+# are fuzzing, and feed them to your fuzzer (if it supports dictionaries).
+
+http_header_body_delimiter = "\x0d\x0a\x0d\x0a"
+http_header_header_delimiter = "\x0d\x0a"
+# multi-character tokens only
+#http_header_value_delimiter = " "
+
+content_length_header = "Content-Length:"
+forwarded_for_header = "Forwarded-For:"
+x_forwarded_for_header = "X-Forwarded-For:"
+
+get_command = "GET"
+post_command = "POST"
diff --git a/src/test/fuzz/dict/iptsv2 b/src/test/fuzz/dict/iptsv2
new file mode 100644
index 0000000000..57791c5e3c
--- /dev/null
+++ b/src/test/fuzz/dict/iptsv2
@@ -0,0 +1,6 @@
+"introduction-point"
+"ip-address"
+"onion-port"
+"onion-key"
+"service-key"
+
diff --git a/src/test/fuzz/dict/microdesc b/src/test/fuzz/dict/microdesc
new file mode 100644
index 0000000000..fdd0567b65
--- /dev/null
+++ b/src/test/fuzz/dict/microdesc
@@ -0,0 +1,7 @@
+"onion-key"
+"ntor-onion-key"
+"id"
+"a"
+"family"
+"p"
+"p6"
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
new file mode 100755
index 0000000000..68efc1abc5
--- /dev/null
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+if [ ! -d "$1" ] ; then
+    echo "I need a directory"
+    exit 1
+fi
+
+for fn in "$1"/* ; do
+    prev=`basename "$fn"`
+    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
+    if [ "$prev" == "$post" ] ; then
+      echo "OK $prev"
+    else
+      echo "mv $prev $post"
+      mv "$fn" "$1/$post"
+    fi
+done
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
new file mode 100644
index 0000000000..6610ade7ad
--- /dev/null
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -0,0 +1,78 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "networkstatus.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+static int
+mock_signed_digest_equals__yes(const uint8_t *d1, const uint8_t *d2,
+                               size_t len)
+{
+  (void) tor_memeq(d1, d2, len);
+  return 1;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  MOCK(signed_digest_equals, mock_signed_digest_equals__yes);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  networkstatus_t *ns;
+  char *str = tor_memdup_nulterm(data, sz);
+  const char *eos = NULL;
+  networkstatus_type_t tp = NS_TYPE_CONSENSUS;
+  if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
+    tp = NS_TYPE_VOTE;
+  const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
+  ns = networkstatus_parse_vote_from_string(str,
+                                            &eos,
+                                            tp);
+  if (ns) {
+    log_debug(LD_GENERAL, "Parsing as %s okay", what);
+    networkstatus_vote_free(ns);
+  } else {
+    log_debug(LD_GENERAL, "Parsing as %s failed", what);
+  }
+  tor_free(str);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c
new file mode 100644
index 0000000000..1a50beae17
--- /dev/null
+++ b/src/test/fuzz/fuzz_descriptor.c
@@ -0,0 +1,79 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "routerlist.h"
+#include "routerkeys.h"
+#include "fuzzing.h"
+
+static int
+mock_check_tap_onion_key_crosscert__nocheck(const uint8_t *crosscert,
+                                   int crosscert_len,
+                                   const crypto_pk_t *onion_pkey,
+                                   const ed25519_public_key_t *master_id_pkey,
+                                   const uint8_t *rsa_id_digest)
+{
+  tor_assert(crosscert && onion_pkey && master_id_pkey && rsa_id_digest);
+  /* we could look at crosscert[..] */
+  (void) crosscert_len;
+  return 0;
+}
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(check_tap_onion_key_crosscert,
+       mock_check_tap_onion_key_crosscert__nocheck);
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  routerinfo_t *ri;
+  const char *str = (const char*) data;
+  ri = router_parse_entry_from_string((const char *)str,
+                                      str+sz,
+                                      0, 0, 0, NULL);
+  if (ri) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    routerinfo_free(ri);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
new file mode 100644
index 0000000000..642380b512
--- /dev/null
+++ b/src/test/fuzz/fuzz_diff.c
@@ -0,0 +1,69 @@
+/* Copyright (c) 2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "orconfig.h"
+#include "or.h"
+#include "consdiff.h"
+
+#include "fuzzing.h"
+
+static int
+mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+{
+  (void)c;
+  memset(d->sha3_256, 3, sizeof(d->sha3_256));
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  MOCK(consensus_compute_digest, mock_consensus_compute_digest_);
+  MOCK(consensus_compute_digest_as_signed, mock_consensus_compute_digest_);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(consensus_compute_digest);
+  UNMOCK(consensus_compute_digest_as_signed);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+#define SEP "=====\n"
+#define SEPLEN strlen(SEP)
+  const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN);
+  if (! separator)
+    return 0;
+  size_t c1_len = separator - stdin_buf;
+  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  size_t c2_len = data_size - c1_len - SEPLEN;
+  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+
+  char *c3 = consensus_diff_generate(c1, c2);
+
+  if (c3) {
+    char *c4 = consensus_diff_apply(c1, c3);
+    tor_assert(c4);
+    if (strcmp(c2, c4)) {
+      printf("%s\n", escaped(c1));
+      printf("%s\n", escaped(c2));
+      printf("%s\n", escaped(c3));
+      printf("%s\n", escaped(c4));
+    }
+    tor_assert(! strcmp(c2, c4));
+    tor_free(c3);
+    tor_free(c4);
+  }
+  tor_free(c1);
+  tor_free(c2);
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
new file mode 100644
index 0000000000..8d7bf751bf
--- /dev/null
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "orconfig.h"
+#include "or.h"
+#include "consdiff.h"
+
+#include "fuzzing.h"
+
+static int
+mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+{
+  (void)c;
+  memset(d->sha3_256, 3, sizeof(d->sha3_256));
+  return 0;
+}
+
+static int
+mock_consensus_digest_eq_(const uint8_t *a, const uint8_t *b)
+{
+  (void)a;
+  (void)b;
+  return 1;
+}
+
+int
+fuzz_init(void)
+{
+  MOCK(consensus_compute_digest, mock_consensus_compute_digest_);
+  MOCK(consensus_digest_eq, mock_consensus_digest_eq_);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(consensus_compute_digest);
+  UNMOCK(consensus_digest_eq);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+#define SEP "=====\n"
+#define SEPLEN strlen(SEP)
+  const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN);
+  if (! separator)
+    return 0;
+  size_t c1_len = separator - stdin_buf;
+  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  size_t c2_len = data_size - c1_len - SEPLEN;
+  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+
+  char *c3 = consensus_diff_apply(c1, c2);
+
+  tor_free(c1);
+  tor_free(c2);
+  tor_free(c3);
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c
new file mode 100644
index 0000000000..2a3de7ecf7
--- /dev/null
+++ b/src/test/fuzz/fuzz_extrainfo.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "routerlist.h"
+#include "routerkeys.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  extrainfo_t *ei;
+  const char *str = (const char*) data;
+  int again = 0;
+  ei = extrainfo_parse_entry_from_string((const char *)str,
+                                         str+sz,
+                                         0, NULL, &again);
+  if (ei) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    extrainfo_free(ei);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c
new file mode 100644
index 0000000000..19db265716
--- /dev/null
+++ b/src/test/fuzz/fuzz_hsdescv2.c
@@ -0,0 +1,52 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "rendcommon.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  rend_service_descriptor_t *desc = NULL;
+  char desc_id[64];
+  char *ipts = NULL;
+  size_t ipts_size, esize;
+  const char *next;
+  char *str = tor_memdup_nulterm(data, sz);
+  (void) rend_parse_v2_service_descriptor(&desc, desc_id, &ipts, &ipts_size,
+                                          &esize, &next, str, 1);
+  if (desc) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    rend_service_descriptor_free(desc);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  tor_free(ipts);
+  tor_free(str);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
new file mode 100644
index 0000000000..428774e330
--- /dev/null
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -0,0 +1,99 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define ROUTERPARSE_PRIVATE
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "or.h"
+#include "ed25519_cert.h" /* Trunnel interface. */
+#include "crypto_ed25519.h"
+#include "hs_descriptor.h"
+#include "routerparse.h"
+#include "util.h"
+
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert,
+                                 const size_t crosscert_len,
+                                 const crypto_pk_t *rsa_id_key,
+                                 const ed25519_public_key_t *master_key,
+                                 const time_t reject_if_expired_before)
+{
+  (void) crosscert;
+  (void) crosscert_len;
+  (void) rsa_id_key;
+  (void) master_key;
+  (void) reject_if_expired_before;
+  return 0;
+}
+
+static size_t
+mock_decrypt_desc_layer(const hs_descriptor_t *desc,
+                        const uint8_t *encrypted_blob,
+                        size_t encrypted_blob_size,
+                        int is_superencrypted_layer,
+                        char **decrypted_out)
+{
+  (void)is_superencrypted_layer;
+  (void)desc;
+  const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN;
+  if (encrypted_blob_size < overhead)
+    return 0;
+  *decrypted_out = tor_memdup_nulterm(
+                   encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN,
+                   encrypted_blob_size - overhead);
+  size_t result = strlen(*decrypted_out);
+  if (result) {
+    return result;
+  } else {
+    tor_free(*decrypted_out);
+    return 0;
+  }
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(rsa_ed25519_crosscert_check, mock_rsa_ed25519_crosscert_check);
+  MOCK(decrypt_desc_layer, mock_decrypt_desc_layer);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  hs_descriptor_t *desc = NULL;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  char *fuzzing_data = tor_memdup_nulterm(data, sz);
+  memset(subcredential, 'A', sizeof(subcredential));
+
+  hs_desc_decode_descriptor(fuzzing_data, subcredential, &desc);
+  if (desc) {
+    log_debug(LD_GENERAL, "Decoding okay");
+    hs_descriptor_free(desc);
+  } else {
+    log_debug(LD_GENERAL, "Decoding failed");
+  }
+
+  tor_free(fuzzing_data);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
new file mode 100644
index 0000000000..2ffeb60244
--- /dev/null
+++ b/src/test/fuzz/fuzz_http.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define BUFFERS_PRIVATE
+#define DIRECTORY_PRIVATE
+
+#include "or.h"
+#include "backtrace.h"
+#include "buffers.h"
+#include "config.h"
+#include "connection.h"
+#include "directory.h"
+#include "torlog.h"
+
+#include "fuzzing.h"
+
+static void
+mock_connection_write_to_buf_impl_(const char *string, size_t len,
+                                   connection_t *conn, int compressed)
+{
+  log_debug(LD_GENERAL, "%sResponse:\n%u\nConnection: %p\n%s\n",
+            compressed ? "Compressed " : "", (unsigned)len, conn, string);
+}
+
+static int
+mock_directory_handle_command_get(dir_connection_t *conn,
+                                      const char *headers,
+                                      const char *body,
+                                      size_t body_len)
+{
+  (void)conn;
+
+  log_debug(LD_GENERAL, "Method:\nGET\n");
+
+  if (headers) {
+    log_debug(LD_GENERAL, "Header-Length:\n%u\n", (unsigned)strlen(headers));
+    log_debug(LD_GENERAL, "Headers:\n%s\n", headers);
+  }
+
+  log_debug(LD_GENERAL, "Body-Length:\n%u\n", (unsigned)body_len);
+  if (body) {
+    log_debug(LD_GENERAL, "Body:\n%s\n", body);
+  }
+
+  /* Always tell the caller we succeeded */
+  return 0;
+}
+
+static int
+mock_directory_handle_command_post(dir_connection_t *conn,
+                                       const char *headers,
+                                       const char *body,
+                                       size_t body_len)
+{
+  (void)conn;
+
+  log_debug(LD_GENERAL, "Method:\nPOST\n");
+
+  if (headers) {
+    log_debug(LD_GENERAL, "Header-Length:\n%u\n", (unsigned)strlen(headers));
+    log_debug(LD_GENERAL, "Headers:\n%s\n", headers);
+  }
+
+  log_debug(LD_GENERAL, "Body-Length:\n%u\n", (unsigned)body_len);
+  if (body) {
+    log_debug(LD_GENERAL, "Body:\n%s\n", body);
+  }
+
+  /* Always tell the caller we succeeded */
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  /* Set up fake response handler */
+  MOCK(connection_write_to_buf_impl_, mock_connection_write_to_buf_impl_);
+  /* Set up the fake handler functions */
+  MOCK(directory_handle_command_get, mock_directory_handle_command_get);
+  MOCK(directory_handle_command_post, mock_directory_handle_command_post);
+
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(directory_handle_command_get);
+  UNMOCK(directory_handle_command_post);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  dir_connection_t dir_conn;
+
+  /* Set up the fake connection */
+  memset(&dir_conn, 0, sizeof(dir_connection_t));
+  dir_conn.base_.type = CONN_TYPE_DIR;
+  /* Apparently tor sets this before directory_handle_command() is called. */
+  dir_conn.base_.address = tor_strdup("replace-this-address.example.com");
+
+  dir_conn.base_.inbuf = buf_new_with_data((char*)stdin_buf, data_size);
+  if (!dir_conn.base_.inbuf) {
+    log_debug(LD_GENERAL, "Zero-Length-Input\n");
+    goto done;
+  }
+
+  /* Parse the headers */
+  int rv = directory_handle_command(&dir_conn);
+
+  /* TODO: check the output is correctly parsed based on the input */
+
+  /* Report the parsed origin address */
+  if (dir_conn.base_.address) {
+    log_debug(LD_GENERAL, "Address:\n%s\n", dir_conn.base_.address);
+  }
+
+  log_debug(LD_GENERAL, "Result:\n%d\n", rv);
+
+ done:
+  /* Reset. */
+  tor_free(dir_conn.base_.address);
+  buf_free(dir_conn.base_.inbuf);
+  dir_conn.base_.inbuf = NULL;
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c
new file mode 100644
index 0000000000..dc674070b2
--- /dev/null
+++ b/src/test/fuzz/fuzz_http_connect.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define BUFFERS_PRIVATE
+#define CONNECTION_EDGE_PRIVATE
+
+#include "or.h"
+#include "backtrace.h"
+#include "buffers.h"
+#include "config.h"
+#include "connection.h"
+#include "connection_edge.h"
+#include "proto_socks.h"
+#include "torlog.h"
+
+#include "fuzzing.h"
+
+static void
+mock_connection_write_to_buf_impl_(const char *string, size_t len,
+                                   connection_t *conn, int compressed)
+{
+  log_debug(LD_GENERAL, "%sResponse:\n%u\nConnection: %p\n%s\n",
+            compressed ? "Compressed " : "", (unsigned)len, conn, string);
+}
+
+static void
+mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
+                                    int line, const char *file)
+{
+  (void)conn;
+  (void)endreason;
+  (void)line;
+  (void)file;
+}
+
+static int
+mock_connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn,
+                                                 origin_circuit_t *circ,
+                                                 crypt_path_t *cpath)
+{
+  (void)conn;
+  (void)circ;
+  (void)cpath;
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  /* Set up fake response handler */
+  MOCK(connection_write_to_buf_impl_, mock_connection_write_to_buf_impl_);
+  /* Set up the fake handler functions */
+  MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_);
+  MOCK(connection_ap_rewrite_and_attach_if_allowed,
+       mock_connection_ap_rewrite_and_attach_if_allowed);
+
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(connection_ap_rewrite_and_attach_if_allowed);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  entry_connection_t conn;
+
+  /* Set up the fake connection */
+  memset(&conn, 0, sizeof(conn));
+  conn.edge_.base_.type = CONN_TYPE_AP;
+  conn.edge_.base_.state = AP_CONN_STATE_HTTP_CONNECT_WAIT;
+  conn.socks_request = tor_malloc_zero(sizeof(socks_request_t));
+  conn.socks_request->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+
+  conn.edge_.base_.inbuf = buf_new_with_data((char*)stdin_buf, data_size);
+  if (!conn.edge_.base_.inbuf) {
+    log_debug(LD_GENERAL, "Zero-Length-Input\n");
+    goto done;
+  }
+
+  /* Parse the headers */
+  int rv = connection_ap_process_http_connect(&conn);
+
+  /* TODO: check the output is correctly parsed based on the input */
+
+  log_debug(LD_GENERAL, "Result:\n%d\n", rv);
+
+  goto done;
+
+ done:
+  /* Reset. */
+  socks_request_free(conn.socks_request);
+  buf_free(conn.edge_.base_.inbuf);
+  conn.edge_.base_.inbuf = NULL;
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c
new file mode 100644
index 0000000000..4abde0c16d
--- /dev/null
+++ b/src/test/fuzz/fuzz_iptsv2.c
@@ -0,0 +1,46 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "rendcommon.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  rend_service_descriptor_t *desc =
+    tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  const char *str = (const char*) data;
+  int r = rend_parse_introduction_points(desc, str, sz);
+  if (r >= 0) {
+    log_debug(LD_GENERAL, "Parsing okay: %d", r);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  rend_service_descriptor_free(desc);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c
new file mode 100644
index 0000000000..396115026e
--- /dev/null
+++ b/src/test/fuzz/fuzz_microdesc.c
@@ -0,0 +1,47 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "microdesc.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  const char *str = (const char*) data;
+  smartlist_t *result = microdescs_parse_from_string((const char *)str,
+                                                     str+sz,
+                                                     0, SAVED_NOWHERE, NULL);
+  if (result) {
+    log_debug(LD_GENERAL, "Parsing okay: %d", smartlist_len(result));
+    SMARTLIST_FOREACH(result, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(result);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
new file mode 100755
index 0000000000..b4a17ed8cb
--- /dev/null
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -0,0 +1,34 @@
+MEMLIMIT_BYTES=21990500990976
+
+N_CPUS=1
+if [ $# -ge 1 ]; then
+    N_CPUS="$1"
+    shift
+fi
+
+FILTER=echo
+
+for i in `seq -w "$N_CPUS"`; do
+    if [ "$i" -eq 1 ]; then
+        if [ "$N_CPUS" -eq 1 ]; then
+            INSTANCE=""
+            NUMBER=""
+        else
+            INSTANCE="-M"
+            NUMBER="$i"
+        fi
+    else
+        INSTANCE="-S"
+        NUMBER="$i"
+    fi
+    # use whatever remains on the command-line to prefix the fuzzer command
+    # you have to copy and paste and run these commands yourself
+    "$FILTER" "$@" \
+        ../afl/afl-fuzz \
+        -i src/test/fuzz/fuzz_dir_testcase \
+        -o src/test/fuzz/fuzz_dir_findings \
+        -x src/test/fuzz/fuzz_dir_dictionary/fuzz_dir_http_header.dct \
+        -m "$MEMLIMIT_BYTES" \
+        "$INSTANCE" "$NUMBER" \
+        -- src/test/fuzz_dir
+done
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
new file mode 100644
index 0000000000..baf0610a0b
--- /dev/null
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -0,0 +1,82 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "memarea.h"
+#include "microdesc.h"
+#include "networkstatus.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static networkstatus_t *dummy_vote = NULL;
+static memarea_t *area = NULL;
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  area = memarea_new();
+  dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
+  dummy_vote->known_flags = smartlist_new();
+  smartlist_split_string(dummy_vote->known_flags,
+                         "Authority BadExit Exit Fast Guard HSDir "
+                         "NoEdConsensus Running Stable V2Dir Valid",
+                         " ", 0, 0);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  SMARTLIST_FOREACH(dummy_vote->known_flags, char *, cp, tor_free(cp));
+  smartlist_free(dummy_vote->known_flags);
+  tor_free(dummy_vote);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  char *str = tor_memdup_nulterm(data, sz);
+  const char *s;
+  routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
+  vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
+  smartlist_t *tokens = smartlist_new();
+
+  s = str;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                               NULL, NULL, 26, FLAV_NS);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  s = str;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                               NULL, NULL, 26, FLAV_MICRODESC);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  s = str;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                              dummy_vote, vrs, 26, FLAV_NS);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  log_debug(LD_GENERAL,
+            "ns=%p, md=%p, vote=%p", rs_ns, rs_md, rs_vote);
+
+  routerstatus_free(rs_md);
+  routerstatus_free(rs_ns);
+  vote_routerstatus_free(vrs);
+  memarea_clear(area);
+  smartlist_free(tokens);
+  tor_free(str);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
new file mode 100644
index 0000000000..aecdbb4e52
--- /dev/null
+++ b/src/test/fuzz/fuzzing.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#ifndef FUZZING_H
+#define FUZZING_H
+
+int fuzz_init(void);
+int fuzz_cleanup(void);
+int fuzz_main(const uint8_t *data, size_t sz);
+
+void disable_signature_checking(void);
+
+#endif /* FUZZING_H */
+
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
new file mode 100644
index 0000000000..a96552f0fc
--- /dev/null
+++ b/src/test/fuzz/fuzzing_common.c
@@ -0,0 +1,192 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define CRYPTO_ED25519_PRIVATE
+#include "orconfig.h"
+#include "or.h"
+#include "backtrace.h"
+#include "config.h"
+#include "fuzzing.h"
+#include "crypto.h"
+#include "crypto_ed25519.h"
+
+static or_options_t *mock_options = NULL;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mock_options;
+}
+
+static int
+mock_crypto_pk_public_checksig__nocheck(const crypto_pk_t *env, char *to,
+                                        size_t tolen,
+                                        const char *from, size_t fromlen)
+{
+  tor_assert(env && to && from);
+  (void)fromlen;
+  /* We could look at from[0..fromlen-1] ... */
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  size_t siglen = MIN(20, crypto_pk_keysize(env));
+  memset(to, 0x01, siglen);
+  return (int)siglen;
+}
+
+static int
+mock_crypto_pk_public_checksig_digest__nocheck(crypto_pk_t *env,
+                                               const char *data,
+                                               size_t datalen,
+                                               const char *sig,
+                                               size_t siglen)
+{
+  tor_assert(env && data && sig);
+  (void)datalen;
+  (void)siglen;
+  /* We could look at data[..] and sig[..] */
+  return 0;
+}
+
+static int
+mock_ed25519_checksig__nocheck(const ed25519_signature_t *signature,
+                      const uint8_t *msg, size_t len,
+                      const ed25519_public_key_t *pubkey)
+{
+  tor_assert(signature && msg && pubkey);
+  /* We could look at msg[0..len-1] ... */
+  (void)len;
+  return 0;
+}
+
+static int
+mock_ed25519_checksig_batch__nocheck(int *okay_out,
+                                     const ed25519_checkable_t *checkable,
+                                     int n_checkable)
+{
+  tor_assert(checkable);
+  int i;
+  for (i = 0; i < n_checkable; ++i) {
+    /* We could look at messages and signatures XXX */
+    tor_assert(checkable[i].pubkey);
+    tor_assert(checkable[i].msg);
+    if (okay_out)
+      okay_out[i] = 1;
+  }
+  return 0;
+}
+
+static int
+mock_ed25519_impl_spot_check__nocheck(void)
+{
+  return 0;
+}
+
+void
+disable_signature_checking(void)
+{
+  MOCK(crypto_pk_public_checksig,
+       mock_crypto_pk_public_checksig__nocheck);
+  MOCK(crypto_pk_public_checksig_digest,
+       mock_crypto_pk_public_checksig_digest__nocheck);
+  MOCK(ed25519_checksig, mock_ed25519_checksig__nocheck);
+  MOCK(ed25519_checksig_batch, mock_ed25519_checksig_batch__nocheck);
+  MOCK(ed25519_impl_spot_check, mock_ed25519_impl_spot_check__nocheck);
+}
+
+static void
+global_init(void)
+{
+  tor_threads_init();
+  tor_compress_init();
+  {
+    struct sipkey sipkey = { 1337, 7331 };
+    siphash_set_global_key(&sipkey);
+  }
+
+  /* Initialise logging first */
+  init_logging(1);
+  configure_backtrace_handler(get_version());
+
+  /* set up the options. */
+  mock_options = tor_malloc_zero(sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  /* Make BUG() and nonfatal asserts crash */
+  tor_set_failed_assertion_callback(abort);
+
+  /* Make protocol warnings handled correctly. */
+  init_protocol_warning_severity_level();
+}
+
+#ifdef LLVM_FUZZ
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+int
+LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+  static int initialized = 0;
+  if (!initialized) {
+    global_init();
+    if (fuzz_init() < 0)
+      abort();
+    initialized = 1;
+  }
+
+  return fuzz_main(Data, Size);
+}
+
+#else /* Not LLVM_FUZZ, so AFL. */
+
+int
+main(int argc, char **argv)
+{
+  size_t size;
+
+  global_init();
+
+  /* Disable logging by default to speed up fuzzing. */
+  int loglevel = LOG_ERR;
+
+  for (int i = 1; i < argc; ++i) {
+    if (!strcmp(argv[i], "--warn")) {
+      loglevel = LOG_WARN;
+    } else if (!strcmp(argv[i], "--notice")) {
+      loglevel = LOG_NOTICE;
+    } else if (!strcmp(argv[i], "--info")) {
+      loglevel = LOG_INFO;
+    } else if (!strcmp(argv[i], "--debug")) {
+      loglevel = LOG_DEBUG;
+    }
+  }
+
+  {
+    log_severity_list_t s;
+    memset(&s, 0, sizeof(s));
+    set_log_severity_config(loglevel, LOG_ERR, &s);
+    /* ALWAYS log bug warnings. */
+    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    add_stream_log(&s, "", fileno(stdout));
+  }
+
+  if (fuzz_init() < 0)
+    abort();
+
+#ifdef __AFL_HAVE_MANUAL_CONTROL
+  /* Tell AFL to pause and fork here - ignored if not using AFL */
+  __AFL_INIT();
+#endif
+
+#define MAX_FUZZ_SIZE (128*1024)
+  char *input = read_file_to_str_until_eof(0, MAX_FUZZ_SIZE, &size);
+  tor_assert(input);
+  char *raw = tor_memdup(input, size); /* Because input is nul-terminated */
+  tor_free(input);
+  fuzz_main((const uint8_t*)raw, size);
+  tor_free(raw);
+
+  if (fuzz_cleanup() < 0)
+    abort();
+
+  tor_free(mock_options);
+  UNMOCK(get_options);
+  return 0;
+}
+
+#endif
+
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
new file mode 100644
index 0000000000..39d6d3c17b
--- /dev/null
+++ b/src/test/fuzz/include.am
@@ -0,0 +1,426 @@
+# This file was generated by fuzzing_include_am.py; do not hand-edit unless
+# you enjoy having your changes erased.
+FUZZING_CPPFLAGS = \
+	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+FUZZING_CFLAGS = \
+	$(AM_CFLAGS) $(TEST_CFLAGS)
+FUZZING_LDFLAG = \
+	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+FUZZING_LIBS = \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a \
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIBEVENT_LIBS@ \
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	@TOR_ZSTD_LIBS@
+
+oss-fuzz-prereqs: \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a
+
+noinst_HEADERS += \
+	src/test/fuzz/fuzzing.h
+
+LIBFUZZER = -lFuzzer
+LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
+LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
+LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
+
+LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
+
+# ===== AFL fuzzers
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_consensus_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_consensus.c
+src_test_fuzz_fuzz_consensus_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_consensus_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_consensus_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_consensus_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_descriptor_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_descriptor.c
+src_test_fuzz_fuzz_descriptor_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_descriptor_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_descriptor_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_descriptor_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_diff_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_diff.c
+src_test_fuzz_fuzz_diff_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_diff_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_diff_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_diff_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_diff_apply_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_diff_apply.c
+src_test_fuzz_fuzz_diff_apply_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_diff_apply_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_diff_apply_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_diff_apply_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_extrainfo_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_extrainfo.c
+src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_hsdescv2_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_hsdescv2.c
+src_test_fuzz_fuzz_hsdescv2_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_hsdescv2_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_hsdescv2_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_hsdescv2_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_hsdescv3_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_hsdescv3.c
+src_test_fuzz_fuzz_hsdescv3_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_hsdescv3_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_hsdescv3_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_hsdescv3_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_http_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_http.c
+src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_http_connect_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_http_connect.c
+src_test_fuzz_fuzz_http_connect_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_http_connect_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_http_connect_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_http_connect_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_iptsv2_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_iptsv2.c
+src_test_fuzz_fuzz_iptsv2_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_iptsv2_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_iptsv2_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_iptsv2_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_microdesc_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_microdesc.c
+src_test_fuzz_fuzz_microdesc_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_microdesc_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_microdesc_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_microdesc_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_vrs_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_vrs.c
+src_test_fuzz_fuzz_vrs_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_vrs_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_vrs_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_vrs_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+FUZZERS = \
+	src/test/fuzz/fuzz-consensus \
+	src/test/fuzz/fuzz-descriptor \
+	src/test/fuzz/fuzz-diff \
+	src/test/fuzz/fuzz-diff-apply \
+	src/test/fuzz/fuzz-extrainfo \
+	src/test/fuzz/fuzz-hsdescv2 \
+	src/test/fuzz/fuzz-hsdescv3 \
+	src/test/fuzz/fuzz-http \
+	src/test/fuzz/fuzz-http-connect \
+	src/test/fuzz/fuzz-iptsv2 \
+	src/test/fuzz/fuzz-microdesc \
+	src/test/fuzz/fuzz-vrs
+endif
+
+# ===== libfuzzer
+
+if LIBFUZZER_ENABLED
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_consensus_SOURCES = \
+	$(src_test_fuzz_fuzz_consensus_SOURCES)
+src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_consensus_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_consensus_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_consensus_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_descriptor_SOURCES = \
+	$(src_test_fuzz_fuzz_descriptor_SOURCES)
+src_test_fuzz_lf_fuzz_descriptor_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_descriptor_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_descriptor_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_descriptor_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_diff_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_SOURCES)
+src_test_fuzz_lf_fuzz_diff_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_diff_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_diff_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_diff_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_diff_apply_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_apply_SOURCES)
+src_test_fuzz_lf_fuzz_diff_apply_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_diff_apply_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_diff_apply_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_diff_apply_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_extrainfo_SOURCES = \
+	$(src_test_fuzz_fuzz_extrainfo_SOURCES)
+src_test_fuzz_lf_fuzz_extrainfo_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_extrainfo_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_extrainfo_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_extrainfo_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_hsdescv2_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv2_SOURCES)
+src_test_fuzz_lf_fuzz_hsdescv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv2_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv2_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_hsdescv2_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_hsdescv3_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv3_SOURCES)
+src_test_fuzz_lf_fuzz_hsdescv3_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv3_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv3_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_hsdescv3_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_http_SOURCES = \
+	$(src_test_fuzz_fuzz_http_SOURCES)
+src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_http_connect_SOURCES = \
+	$(src_test_fuzz_fuzz_http_connect_SOURCES)
+src_test_fuzz_lf_fuzz_http_connect_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_http_connect_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_http_connect_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_http_connect_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \
+	$(src_test_fuzz_fuzz_iptsv2_SOURCES)
+src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_iptsv2_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_iptsv2_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_iptsv2_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_microdesc_SOURCES = \
+	$(src_test_fuzz_fuzz_microdesc_SOURCES)
+src_test_fuzz_lf_fuzz_microdesc_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_microdesc_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_microdesc_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_microdesc_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_vrs_SOURCES = \
+	$(src_test_fuzz_fuzz_vrs_SOURCES)
+src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_vrs_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_vrs_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+LIBFUZZER_FUZZERS = \
+	src/test/fuzz/lf-fuzz-consensus \
+	src/test/fuzz/lf-fuzz-descriptor \
+	src/test/fuzz/lf-fuzz-diff \
+	src/test/fuzz/lf-fuzz-diff-apply \
+	src/test/fuzz/lf-fuzz-extrainfo \
+	src/test/fuzz/lf-fuzz-hsdescv2 \
+	src/test/fuzz/lf-fuzz-hsdescv3 \
+	src/test/fuzz/lf-fuzz-http \
+	src/test/fuzz/lf-fuzz-http-connect \
+	src/test/fuzz/lf-fuzz-iptsv2 \
+	src/test/fuzz/lf-fuzz-microdesc \
+	src/test/fuzz/lf-fuzz-vrs
+
+else
+LIBFUZZER_FUZZERS =
+endif
+
+# ===== oss-fuzz
+
+if OSS_FUZZ_ENABLED
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \
+	$(src_test_fuzz_fuzz_consensus_SOURCES)
+src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_consensus_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_descriptor_a_SOURCES = \
+	$(src_test_fuzz_fuzz_descriptor_SOURCES)
+src_test_fuzz_liboss_fuzz_descriptor_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_descriptor_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_diff_a_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_SOURCES)
+src_test_fuzz_liboss_fuzz_diff_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_diff_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_diff_apply_a_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_apply_SOURCES)
+src_test_fuzz_liboss_fuzz_diff_apply_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_diff_apply_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_extrainfo_a_SOURCES = \
+	$(src_test_fuzz_fuzz_extrainfo_SOURCES)
+src_test_fuzz_liboss_fuzz_extrainfo_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_extrainfo_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_hsdescv2_a_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv2_SOURCES)
+src_test_fuzz_liboss_fuzz_hsdescv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_hsdescv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_hsdescv3_a_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv3_SOURCES)
+src_test_fuzz_liboss_fuzz_hsdescv3_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_hsdescv3_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_http_a_SOURCES = \
+	$(src_test_fuzz_fuzz_http_SOURCES)
+src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_http_connect_a_SOURCES = \
+	$(src_test_fuzz_fuzz_http_connect_SOURCES)
+src_test_fuzz_liboss_fuzz_http_connect_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_http_connect_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \
+	$(src_test_fuzz_fuzz_iptsv2_SOURCES)
+src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_iptsv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_microdesc_a_SOURCES = \
+	$(src_test_fuzz_fuzz_microdesc_SOURCES)
+src_test_fuzz_liboss_fuzz_microdesc_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_microdesc_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
+	$(src_test_fuzz_fuzz_vrs_SOURCES)
+src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+OSS_FUZZ_FUZZERS = \
+	src/test/fuzz/liboss-fuzz-consensus.a \
+	src/test/fuzz/liboss-fuzz-descriptor.a \
+	src/test/fuzz/liboss-fuzz-diff.a \
+	src/test/fuzz/liboss-fuzz-diff-apply.a \
+	src/test/fuzz/liboss-fuzz-extrainfo.a \
+	src/test/fuzz/liboss-fuzz-hsdescv2.a \
+	src/test/fuzz/liboss-fuzz-hsdescv3.a \
+	src/test/fuzz/liboss-fuzz-http.a \
+	src/test/fuzz/liboss-fuzz-http-connect.a \
+	src/test/fuzz/liboss-fuzz-iptsv2.a \
+	src/test/fuzz/liboss-fuzz-microdesc.a \
+	src/test/fuzz/liboss-fuzz-vrs.a
+
+else
+OSS_FUZZ_FUZZERS =
+endif
+
+noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
+noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
+oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
+fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
+
+test-fuzz-corpora: $(FUZZERS)
+	$(top_srcdir)/src/test/fuzz_static_testcases.sh
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
new file mode 100755
index 0000000000..87d3dda13c
--- /dev/null
+++ b/src/test/fuzz/minimize.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+if [ ! -d "$1" ] ; then
+    echo "I need a directory"
+    exit 1
+fi
+
+which=`basename "$1"`
+
+mkdir "$1.out"
+afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
+
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
new file mode 100755
index 0000000000..3cb45ad5e6
--- /dev/null
+++ b/src/test/fuzz_static_testcases.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Copyright (c) 2016-2017, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+set -e
+
+if [ -z "${TOR_FUZZ_CORPORA}" ] || [ ! -d "${TOR_FUZZ_CORPORA}" ] ; then
+    echo "You need to set TOR_FUZZ_CORPORA to point to a checkout of "
+    echo "the 'fuzzing-corpora' repository."
+    exit 77
+fi
+
+
+
+for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
+    f=`basename $fuzzer`
+    case="${f#fuzz-}"
+    if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
+        echo "Running tests for ${case}"
+        for entry in "${TOR_FUZZ_CORPORA}/${case}/"*; do
+	    "${fuzzer}" "--err" < "$entry"
+        done
+    else
+	echo "No tests found for ${case}"
+    fi
+done
diff --git a/src/test/hs_build_address.py b/src/test/hs_build_address.py
new file mode 100644
index 0000000000..7ff22c3a9a
--- /dev/null
+++ b/src/test/hs_build_address.py
@@ -0,0 +1,38 @@
+import sys
+import hashlib
+import struct
+import base64
+
+# Python 3.6+, the SHA3 is available in hashlib natively. Else this requires
+# the pysha3 package (pip install pysha3).
+if sys.version_info < (3, 6):
+    import sha3
+
+# Test vector to make sure the right sha3 version will be used. pysha3 < 1.0
+# used the old Keccak implementation. During the finalization of SHA3, NIST
+# changed the delimiter suffix from 0x01 to 0x06. The Keccak sponge function
+# stayed the same. pysha3 1.0 provides the previous Keccak hash, too.
+TEST_VALUE = "e167f68d6563d75bb25f3aa49c29ef612d41352dc00606de7cbd630bb2665f51"
+if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest():
+    print("pysha3 version is < 1.0. Please install from:")
+    print("https://github.com/tiran/pysha3https://github.com/tiran/pysha3")
+    sys.exit(1)
+
+# Checksum is built like so:
+#   CHECKSUM = SHA3(".onion checksum" || PUBKEY || VERSION)
+PREFIX = ".onion checksum".encode()
+# 32 bytes ed25519 pubkey from first test vector of
+# https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-02#section-6
+PUBKEY = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a".decode('hex')
+# Version 3 is proposal224
+VERSION = 3
+
+data = struct.pack('15s32sb', PREFIX, PUBKEY, VERSION)
+checksum = hashlib.sha3_256(data).digest()
+
+# Onion address is built like so:
+#   onion_address = base32(PUBKEY || CHECKSUM || VERSION) + ".onion"
+address = struct.pack('!32s2sb', PUBKEY, checksum, VERSION)
+onion_addr = base64.b32encode(address).decode().lower()
+
+print("%s" % (onion_addr))
diff --git a/src/test/hs_indexes.py b/src/test/hs_indexes.py
new file mode 100644
index 0000000000..af0b81f8de
--- /dev/null
+++ b/src/test/hs_indexes.py
@@ -0,0 +1,70 @@
+#
+# The hidden service subsystem has two type of index. The first type is a
+# value that each node in the network gets assigned to using their identity
+# key which is their position in the hashring. (hs_build_hsdir_index()).
+#
+# The second type is a value that both the client and service computes to
+# store/fetch the descriptor on the hashring. (hs_build_hs_index()).
+#
+
+import sys
+import hashlib
+import struct
+import base64
+
+# Python 3.6+, the SHA3 is available in hashlib natively. Else this requires
+# the pysha3 package (pip install pysha3).
+if sys.version_info < (3, 6):
+    import sha3
+    # Test vector to make sure the right sha3 version will be used. pysha3 < 1.0
+    # used the old Keccak implementation. During the finalization of SHA3, NIST
+    # changed the delimiter suffix from 0x01 to 0x06. The Keccak sponge function
+    # stayed the same. pysha3 1.0 provides the previous Keccak hash, too.
+    TEST_VALUE = "e167f68d6563d75bb25f3aa49c29ef612d41352dc00606de7cbd630bb2665f51"
+    if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest():
+        print("pysha3 version is < 1.0. Please install from:")
+        print("https://github.com/tiran/pysha3https://github.com/tiran/pysha3")
+        sys.exit(1)
+
+# The first index we'll build is the position index in the hashring that is
+# constructed by the hs_build_hsdir_index() function. Construction is:
+#   SHA3-256("node-idx" | node_identity |
+#            shared_random_value | INT_8(period_length) | INT_8(period_num) )
+
+PREFIX = "node-idx".encode()
+# 32 bytes ed25519 pubkey.
+IDENTITY = ("\x42" * 32).encode()
+# SRV is 32 bytes.
+SRV = ("\x43" * 32).encode()
+# Time period length is a 8 bytes value.
+PERIOD_LEN = 1440
+# Period number is a 8 bytes value.
+PERIOD_NUM = 42
+
+data = struct.pack('!8s32s32sQQ', PREFIX, IDENTITY, SRV, PERIOD_NUM,
+                                  PERIOD_LEN)
+hsdir_index = hashlib.sha3_256(data).hexdigest()
+
+print("[hs_build_hsdir_index] %s" % (hsdir_index))
+
+# The second index we'll build is where the HS stores and the client fetches
+# the descriptor on the hashring. It is constructed by the hs_build_hs_index()
+# function and the construction is:
+#   SHA3-256("store-at-idx" | blinded_public_key |
+#            INT_8(replicanum) | INT_8(period_num) | INT_8(period_length) )
+
+PREFIX = "store-at-idx".encode()
+# 32 bytes ed25519 pubkey.
+PUBKEY = ("\x42" * 32).encode()
+# Replica number is a 8 bytes value.
+REPLICA_NUM = 1
+# Time period length is a 8 bytes value.
+PERIOD_LEN = 1440
+# Period number is a 8 bytes value.
+PERIOD_NUM = 42
+
+data = struct.pack('!12s32sQQQ', PREFIX, PUBKEY, REPLICA_NUM, PERIOD_LEN,
+                                   PERIOD_NUM)
+hs_index = hashlib.sha3_256(data).hexdigest()
+
+print("[hs_build_hs_index]   %s" % (hs_index))
diff --git a/src/test/hs_ntor_ref.py b/src/test/hs_ntor_ref.py
new file mode 100644
index 0000000000..542b02d2e0
--- /dev/null
+++ b/src/test/hs_ntor_ref.py
@@ -0,0 +1,428 @@
+#!/usr/bin/python
+# Copyright 2017, The Tor Project, Inc
+# See LICENSE for licensing information
+
+"""
+hs_ntor_ref.py
+
+This module is a reference implementation of the modified ntor protocol
+proposed for Tor hidden services in proposal 224 (Next Generation Hidden
+Services) in section [NTOR-WITH-EXTRA-DATA].
+
+The modified ntor protocol is a single-round protocol, with three steps in total:
+
+    1: Client generates keys and sends them to service via INTRODUCE cell
+
+    2: Service computes key material based on client's keys, and sends its own
+       keys to client via RENDEZVOUS cell
+
+    3: Client computes key material as well.
+
+It's meant to be used to validate Tor's HS ntor implementation by conducting
+various integration tests. Specifically it conducts the following three tests:
+
+- Tests our Python implementation by running the whole protocol in Python and
+  making sure that results are consistent.
+
+- Tests little-t-tor ntor implementation. We use this Python code to instrument
+  little-t-tor and carry out the handshake by using little-t-tor code. The
+  small C wrapper at src/test/test-hs-ntor-cl is used for this Python module to
+  interface with little-t-tor.
+
+- Cross-tests Python and little-t-tor implementation by running half of the
+  protocol in Python code and the other in little-t-tor. This is actually two
+  tests so that all parts of the protocol are run both by little-t-tor and
+  Python.
+
+It requires the curve25519 python module from the curve25519-donna package.
+
+The whole logic and concept for this test suite was taken from ntor_ref.py.
+
+                *** DO NOT USE THIS IN PRODUCTION. ***
+"""
+
+import struct
+import os, sys
+import binascii
+import subprocess
+
+try:
+    import curve25519
+    curve25519mod = curve25519.keys
+except ImportError:
+    curve25519 = None
+    import slownacl_curve25519
+    curve25519mod = slownacl_curve25519
+
+import hashlib
+try:
+    import sha3
+except ImportError:
+    # In python 3.6, the sha3 functions are in hashlib whether we
+    # import sha3 or not.
+    sha3 = None
+
+try:
+    # Pull the sha3 functions in.
+    from hashlib import sha3_256, shake_256
+    shake_squeeze = shake_256.digest
+except ImportError:
+    if hasattr(sha3, "SHA3256"):
+        # If this happens, then we have the old "sha3" module which
+        # hashlib and pysha3 superseded.
+        sha3_256 = sha3.SHA3256
+        shake_256 = sha3.SHAKE256
+        shake_squeeze = shake_256.squeeze
+    else:
+        # error code 77 tells automake to skip this test
+        sys.exit(77)
+
+# Import Nick's ntor reference implementation in Python
+# We are gonna use a few of its utilities.
+from ntor_ref import hash_nil
+from ntor_ref import PrivateKey
+
+# String constants used in this protocol
+PROTOID = b"tor-hs-ntor-curve25519-sha3-256-1"
+T_HSENC    = PROTOID + b":hs_key_extract"
+T_HSVERIFY = PROTOID + b":hs_verify"
+T_HSMAC    = PROTOID + b":hs_mac"
+M_HSEXPAND = PROTOID + b":hs_key_expand"
+
+INTRO_SECRET_LEN = 161
+REND_SECRET_LEN = 225
+AUTH_INPUT_LEN = 199
+
+# Implements MAC(k,m) = H(htonll(len(k)) | k | m)
+def mac(k,m):
+    def htonll(num):
+        return struct.pack('!q', num)
+
+    s = sha3_256()
+    s.update(htonll(len(k)))
+    s.update(k)
+    s.update(m)
+    return s.digest()
+
+######################################################################
+
+# Functions that implement the modified HS ntor protocol
+
+"""As client compute key material for INTRODUCE cell as follows:
+
+  intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+  info = m_hsexpand | subcredential
+  hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+  ENC_KEY = hs_keys[0:S_KEY_LEN]
+  MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+"""
+def intro2_ntor_client(intro_auth_pubkey_str, intro_enc_pubkey,
+                       client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential):
+
+    dh_result = client_ephemeral_enc_privkey.get_shared_key(intro_enc_pubkey, hash_nil)
+    secret =  dh_result + intro_auth_pubkey_str + client_ephemeral_enc_pubkey.serialize() + intro_enc_pubkey.serialize() + PROTOID
+    assert(len(secret) == INTRO_SECRET_LEN)
+    info = M_HSEXPAND + subcredential
+
+    kdf = shake_256()
+    kdf.update(secret + T_HSENC + info)
+    key_material = shake_squeeze(kdf, 64*8)
+
+    enc_key = key_material[0:32]
+    mac_key = key_material[32:64]
+
+    return enc_key, mac_key
+
+"""Wrapper over intro2_ntor_client()"""
+def client_part1(intro_auth_pubkey_str, intro_enc_pubkey,
+                 client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential):
+    enc_key, mac_key = intro2_ntor_client(intro_auth_pubkey_str, intro_enc_pubkey, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential)
+    assert(enc_key)
+    assert(mac_key)
+
+    return enc_key, mac_key
+
+"""As service compute key material for INTRODUCE cell as follows:
+
+  intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+  info = m_hsexpand | subcredential
+  hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+  HS_DEC_KEY = hs_keys[0:S_KEY_LEN]
+  HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+"""
+def intro2_ntor_service(intro_auth_pubkey_str, client_enc_pubkey, service_enc_privkey, service_enc_pubkey, subcredential):
+    dh_result = service_enc_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    secret = dh_result + intro_auth_pubkey_str + client_enc_pubkey.serialize() + service_enc_pubkey.serialize() + PROTOID
+    assert(len(secret) == INTRO_SECRET_LEN)
+    info = M_HSEXPAND + subcredential
+
+    kdf = shake_256()
+    kdf.update(secret + T_HSENC + info)
+    key_material = shake_squeeze(kdf, 64*8)
+
+    enc_key = key_material[0:32]
+    mac_key = key_material[32:64]
+
+    return enc_key, mac_key
+
+"""As service compute key material for INTRODUCE and REDNEZVOUS cells.
+
+  Use intro2_ntor_service() to calculate the INTRODUCE key material, and use
+  the following computations to do the RENDEZVOUS ones:
+
+      rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+      NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+      verify = MAC(rend_secret_hs_input, t_hsverify)
+      auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+      AUTH_INPUT_MAC = MAC(auth_input, t_hsmac)
+"""
+def service_part1(intro_auth_pubkey_str, client_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential):
+    intro_enc_key, intro_mac_key = intro2_ntor_service(intro_auth_pubkey_str, client_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+    assert(intro_enc_key)
+    assert(intro_mac_key)
+
+    service_ephemeral_privkey = PrivateKey()
+    service_ephemeral_pubkey = service_ephemeral_privkey.get_public()
+
+    dh_result1 = service_ephemeral_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    dh_result2 = intro_enc_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    rend_secret_hs_input = dh_result1 + dh_result2 + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + client_enc_pubkey.serialize() + service_ephemeral_pubkey.serialize() + PROTOID
+    assert(len(rend_secret_hs_input) == REND_SECRET_LEN)
+
+    ntor_key_seed = mac(rend_secret_hs_input, T_HSENC)
+    verify = mac(rend_secret_hs_input, T_HSVERIFY)
+    auth_input = verify + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + service_ephemeral_pubkey.serialize() + client_enc_pubkey.serialize() + PROTOID + b"Server"
+    assert(len(auth_input) == AUTH_INPUT_LEN)
+    auth_input_mac = mac(auth_input, T_HSMAC)
+
+    assert(ntor_key_seed)
+    assert(auth_input_mac)
+    assert(service_ephemeral_pubkey)
+
+    return intro_enc_key, intro_mac_key, ntor_key_seed, auth_input_mac, service_ephemeral_pubkey
+
+"""As client compute key material for rendezvous cells as follows:
+
+  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+  NTOR_KEY_SEED = MAC(ntor_secret_input, t_hsenc)
+  verify = MAC(ntor_secret_input, t_hsverify)
+  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+  AUTH_INPUT_MAC = MAC(auth_input, t_hsmac)
+"""
+def client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                 intro_enc_pubkey, service_ephemeral_rend_pubkey):
+    dh_result1 = client_ephemeral_enc_privkey.get_shared_key(service_ephemeral_rend_pubkey, hash_nil)
+    dh_result2 = client_ephemeral_enc_privkey.get_shared_key(intro_enc_pubkey, hash_nil)
+    rend_secret_hs_input = dh_result1 + dh_result2 + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + client_ephemeral_enc_pubkey.serialize() + service_ephemeral_rend_pubkey.serialize() + PROTOID
+    assert(len(rend_secret_hs_input) == REND_SECRET_LEN)
+
+    ntor_key_seed = mac(rend_secret_hs_input, T_HSENC)
+    verify = mac(rend_secret_hs_input, T_HSVERIFY)
+    auth_input = verify + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + service_ephemeral_rend_pubkey.serialize() + client_ephemeral_enc_pubkey.serialize() + PROTOID + b"Server"
+    assert(len(auth_input) == AUTH_INPUT_LEN)
+    auth_input_mac = mac(auth_input, T_HSMAC)
+
+    assert(ntor_key_seed)
+    assert(auth_input_mac)
+
+    return ntor_key_seed, auth_input_mac
+
+#################################################################################
+
+"""
+Utilities for communicating with the little-t-tor ntor wrapper to conduct the
+integration tests
+"""
+
+PROG = "./src/test/test-hs-ntor-cl"
+if sys.version_info[0] >= 3:
+    enhex=lambda s: binascii.b2a_hex(s).decode("ascii")
+else:
+    enhex=lambda s: binascii.b2a_hex(s)
+dehex=lambda s: binascii.a2b_hex(s.strip())
+
+def tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                client_ephemeral_enc_privkey, subcredential):
+    p = subprocess.Popen([PROG, "client1",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(intro_enc_pubkey.serialize()),
+                          enhex(client_ephemeral_enc_privkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+def tor_server1(intro_auth_pubkey_str, intro_enc_privkey,
+                client_ephemeral_enc_pubkey, subcredential):
+    p = subprocess.Popen([PROG, "server1",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(intro_enc_privkey.serialize()),
+                          enhex(client_ephemeral_enc_pubkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+def tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                intro_enc_pubkey, service_ephemeral_rend_pubkey, subcredential):
+    p = subprocess.Popen([PROG, "client2",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(client_ephemeral_enc_privkey.serialize()),
+                          enhex(intro_enc_pubkey.serialize()),
+                          enhex(service_ephemeral_rend_pubkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+##################################################################################
+
+# Perform a pure python ntor test
+def do_pure_python_ntor_test():
+    # Initialize all needed key material
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public()
+    intro_auth_pubkey_str = os.urandom(32)
+    subcredential = os.urandom(32)
+
+    client_enc_key, client_mac_key = client_part1(intro_auth_pubkey_str, intro_enc_pubkey, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential)
+
+    service_enc_key, service_mac_key, service_ntor_key_seed, service_auth_input_mac, service_ephemeral_pubkey = service_part1(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    client_ntor_key_seed, client_auth_input_mac = client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                               intro_enc_pubkey, service_ephemeral_pubkey)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_auth_input_mac == service_auth_input_mac)
+
+    print("DONE: python dance [%s]" % repr(client_auth_input_mac))
+
+# Perform a pure little-t-tor integration test.
+def do_little_t_tor_ntor_test():
+    # Initialize all needed key material
+    subcredential = os.urandom(32)
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+    intro_auth_pubkey_str = os.urandom(32)
+
+    client_enc_key, client_mac_key = tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                 client_ephemeral_enc_privkey, subcredential)
+    assert(client_enc_key)
+    assert(client_mac_key)
+
+    service_enc_key, service_mac_key, service_ntor_auth_mac, service_ntor_key_seed, service_eph_pubkey = tor_server1(intro_auth_pubkey_str,
+                                                                                                                     intro_enc_privkey,
+                                                                                                                     client_ephemeral_enc_pubkey,
+                                                                                                                     subcredential)
+    assert(service_enc_key)
+    assert(service_mac_key)
+    assert(service_ntor_auth_mac)
+    assert(service_ntor_key_seed)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    # Turn from bytes to key
+    service_eph_pubkey = curve25519mod.Public(service_eph_pubkey)
+
+    client_ntor_auth_mac, client_ntor_key_seed  = tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                                                              intro_enc_pubkey, service_eph_pubkey, subcredential)
+    assert(client_ntor_auth_mac)
+    assert(client_ntor_key_seed)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_ntor_auth_mac == service_ntor_auth_mac)
+
+    print("DONE: tor dance [%s]" % repr(client_ntor_auth_mac))
+
+"""
+Do mixed test as follows:
+    1. C -> S (python mode)
+    2. C <- S (tor mode)
+    3. Client computes keys (python mode)
+"""
+def do_first_mixed_test():
+    subcredential = os.urandom(32)
+
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+
+    intro_auth_pubkey_str = os.urandom(32)
+
+    # Let's do mixed
+    client_enc_key, client_mac_key = client_part1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                  client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                  subcredential)
+
+    service_enc_key, service_mac_key, service_ntor_auth_mac, service_ntor_key_seed, service_eph_pubkey = tor_server1(intro_auth_pubkey_str,
+                                                                                                                     intro_enc_privkey,
+                                                                                                                     client_ephemeral_enc_pubkey,
+                                                                                                                     subcredential)
+    assert(service_enc_key)
+    assert(service_mac_key)
+    assert(service_ntor_auth_mac)
+    assert(service_ntor_key_seed)
+    assert(service_eph_pubkey)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    # Turn from bytes to key
+    service_eph_pubkey = curve25519mod.Public(service_eph_pubkey)
+
+    client_ntor_key_seed, client_auth_input_mac = client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                               intro_enc_pubkey, service_eph_pubkey)
+
+    assert(client_auth_input_mac == service_ntor_auth_mac)
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+
+    print("DONE: 1st mixed dance [%s]" % repr(client_auth_input_mac))
+
+"""
+Do mixed test as follows:
+    1. C -> S (tor mode)
+    2. C <- S (python mode)
+    3. Client computes keys (tor mode)
+"""
+def do_second_mixed_test():
+    subcredential = os.urandom(32)
+
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+
+    intro_auth_pubkey_str = os.urandom(32)
+
+    # Let's do mixed
+    client_enc_key, client_mac_key = tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                 client_ephemeral_enc_privkey, subcredential)
+    assert(client_enc_key)
+    assert(client_mac_key)
+
+    service_enc_key, service_mac_key, service_ntor_key_seed, service_ntor_auth_mac, service_ephemeral_pubkey = service_part1(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+
+    client_ntor_auth_mac, client_ntor_key_seed  = tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                                                              intro_enc_pubkey, service_ephemeral_pubkey, subcredential)
+    assert(client_ntor_auth_mac)
+    assert(client_ntor_key_seed)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_ntor_auth_mac == service_ntor_auth_mac)
+
+    print("DONE: 2nd mixed dance [%s]" % repr(client_ntor_auth_mac))
+
+def do_mixed_tests():
+    do_first_mixed_test()
+    do_second_mixed_test()
+
+if __name__ == '__main__':
+    do_pure_python_ntor_test()
+    do_little_t_tor_ntor_test()
+    do_mixed_tests()
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
new file mode 100644
index 0000000000..5c1b9123d8
--- /dev/null
+++ b/src/test/hs_test_helpers.c
@@ -0,0 +1,282 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "crypto_ed25519.h"
+#include "test.h"
+#include "torcert.h"
+
+#include "hs_common.h"
+#include "hs_test_helpers.h"
+
+hs_desc_intro_point_t *
+hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
+                            const char *addr, int legacy)
+{
+  int ret;
+  ed25519_keypair_t auth_kp;
+  hs_desc_intro_point_t *intro_point = NULL;
+  hs_desc_intro_point_t *ip = hs_desc_intro_point_new();
+
+  /* For a usable intro point we need at least two link specifiers: One legacy
+   * keyid and one ipv4 */
+  {
+    hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy));
+    hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4));
+    ls_legacy->type = LS_LEGACY_ID;
+    memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
+           DIGEST_LEN);
+    ls_v4->u.ap.port = 9001;
+    int family = tor_addr_parse(&ls_v4->u.ap.addr, addr);
+    switch (family) {
+    case AF_INET:
+          ls_v4->type = LS_IPV4;
+          break;
+        case AF_INET6:
+          ls_v4->type = LS_IPV6;
+          break;
+        default:
+          /* Stop the test, not suppose to have an error. */
+          tt_int_op(family, OP_EQ, AF_INET);
+    }
+    smartlist_add(ip->link_specifiers, ls_legacy);
+    smartlist_add(ip->link_specifiers, ls_v4);
+  }
+
+  ret = ed25519_keypair_generate(&auth_kp, 0);
+  tt_int_op(ret, ==, 0);
+  ip->auth_key_cert = tor_cert_create(signing_kp, CERT_TYPE_AUTH_HS_IP_KEY,
+                                      &auth_kp.pubkey, now,
+                                      HS_DESC_CERT_LIFETIME,
+                                      CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(ip->auth_key_cert);
+
+  if (legacy) {
+    ip->legacy.key = crypto_pk_new();
+    tt_assert(ip->legacy.key);
+    ret = crypto_pk_generate_key(ip->legacy.key);
+    tt_int_op(ret, ==, 0);
+    ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
+                                    &signing_kp->pubkey, ip->legacy.key,
+                                    now + HS_DESC_CERT_LIFETIME,
+                                    &ip->legacy.cert.encoded);
+    tt_assert(ip->legacy.cert.encoded);
+    tt_u64_op(cert_len, OP_GT, 0);
+    ip->legacy.cert.len = cert_len;
+  }
+
+  /* Encryption key. */
+  {
+    int signbit;
+    curve25519_keypair_t curve25519_kp;
+    ed25519_keypair_t ed25519_kp;
+    tor_cert_t *cross_cert;
+
+    ret = curve25519_keypair_generate(&curve25519_kp, 0);
+    tt_int_op(ret, ==, 0);
+    ed25519_keypair_from_curve25519_keypair(&ed25519_kp, &signbit,
+                                            &curve25519_kp);
+    cross_cert = tor_cert_create(signing_kp, CERT_TYPE_CROSS_HS_IP_KEYS,
+                                 &ed25519_kp.pubkey, time(NULL),
+                                 HS_DESC_CERT_LIFETIME,
+                                 CERT_FLAG_INCLUDE_SIGNING_KEY);
+    tt_assert(cross_cert);
+    ip->enc_key_cert = cross_cert;
+  }
+
+  intro_point = ip;
+ done:
+  if (intro_point == NULL)
+    tor_free(ip);
+
+  return intro_point;
+}
+
+/* Return a valid hs_descriptor_t object. If no_ip is set, no introduction
+ * points are added. */
+static hs_descriptor_t *
+hs_helper_build_hs_desc_impl(unsigned int no_ip,
+                             const ed25519_keypair_t *signing_kp)
+{
+  time_t now = approx_time();
+  ed25519_keypair_t blinded_kp;
+  hs_descriptor_t *descp = NULL, *desc = tor_malloc_zero(sizeof(*desc));
+
+  desc->plaintext_data.version = HS_DESC_SUPPORTED_FORMAT_VERSION_MAX;
+
+  /* Copy only the public key into the descriptor. */
+  memcpy(&desc->plaintext_data.signing_pubkey, &signing_kp->pubkey,
+         sizeof(ed25519_public_key_t));
+
+  uint64_t current_time_period = hs_get_time_period_num(0);
+  hs_build_blinded_keypair(signing_kp, NULL, 0,
+                           current_time_period, &blinded_kp);
+  /* Copy only the public key into the descriptor. */
+  memcpy(&desc->plaintext_data.blinded_pubkey, &blinded_kp.pubkey,
+         sizeof(ed25519_public_key_t));
+
+  desc->plaintext_data.signing_key_cert =
+    tor_cert_create(&blinded_kp, CERT_TYPE_SIGNING_HS_DESC,
+                    &signing_kp->pubkey, now, 3600,
+                    CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(desc->plaintext_data.signing_key_cert);
+  desc->plaintext_data.revision_counter = 42;
+  desc->plaintext_data.lifetime_sec = 3 * 60 * 60;
+
+  hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey,
+                    desc->subcredential);
+
+  /* Setup encrypted data section. */
+  desc->encrypted_data.create2_ntor = 1;
+  desc->encrypted_data.intro_auth_types = smartlist_new();
+  desc->encrypted_data.single_onion_service = 1;
+  smartlist_add(desc->encrypted_data.intro_auth_types, tor_strdup("ed25519"));
+  desc->encrypted_data.intro_points = smartlist_new();
+  if (!no_ip) {
+    /* Add four intro points. */
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "5.6.7.8", 1));
+  }
+
+  descp = desc;
+ done:
+  if (descp == NULL)
+    tor_free(desc);
+
+  return descp;
+}
+
+/** Helper function to get the HS subcredential using the identity keypair of
+ *  an HS. Used to decrypt descriptors in unittests. */
+void
+hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
+                                            uint8_t *subcred_out)
+{
+  ed25519_keypair_t blinded_kp;
+  uint64_t current_time_period = hs_get_time_period_num(approx_time());
+  hs_build_blinded_keypair(signing_kp, NULL, 0,
+                           current_time_period, &blinded_kp);
+
+  hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey,
+                       subcred_out);
+}
+
+/* Build a descriptor with introduction points. */
+hs_descriptor_t *
+hs_helper_build_hs_desc_with_ip(const ed25519_keypair_t *signing_kp)
+{
+  return hs_helper_build_hs_desc_impl(0, signing_kp);
+}
+
+/* Build a descriptor without any introduction points. */
+hs_descriptor_t *
+hs_helper_build_hs_desc_no_ip(const ed25519_keypair_t *signing_kp)
+{
+  return hs_helper_build_hs_desc_impl(1, signing_kp);
+}
+
+void
+hs_helper_desc_equal(const hs_descriptor_t *desc1,
+                     const hs_descriptor_t *desc2)
+{
+  char *addr1 = NULL, *addr2 = NULL;
+  /* Plaintext data section. */
+  tt_int_op(desc1->plaintext_data.version, OP_EQ,
+            desc2->plaintext_data.version);
+  tt_uint_op(desc1->plaintext_data.lifetime_sec, OP_EQ,
+             desc2->plaintext_data.lifetime_sec);
+  tt_assert(tor_cert_eq(desc1->plaintext_data.signing_key_cert,
+                        desc2->plaintext_data.signing_key_cert));
+  tt_mem_op(desc1->plaintext_data.signing_pubkey.pubkey, OP_EQ,
+            desc2->plaintext_data.signing_pubkey.pubkey,
+            ED25519_PUBKEY_LEN);
+  tt_mem_op(desc1->plaintext_data.blinded_pubkey.pubkey, OP_EQ,
+            desc2->plaintext_data.blinded_pubkey.pubkey,
+            ED25519_PUBKEY_LEN);
+  tt_u64_op(desc1->plaintext_data.revision_counter, ==,
+            desc2->plaintext_data.revision_counter);
+
+  /* NOTE: We can't compare the encrypted blob because when encoding the
+   * descriptor, the object is immutable thus we don't update it with the
+   * encrypted blob. As contrast to the decoding process where we populate a
+   * descriptor object. */
+
+  /* Encrypted data section. */
+  tt_uint_op(desc1->encrypted_data.create2_ntor, ==,
+             desc2->encrypted_data.create2_ntor);
+
+  /* Authentication type. */
+  tt_int_op(!!desc1->encrypted_data.intro_auth_types, ==,
+            !!desc2->encrypted_data.intro_auth_types);
+  if (desc1->encrypted_data.intro_auth_types &&
+      desc2->encrypted_data.intro_auth_types) {
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), ==,
+              smartlist_len(desc2->encrypted_data.intro_auth_types));
+    for (int i = 0;
+         i < smartlist_len(desc1->encrypted_data.intro_auth_types);
+         i++) {
+      tt_str_op(smartlist_get(desc1->encrypted_data.intro_auth_types, i),OP_EQ,
+                smartlist_get(desc2->encrypted_data.intro_auth_types, i));
+    }
+  }
+
+  /* Introduction points. */
+  {
+    tt_assert(desc1->encrypted_data.intro_points);
+    tt_assert(desc2->encrypted_data.intro_points);
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), ==,
+              smartlist_len(desc2->encrypted_data.intro_points));
+    for (int i=0; i < smartlist_len(desc1->encrypted_data.intro_points); i++) {
+      hs_desc_intro_point_t *ip1 = smartlist_get(desc1->encrypted_data
+                                                 .intro_points, i),
+                            *ip2 = smartlist_get(desc2->encrypted_data
+                                                 .intro_points, i);
+      tt_assert(tor_cert_eq(ip1->auth_key_cert, ip2->auth_key_cert));
+      if (ip1->legacy.key) {
+        tt_int_op(crypto_pk_cmp_keys(ip1->legacy.key, ip2->legacy.key),
+                  OP_EQ, 0);
+      } else {
+        tt_mem_op(&ip1->enc_key, OP_EQ, &ip2->enc_key, CURVE25519_PUBKEY_LEN);
+      }
+
+      tt_int_op(smartlist_len(ip1->link_specifiers), ==,
+                smartlist_len(ip2->link_specifiers));
+      for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
+        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(ls1->type, ==, ls2->type);
+        switch (ls1->type) {
+          case LS_IPV4:
+          case LS_IPV6:
+            {
+              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
+              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
+              tt_str_op(addr1, OP_EQ, addr2);
+              tor_free(addr1);
+              tor_free(addr2);
+              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+            }
+            break;
+          case LS_LEGACY_ID:
+            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
+                      sizeof(ls1->u.legacy_id));
+            break;
+          default:
+            /* Unknown type, caught it and print its value. */
+            tt_int_op(ls1->type, OP_EQ, -1);
+        }
+      }
+    }
+  }
+
+ done:
+  tor_free(addr1);
+  tor_free(addr2);
+}
+
diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h
new file mode 100644
index 0000000000..b1b0490f05
--- /dev/null
+++ b/src/test/hs_test_helpers.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_HS_TEST_HELPERS_H
+#define TOR_HS_TEST_HELPERS_H
+
+#include "ed25519_cert.h"
+#include "hs_descriptor.h"
+
+/* Set of functions to help build and test descriptors. */
+hs_desc_intro_point_t *hs_helper_build_intro_point(
+                          const ed25519_keypair_t *signing_kp, time_t now,
+                          const char *addr, int legacy);
+hs_descriptor_t *hs_helper_build_hs_desc_no_ip(
+                                 const ed25519_keypair_t *signing_kp);
+hs_descriptor_t *hs_helper_build_hs_desc_with_ip(
+                                 const ed25519_keypair_t *signing_kp);
+void hs_helper_desc_equal(const hs_descriptor_t *desc1,
+                          const hs_descriptor_t *desc2);
+void
+hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
+                                            uint8_t *subcred_out);
+
+#endif /* !defined(TOR_HS_TEST_HELPERS_H) */
+
diff --git a/src/test/include.am b/src/test/include.am
index 7864d7d9fd..70df3aac14 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -5,10 +5,19 @@ TESTS_ENVIRONMENT = \
 	export PYTHON="$(PYTHON)"; \
 	export SHELL="$(SHELL)"; \
 	export abs_top_srcdir="$(abs_top_srcdir)"; \
+	export abs_top_builddir="$(abs_top_builddir)"; \
 	export builddir="$(builddir)"; \
-	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)";
+	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \
+	export CARGO="$(CARGO)"; \
+	export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \
+	export CARGO_ONLINE="$(CARGO_ONLINE)"; \
+        export CCLD="$(CCLD)"; \
+        chmod +x "$(abs_top_builddir)/link_rust.sh"; \
+        export RUSTFLAGS="-C linker=$(abs_top_builddir)/link_rust.sh";
 
-TESTSCRIPTS = src/test/test_zero_length_keys.sh \
+TESTSCRIPTS = \
+	src/test/fuzz_static_testcases.sh \
+	src/test/test_zero_length_keys.sh \
 	src/test/test_workqueue_cancel.sh \
 	src/test/test_workqueue_efd.sh \
 	src/test/test_workqueue_efd2.sh \
@@ -17,23 +26,32 @@ TESTSCRIPTS = src/test/test_zero_length_keys.sh \
 	src/test/test_workqueue_socketpair.sh \
 	src/test/test_switch_id.sh
 
+if USE_RUST
+TESTSCRIPTS += \
+	src/test/test_rust.sh
+endif
+
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
 endif
 
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
 	src/test/test_keygen.sh \
+	src/test/test_key_expiration.sh \
 	src/test/test-timers \
 	$(TESTSCRIPTS)
 
 # These flavors are run using automake's test-driver and test-network.sh
-TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-min single-onion
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
+	single-onion-v23
 # only run if we can ping6 ::1 (localhost)
-TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-ipv6 \
-	single-onion-ipv6
+# IPv6-only v3 single onion services don't work yet, so we don't test the
+# single-onion-v23-ipv6-md flavor
+TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
+	single-onion-ipv6-md
 # only run if we can find a stable (or simply another) version of tor
-TEST_CHUTNEY_FLAVORS_MIXED = mixed
+TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2
 
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.
@@ -65,56 +83,86 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
 # This seems to matter nowhere but on Windows, but I assure you that it
 # matters a lot there, and is quite hard to debug if you forget to do it.
 
-src_test_test_SOURCES = \
+src_test_test_SOURCES =
+
+if UNITTESTS_ENABLED
+src_test_test_SOURCES += \
 	src/test/log_test_helpers.c \
+	src/test/hs_test_helpers.c \
 	src/test/rend_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
 	src/test/test_addr.c \
 	src/test/test_address.c \
 	src/test/test_address_set.c \
+	src/test/test_bridges.c \
 	src/test/test_buffers.c \
+	src/test/test_bwmgt.c \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
+	src/test/test_channelpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
 	src/test/test_circuitmux.c \
+	src/test/test_circuitbuild.c \
+	src/test/test_circuituse.c \
+	src/test/test_circuitstats.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
 	src/test/test_connection.c \
+	src/test/test_conscache.c \
+	src/test/test_consdiff.c \
+	src/test/test_consdiffmgr.c \
 	src/test/test_containers.c \
 	src/test/test_controller.c \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
-	src/test/test_dos.c \
+	src/test/test_crypto_openssl.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
 	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
+	src/test/test_dos.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \
+	src/test/test_geoip.c \
 	src/test/test_guardfraction.c \
 	src/test/test_extorport.c \
 	src/test/test_hs.c \
+	src/test/test_hs_common.c \
+	src/test/test_hs_config.c \
+	src/test/test_hs_cell.c \
+	src/test/test_hs_ntor.c \
+	src/test/test_hs_service.c \
+	src/test/test_hs_client.c  \
+	src/test/test_hs_intropoint.c \
+	src/test/test_hs_control.c \
 	src/test/test_handles.c \
+	src/test/test_hs_cache.c \
+	src/test/test_hs_descriptor.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
 	src/test/test_logging.c \
+	src/test/test_mainloop.c \
 	src/test/test_microdesc.c \
 	src/test/test_nodelist.c \
 	src/test/test_oom.c \
 	src/test/test_oos.c \
 	src/test/test_options.c \
+	src/test/test_periodic_event.c \
 	src/test/test_policy.c \
 	src/test/test_procmon.c \
+	src/test/test_proto_http.c \
+	src/test/test_proto_misc.c \
 	src/test/test_protover.c \
 	src/test/test_pt.c \
 	src/test/test_pubsub.c \
 	src/test/test_relay.c \
 	src/test/test_relaycell.c \
+	src/test/test_relaycrypt.c \
 	src/test/test_rendcache.c \
 	src/test/test_replay.c \
 	src/test/test_router.c \
@@ -125,22 +173,30 @@ src_test_test_SOURCES = \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
 	src/test/test_status.c \
+	src/test/test_storagedir.c \
 	src/test/test_threads.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
 	src/test/test_util_format.c \
 	src/test/test_util_process.c \
+	src/test/test_voting_schedule.c \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
 	src/test/testing_common.c \
+	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
+endif
 
-src_test_test_slow_SOURCES = \
+src_test_test_slow_SOURCES =
+if UNITTESTS_ENABLED
+src_test_test_slow_SOURCES += \
 	src/test/test_slow.c \
 	src/test/test_crypto_slow.c \
 	src/test/test_util_slow.c \
 	src/test/testing_common.c \
+	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
+endif
 
 src_test_test_memwipe_SOURCES = \
 	src/test/test-memwipe.c
@@ -168,7 +224,10 @@ src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@
 src_test_test_switch_id_LDADD = \
 	src/common/libor-testing.a \
 	src/common/libor-ctime-testing.a \
-	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@
+	$(rust_ldadd) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIB_WS32@ @TOR_LIB_USERENV@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
@@ -180,9 +239,12 @@ src_test_test_LDADD = src/or/libtor-testing.a \
 	src/common/libor-ctime-testing.a \
 	src/common/libor-event-testing.a \
 	src/trunnel/libor-trunnel-testing.a \
+	src/trace/libor-trace.a \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS)
 src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
@@ -203,9 +265,12 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \
 	src/common/libor-ctime.a \
 	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event.a src/trunnel/libor-trunnel.a \
+	src/trace/libor-trace.a \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
@@ -214,8 +279,12 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \
 	src/common/libor-ctime-testing.a \
 	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event-testing.a \
+	src/trace/libor-trace.a \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 
 src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS)
 src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS)
@@ -224,53 +293,85 @@ src_test_test_timers_LDADD = \
 	src/common/libor-ctime-testing.a \
 	src/common/libor-event-testing.a \
 	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
 noinst_HEADERS+= \
 	src/test/fakechans.h \
+	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
 	src/test/rend_test_helpers.h \
 	src/test/test.h \
 	src/test/test_helpers.h \
 	src/test/test_dir_common.h \
+	src/test/test_connection.h \
 	src/test/test_descriptors.inc \
 	src/test/example_extrainfo.inc \
 	src/test/failing_routerdescs.inc \
 	src/test/ed25519_vectors.inc \
 	src/test/test_descriptors.inc \
+	src/test/test_hs_descriptor.inc \
 	src/test/vote_descriptors.inc
 
 noinst_PROGRAMS+= src/test/test-ntor-cl
+noinst_PROGRAMS+= src/test/test-hs-ntor-cl
 src_test_test_ntor_cl_SOURCES  = src/test/test_ntor_cl.c
 src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \
 	src/common/libor-ctime.a \
 	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	src/trace/libor-trace.a \
+	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@CURVE25519_LIBS@ @TOR_LZMA_LIBS@
 src_test_test_ntor_cl_AM_CPPFLAGS =	       \
 	-I"$(top_srcdir)/src/or"
 
+src_test_test_hs_ntor_cl_SOURCES  = src/test/test_hs_ntor_cl.c
+src_test_test_hs_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
+src_test_test_hs_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \
+	src/common/libor-ctime.a \
+	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+src_test_test_hs_ntor_cl_AM_CPPFLAGS =	       \
+	-I"$(top_srcdir)/src/or"
+
+
+if UNITTESTS_ENABLED
 noinst_PROGRAMS += src/test/test-bt-cl
 src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c
 src_test_test_bt_cl_LDADD = src/common/libor-testing.a \
 	src/common/libor-ctime-testing.a \
+	src/trace/libor-trace.a \
+	$(rust_ldadd) \
 	@TOR_LIB_MATH@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@
 src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+endif
 
 EXTRA_DIST += \
 	src/test/bt_test.py \
 	src/test/ntor_ref.py \
+	src/test/hs_ntor_ref.py \
+	src/test/hs_build_address.py \
+	src/test/hs_indexes.py \
+	src/test/fuzz_static_testcases.sh \
 	src/test/slownacl_curve25519.py \
 	src/test/zero_length_keys.sh \
+        src/test/rust_supp.txt \
 	src/test/test_keygen.sh \
+	src/test/test_key_expiration.sh \
 	src/test/test_zero_length_keys.sh \
-	src/test/test_ntor.sh src/test/test_bt.sh \
+	src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh \
 	src/test/test-network.sh \
+	src/test/test_rust.sh \
 	src/test/test_switch_id.sh \
 	src/test/test_workqueue_cancel.sh \
 	src/test/test_workqueue_efd.sh \
@@ -279,3 +380,5 @@ EXTRA_DIST += \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh
 
+test-rust:
+	$(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh"
diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c
index c5368b6cbc..1ad01afc8d 100644
--- a/src/test/log_test_helpers.c
+++ b/src/test/log_test_helpers.c
@@ -258,4 +258,3 @@ mock_dump_saved_logs(void)
            escaped(m->generated_msg));
   } SMARTLIST_FOREACH_END(m);
 }
-
diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h
index a087b913f4..8057995070 100644
--- a/src/test/log_test_helpers.h
+++ b/src/test/log_test_helpers.h
@@ -76,20 +76,24 @@ void mock_dump_saved_logs(void);
                                                                         \
     assert_log_predicate(mock_saved_log_has_message_containing(str) &&  \
                          mock_saved_log_n_entries() == 1,               \
-                  "expected log to contain exactly 1 message: " # str); \
+                  "expected log to contain exactly 1 message " # str); \
   } while (0);
 
 #define expect_single_log_msg_containing(str) \
   do {                                                                  \
     assert_log_predicate(mock_saved_log_has_message_containing(str)&&   \
                          mock_saved_log_n_entries() == 1 ,              \
-            "expected log to contain 1 message, containing" # str);     \
+            "expected log to contain 1 message, containing " # str);     \
   } while (0);
 
 #define expect_no_log_msg(str) \
   assert_log_predicate(!mock_saved_log_has_message(str), \
                 "expected log to not contain " # str);
 
+#define expect_no_log_msg_containing(str) \
+  assert_log_predicate(!mock_saved_log_has_message_containing(str), \
+                "expected log to not contain " # str);
+
 #define expect_log_severity(severity) \
   assert_log_predicate(mock_saved_log_has_severity(severity), \
                 "expected log to contain severity " # severity);
@@ -106,5 +110,4 @@ void mock_dump_saved_logs(void);
   assert_log_predicate(!mock_saved_log_has_entry(), \
                 "expected log to not contain entries");
 
-#endif
-
+#endif /* !defined(TOR_LOG_TEST_HELPERS_H) */
diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py
index 5ec117f2bd..51f218f512 100755
--- a/src/test/ntor_ref.py
+++ b/src/test/ntor_ref.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2012-2015, The Tor Project, Inc
+# Copyright 2012-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c
index 377337bcb9..9ac3894b0b 100644
--- a/src/test/rend_test_helpers.c
+++ b/src/test/rend_test_helpers.c
@@ -1,7 +1,8 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
+#include "crypto_rand.h"
 #include "test.h"
 #include "rendcommon.h"
 #include "rend_test_helpers.h"
@@ -71,3 +72,19 @@ create_descriptor(rend_service_descriptor_t **generated, char **service_id,
   crypto_pk_free(pk2);
 }
 
+rend_data_t *
+mock_rend_data(const char *onion_address)
+{
+  rend_data_v2_t *v2_data = tor_malloc_zero(sizeof(*v2_data));
+  rend_data_t *rend_query = &v2_data->base_;
+  rend_query->version = 2;
+
+  strlcpy(v2_data->onion_address, onion_address,
+          sizeof(v2_data->onion_address));
+  v2_data->auth_type = REND_NO_AUTH;
+  rend_query->hsdirs_fp = smartlist_new();
+  smartlist_add(rend_query->hsdirs_fp, tor_memdup("aaaaaaaaaaaaaaaaaaaaaaaa",
+                                                 DIGEST_LEN));
+  return rend_query;
+}
+
diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h
index 180a4e8fde..abf4324988 100644
--- a/src/test/rend_test_helpers.h
+++ b/src/test/rend_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -10,6 +10,7 @@ void generate_desc(int time_diff, rend_encoded_v2_service_descriptor_t **desc,
                    char **service_id, int intro_points);
 void create_descriptor(rend_service_descriptor_t **generated,
                        char **service_id, int intro_points);
+rend_data_t *mock_rend_data(const char *onion_address);
 
-#endif
+#endif /* !defined(TOR_REND_TEST_HELPERS_H) */
 
diff --git a/src/test/rust_supp.txt b/src/test/rust_supp.txt
new file mode 100644
index 0000000000..7fa50f3fb1
--- /dev/null
+++ b/src/test/rust_supp.txt
@@ -0,0 +1 @@
+leak:backtrace_alloc
diff --git a/src/test/test-child.c b/src/test/test-child.c
index fdf3ccec0a..f78a829107 100644
--- a/src/test/test-child.c
+++ b/src/test/test-child.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,7 +8,7 @@
 #include <windows.h>
 #else
 #include <unistd.h>
-#endif
+#endif /* defined(_WIN32) */
 #include <string.h>
 
 #ifdef _WIN32
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index c650b99c85..9e7dff5f63 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -1,10 +1,13 @@
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #include "orconfig.h"
 #include <string.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <stdlib.h>
 
-#include "crypto.h"
+#include "crypto_util.h"
 #include "compat.h"
 #include "util.h"
 
@@ -36,7 +39,7 @@ const char *s = NULL;
     sum += (unsigned char)buf[i];                                       \
   }
 
-#ifdef __OpenBSD__
+#ifdef OpenBSD
 /* Disable some of OpenBSD's malloc protections for this test. This helps
  * us do bad things, such as access freed buffers, without crashing. */
 extern const char *malloc_options;
diff --git a/src/test/test-timers.c b/src/test/test-timers.c
index b5fcade7f8..f20f29578b 100644
--- a/src/test/test-timers.c
+++ b/src/test/test-timers.c
@@ -1,4 +1,4 @@
-/* Copyright 2016, The Tor Project, Inc. */
+/* Copyright 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -7,11 +7,9 @@
 #include <stdio.h>
 #include <string.h>
 
-#include <event2/event.h>
-
 #include "compat.h"
 #include "compat_libevent.h"
-#include "crypto.h"
+#include "crypto_rand.h"
 #include "timers.h"
 #include "util.h"
 
@@ -50,7 +48,7 @@ timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono)
 
   // printf("%d / %d\n",n_fired, N_TIMERS);
   if (n_fired == n_active_timers) {
-    event_base_loopbreak(tor_libevent_get_base());
+    tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
   }
 }
 
@@ -90,7 +88,7 @@ main(int argc, char **argv)
     --n_active_timers;
   }
 
-  event_base_loop(tor_libevent_get_base(), 0);
+  tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
 
   int64_t total_difference = 0;
   uint64_t total_square_difference = 0;
@@ -133,7 +131,7 @@ main(int argc, char **argv)
     ret = 0;
   }
 
-  timer_free(NULL);
+  timer_free_(NULL);
 
   for (i = 0; i < N_TIMERS; ++i) {
     timer_free(timers[i]);
diff --git a/src/test/test.c b/src/test/test.c
index 4060ae1371..aea3ad6595 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +9,7 @@
  **/
 
 #include "orconfig.h"
+#include "crypto_rand.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -20,13 +21,12 @@
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include <math.h>
 
 /* These macros pull in declarations for some functions and structures that
  * are typically file-private. */
-#define GEOIP_PRIVATE
 #define ROUTER_PRIVATE
 #define CIRCUITSTATS_PRIVATE
 #define CIRCUITLIST_PRIVATE
@@ -38,13 +38,12 @@
 #include "buffers.h"
 #include "circuitlist.h"
 #include "circuitstats.h"
+#include "compress.h"
 #include "config.h"
 #include "connection_edge.h"
-#include "geoip.h"
 #include "rendcommon.h"
 #include "rendcache.h"
 #include "test.h"
-#include "torgzip.h"
 #include "main.h"
 #include "memarea.h"
 #include "onion.h"
@@ -56,7 +55,6 @@
 #include "routerparse.h"
 #include "statefile.h"
 #include "crypto_curve25519.h"
-#include "onion_ntor.h"
 
 /** Run unit tests for the onion handshake code. */
 static void
@@ -136,7 +134,8 @@ test_bad_onion_handshake(void *arg)
 
   /* Server: Case 1: the encrypted data is degenerate. */
   memset(junk_buf, 0, sizeof(junk_buf));
-  crypto_pk_public_hybrid_encrypt(pk, junk_buf2, TAP_ONIONSKIN_CHALLENGE_LEN,
+  crypto_pk_obsolete_public_hybrid_encrypt(pk,
+                               junk_buf2, TAP_ONIONSKIN_CHALLENGE_LEN,
                                junk_buf, DH_KEY_LEN, PK_PKCS1_OAEP_PADDING, 1);
   tt_int_op(-1, OP_EQ,
             onion_skin_TAP_server_handshake(junk_buf2, pk, NULL,
@@ -167,7 +166,7 @@ test_bad_onion_handshake(void *arg)
                                             s_buf, s_keys, 40));
   c_buf[64] ^= 33;
 
-  /* (Let the server procede) */
+  /* (Let the server proceed) */
   tt_int_op(0, OP_EQ,
             onion_skin_TAP_server_handshake(c_buf, pk, NULL,
                                             s_buf, s_keys, 40));
@@ -338,13 +337,25 @@ test_onion_queues(void *arg)
   tt_int_op(0,OP_EQ, onion_num_pending(ONION_HANDSHAKE_TYPE_NTOR));
 
  done:
-  circuit_free(TO_CIRCUIT(circ1));
-  circuit_free(TO_CIRCUIT(circ2));
+  circuit_free_(TO_CIRCUIT(circ1));
+  circuit_free_(TO_CIRCUIT(circ2));
   tor_free(create1);
   tor_free(create2);
   tor_free(onionskin);
 }
 
+static crypto_cipher_t *crypto_rand_aes_cipher = NULL;
+
+// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR
+// cipher in <b>crypto_rand_aes_cipher</b>.
+static void
+crypto_rand_deterministic_aes(char *out, size_t n)
+{
+  tor_assert(crypto_rand_aes_cipher);
+  memset(out, 0, n);
+  crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n);
+}
+
 static void
 test_circuit_timeout(void *arg)
 {
@@ -374,6 +385,11 @@ test_circuit_timeout(void *arg)
 
   state = or_state_new();
 
+  // Use a deterministic RNG here, or else we'll get nondeterministic
+  // coverage in some of the circuitstats functions.
+  MOCK(crypto_rand, crypto_rand_deterministic_aes);
+  crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover");
+
   circuitbuild_running_unit_tests();
 #define timeout0 (build_time_t)(30*1000.0)
   initial.Xm = 3000;
@@ -402,11 +418,11 @@ test_circuit_timeout(void *arg)
   } while (fabs(circuit_build_times_cdf(&initial, timeout0) -
                 circuit_build_times_cdf(&initial, timeout1)) > 0.02);
 
-  tt_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE);
+  tt_int_op(estimate.total_build_times, OP_LE, CBT_NCIRCUITS_TO_OBSERVE);
 
   circuit_build_times_update_state(&estimate, state);
   circuit_build_times_free_timeouts(&final);
-  tt_assert(circuit_build_times_parse_state(&final, state) == 0);
+  tt_int_op(circuit_build_times_parse_state(&final, state), OP_EQ, 0);
 
   circuit_build_times_update_alpha(&final);
   timeout2 = circuit_build_times_calculate_timeout(&final,
@@ -484,7 +500,7 @@ test_circuit_timeout(void *arg)
       }
     }
 
-    tt_assert(estimate.liveness.after_firsthop_idx == 0);
+    tt_int_op(estimate.liveness.after_firsthop_idx, OP_EQ, 0);
     tt_assert(final.liveness.after_firsthop_idx ==
                 CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1);
 
@@ -508,6 +524,8 @@ test_circuit_timeout(void *arg)
   circuit_build_times_free_timeouts(&final);
   or_state_free(state);
   teardown_periodic_events();
+  UNMOCK(crypto_rand);
+  crypto_cipher_free(crypto_rand_aes_cipher);
 }
 
 /** Test encoding and parsing of rendezvous service descriptors. */
@@ -527,25 +545,8 @@ test_rend_fns(void *arg)
   size_t intro_points_size;
   size_t encoded_size;
   int i;
-  char address1[] = "fooaddress.onion";
-  char address2[] = "aaaaaaaaaaaaaaaa.onion";
-  char address3[] = "fooaddress.exit";
-  char address4[] = "www.torproject.org";
-  char address5[] = "foo.abcdefghijklmnop.onion";
-  char address6[] = "foo.bar.abcdefghijklmnop.onion";
-  char address7[] = ".abcdefghijklmnop.onion";
 
   (void)arg;
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address1));
-  tt_assert(ONION_HOSTNAME == parse_extended_hostname(address2));
-  tt_str_op(address2,OP_EQ, "aaaaaaaaaaaaaaaa");
-  tt_assert(EXIT_HOSTNAME == parse_extended_hostname(address3));
-  tt_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4));
-  tt_assert(ONION_HOSTNAME == parse_extended_hostname(address5));
-  tt_str_op(address5,OP_EQ, "abcdefghijklmnop");
-  tt_assert(ONION_HOSTNAME == parse_extended_hostname(address6));
-  tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
 
   /* Initialize the service cache. */
   rend_cache_init();
@@ -581,20 +582,21 @@ test_rend_fns(void *arg)
     intro->intro_key = crypto_pk_dup_key(pk2);
     smartlist_add(generated->intro_nodes, intro);
   }
-  tt_assert(rend_encode_v2_descriptors(descs, generated, now, 0,
-                                         REND_NO_AUTH, NULL, NULL) > 0);
-  tt_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32,
-                                      NULL, now, 0) == 0);
+  int rv = rend_encode_v2_descriptors(descs, generated, now, 0,
+                                      REND_NO_AUTH, NULL, NULL);
+  tt_int_op(rv, OP_GT, 0);
+  rv = rend_compute_v2_desc_id(computed_desc_id, service_id_base32, NULL,
+                               now, 0);
+  tt_int_op(rv, OP_EQ, 0);
   tt_mem_op(((rend_encoded_v2_service_descriptor_t *)
              smartlist_get(descs, 0))->desc_id, OP_EQ,
             computed_desc_id, DIGEST_LEN);
-  tt_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id,
-                                             &intro_points_encrypted,
-                                             &intro_points_size,
-                                             &encoded_size,
-                                              &next_desc,
-                             ((rend_encoded_v2_service_descriptor_t *)
-                                 smartlist_get(descs, 0))->desc_str, 1) == 0);
+  rv = rend_parse_v2_service_descriptor(&parsed, parsed_desc_id,
+               &intro_points_encrypted, &intro_points_size, &encoded_size,
+               &next_desc,
+          ((rend_encoded_v2_service_descriptor_t *)smartlist_get(descs, 0))
+                                        ->desc_str, 1);
+  tt_int_op(rv, OP_EQ, 0);
   tt_assert(parsed);
   tt_mem_op(((rend_encoded_v2_service_descriptor_t *)
          smartlist_get(descs, 0))->desc_id,OP_EQ, parsed_desc_id, DIGEST_LEN);
@@ -625,7 +627,7 @@ test_rend_fns(void *arg)
  done:
   if (descs) {
     for (i = 0; i < smartlist_len(descs); i++)
-      rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i));
+      rend_encoded_v2_service_descriptor_free_(smartlist_get(descs, i));
     smartlist_free(descs);
   }
   if (parsed)
@@ -639,376 +641,6 @@ test_rend_fns(void *arg)
   tor_free(intro_points_encrypted);
 }
 
-  /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs
-   * using ipv4.  Since our fake geoip database is the same between
-   * ipv4 and ipv6, we should get the same result no matter which
-   * address family we pick for each IP. */
-#define SET_TEST_ADDRESS(i) do {                \
-    if ((i) & 1) {                              \
-      SET_TEST_IPV6(i);                         \
-      tor_addr_from_in6(&addr, &in6);           \
-    } else {                                    \
-      tor_addr_from_ipv4h(&addr, (uint32_t) i); \
-    }                                           \
-  } while (0)
-
-  /* Make sure that country ID actually works. */
-#define SET_TEST_IPV6(i) \
-  do {                                                          \
-    set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i)));        \
-  } while (0)
-#define CHECK_COUNTRY(country, val) do {                                \
-    /* test ipv4 country lookup */                                      \
-    tt_str_op(country, OP_EQ,                                              \
-               geoip_get_country_name(geoip_get_country_by_ipv4(val))); \
-    /* test ipv6 country lookup */                                      \
-    SET_TEST_IPV6(val);                                                 \
-    tt_str_op(country, OP_EQ,                                              \
-               geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \
-  } while (0)
-
-/** Run unit tests for GeoIP code. */
-static void
-test_geoip(void *arg)
-{
-  int i, j;
-  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
-  char *s = NULL, *v = NULL;
-  const char *bridge_stats_1 =
-      "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "bridge-ips zz=24,xy=8\n"
-      "bridge-ip-versions v4=16,v6=16\n"
-      "bridge-ip-transports <OR>=24\n",
-  *dirreq_stats_1 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips ab=8\n"
-      "dirreq-v3-reqs ab=8\n"
-      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
-  *dirreq_stats_2 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips \n"
-      "dirreq-v3-reqs \n"
-      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
-  *dirreq_stats_3 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips \n"
-      "dirreq-v3-reqs \n"
-      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
-  *dirreq_stats_4 =
-      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "dirreq-v3-ips \n"
-      "dirreq-v3-reqs \n"
-      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
-          "not-modified=0,busy=0\n"
-      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
-      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n",
-  *entry_stats_1 =
-      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "entry-ips ab=8\n",
-  *entry_stats_2 =
-      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-      "entry-ips \n";
-  tor_addr_t addr;
-  struct in6_addr in6;
-
-  /* Populate the DB a bit.  Add these in order, since we can't do the final
-   * 'sort' step.  These aren't very good IP addresses, but they're perfectly
-   * fine uint32_t values. */
-  (void)arg;
-  tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET));
-
-  /* Populate the IPv6 DB equivalently with fake IPs in the same range */
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6));
-  tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6));
-
-  /* We should have 4 countries: ??, ab, xy, zz. */
-  tt_int_op(4,OP_EQ, geoip_get_n_countries());
-  memset(&in6, 0, sizeof(in6));
-
-  CHECK_COUNTRY("??", 3);
-  CHECK_COUNTRY("ab", 32);
-  CHECK_COUNTRY("??", 5);
-  CHECK_COUNTRY("??", 51);
-  CHECK_COUNTRY("xy", 150);
-  CHECK_COUNTRY("xy", 190);
-  CHECK_COUNTRY("??", 2000);
-
-  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3));
-  SET_TEST_IPV6(3);
-  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6));
-
-  get_options_mutable()->BridgeRelay = 1;
-  get_options_mutable()->BridgeRecordUsageByCountry = 1;
-  /* Put 9 observations in AB... */
-  for (i=32; i < 40; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
-  }
-  SET_TEST_ADDRESS(225);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
-  /* and 3 observations in XY, several times. */
-  for (j=0; j < 10; ++j)
-    for (i=52; i < 55; ++i) {
-      SET_TEST_ADDRESS(i);
-      geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600);
-    }
-  /* and 17 observations in ZZ... */
-  for (i=110; i < 127; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  }
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
-  tt_assert(s);
-  tt_assert(v);
-  tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s);
-  tt_str_op("v4=16,v6=16",OP_EQ, v);
-  tor_free(s);
-  tor_free(v);
-
-  /* Now clear out all the AB observations. */
-  geoip_remove_old_clients(now-6000);
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
-  tt_assert(s);
-  tt_assert(v);
-  tt_str_op("zz=24,xy=8",OP_EQ, s);
-  tt_str_op("v4=16,v6=16",OP_EQ, v);
-  tor_free(s);
-  tor_free(v);
-
-  /* Start testing bridge statistics by making sure that we don't output
-   * bridge stats without initializing them. */
-  s = geoip_format_bridge_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Initialize stats and generate the bridge-stats history string out of
-   * the connecting clients added above. */
-  geoip_bridge_stats_init(now);
-  s = geoip_format_bridge_stats(now + 86400);
-  tt_assert(s);
-  tt_str_op(bridge_stats_1,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting bridge stats and make sure we don't write a history
-   * string anymore. */
-  geoip_bridge_stats_term();
-  s = geoip_format_bridge_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Stop being a bridge and start being a directory mirror that gathers
-   * directory request statistics. */
-  geoip_bridge_stats_term();
-  get_options_mutable()->BridgeRelay = 0;
-  get_options_mutable()->BridgeRecordUsageByCountry = 0;
-  get_options_mutable()->DirReqStatistics = 1;
-
-  /* Start testing dirreq statistics by making sure that we don't collect
-   * dirreq stats without initializing them. */
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Initialize stats, note one connecting client, and generate the
-   * dirreq-stats history string. */
-  geoip_dirreq_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_1,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add another connecting client, and ensure we
-   * don't generate a history string. */
-  geoip_dirreq_stats_term();
-  SET_TEST_ADDRESS(101);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Re-start stats, add a connecting client, reset stats, and make sure
-   * that we get an all empty history string. */
-  geoip_dirreq_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
-  geoip_reset_dirreq_stats(now);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_2,OP_EQ, s);
-  tor_free(s);
-
-  /* Note a successful network status response and make sure that it
-   * appears in the history string. */
-  geoip_note_ns_response(GEOIP_SUCCESS);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_3,OP_EQ, s);
-  tor_free(s);
-
-  /* Start a tunneled directory request. */
-  geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED);
-  s = geoip_format_dirreq_stats(now + 86400);
-  tt_str_op(dirreq_stats_4,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting directory request statistics and start gathering
-   * entry stats. */
-  geoip_dirreq_stats_term();
-  get_options_mutable()->DirReqStatistics = 0;
-  get_options_mutable()->EntryStatistics = 1;
-
-  /* Start testing entry statistics by making sure that we don't collect
-   * anything without initializing entry stats. */
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Initialize stats, note one connecting client, and generate the
-   * entry-stats history string. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_str_op(entry_stats_1,OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add another connecting client, and ensure we
-   * don't generate a history string. */
-  geoip_entry_stats_term();
-  SET_TEST_ADDRESS(101);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_assert(!s);
-
-  /* Re-start stats, add a connecting client, reset stats, and make sure
-   * that we get an all empty history string. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
-  geoip_reset_entry_stats(now);
-  s = geoip_format_entry_stats(now + 86400);
-  tt_str_op(entry_stats_2,OP_EQ, s);
-  tor_free(s);
-
-  /* Test the OOM handler. Add a client, run the OOM. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
-                         now - (12 * 60 * 60));
-  /* We've seen this 12 hours ago. Run the OOM, it should clean the entry
-   * because it is above the minimum cutoff of 4 hours. */
-  size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000);
-  tt_size_op(bytes_removed, OP_GT, 0);
-
-  /* Do it again but this time with an entry with a lower cutoff. */
-  geoip_entry_stats_init(now);
-  SET_TEST_ADDRESS(100);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
-                         now - (3 * 60 * 60));
-  bytes_removed = geoip_client_cache_handle_oom(now, 1000);
-  tt_size_op(bytes_removed, OP_EQ, 0);
-
-  /* Stop collecting entry statistics. */
-  geoip_entry_stats_term();
-  get_options_mutable()->EntryStatistics = 0;
-
- done:
-  tor_free(s);
-  tor_free(v);
-}
-
-static void
-test_geoip_with_pt(void *arg)
-{
-  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
-  char *s = NULL;
-  int i;
-  tor_addr_t addr;
-  struct in6_addr in6;
-
-  (void)arg;
-  get_options_mutable()->BridgeRelay = 1;
-  get_options_mutable()->BridgeRecordUsageByCountry = 1;
-
-  memset(&in6, 0, sizeof(in6));
-
-  /* No clients seen yet. */
-  s = geoip_get_transport_history();
-  tor_assert(!s);
-
-  /* 4 connections without a pluggable transport */
-  for (i=0; i < 4; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
-  }
-
-  /* 9 connections with "alpha" */
-  for (i=4; i < 13; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200);
-  }
-
-  /* one connection with "beta" */
-  SET_TEST_ADDRESS(13);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200);
-
-  /* 14 connections with "charlie" */
-  for (i=14; i < 28; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200);
-  }
-
-  /* 131 connections with "ddr" */
-  for (i=28; i < 159; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200);
-  }
-
-  /* 8 connections with "entropy" */
-  for (i=159; i < 167; ++i) {
-    SET_TEST_ADDRESS(i);
-    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200);
-  }
-
-  /* 2 connections from the same IP with two different transports. */
-  SET_TEST_ADDRESS(++i);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200);
-  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200);
-
-  /* Test the transport history string. */
-  s = geoip_get_transport_history();
-  tor_assert(s);
-  tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136,"
-             "entropy=8,fire=8,google=8");
-
-  /* Stop collecting entry statistics. */
-  geoip_entry_stats_term();
-  get_options_mutable()->EntryStatistics = 0;
-
- done:
-  tor_free(s);
-}
-
-#undef SET_TEST_ADDRESS
-#undef SET_TEST_IPV6
-#undef CHECK_COUNTRY
-
 /** Run unit tests for stats code. */
 static void
 test_stats(void *arg)
@@ -1023,7 +655,7 @@ test_stats(void *arg)
   rep_hist_note_exit_stream_opened(80);
   rep_hist_note_exit_bytes(80, 100, 10000);
   s = rep_hist_format_exit_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, note some streams and bytes, and generate history
    * string. */
@@ -1061,7 +693,7 @@ test_stats(void *arg)
   rep_hist_exit_stats_term();
   rep_hist_note_exit_bytes(80, 100, 10000);
   s = rep_hist_format_exit_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add some bytes, reset stats, and see what history we
    * get when observing no streams or bytes at all. */
@@ -1080,7 +712,7 @@ test_stats(void *arg)
    * conn stats without initializing them. */
   rep_hist_note_or_conn_bytes(1, 20, 400, now);
   s = rep_hist_format_conn_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, note bytes, and generate history string. */
   rep_hist_conn_stats_init(now);
@@ -1097,7 +729,7 @@ test_stats(void *arg)
   rep_hist_conn_stats_term();
   rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
   s = rep_hist_format_conn_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add some bytes, reset stats, and see what history we
    * get when observing no bytes at all. */
@@ -1115,7 +747,7 @@ test_stats(void *arg)
    * stats without initializing them. */
   rep_hist_add_buffer_stats(2.0, 2.0, 20);
   s = rep_hist_format_buffer_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Initialize stats, add statistics for a single circuit, and generate
    * the history string. */
@@ -1150,7 +782,7 @@ test_stats(void *arg)
   rep_hist_buffer_stats_term();
   rep_hist_add_buffer_stats(2.0, 2.0, 20);
   s = rep_hist_format_buffer_stats(now + 86400);
-  tt_assert(!s);
+  tt_ptr_op(s, OP_EQ, NULL);
 
   /* Re-start stats, add statistics for one circuit, reset stats, and make
    * sure that the history has all zeros. */
@@ -1182,8 +814,6 @@ static struct testcase_t test_array[] = {
   { "fast_handshake", test_fast_handshake, 0, NULL, NULL },
   FORK(circuit_timeout),
   FORK(rend_fns),
-  ENT(geoip),
-  FORK(geoip_with_pt),
   FORK(stats),
 
   END_OF_TESTCASES
@@ -1195,43 +825,70 @@ struct testgroup_t testgroups[] = {
   { "addr/", addr_tests },
   { "address/", address_tests },
   { "address_set/", address_set_tests },
+  { "bridges/", bridges_tests },
   { "buffer/", buffer_tests },
+  { "bwmgt/", bwmgt_tests },
   { "cellfmt/", cell_format_tests },
   { "cellqueue/", cell_queue_tests },
   { "channel/", channel_tests },
+  { "channelpadding/", channelpadding_tests },
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
+  { "circuitbuild/", circuitbuild_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
+  { "circuituse/", circuituse_tests },
+  { "circuitstats/", circuitstats_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
   { "connection/", connection_tests },
+  { "conscache/", conscache_tests },
+  { "consdiff/", consdiff_tests },
+  { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
   { "control/", controller_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
-  { "dos/", dos_tests },
+  { "crypto/openssl/", crypto_openssl_tests },
   { "dir/", dir_tests },
   { "dir_handle_get/", dir_handle_get_tests },
   { "dir/md/", microdesc_tests },
+  { "dir/voting-schedule/", voting_schedule_tests },
+  { "dos/", dos_tests },
   { "entryconn/", entryconn_tests },
   { "entrynodes/", entrynodes_tests },
   { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
-  { "hs/", hs_tests },
+  { "geoip/", geoip_tests },
+  { "legacy_hs/", hs_tests },
+  { "hs_cache/", hs_cache },
+  { "hs_cell/", hs_cell_tests },
+  { "hs_common/", hs_common_tests },
+  { "hs_config/", hs_config_tests },
+  { "hs_control/", hs_control_tests },
+  { "hs_descriptor/", hs_descriptor },
+  { "hs_ntor/", hs_ntor_tests },
+  { "hs_service/", hs_service_tests },
+  { "hs_client/", hs_client_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
   { "link-handshake/", link_handshake_tests },
+  { "mainloop/", mainloop_tests },
   { "nodelist/", nodelist_tests },
   { "oom/", oom_tests },
   { "oos/", oos_tests },
   { "options/", options_tests },
+  { "periodic-event/" , periodic_event_tests },
   { "policy/" , policy_tests },
   { "procmon/", procmon_tests },
+  { "proto/http/", proto_http_tests },
+  { "proto/misc/", proto_misc_tests },
   { "protover/", protover_tests },
   { "pt/", pt_tests },
   { "relay/" , relay_tests },
   { "relaycell/", relaycell_tests },
+  { "relaycrypt/", relaycrypt_tests },
   { "rend_cache/", rend_cache_tests },
   { "replaycache/", replaycache_tests },
   { "router/", router_tests },
@@ -1242,6 +899,7 @@ struct testgroup_t testgroups[] = {
   { "socks/", socks_tests },
   { "shared-random/", sr_tests },
   { "status/" , status_tests },
+  { "storagedir/", storagedir_tests },
   { "tortls/", tortls_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
diff --git a/src/test/test.h b/src/test/test.h
index 028082386e..63b2b30746 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_H
@@ -11,6 +11,8 @@
  * \brief Macros and functions used by unit tests.
  */
 
+#define DEBUG_SMARTLIST 1
+
 #include "compat.h"
 #include "tinytest.h"
 #define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END
@@ -45,8 +47,8 @@
  * you're doing. */
 #define tt_double_eq(a,b)     \
   STMT_BEGIN                  \
-  tt_double_op((a), >=, (b)); \
-  tt_double_op((a), <=, (b)); \
+  tt_double_op((a), OP_GE, (b)); \
+  tt_double_op((a), OP_LE, (b)); \
   STMT_END
 
 #ifdef _MSC_VER
@@ -55,7 +57,7 @@
 #else
 #define U64_PRINTF_TYPE unsigned long long
 #define I64_PRINTF_TYPE long long
-#endif
+#endif /* defined(_MSC_VER) */
 
 #define tt_size_op(a,op,b)                                              \
   tt_assert_test_fmt_type(a,b,#a" "#op" "#b,size_t,(val1_ op val2_),    \
@@ -72,9 +74,19 @@
     I64_PRINTF_TYPE, I64_FORMAT,                                       \
     {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION)
 
+/**
+ * Declare that the test is done, even though no tt___op() calls were made.
+ *
+ * For use when you only want to test calling something, but not check
+ * any values/pointers/etc afterwards.
+ */
+#define tt_finished() TT_EXIT_TEST_FUNCTION
+
 const char *get_fname(const char *name);
 const char *get_fname_rnd(const char *name);
 struct crypto_pk_t *pk_generate(int idx);
+void init_pregenerated_keys(void);
+void free_pregenerated_keys(void);
 
 #define US2_CONCAT_2__(a, b) a ## __ ## b
 #define US_CONCAT_2__(a, b) a ## _ ## b
@@ -176,45 +188,71 @@ extern struct testcase_t accounting_tests[];
 extern struct testcase_t addr_tests[];
 extern struct testcase_t address_tests[];
 extern struct testcase_t address_set_tests[];
+extern struct testcase_t bridges_tests[];
+extern struct testcase_t bwmgt_tests[];
 extern struct testcase_t buffer_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
+extern struct testcase_t channelpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
+extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
+extern struct testcase_t circuituse_tests[];
+extern struct testcase_t circuitstats_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
 extern struct testcase_t connection_tests[];
+extern struct testcase_t conscache_tests[];
+extern struct testcase_t consdiff_tests[];
+extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
 extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
 extern struct testcase_t crypto_tests[];
-extern struct testcase_t dos_tests[];
+extern struct testcase_t crypto_openssl_tests[];
 extern struct testcase_t dir_tests[];
 extern struct testcase_t dir_handle_get_tests[];
+extern struct testcase_t dos_tests[];
 extern struct testcase_t entryconn_tests[];
 extern struct testcase_t entrynodes_tests[];
 extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
+extern struct testcase_t geoip_tests[];
 extern struct testcase_t hs_tests[];
+extern struct testcase_t hs_cache[];
+extern struct testcase_t hs_cell_tests[];
+extern struct testcase_t hs_common_tests[];
+extern struct testcase_t hs_config_tests[];
+extern struct testcase_t hs_control_tests[];
+extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_ntor_tests[];
+extern struct testcase_t hs_service_tests[];
+extern struct testcase_t hs_client_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
 extern struct testcase_t logging_tests[];
+extern struct testcase_t mainloop_tests[];
 extern struct testcase_t microdesc_tests[];
 extern struct testcase_t nodelist_tests[];
 extern struct testcase_t oom_tests[];
 extern struct testcase_t oos_tests[];
 extern struct testcase_t options_tests[];
+extern struct testcase_t periodic_event_tests[];
 extern struct testcase_t policy_tests[];
 extern struct testcase_t procmon_tests[];
+extern struct testcase_t proto_http_tests[];
+extern struct testcase_t proto_misc_tests[];
 extern struct testcase_t protover_tests[];
 extern struct testcase_t pubsub_tests[];
 extern struct testcase_t pt_tests[];
 extern struct testcase_t relay_tests[];
 extern struct testcase_t relaycell_tests[];
+extern struct testcase_t relaycrypt_tests[];
 extern struct testcase_t rend_cache_tests[];
 extern struct testcase_t replaycache_tests[];
 extern struct testcase_t router_tests[];
@@ -222,6 +260,7 @@ extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
 extern struct testcase_t scheduler_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t socks_tests[];
 extern struct testcase_t status_tests[];
 extern struct testcase_t thread_tests[];
@@ -229,6 +268,7 @@ extern struct testcase_t tortls_tests[];
 extern struct testcase_t util_tests[];
 extern struct testcase_t util_format_tests[];
 extern struct testcase_t util_process_tests[];
+extern struct testcase_t voting_schedule_tests[];
 extern struct testcase_t dns_tests[];
 extern struct testcase_t handle_tests[];
 extern struct testcase_t sr_tests[];
@@ -251,5 +291,5 @@ extern const char AUTHORITY_SIGNKEY_3[];
 extern const char AUTHORITY_SIGNKEY_C_DIGEST[];
 extern const char AUTHORITY_SIGNKEY_C_DIGEST256[];
 
-#endif
+#endif /* !defined(TOR_TEST_H) */
 
diff --git a/src/test/test_accounting.c b/src/test/test_accounting.c
index 7edba988a6..b0d37b2989 100644
--- a/src/test/test_accounting.c
+++ b/src/test/test_accounting.c
@@ -1,3 +1,6 @@
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #include "or.h"
 #include "test.h"
 #define HIBERNATE_PRIVATE
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index be440a0925..40db31320f 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -1,13 +1,33 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESSMAP_PRIVATE
 #include "orconfig.h"
 #include "or.h"
+#include "crypto_rand.h"
 #include "test.h"
 #include "addressmap.h"
+#include "log_test_helpers.h"
+
+/** Mocking replacement: only handles localhost. */
+static int
+mock_tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out)
+{
+  if (!strcmp(name, "localhost")) {
+    if (family == AF_INET || family == AF_UNSPEC) {
+      tor_addr_from_ipv4h(addr_out, 0x7f000001);
+      return 0;
+    } else if (family == AF_INET6) {
+      char bytes[16] = { 0, 0, 0, 0, 0, 0, 0, 0,
+                         0, 0, 0, 0, 0, 0, 0, 1 };
+      tor_addr_from_ipv6_bytes(addr_out, bytes);
+      return 0;
+    }
+  }
+  return -1;
+}
 
 static void
 test_addr_basic(void *arg)
@@ -29,6 +49,9 @@ test_addr_basic(void *arg)
   tt_int_op(u32,OP_EQ, 0x04030201u);
   tt_int_op(u16,OP_EQ, 99);
   tor_free(cp);
+
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup);
+
   tt_assert(!addr_port_lookup(LOG_WARN, "nonexistent.address:4040",
                                &cp, NULL, &u16));
   tt_str_op(cp,OP_EQ, "nonexistent.address");
@@ -36,8 +59,8 @@ test_addr_basic(void *arg)
   tor_free(cp);
   tt_assert(!addr_port_lookup(LOG_WARN, "localhost:9999", &cp, &u32, &u16));
   tt_str_op(cp,OP_EQ, "localhost");
-  tt_int_op(u32,OP_EQ, 0x7f000001u);
   tt_int_op(u16,OP_EQ, 9999);
+  tt_int_op(u32,OP_EQ, 0x7f000001u);
   tor_free(cp);
   u32 = 3;
   tt_assert(!addr_port_lookup(LOG_WARN, "localhost", NULL, &u32, &u16));
@@ -75,6 +98,7 @@ test_addr_basic(void *arg)
   }
 
  done:
+  UNMOCK(tor_addr_lookup);
   tor_free(cp);
 }
 
@@ -429,10 +453,10 @@ test_addr_ip6_helpers(void *arg)
                     "::ffff:6.0.0.0"); /* XXXX wrong. */
   tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", 0, &t1, NULL, NULL, NULL);
   tor_addr_parse_mask_ports("2.3.4.5", 0, &t2, NULL, NULL, NULL);
-  tt_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0);
+  tt_int_op(tor_addr_compare(&t1, &t2, CMP_SEMANTIC), OP_EQ, 0);
   tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", 0, &t1, NULL, NULL, NULL);
   tor_addr_parse_mask_ports("2.3.4.5", 0, &t2, NULL, NULL, NULL);
-  tt_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0);
+  tt_int_op(tor_addr_compare(&t1, &t2, CMP_SEMANTIC), OP_LT, 0);
 
   /* test compare_masked */
   test_addr_compare_masked("ffff::", OP_EQ, "ffff::0", 128);
@@ -615,7 +639,7 @@ test_addr_ip6_helpers(void *arg)
   /* Try some long addresses. */
   r=tor_addr_parse_mask_ports("[ffff:1111:1111:1111:1111:1111:1111:1111]",
                               0, &t1, NULL, NULL, NULL);
-  tt_assert(r == AF_INET6);
+  tt_int_op(r, OP_EQ, AF_INET6);
   r=tor_addr_parse_mask_ports("[ffff:1111:1111:1111:1111:1111:1111:11111]",
                               0, &t1, NULL, NULL, NULL);
   tt_int_op(r, OP_EQ, -1);
@@ -664,38 +688,38 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("*6",0,&t1, &mask, NULL, NULL);
   tt_int_op(r, OP_EQ, -1);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   /* Try a mask with a wildcard. */
   r=tor_addr_parse_mask_ports("*/16",0,&t1, &mask, NULL, NULL);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("*4/16",TAPMP_EXTENDED_STAR,
                               &t1, &mask, NULL, NULL);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("*6/30",TAPMP_EXTENDED_STAR,
                               &t1, &mask, NULL, NULL);
-  tt_assert(r == -1);
+  tt_int_op(r, OP_EQ, -1);
   /* Basic mask tests*/
   r=tor_addr_parse_mask_ports("1.1.2.2/31",0,&t1, &mask, NULL, NULL);
-  tt_assert(r == AF_INET);
+  tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,31);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x01010202);
   r=tor_addr_parse_mask_ports("3.4.16.032:1-2",0,&t1, &mask, &port1, &port2);
-  tt_assert(r == AF_INET);
+  tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,32);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x03041020);
-  tt_assert(port1 == 1);
-  tt_assert(port2 == 2);
+  tt_uint_op(port1, OP_EQ, 1);
+  tt_uint_op(port2, OP_EQ, 2);
   r=tor_addr_parse_mask_ports("1.1.2.3/255.255.128.0",0,&t1, &mask,NULL,NULL);
-  tt_assert(r == AF_INET);
+  tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,17);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x01010203);
   r=tor_addr_parse_mask_ports("[efef::]/112",0,&t1, &mask, &port1, &port2);
-  tt_assert(r == AF_INET6);
-  tt_assert(port1 == 1);
-  tt_assert(port2 == 65535);
+  tt_int_op(r, OP_EQ, AF_INET6);
+  tt_uint_op(port1, OP_EQ, 1);
+  tt_uint_op(port2, OP_EQ, 65535);
   /* Try regular wildcard behavior without TAPMP_EXTENDED_STAR */
   r=tor_addr_parse_mask_ports("*:80-443",0,&t1,&mask,&port1,&port2);
   tt_int_op(r,OP_EQ,AF_INET); /* Old users of this always get inet */
@@ -730,15 +754,17 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(port2,OP_EQ,65535);
 
   /* make sure inet address lengths >= max */
-  tt_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255"));
-  tt_assert(TOR_ADDR_BUF_LEN >=
-              sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255"));
+  tt_int_op(INET_NTOA_BUF_LEN, OP_GE, sizeof("255.255.255.255"));
+  tt_int_op(TOR_ADDR_BUF_LEN, OP_GE,
+            sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255"));
 
   tt_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr));
 
   /* get interface addresses */
   r = get_interface_address6(LOG_DEBUG, AF_INET, &t1);
+  tt_int_op(r, OP_LE, 0); // "it worked or it didn't"
   i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2);
+  tt_int_op(i, OP_LE, 0); // "it worked or it didn't"
 
   TT_BLATHER(("v4 address: %s (family=%d)", fmt_addr(&t1),
               tor_addr_family(&t1)));
@@ -917,6 +943,158 @@ test_virtaddrmap(void *data)
   ;
 }
 
+static const char *canned_data = NULL;
+static size_t canned_data_len = 0;
+
+/* Mock replacement for crypto_rand() that returns canned data from
+ * canned_data above. */
+static void
+crypto_canned(char *ptr, size_t n)
+{
+  if (canned_data_len) {
+    size_t to_copy = MIN(n, canned_data_len);
+    memcpy(ptr, canned_data, to_copy);
+    canned_data += to_copy;
+    canned_data_len -= to_copy;
+    n -= to_copy;
+    ptr += to_copy;
+  }
+  if (n) {
+    crypto_rand_unmocked(ptr, n);
+  }
+}
+
+static void
+test_virtaddrmap_persist(void *data)
+{
+  (void)data;
+  const char *a, *b, *c;
+  tor_addr_t addr;
+  char *ones = NULL;
+
+  addressmap_init();
+
+  // Try a hostname.
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
+                                          tor_strdup("foobar.baz"));
+  tt_assert(a);
+  tt_assert(!strcmpend(a, ".virtual"));
+
+  // mock crypto_rand to repeat the same result twice; make sure we get
+  // different outcomes.  (Because even though the odds for receiving the
+  // same 80-bit address twice is only 1/2^40, it could still happen for
+  // some user -- but running our test through 2^40 iterations isn't
+  // reasonable.)
+  canned_data = "1234567890" // the first call returns this.
+                "1234567890" // the second call returns this.
+                "abcdefghij"; // the third call returns this.
+  canned_data_len = 30;
+  MOCK(crypto_rand, crypto_canned);
+
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
+                                          tor_strdup("quuxit.baz"));
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
+                                          tor_strdup("nescio.baz"));
+  tt_assert(a);
+  tt_assert(b);
+  tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual");
+  tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual");
+
+  // Now try something to get us an ipv4 address
+  UNMOCK(crypto_rand);
+  tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16",
+                                                AF_INET, 0, NULL));
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("foobar.baz"));
+  tt_assert(a);
+  tt_assert(!strcmpstart(a, "192.168."));
+  tor_addr_parse(&addr, a);
+  tt_int_op(AF_INET, OP_EQ, tor_addr_family(&addr));
+
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("quuxit.baz"));
+  tt_str_op(b, OP_NE, a);
+  tt_assert(!strcmpstart(b, "192.168."));
+
+  // Try some canned entropy and verify all the we discard duplicates,
+  // addresses that end with 0, and addresses that end with 255.
+  MOCK(crypto_rand, crypto_canned);
+  canned_data = "\x01\x02\x03\x04" // okay
+                "\x01\x02\x03\x04" // duplicate
+                "\x03\x04\x00\x00" // bad ending 1
+                "\x05\x05\x00\xff" // bad ending 2
+                "\x05\x06\x07\xf0"; // okay
+  canned_data_len = 20;
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("wumble.onion"));
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("wumpus.onion"));
+  tt_str_op(a, OP_EQ, "192.168.3.4");
+  tt_str_op(b, OP_EQ, "192.168.7.240");
+
+  // Now try IPv6!
+  UNMOCK(crypto_rand);
+  tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20",
+                                                AF_INET6, 0, NULL));
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("foobar.baz"));
+  tt_assert(a);
+  tt_assert(!strcmpstart(a, "[1010:f"));
+  tor_addr_parse(&addr, a);
+  tt_int_op(AF_INET6, OP_EQ, tor_addr_family(&addr));
+
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("quuxit.baz"));
+  tt_str_op(b, OP_NE, a);
+  tt_assert(!strcmpstart(b, "[1010:f"));
+
+  // Try IPv6 with canned entropy, to make sure we detect duplicates.
+  MOCK(crypto_rand, crypto_canned);
+  canned_data = "acanthopterygian" // okay
+                "cinematographist" // okay
+                "acanthopterygian" // duplicate
+                "acanthopterygian" // duplicate
+                "acanthopterygian" // duplicate
+                "cinematographist" // duplicate
+                "coadministration"; // okay
+  canned_data_len = 16 * 7;
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("wuffle.baz"));
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                          tor_strdup("gribble.baz"));
+  c = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
+                                      tor_strdup("surprisingly-legible.baz"));
+  tt_str_op(a, OP_EQ, "[1010:f16e:7468:6f70:7465:7279:6769:616e]");
+  tt_str_op(b, OP_EQ, "[1010:fe65:6d61:746f:6772:6170:6869:7374]");
+  tt_str_op(c, OP_EQ, "[1010:f164:6d69:6e69:7374:7261:7469:6f6e]");
+
+  // Try address exhaustion: make sure we can actually fail if we
+  // get too many already-existing addresses.
+  canned_data_len = 128*1024;
+  canned_data = ones = tor_malloc(canned_data_len);
+  memset(ones, 1, canned_data_len);
+  // There is some chance this one will fail if a previous random
+  // allocation gave out the address already.
+  a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("might-work.onion"));
+  if (a) {
+    tt_str_op(a, OP_EQ, "192.168.1.1");
+  }
+  setup_capture_of_logs(LOG_WARN);
+  // This one will definitely fail, since we've set up the RNG to hand
+  // out "1" forever.
+  b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
+                                          tor_strdup("wont-work.onion"));
+  tt_assert(b == NULL);
+  expect_single_log_msg_containing("Ran out of virtual addresses!");
+
+ done:
+  UNMOCK(crypto_rand);
+  tor_free(ones);
+  addressmap_free_all();
+  teardown_capture_of_logs();
+}
+
 static void
 test_addr_localname(void *arg)
 {
@@ -988,7 +1166,7 @@ test_addr_sockaddr_to_str(void *arg)
   s_un.sun_family = AF_UNIX;
   strlcpy(s_un.sun_path, "/here/is/a/path", sizeof(s_un.sun_path));
   CHECK(s_un, "unix:/here/is/a/path");
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
 
   memset(&sin6,0,sizeof(sin6));
   sin6.sin6_family = AF_INET6;
@@ -1071,6 +1249,7 @@ struct testcase_t addr_tests[] = {
   ADDR_LEGACY(ip6_helpers),
   ADDR_LEGACY(parse),
   { "virtaddr", test_virtaddrmap, 0, NULL, NULL },
+  { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL },
   { "localname", test_addr_localname, 0, NULL, NULL },
   { "dup_ip", test_addr_dup_ip, 0, NULL, NULL },
   { "sockaddr_to_str", test_addr_sockaddr_to_str, 0, NULL, NULL },
diff --git a/src/test/test_address.c b/src/test/test_address.c
index 0d142ad483..9c88d37a41 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESS_PRIVATE
@@ -21,7 +21,7 @@
 #include <sys/ioctl.h>
 #endif
 #include <net/if.h>
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #include "or.h"
 #include "address.h"
@@ -224,7 +224,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    smartlist = ifaddrs_to_smartlist(ifa, AF_UNSPEC);
 
    tt_assert(smartlist);
-   tt_assert(smartlist_len(smartlist) == 3);
+   tt_int_op(smartlist_len(smartlist), OP_EQ, 3);
 
    sockaddr_to_check = tor_malloc(sizeof(struct sockaddr_in6));
 
@@ -233,7 +233,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    tor_addr_to_sockaddr(tor_addr,0,sockaddr_to_check,
                         sizeof(struct sockaddr_in));
 
-   tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+   tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
    tt_assert(sockaddr_in_are_equal((struct sockaddr_in *)sockaddr_to_check,
                                    ipv4_sockaddr_local));
 
@@ -242,7 +242,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    tor_addr_to_sockaddr(tor_addr,0,sockaddr_to_check,
                         sizeof(struct sockaddr_in));
 
-   tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+   tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
    tt_assert(sockaddr_in_are_equal((struct sockaddr_in *)sockaddr_to_check,
                                    ipv4_sockaddr_remote));
 
@@ -251,7 +251,7 @@ test_address_ifaddrs_to_smartlist(void *arg)
    tor_addr_to_sockaddr(tor_addr,0,sockaddr_to_check,
                         sizeof(struct sockaddr_in6));
 
-   tt_int_op(addr_len,==,sizeof(struct sockaddr_in6));
+   tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in6));
    tt_assert(sockaddr_in6_are_equal((struct sockaddr_in6*)sockaddr_to_check,
                                     ipv6_sockaddr));
 
@@ -305,7 +305,7 @@ test_address_get_if_addrs_ifaddrs(void *arg)
   return;
 }
 
-#endif
+#endif /* defined(HAVE_IFADDRS_TO_SMARTLIST) */
 
 #ifdef HAVE_IP_ADAPTER_TO_SMARTLIST
 
@@ -319,7 +319,7 @@ test_address_get_if_addrs_win32(void *arg)
 
   results = get_interface_addresses_win32(LOG_ERR, AF_UNSPEC);
 
-  tt_int_op(smartlist_len(results),>=,1);
+  tt_int_op(smartlist_len(results),OP_GE,1);
   tt_assert(smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_null_tor_addr(results));
 
@@ -384,7 +384,7 @@ test_address_ip_adapter_addresses_to_smartlist(void *arg)
   result = ip_adapter_addresses_to_smartlist(addrs1);
 
   tt_assert(result);
-  tt_assert(smartlist_len(result) == 3);
+  tt_int_op(smartlist_len(result), OP_EQ, 3);
 
   tor_addr = smartlist_get(result,0);
 
@@ -421,7 +421,7 @@ test_address_ip_adapter_addresses_to_smartlist(void *arg)
   tor_free(sockaddr_to_check);
   return;
 }
-#endif
+#endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
 
@@ -456,14 +456,14 @@ test_address_ifreq_to_smartlist(void *arg)
   ifc->ifc_ifcu.ifcu_req = ifr;
 
   results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
-  tt_int_op(smartlist_len(results),==,1);
+  tt_int_op(smartlist_len(results),OP_EQ,1);
 
   tor_addr = smartlist_get(results, 0);
   addr_len =
   tor_addr_to_sockaddr(tor_addr,0,(struct sockaddr *)sockaddr_to_check,
                        sizeof(struct sockaddr_in));
 
-  tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+  tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
   tt_assert(sockaddr_in_are_equal(sockaddr,sockaddr_to_check));
 
   ifr = tor_realloc(ifr,2*sizeof(struct ifreq));
@@ -479,14 +479,14 @@ test_address_ifreq_to_smartlist(void *arg)
   smartlist_free(results);
 
   results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
-  tt_int_op(smartlist_len(results),==,2);
+  tt_int_op(smartlist_len(results),OP_EQ,2);
 
   tor_addr = smartlist_get(results, 0);
   addr_len =
   tor_addr_to_sockaddr(tor_addr,0,(struct sockaddr *)sockaddr_to_check,
                        sizeof(struct sockaddr_in));
 
-  tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+  tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
   tt_assert(sockaddr_in_are_equal(sockaddr,sockaddr_to_check));
 
   tor_addr = smartlist_get(results, 1);
@@ -494,7 +494,7 @@ test_address_ifreq_to_smartlist(void *arg)
   tor_addr_to_sockaddr(tor_addr,0,(struct sockaddr *)sockaddr_to_check,
                        sizeof(struct sockaddr_in));
 
-  tt_int_op(addr_len,==,sizeof(struct sockaddr_in));
+  tt_int_op(addr_len,OP_EQ,sizeof(struct sockaddr_in));
   tt_assert(sockaddr_in_are_equal(sockaddr_eth1,sockaddr_to_check));
 
   done:
@@ -543,7 +543,7 @@ test_address_get_if_addrs_ioctl(void *arg)
   return;
 }
 
-#endif
+#endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #define FAKE_SOCKET_FD (42)
 
@@ -633,7 +633,7 @@ test_address_udp_socket_trick_whitebox(void *arg)
   get_interface_address6_via_udp_socket_hack(LOG_DEBUG,
                                              AF_INET, addr_from_hack);
 
-  tt_int_op(hack_retval,==,0);
+  tt_int_op(hack_retval,OP_EQ,0);
   tt_assert(tor_addr_eq_ipv4h(addr_from_hack, 0x1720f676));
 
   /* Now, lets do an IPv6 case. */
@@ -648,7 +648,7 @@ test_address_udp_socket_trick_whitebox(void *arg)
   get_interface_address6_via_udp_socket_hack(LOG_DEBUG,
                                              AF_INET6, addr_from_hack);
 
-  tt_int_op(hack_retval,==,0);
+  tt_int_op(hack_retval,OP_EQ,0);
 
   tor_addr_to_sockaddr(addr_from_hack,0,(struct sockaddr *)ipv6_to_check,
                        sizeof(struct sockaddr_in6));
@@ -693,7 +693,7 @@ test_address_udp_socket_trick_blackbox(void *arg)
                                                       AF_INET,
                                                       &addr4_to_check);
 
-  tt_int_op(retval,==,retval_reference);
+  tt_int_op(retval,OP_EQ,retval_reference);
   tt_assert( (retval == -1 && retval_reference == -1) ||
              (tor_addr_compare(&addr4,&addr4_to_check,CMP_EXACT) == 0) );
 
@@ -702,11 +702,11 @@ test_address_udp_socket_trick_blackbox(void *arg)
                                                       AF_INET6,
                                                       &addr6_to_check);
 
-  tt_int_op(retval,==,retval_reference);
+  tt_int_op(retval,OP_EQ,retval_reference);
   tt_assert( (retval == -1 && retval_reference == -1) ||
              (tor_addr_compare(&addr6,&addr6_to_check,CMP_EXACT) == 0) );
 
-#else
+#else /* !(0) */
   /* Both of the blackbox test cases fail horribly if:
    *  * The host has no external addreses.
    *  * There are multiple interfaces with either AF_INET or AF_INET6.
@@ -721,7 +721,7 @@ test_address_udp_socket_trick_blackbox(void *arg)
   (void)addr6_to_check;
   (void)addr6;
   (void) retval_reference;
-#endif
+#endif /* 0 */
 
   /* When family is neither AF_INET nor AF_INET6, we want _hack to
    * fail and return -1.
@@ -730,7 +730,7 @@ test_address_udp_socket_trick_blackbox(void *arg)
   retval = get_interface_address6_via_udp_socket_hack(LOG_DEBUG,
                                                       AF_INET+AF_INET6,&addr4);
 
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   done:
   return;
@@ -745,11 +745,11 @@ test_address_get_if_addrs_list_internal(void *arg)
 
   results = get_interface_address_list(LOG_ERR, 1);
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* When the network is down, a system might not have any non-local
    * non-multicast addresseses, not even internal ones.
    * Unit tests shouldn't fail because of this. */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -763,7 +763,7 @@ test_address_get_if_addrs_list_internal(void *arg)
   tt_assert(!smartlist_contains_ipv6_tor_addr(results));
 
  done:
-  free_interface_address_list(results);
+  interface_address_list_free(results);
   return;
 }
 
@@ -776,9 +776,9 @@ test_address_get_if_addrs_list_no_internal(void *arg)
 
   results = get_interface_address_list(LOG_ERR, 0);
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* Work even on systems with only internal IPv4 addresses */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -792,7 +792,7 @@ test_address_get_if_addrs_list_no_internal(void *arg)
   tt_assert(!smartlist_contains_ipv6_tor_addr(results));
 
  done:
-  free_interface_address_list(results);
+  interface_address_list_free(results);
   return;
 }
 
@@ -818,9 +818,9 @@ test_address_get_if_addrs6_list_internal(void *arg)
   }
   teardown_capture_of_logs();
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* Work even on systems without IPv6 interfaces */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -834,7 +834,7 @@ test_address_get_if_addrs6_list_internal(void *arg)
   }
 
  done:
-  free_interface_address6_list(results);
+  interface_address6_list_free(results);
   teardown_capture_of_logs();
   return;
 }
@@ -861,9 +861,9 @@ test_address_get_if_addrs6_list_no_internal(void *arg)
   }
   teardown_capture_of_logs();
 
-  tt_assert(results != NULL);
+  tt_ptr_op(results, OP_NE, NULL);
   /* Work even on systems without IPv6 interfaces */
-  tt_int_op(smartlist_len(results),>=,0);
+  tt_int_op(smartlist_len(results),OP_GE,0);
 
   tt_assert(!smartlist_contains_localhost_tor_addr(results));
   tt_assert(!smartlist_contains_multicast_tor_addr(results));
@@ -878,7 +878,7 @@ test_address_get_if_addrs6_list_no_internal(void *arg)
 
  done:
   teardown_capture_of_logs();
-  free_interface_address6_list(results);
+  interface_address6_list_free(results);
   return;
 }
 
@@ -927,24 +927,24 @@ test_address_get_if_addrs_internal_fail(void *arg)
        mock_get_interface_address6_via_udp_socket_hack_fail);
 
   results1 = get_interface_address6_list(LOG_ERR, AF_INET6, 1);
-  tt_assert(results1 != NULL);
-  tt_int_op(smartlist_len(results1),==,0);
+  tt_ptr_op(results1, OP_NE, NULL);
+  tt_int_op(smartlist_len(results1),OP_EQ,0);
 
   results2 = get_interface_address_list(LOG_ERR, 1);
-  tt_assert(results2 != NULL);
-  tt_int_op(smartlist_len(results2),==,0);
+  tt_ptr_op(results2, OP_NE, NULL);
+  tt_int_op(smartlist_len(results2),OP_EQ,0);
 
   rv = get_interface_address6(LOG_ERR, AF_INET6, &ipv6_addr);
-  tt_assert(rv == -1);
+  tt_int_op(rv, OP_EQ, -1);
 
   rv = get_interface_address(LOG_ERR, &ipv4h_addr);
-  tt_assert(rv == -1);
+  tt_int_op(rv, OP_EQ, -1);
 
  done:
   UNMOCK(get_interface_addresses_raw);
   UNMOCK(get_interface_address6_via_udp_socket_hack);
-  free_interface_address6_list(results1);
-  free_interface_address6_list(results2);
+  interface_address6_list_free(results1);
+  interface_address6_list_free(results2);
   return;
 }
 
@@ -961,18 +961,18 @@ test_address_get_if_addrs_no_internal_fail(void *arg)
        mock_get_interface_address6_via_udp_socket_hack_fail);
 
   results1 = get_interface_address6_list(LOG_ERR, AF_INET6, 0);
-  tt_assert(results1 != NULL);
-  tt_int_op(smartlist_len(results1),==,0);
+  tt_ptr_op(results1, OP_NE, NULL);
+  tt_int_op(smartlist_len(results1),OP_EQ,0);
 
   results2 = get_interface_address_list(LOG_ERR, 0);
-  tt_assert(results2 != NULL);
-  tt_int_op(smartlist_len(results2),==,0);
+  tt_ptr_op(results2, OP_NE, NULL);
+  tt_int_op(smartlist_len(results2),OP_EQ,0);
 
  done:
   UNMOCK(get_interface_addresses_raw);
   UNMOCK(get_interface_address6_via_udp_socket_hack);
-  free_interface_address6_list(results1);
-  free_interface_address6_list(results2);
+  interface_address6_list_free(results1);
+  interface_address6_list_free(results2);
   return;
 }
 
diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c
index df022f539a..f7441a6491 100644
--- a/src/test/test_address_set.c
+++ b/src/test/test_address_set.c
@@ -2,6 +2,7 @@
 /* See LICENSE for licensing information */
 
 #include "or.h"
+#include "crypto_rand.h"
 #include "address_set.h"
 #include "microdesc.h"
 #include "networkstatus.h"
diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c
new file mode 100644
index 0000000000..c44f791e0d
--- /dev/null
+++ b/src/test/test_bridges.c
@@ -0,0 +1,614 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information     */
+
+/**
+ * \file test_bridges.c
+ * \brief Unittests for code in src/or/bridges.c
+ **/
+
+#define TOR_BRIDGES_PRIVATE
+#define PT_PRIVATE /* Only needed for the mock_* items below */
+
+#include <stdbool.h>
+
+#include "or.h"
+#include "address.h"
+#include "bridges.h"
+#include "config.h"
+#include "container.h"
+#include "transports.h"
+#include "util.h"
+
+/* Test suite stuff */
+#include "test.h"
+
+/**
+ * A mocked transport_t, constructed via mock_transport_get_by_name().
+ */
+static transport_t *mock_transport = NULL;
+
+/**
+ * Mock transport_get_by_name() to simply return a transport_t for the
+ * transport name that was input to it.
+ */
+static transport_t *
+mock_transport_get_by_name(const char *name)
+{
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 9999;
+  int socksv = 9;
+  char *args = tor_strdup("foo=bar");
+
+  if (!mock_transport) {
+    tor_addr_parse(addr, "99.99.99.99");
+    mock_transport = transport_new(addr, port, name, socksv, args);
+  }
+
+  tor_free(addr);
+  tor_free(args);
+
+  return mock_transport;
+}
+
+#undef PT_PRIVATE /* defined(PT_PRIVATE) */
+
+/**
+ * Test helper: Add a variety of bridges to our global bridgelist.
+ */
+static void
+helper_add_bridges_to_bridgelist(void *arg)
+{
+  /* Note: the two bridges which do not have specified fingerprints will be
+   * internally stored as both having the same fingerprint of all-zero bytes.
+   */
+
+  (void)arg;
+  char *bridge0 = tor_strdup("6.6.6.6:6666");
+  char *bridge1 = tor_strdup("6.6.6.7:6667 "
+                             "A10C4F666D27364036B562823E5830BC448E046A");
+  char *bridge2 = tor_strdup("obfs4 198.245.60.51:443 "
+                             "752CF7825B3B9EA6A98C83AC41F7099D67007EA5 "
+                  "cert=xpmQtKUqQ/6v5X7ijgYE/f03+l2/EuQ1dexjyUhh16wQlu/"
+                             "cpXUGalmhDIlhuiQPNEKmKw iat-mode=0");
+  char *bridge3 = tor_strdup("banana 5.5.5.5:5555 "
+                             "9D6AE1BD4FDF39721CE908966E79E16F9BFCCF2F");
+  char *bridge4 = tor_strdup("obfs4 1.2.3.4:1234 "
+                             "foo=abcdefghijklmnopqrstuvwxyz");
+  char *bridge5 = tor_strdup("apple 4.4.4.4:4444 "
+                             "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+                             "foo=abcdefghijklmnopqrstuvwxyz");
+
+  mark_bridge_list();
+
+#define ADD_BRIDGE(bridge)                                          \
+  bridge_line_t *bridge_line_ ##bridge = parse_bridge_line(bridge); \
+  if (!bridge_line_ ##bridge) {                                     \
+    printf("Unparseable bridge line: '%s'", #bridge);               \
+  } else {                                                          \
+    bridge_add_from_config(bridge_line_ ##bridge);                  \
+  }                                                                 \
+  tor_free(bridge);
+
+  ADD_BRIDGE(bridge0);
+  ADD_BRIDGE(bridge1);
+  ADD_BRIDGE(bridge2);
+  ADD_BRIDGE(bridge3);
+  ADD_BRIDGE(bridge4);
+  ADD_BRIDGE(bridge5);
+#undef ADD_BRIDGES
+
+  sweep_bridge_list();
+}
+
+/**
+ * Make sure our test helper works too.
+ */
+static void
+test_bridges_helper_func_add_bridges_to_bridgelist(void *arg)
+{
+  helper_add_bridges_to_bridgelist(arg);
+  tt_finished();
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling bridge_list_get() should create a new bridgelist if we
+ * didn't have one before.
+ */
+static void
+test_bridges_bridge_list_get_creates_new_bridgelist(void *arg)
+{
+  const smartlist_t *bridgelist = bridge_list_get();
+
+  (void)arg;
+
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+ done:
+  return;
+}
+
+/**
+ * Calling clear_bridge_list() should remove all bridges from the bridgelist.
+ */
+static void
+test_bridges_clear_bridge_list(void *arg)
+{
+  const smartlist_t *bridgelist;
+  const smartlist_t *bridgelist_after;
+  const bridge_info_t *bridge;
+
+  helper_add_bridges_to_bridgelist(arg);
+  bridgelist = bridge_list_get();
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+  bridge = smartlist_get(bridgelist, 0);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  clear_bridge_list();
+  bridgelist_after = bridge_list_get();
+  tt_ptr_op(bridgelist_after, OP_NE, NULL);
+  tt_int_op(smartlist_len(bridgelist_after), OP_EQ, 0);
+
+ done:
+  return;
+}
+
+/**
+ * Calling bridge_get_addrport() should give me the address and port
+ * of the bridge.  In this case, we sort the smartlist of bridges on
+ * fingerprints and choose the first one.
+ */
+static void
+test_bridges_bridge_get_addrport(void *arg)
+{
+  smartlist_t *bridgelist;
+  const bridge_info_t *bridge;
+  const tor_addr_port_t *addrport;
+
+  helper_add_bridges_to_bridgelist(arg);
+  bridgelist = (smartlist_t*)bridge_list_get();
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+  // This should be the bridge at 6.6.6.6:6666 with fingerprint
+  // 0000000000000000000000000000000000000000
+  bridge = smartlist_get(bridgelist, 0);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  addrport = bridge_get_addr_port(bridge);
+  tt_int_op(addrport->port, OP_EQ, 6666);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_orports_digest() with two
+ * configured bridge orports and an invalid digest should return the
+ * bridge of the first addrport in the list.
+ */
+static void
+test_bridges_get_configured_bridge_by_orports_digest(void *arg)
+{
+  smartlist_t *orports = NULL;
+  const smartlist_t *bridgelist;
+  const bridge_info_t *bridge1;
+  const bridge_info_t *bridge2;
+  const bridge_info_t *ret;
+  tor_addr_port_t *addrport1;
+  tor_addr_port_t *addrport2;
+  const char *digest;
+
+  helper_add_bridges_to_bridgelist(arg);
+  bridgelist = bridge_list_get();
+  tt_ptr_op(bridgelist, OP_NE, NULL);
+
+  // This should be the bridge at 6.6.6.6:6666 with fingerprint
+  // 0000000000000000000000000000000000000000
+  bridge1 = smartlist_get(bridgelist, 0);
+  tt_ptr_op(bridge1, OP_NE, NULL);
+  // This should be the bridge at 6.6.6.7:6667 with fingerprint
+  // A10C4F666D27364036B562823E5830BC448E046A
+  bridge2 = smartlist_get(bridgelist, 1);
+  tt_ptr_op(bridge2, OP_NE, NULL);
+
+  addrport1 = (tor_addr_port_t*)bridge_get_addr_port(bridge1);
+  tt_int_op(addrport1->port, OP_EQ, 6666);
+  addrport2 = (tor_addr_port_t*)bridge_get_addr_port(bridge2);
+  tt_int_op(addrport2->port, OP_EQ, 6667);
+
+  orports = smartlist_new();
+  smartlist_add(orports, addrport1);
+  smartlist_add(orports, addrport2);
+
+  digest = "zzzzzzzzzzzzzzzz";
+
+  ret = get_configured_bridge_by_orports_digest(digest, orports);
+  tt_ptr_op(ret, OP_NE, NULL);
+
+  tt_assert(tor_addr_port_eq(addrport1, bridge_get_addr_port(ret)));
+
+ done:
+  smartlist_free(orports);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_addr_port_digest() with a digest that we do
+ * have and an addr:port pair we don't should return the bridge for that
+ * digest.
+ */
+static void
+test_bridges_get_configured_bridge_by_addr_port_digest_digest_only(void *arg)
+{
+  char digest[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  char ret_addr[16];
+  uint16_t port = 11111;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  // We don't actually have a bridge with this addr:port pair
+  base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  ret = tor_addr_parse(addr, "111.111.111.111");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_addr_port_digest(addr, port, digest);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("4.4.4.4", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_addr_port_digest() with only an
+ * addr:port (i.e. digest set to NULL) should return the bridge for
+ * that digest when there is such a bridge.
+ */
+static void
+test_bridges_get_configured_bridge_by_addr_port_digest_address_only(void *arg)
+{
+  bridge_info_t *bridge;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  char ret_addr[16];
+  uint16_t port = 6666;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "6.6.6.6");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_addr_port_digest(addr, port, NULL);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("6.6.6.6", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that
+ * we do have, and an addr:port pair we don't have, should return NULL.
+ */
+static void
+test_bridges_get_configured_bridge_by_exact_addr_port_digest_donly(void *arg)
+{
+  char digest[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 11111;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  // We don't actually have a bridge with this addr:port pair
+  base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  ret = tor_addr_parse(addr, "111.111.111.111");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  tt_ptr_op(bridge, OP_EQ, NULL);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that
+ * we do have, and an addr:port pair we do have, should return the bridge.
+ */
+static void
+test_bridges_get_configured_bridge_by_exact_addr_port_digest_both(void *arg)
+{
+  char digest[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 4444;
+  char ret_addr[16];
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  ret = tor_addr_parse(addr, "4.4.4.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("4.4.4.4", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_configured_bridge_by_exact_addr_port_digest() with no digest,
+ * and an addr:port pair we do have, should return the bridge.
+ */
+static void
+test_bridges_get_configured_bridge_by_exact_addr_port_digest_aonly(void *arg)
+{
+  bridge_info_t *bridge;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 4444;
+  char ret_addr[16];
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "4.4.4.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, NULL);
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0);
+  tt_str_op("4.4.4.4", OP_EQ, ret_addr);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling find_bridge_by_digest() when we have a bridge with a known
+ * identity digest should return the bridge's information.
+ */
+static void
+test_bridges_find_bridge_by_digest_known(void *arg)
+{
+  char digest1[DIGEST_LEN];
+  bridge_info_t *bridge;
+  const char fingerprint[HEX_DIGEST_LEN] =
+    "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  base16_decode(digest1, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN);
+  bridge = find_bridge_by_digest(digest1);
+
+  tt_ptr_op(bridge, OP_NE, NULL);
+
+  /* We have to call bridge_get_rsa_id_digest() here because the bridge_info_t
+   * struct is opaquely defined in bridges.h. */
+  const uint8_t *digest2 = bridge_get_rsa_id_digest(bridge);
+
+  tt_mem_op((char*)digest2, OP_EQ, digest1, DIGEST_LEN);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling find_bridge_by_digest() when we do NOT have a bridge with that
+ * identity digest should return NULL.
+ */
+static void
+test_bridges_find_bridge_by_digest_unknown(void *arg)
+{
+  const char *fingerprint = "cccccccccccccccccccccccccccccccccccccccc";
+  bridge_info_t *bridge;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  bridge = find_bridge_by_digest(fingerprint);
+
+  tt_ptr_op(bridge, OP_EQ, NULL);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling bridge_resolve_conflicts() with an identical bridge to one we've
+ * already configure should mark the pre-configured bridge for removal.
+ */
+static void
+test_bridges_bridge_resolve_conflicts(void *arg)
+{
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 4444;
+  const char *digest = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+  const char *transport = "apple";
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "4.4.4.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success
+
+  bridge_resolve_conflicts((const tor_addr_t*)addr, port, digest, transport);
+
+  /* The bridge should now be marked for removal, and removed when we sweep the
+   * bridge_list */
+  sweep_bridge_list();
+  ret = addr_is_a_configured_bridge((const tor_addr_t*)addr, port, digest);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling transport_is_needed() with a transport we do need ("obfs4") and a
+ * bogus transport that we don't need should return 1 and 0, respectively.
+ */
+static void
+test_bridges_transport_is_needed(void *arg)
+{
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = transport_is_needed("obfs4");
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = transport_is_needed("apowefjaoewpaief");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+/**
+ * Calling get_transport_by_bridge_addrport() with the address and port of a
+ * configured bridge which uses a pluggable transport when there is no global
+ * transport_list should return -1 and the transport_t should be NULL.
+ */
+static void
+test_bridges_get_transport_by_bridge_addrport_no_ptlist(void *arg)
+{
+  transport_t *transport = NULL;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 1234;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+
+  ret = tor_addr_parse(addr, "1.2.3.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success?
+
+  /* This will fail because the global transport_list has nothing in it, and so
+   * transport_get_by_name() has nothing to return, even the the bridge *did*
+   * say it had an obfs4 transport.
+   */
+  ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port,
+                                         (const transport_t**)&transport);
+  tt_int_op(ret, OP_EQ, -1); // returns -1 on failure
+  tt_ptr_op(transport, OP_EQ, NULL);
+
+ done:
+  tor_free(addr);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+#define PT_PRIVATE
+
+/**
+ * Calling get_transport_by_bridge_addrport() with the address and port of a
+ * configured bridge which uses a pluggable transport should return 0 and set
+ * appropriate transport_t.
+ */
+static void
+test_bridges_get_transport_by_bridge_addrport(void *arg)
+{
+  transport_t *transport = NULL;
+  tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t));
+  uint16_t port = 1234;
+  int ret;
+
+  helper_add_bridges_to_bridgelist(arg);
+  mark_transport_list(); // Also initialise our transport_list
+
+  ret = tor_addr_parse(addr, "1.2.3.4");
+  tt_int_op(ret, OP_EQ, 2); // it returns the address family on success?
+
+  /* After we mock transport_get_by_name() to return a bogus transport_t with
+   * the name it was asked for, the call should succeed.
+   */
+  MOCK(transport_get_by_name, mock_transport_get_by_name);
+  ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port,
+                                         (const transport_t**)&transport);
+  tt_int_op(ret, OP_EQ, 0); // returns 0 on success
+  tt_ptr_op(transport, OP_NE, NULL);
+  tt_str_op(transport->name, OP_EQ, "obfs4");
+
+ done:
+  UNMOCK(transport_get_by_name);
+
+  tor_free(addr);
+  transport_free(transport);
+
+  mark_bridge_list();
+  sweep_bridge_list();
+}
+
+#undef PT_PRIVATE /* defined(PT_PRIVATE) */
+
+#define B_TEST(name, flags) \
+  { #name, test_bridges_ ##name, (flags), NULL, NULL }
+
+struct testcase_t bridges_tests[] = {
+  B_TEST(helper_func_add_bridges_to_bridgelist, 0),
+  B_TEST(bridge_list_get_creates_new_bridgelist, 0),
+  B_TEST(clear_bridge_list, 0),
+  B_TEST(bridge_get_addrport, 0),
+  B_TEST(get_configured_bridge_by_orports_digest, 0),
+  B_TEST(get_configured_bridge_by_addr_port_digest_digest_only, 0),
+  B_TEST(get_configured_bridge_by_addr_port_digest_address_only, 0),
+  B_TEST(get_configured_bridge_by_exact_addr_port_digest_donly, 0),
+  B_TEST(get_configured_bridge_by_exact_addr_port_digest_both, 0),
+  B_TEST(get_configured_bridge_by_exact_addr_port_digest_aonly, 0),
+  B_TEST(find_bridge_by_digest_known, 0),
+  B_TEST(find_bridge_by_digest_unknown, 0),
+  B_TEST(bridge_resolve_conflicts, 0),
+  B_TEST(get_transport_by_bridge_addrport_no_ptlist, 0),
+  B_TEST(get_transport_by_bridge_addrport, 0),
+  B_TEST(transport_is_needed, 0),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 95b4f48f11..b5c8d7cf9e 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -19,14 +19,12 @@ static int crashtype = 0;
 
 #ifdef __GNUC__
 #define NOINLINE __attribute__((noinline))
-#define NORETURN __attribute__((noreturn))
 #endif
 
 int crash(int x) NOINLINE;
 int oh_what(int x) NOINLINE;
 int a_tangled_web(int x) NOINLINE;
 int we_weave(int x) NOINLINE;
-static void abort_handler(int s) NORETURN;
 
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
 DISABLE_GCC_WARNING(null-dereference)
@@ -40,7 +38,7 @@ crash(int x)
                          * don't need to see us dereference NULL. */
 #else
     *(volatile int *)0 = 0;
-#endif
+#endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */
   } else if (crashtype == 1) {
     tor_assert(1 == 0);
   } else if (crashtype == -1) {
@@ -76,13 +74,6 @@ we_weave(int x)
   return a_tangled_web(x) + a_tangled_web(x+1);
 }
 
-static void
-abort_handler(int s)
-{
-  (void)s;
-  exit(0);
-}
-
 int
 main(int argc, char **argv)
 {
@@ -120,8 +111,6 @@ main(int argc, char **argv)
 
   configure_backtrace_handler(NULL);
 
-  signal(SIGABRT, abort_handler);
-
   printf("%d\n", we_weave(2));
 
   clean_up_backtrace_handler();
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3408da3aa9..868f6a8ba4 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -1,12 +1,16 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
+#define PROTO_HTTP_PRIVATE
 #include "or.h"
 #include "buffers.h"
-#include "ext_orport.h"
+#include "buffers_tls.h"
+#include "crypto_rand.h"
+#include "proto_http.h"
+#include "proto_socks.h"
 #include "test.h"
 
 /** Run unit tests for buffers.c */
@@ -38,15 +42,15 @@ test_buffers_basic(void *arg)
   for (j=0;j<256;++j) {
     str[j] = (char)j;
   }
-  write_to_buf(str, 256, buf);
-  write_to_buf(str, 256, buf);
+  buf_add(buf, str, 256);
+  buf_add(buf, str, 256);
   tt_int_op(buf_datalen(buf),OP_EQ, 512);
-  fetch_from_buf(str2, 200, buf);
+  buf_get_bytes(buf, str2, 200);
   tt_mem_op(str,OP_EQ, str2, 200);
   tt_int_op(buf_datalen(buf),OP_EQ, 312);
   memset(str2, 0, sizeof(str2));
 
-  fetch_from_buf(str2, 256, buf);
+  buf_get_bytes(buf, str2, 256);
   tt_mem_op(str+200,OP_EQ, str2, 56);
   tt_mem_op(str,OP_EQ, str2+56, 200);
   tt_int_op(buf_datalen(buf),OP_EQ, 56);
@@ -54,16 +58,16 @@ test_buffers_basic(void *arg)
   /* Okay, now we should be 512 bytes into the 4096-byte buffer.  If we add
    * another 3584 bytes, we hit the end. */
   for (j=0;j<15;++j) {
-    write_to_buf(str, 256, buf);
+    buf_add(buf, str, 256);
   }
-  assert_buf_ok(buf);
+  buf_assert_ok(buf);
   tt_int_op(buf_datalen(buf),OP_EQ, 3896);
-  fetch_from_buf(str2, 56, buf);
+  buf_get_bytes(buf, str2, 56);
   tt_int_op(buf_datalen(buf),OP_EQ, 3840);
   tt_mem_op(str+200,OP_EQ, str2, 56);
   for (j=0;j<15;++j) {
     memset(str2, 0, sizeof(str2));
-    fetch_from_buf(str2, 256, buf);
+    buf_get_bytes(buf, str2, 256);
     tt_mem_op(str,OP_EQ, str2, 256);
   }
   tt_int_op(buf_datalen(buf),OP_EQ, 0);
@@ -73,38 +77,38 @@ test_buffers_basic(void *arg)
   /* Okay, now make sure growing can work. */
   buf = buf_new_with_capacity(16);
   //test_eq(buf_capacity(buf), 16);
-  write_to_buf(str+1, 255, buf);
+  buf_add(buf, str+1, 255);
   //test_eq(buf_capacity(buf), 256);
-  fetch_from_buf(str2, 254, buf);
+  buf_get_bytes(buf, str2, 254);
   tt_mem_op(str+1,OP_EQ, str2, 254);
   //test_eq(buf_capacity(buf), 256);
-  assert_buf_ok(buf);
-  write_to_buf(str, 32, buf);
+  buf_assert_ok(buf);
+  buf_add(buf, str, 32);
   //test_eq(buf_capacity(buf), 256);
-  assert_buf_ok(buf);
-  write_to_buf(str, 256, buf);
-  assert_buf_ok(buf);
+  buf_assert_ok(buf);
+  buf_add(buf, str, 256);
+  buf_assert_ok(buf);
   //test_eq(buf_capacity(buf), 512);
   tt_int_op(buf_datalen(buf),OP_EQ, 33+256);
-  fetch_from_buf(str2, 33, buf);
+  buf_get_bytes(buf, str2, 33);
   tt_int_op(*str2,OP_EQ, str[255]);
 
   tt_mem_op(str2+1,OP_EQ, str, 32);
   //test_eq(buf_capacity(buf), 512);
   tt_int_op(buf_datalen(buf),OP_EQ, 256);
-  fetch_from_buf(str2, 256, buf);
+  buf_get_bytes(buf, str2, 256);
   tt_mem_op(str,OP_EQ, str2, 256);
 
   /* now try shrinking: case 1. */
   buf_free(buf);
   buf = buf_new_with_capacity(33668);
   for (j=0;j<67;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str,255);
   }
   //test_eq(buf_capacity(buf), 33668);
   tt_int_op(buf_datalen(buf),OP_EQ, 17085);
   for (j=0; j < 40; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
 
@@ -112,18 +116,18 @@ test_buffers_basic(void *arg)
   buf_free(buf);
   buf = buf_new_with_capacity(33668);
   for (j=0;j<67;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str, 255);
   }
   for (j=0; j < 20; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   for (j=0;j<80;++j) {
-    write_to_buf(str,255, buf);
+    buf_add(buf, str, 255);
   }
   //test_eq(buf_capacity(buf),33668);
   for (j=0; j < 120; ++j) {
-    fetch_from_buf(str2, 255,buf);
+    buf_get_bytes(buf, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
 
@@ -132,27 +136,27 @@ test_buffers_basic(void *arg)
   buf = buf_new_with_capacity(4096);
   buf2 = buf_new_with_capacity(4096);
   for (j=0;j<100;++j)
-    write_to_buf(str, 255, buf);
+    buf_add(buf, str, 255);
   tt_int_op(buf_datalen(buf),OP_EQ, 25500);
   for (j=0;j<100;++j) {
     r = 10;
-    move_buf_to_buf(buf2, buf, &r);
+    buf_move_to_buf(buf2, buf, &r);
     tt_int_op(r,OP_EQ, 0);
   }
   tt_int_op(buf_datalen(buf),OP_EQ, 24500);
   tt_int_op(buf_datalen(buf2),OP_EQ, 1000);
   for (j=0;j<3;++j) {
-    fetch_from_buf(str2, 255, buf2);
+    buf_get_bytes(buf2, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   r = 8192; /*big move*/
-  move_buf_to_buf(buf2, buf, &r);
+  buf_move_to_buf(buf2, buf, &r);
   tt_int_op(r,OP_EQ, 0);
   r = 30000; /* incomplete move */
-  move_buf_to_buf(buf2, buf, &r);
+  buf_move_to_buf(buf2, buf, &r);
   tt_int_op(r,OP_EQ, 13692);
   for (j=0;j<97;++j) {
-    fetch_from_buf(str2, 255, buf2);
+    buf_get_bytes(buf2, str2, 255);
     tt_mem_op(str2,OP_EQ, str, 255);
   }
   buf_free(buf);
@@ -162,7 +166,7 @@ test_buffers_basic(void *arg)
   buf = buf_new_with_capacity(5);
   cp = "Testing. This is a moderately long Testing string.";
   for (j = 0; cp[j]; j++)
-    write_to_buf(cp+j, 1, buf);
+    buf_add(buf, cp+j, 1);
   tt_int_op(0,OP_EQ, buf_find_string_offset(buf, "Testing", 7));
   tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "esting", 6));
   tt_int_op(1,OP_EQ, buf_find_string_offset(buf, "est", 3));
@@ -180,7 +184,7 @@ test_buffers_basic(void *arg)
   {
     char *mem = tor_malloc_zero(65536);
     buf = buf_new();
-    write_to_buf(mem, 65536, buf);
+    buf_add(buf, mem, 65536);
     tor_free(mem);
 
     tt_int_op(buf_datalen(buf), OP_EQ, 65536);
@@ -215,8 +219,7 @@ test_buffer_pullup(void *arg)
 
   /* There are a bunch of cases for pullup.  One is the trivial case. Let's
      mess around with an empty buffer. */
-  buf_pullup(buf, 16);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 16, &cp, &sz);
   tt_ptr_op(cp, OP_EQ, NULL);
   tt_uint_op(sz, OP_EQ, 0);
 
@@ -227,65 +230,62 @@ test_buffer_pullup(void *arg)
 
   /* Let's add some data. */
   crypto_rand(stuff, 16384);
-  write_to_buf(stuff, 3000, buf);
-  write_to_buf(stuff+3000, 3000, buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_add(buf, stuff, 3000);
+  buf_add(buf, stuff+3000, 3000);
+  buf_pullup(buf, 0, &cp, &sz);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_LE, 4096);
 
   /* Make room for 3000 bytes in the first chunk, so that the pullup-move code
    * can get tested. */
-  tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 3000);
+  tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 3000);
   tt_mem_op(tmp,OP_EQ, stuff, 3000);
-  buf_pullup(buf, 2048);
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 2048, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_GE, 2048);
   tt_mem_op(cp,OP_EQ, stuff+3000, 2048);
   tt_int_op(3000, OP_EQ, buf_datalen(buf));
-  tt_int_op(fetch_from_buf(tmp, 3000, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, tmp, 3000), OP_EQ, 0);
   tt_mem_op(tmp,OP_EQ, stuff+3000, 2048);
 
   buf_free(buf);
 
   /* Now try the large-chunk case. */
   buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */
-  write_to_buf(stuff, 4000, buf);
-  write_to_buf(stuff+4000, 4000, buf);
-  write_to_buf(stuff+8000, 4000, buf);
-  write_to_buf(stuff+12000, 4000, buf);
+  buf_add(buf, stuff, 4000);
+  buf_add(buf, stuff+4000, 4000);
+  buf_add(buf, stuff+8000, 4000);
+  buf_add(buf, stuff+12000, 4000);
   tt_int_op(buf_datalen(buf), OP_EQ, 16000);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 0, &cp, &sz);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_LE, 4096);
 
-  buf_pullup(buf, 12500);
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_pullup(buf, 12500, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_GE, 12500);
   tt_mem_op(cp,OP_EQ, stuff, 12500);
   tt_int_op(buf_datalen(buf), OP_EQ, 16000);
 
-  fetch_from_buf(tmp, 12400, buf);
+  buf_get_bytes(buf, tmp, 12400);
   tt_mem_op(tmp,OP_EQ, stuff, 12400);
   tt_int_op(buf_datalen(buf), OP_EQ, 3600);
-  fetch_from_buf(tmp, 3500, buf);
+  buf_get_bytes(buf, tmp, 3500);
   tt_mem_op(tmp,OP_EQ, stuff+12400, 3500);
-  fetch_from_buf(tmp, 100, buf);
+  buf_get_bytes(buf, tmp, 100);
   tt_mem_op(tmp,OP_EQ, stuff+15900, 10);
 
   buf_free(buf);
 
   /* Make sure that the pull-up-whole-buffer case works */
   buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */
-  write_to_buf(stuff, 4000, buf);
-  write_to_buf(stuff+4000, 4000, buf);
-  fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */
-  buf_pullup(buf, 16000); /* Way too much. */
-  assert_buf_ok(buf);
-  buf_get_first_chunk_data(buf, &cp, &sz);
+  buf_add(buf, stuff, 4000);
+  buf_add(buf, stuff+4000, 4000);
+  buf_get_bytes(buf, tmp, 100); /* dump 100 bytes from first chunk */
+  buf_pullup(buf, 16000, &cp, &sz);
+  buf_assert_ok(buf);
   tt_ptr_op(cp, OP_NE, NULL);
   tt_int_op(sz, OP_EQ, 7900);
   tt_mem_op(cp,OP_EQ, stuff+100, 7900);
@@ -321,23 +321,23 @@ test_buffer_copy(void *arg)
   /* Now try with a short buffer. */
   s = "And now comes an act of enormous enormance!";
   len = strlen(s);
-  write_to_buf(s, len, buf);
+  buf_add(buf, s, len);
   tt_int_op(len, OP_EQ, buf_datalen(buf));
   /* Add junk to buf2 so we can test replacing.*/
-  write_to_buf("BLARG", 5, buf2);
+  buf_add(buf2, "BLARG", 5);
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(len, OP_EQ, buf_datalen(buf2));
-  fetch_from_buf(b, len, buf2);
+  buf_get_bytes(buf2, b, len);
   tt_mem_op(b, OP_EQ, s, len);
   /* Now free buf2 and retry so we can test allocating */
   buf_free(buf2);
   buf2 = NULL;
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(len, OP_EQ, buf_datalen(buf2));
-  fetch_from_buf(b, len, buf2);
+  buf_get_bytes(buf2, b, len);
   tt_mem_op(b, OP_EQ, s, len);
   /* Clear buf for next test */
-  fetch_from_buf(b, len, buf);
+  buf_get_bytes(buf, b, len);
   tt_int_op(buf_datalen(buf),OP_EQ,0);
 
   /* Okay, now let's try a bigger buffer. */
@@ -347,13 +347,13 @@ test_buffer_copy(void *arg)
   len = strlen(s);
   for (i = 0; i < 256; ++i) {
     b[0]=i;
-    write_to_buf(b, 1, buf);
-    write_to_buf(s, len, buf);
+    buf_add(buf, b, 1);
+    buf_add(buf, s, len);
   }
   tt_int_op(0, OP_EQ, buf_set_to_copy(&buf2, buf));
   tt_int_op(buf_datalen(buf2), OP_EQ, buf_datalen(buf));
   for (i = 0; i < 256; ++i) {
-    fetch_from_buf(b, len+1, buf2);
+    buf_get_bytes(buf2, b, len+1);
     tt_int_op((unsigned char)b[0],OP_EQ,i);
     tt_mem_op(b+1, OP_EQ, s, len);
   }
@@ -366,79 +366,6 @@ test_buffer_copy(void *arg)
 }
 
 static void
-test_buffer_ext_or_cmd(void *arg)
-{
-  ext_or_cmd_t *cmd = NULL;
-  buf_t *buf = buf_new();
-  char *tmp = NULL;
-  (void) arg;
-
-  /* Empty -- should give "not there. */
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_EQ, cmd);
-
-  /* Three bytes: shouldn't work. */
-  write_to_buf("\x00\x20\x00", 3, buf);
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_EQ, cmd);
-  tt_int_op(3, OP_EQ, buf_datalen(buf));
-
-  /* 0020 0000: That's a nil command. It should work. */
-  write_to_buf("\x00", 1, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0x20, OP_EQ, cmd->cmd);
-  tt_int_op(0, OP_EQ, cmd->len);
-  tt_int_op(0, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
-  /* Now try a length-6 command with one byte missing. */
-  write_to_buf("\x10\x21\x00\x06""abcde", 9, buf);
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_EQ, cmd);
-  write_to_buf("f", 1, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0x1021, OP_EQ, cmd->cmd);
-  tt_int_op(6, OP_EQ, cmd->len);
-  tt_mem_op("abcdef", OP_EQ, cmd->body, 6);
-  tt_int_op(0, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
-  /* Now try a length-10 command with 4 extra bytes. */
-  write_to_buf("\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0xffff, OP_EQ, cmd->cmd);
-  tt_int_op(10, OP_EQ, cmd->len);
-  tt_mem_op("loremipsum", OP_EQ, cmd->body, 10);
-  tt_int_op(4, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
-  /* Finally, let's try a maximum-length command. We already have the header
-   * waiting. */
-  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tmp = tor_malloc_zero(65535);
-  write_to_buf(tmp, 65535, buf);
-  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
-  tt_ptr_op(NULL, OP_NE, cmd);
-  tt_int_op(0x1000, OP_EQ, cmd->cmd);
-  tt_int_op(0xffff, OP_EQ, cmd->len);
-  tt_mem_op(tmp, OP_EQ, cmd->body, 65535);
-  tt_int_op(0, OP_EQ, buf_datalen(buf));
-  ext_or_cmd_free(cmd);
-  cmd = NULL;
-
- done:
-  ext_or_cmd_free(cmd);
-  buf_free(buf);
-  tor_free(tmp);
-}
-
-static void
 test_buffer_allocation_tracking(void *arg)
 {
   char *junk = tor_malloc(16384);
@@ -458,36 +385,36 @@ test_buffer_allocation_tracking(void *arg)
   tt_int_op(buf_allocation(buf1), OP_EQ, 0);
   tt_int_op(buf_get_total_allocation(), OP_EQ, 0);
 
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
-  write_to_buf(junk, 4000, buf1);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
+  buf_add(buf1, junk, 4000);
   tt_int_op(buf_allocation(buf1), OP_EQ, 16384);
-  fetch_from_buf(junk, 100, buf1);
+  buf_get_bytes(buf1, junk, 100);
   tt_int_op(buf_allocation(buf1), OP_EQ, 16384); /* still 4 4k chunks */
 
   tt_int_op(buf_get_total_allocation(), OP_EQ, 16384);
 
-  fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */
+  buf_get_bytes(buf1, junk, 4096); /* drop a 1k chunk... */
   tt_int_op(buf_allocation(buf1), OP_EQ, 3*4096); /* now 3 4k chunks */
 
   tt_int_op(buf_get_total_allocation(), OP_EQ, 12288); /* that chunk was really
                                                        freed. */
 
-  write_to_buf(junk, 4000, buf2);
+  buf_add(buf2, junk, 4000);
   tt_int_op(buf_allocation(buf2), OP_EQ, 4096); /* another 4k chunk. */
   /*
    * We bounce back up to 16384 by allocating a new chunk.
    */
   tt_int_op(buf_get_total_allocation(), OP_EQ, 16384);
-  write_to_buf(junk, 4000, buf2);
+  buf_add(buf2, junk, 4000);
   tt_int_op(buf_allocation(buf2), OP_EQ, 8192); /* another 4k chunk. */
   tt_int_op(buf_get_total_allocation(),
             OP_EQ, 5*4096); /* that chunk was new. */
 
   /* Make a really huge buffer */
   for (i = 0; i < 1000; ++i) {
-    write_to_buf(junk, 4000, buf2);
+    buf_add(buf2, junk, 4000);
   }
   tt_int_op(buf_allocation(buf2), OP_GE, 4008000);
   tt_int_op(buf_get_total_allocation(), OP_GE, 4008000);
@@ -524,52 +451,54 @@ test_buffer_time_tracking(void *arg)
   tt_assert(buf);
 
   monotime_coarse_set_mock_time_nsec(START_NSEC);
-  const uint32_t START_MSEC = (uint32_t)monotime_coarse_absolute_msec();
+  const uint32_t START_TS = monotime_coarse_get_stamp();
 
   /* Empty buffer means the timestamp is 0. */
-  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC));
-  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000));
+  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS));
+  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+1000));
 
-  write_to_buf("ABCDEFG", 7, buf);
-  tt_int_op(1000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000));
+  buf_add(buf, "ABCDEFG", 7);
+  tt_int_op(1000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+1000));
 
   buf2 = buf_copy(buf);
   tt_assert(buf2);
   tt_int_op(1234, OP_EQ,
-            buf_get_oldest_chunk_timestamp(buf2, START_MSEC+1234));
+            buf_get_oldest_chunk_timestamp(buf2, START_TS+1234));
 
   /* Now add more bytes; enough to overflow the first chunk. */
   monotime_coarse_set_mock_time_nsec(START_NSEC + 123 * (uint64_t)1000000);
+  const uint32_t TS2 = monotime_coarse_get_stamp();
   for (i = 0; i < 600; ++i)
-    write_to_buf("ABCDEFG", 7, buf);
+    buf_add(buf, "ABCDEFG", 7);
   tt_int_op(4207, OP_EQ, buf_datalen(buf));
 
   /* The oldest bytes are still in the front. */
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000));
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+2000));
 
   /* Once those bytes are dropped, the chunk is still on the first
    * timestamp. */
-  fetch_from_buf(tmp, 100, buf);
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000));
+  buf_get_bytes(buf, tmp, 100);
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_TS+2000));
 
   /* But once we discard the whole first chunk, we get the data in the second
    * chunk. */
-  fetch_from_buf(tmp, 4000, buf);
+  buf_get_bytes(buf, tmp, 4000);
   tt_int_op(107, OP_EQ, buf_datalen(buf));
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123));
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS2+2000));
 
   /* This time we'll be grabbing a chunk from the freelist, and making sure
      its time gets updated */
   monotime_coarse_set_mock_time_nsec(START_NSEC + 5617 * (uint64_t)1000000);
+  const uint32_t TS3 = monotime_coarse_get_stamp();
   for (i = 0; i < 600; ++i)
-    write_to_buf("ABCDEFG", 7, buf);
+    buf_add(buf, "ABCDEFG", 7);
   tt_int_op(4307, OP_EQ, buf_datalen(buf));
 
-  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123));
-  fetch_from_buf(tmp, 4000, buf);
-  fetch_from_buf(tmp, 306, buf);
-  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617));
-  tt_int_op(383, OP_EQ, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000));
+  tt_int_op(2000, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS2+2000));
+  buf_get_bytes(buf, tmp, 4000);
+  buf_get_bytes(buf, tmp, 306);
+  tt_int_op(0, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS3));
+  tt_int_op(383, OP_EQ, buf_get_oldest_chunk_timestamp(buf, TS3+383));
 
  done:
   buf_free(buf);
@@ -578,120 +507,150 @@ test_buffer_time_tracking(void *arg)
 }
 
 static void
-test_buffers_zlib_impl(int finalize_with_nil)
+test_buffers_compress_fin_at_chunk_end_impl(compress_method_t method,
+                                            compression_level_t level)
 {
   char *msg = NULL;
   char *contents = NULL;
   char *expanded = NULL;
   buf_t *buf = NULL;
-  tor_zlib_state_t *zlib_state = NULL;
+  tor_compress_state_t *compress_state = NULL;
   size_t out_len, in_len;
-  int done;
+  size_t sz, headerjunk;
 
   buf = buf_new_with_capacity(128); /* will round up */
-  zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
+  sz = buf_get_default_chunk_size(buf);
+  msg = tor_malloc_zero(sz);
 
-  msg = tor_malloc(512);
-  crypto_rand(msg, 512);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+128, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+256, 256, 0), OP_EQ, 0);
-  done = !finalize_with_nil;
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, "all done", 9, done), OP_EQ, 0);
-  if (finalize_with_nil) {
-    tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
-  }
+  buf_add(buf, msg, 1);
+  tt_assert(buf->head);
+
+  /* Fill up the chunk so the compression stuff won't fit in one chunk. */
+  tt_uint_op(buf->head->memlen, OP_LT, sz);
+  headerjunk = buf->head->memlen - 7;
+  buf_add(buf, msg, headerjunk-1);
+  tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
+  tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
+  /* Write an empty string, with finalization on. */
+  compress_state = tor_compress_new(1, method, level);
+  tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
-  tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0);
 
-  tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len,
-                                       contents, in_len,
-                                       ZLIB_METHOD, 1,
-                                       LOG_WARN));
+  if (method == NO_METHOD) {
+    tt_uint_op(in_len, OP_EQ, headerjunk);
+  } else {
+    tt_uint_op(in_len, OP_GT, headerjunk);
+  }
 
-  tt_int_op(out_len, OP_GE, 128);
-  tt_mem_op(msg, OP_EQ, expanded, 128);
-  tt_int_op(out_len, OP_GE, 512);
-  tt_mem_op(msg, OP_EQ, expanded, 512);
-  tt_int_op(out_len, OP_EQ, 512+9);
-  tt_mem_op("all done", OP_EQ, expanded+512, 9);
+  tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
+                                     contents + headerjunk,
+                                     in_len - headerjunk,
+                                     method, 1,
+                                     LOG_WARN));
+
+  tt_int_op(out_len, OP_EQ, 0);
+  tt_assert(expanded);
 
  done:
   buf_free(buf);
-  tor_zlib_free(zlib_state);
+  tor_compress_free(compress_state);
   tor_free(contents);
   tor_free(expanded);
   tor_free(msg);
 }
 
 static void
-test_buffers_zlib(void *arg)
-{
-  (void) arg;
-  test_buffers_zlib_impl(0);
-}
-static void
-test_buffers_zlib_fin_with_nil(void *arg)
-{
-  (void) arg;
-  test_buffers_zlib_impl(1);
-}
-
-static void
-test_buffers_zlib_fin_at_chunk_end(void *arg)
+test_buffers_compress_impl(compress_method_t method,
+                           compression_level_t level,
+                           int finalize_with_nil)
 {
   char *msg = NULL;
   char *contents = NULL;
   char *expanded = NULL;
   buf_t *buf = NULL;
-  tor_zlib_state_t *zlib_state = NULL;
+  tor_compress_state_t *compress_state = NULL;
   size_t out_len, in_len;
-  size_t sz, headerjunk;
-  (void) arg;
+  int done;
 
   buf = buf_new_with_capacity(128); /* will round up */
-  sz = buf_get_default_chunk_size(buf);
-  msg = tor_malloc_zero(sz);
-
-  write_to_buf(msg, 1, buf);
-  tt_assert(buf->head);
+  compress_state = tor_compress_new(1, method, level);
 
-  /* Fill up the chunk so the zlib stuff won't fit in one chunk. */
-  tt_uint_op(buf->head->memlen, OP_LT, sz);
-  headerjunk = buf->head->memlen - 7;
-  write_to_buf(msg, headerjunk-1, buf);
-  tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
-  tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
-  /* Write an empty string, with finalization on. */
-  zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
+  msg = tor_malloc(512);
+  crypto_rand(msg, 512);
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  msg, 128, 0), OP_EQ, 0);
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  msg+128, 128, 0), OP_EQ, 0);
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  msg+256, 256, 0), OP_EQ, 0);
+  done = !finalize_with_nil;
+  tt_int_op(buf_add_compress(buf, compress_state,
+                                  "all done", 9, done), OP_EQ, 0);
+  if (finalize_with_nil) {
+    tt_int_op(buf_add_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
+  }
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
-  tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
+  tt_int_op(buf_get_bytes(buf, contents, in_len), OP_EQ, 0);
 
-  tt_uint_op(in_len, OP_GT, headerjunk);
+  tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
+                                     contents, in_len,
+                                     method, 1,
+                                     LOG_WARN));
 
-  tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len,
-                                  contents + headerjunk, in_len - headerjunk,
-                                  ZLIB_METHOD, 1,
-                                  LOG_WARN));
-
-  tt_int_op(out_len, OP_EQ, 0);
-  tt_assert(expanded);
+  tt_int_op(out_len, OP_GE, 128);
+  tt_mem_op(msg, OP_EQ, expanded, 128);
+  tt_int_op(out_len, OP_GE, 512);
+  tt_mem_op(msg, OP_EQ, expanded, 512);
+  tt_int_op(out_len, OP_EQ, 512+9);
+  tt_mem_op("all done", OP_EQ, expanded+512, 9);
 
  done:
   buf_free(buf);
-  tor_zlib_free(zlib_state);
+  tor_compress_free(compress_state);
   tor_free(contents);
   tor_free(expanded);
   tor_free(msg);
 }
 
+static void
+test_buffers_compress(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  compression_level_t levels[] = {
+    BEST_COMPRESSION,
+    HIGH_COMPRESSION,
+    MEDIUM_COMPRESSION,
+    LOW_COMPRESSION
+  };
+
+  for (unsigned l = 0; l < ARRAY_LENGTH(levels); ++l) {
+    compression_level_t level = levels[l];
+
+    test_buffers_compress_impl(method, level, 0);
+    test_buffers_compress_impl(method, level, 1);
+    test_buffers_compress_fin_at_chunk_end_impl(method, level);
+  }
+
+ done:
+  ;
+}
+
 static const uint8_t *tls_read_ptr;
 static int n_remaining;
 static int next_reply_val[16];
@@ -732,11 +691,11 @@ test_buffers_tls_read_mocked(void *arg)
   buf = buf_new();
 
   next_reply_val[0] = 1024;
-  tt_int_op(128, ==, read_to_buf_tls(NULL, 128, buf));
+  tt_int_op(128, OP_EQ, buf_read_from_tls(buf, NULL, 128));
 
   next_reply_val[0] = 5000;
   next_reply_val[1] = 5000;
-  tt_int_op(6000, ==, read_to_buf_tls(NULL, 6000, buf));
+  tt_int_op(6000, OP_EQ, buf_read_from_tls(buf, NULL, 6000));
 
  done:
   UNMOCK(tor_tls_read);
@@ -750,36 +709,114 @@ test_buffers_chunk_size(void *arg)
   (void)arg;
   const int min = 256;
   const int max = 65536;
-  tt_uint_op(preferred_chunk_size(3), OP_EQ, min);
-  tt_uint_op(preferred_chunk_size(25), OP_EQ, min);
-  tt_uint_op(preferred_chunk_size(0), OP_EQ, min);
-  tt_uint_op(preferred_chunk_size(256), OP_EQ, 512);
-  tt_uint_op(preferred_chunk_size(65400), OP_EQ, max);
+  tt_uint_op(buf_preferred_chunk_size(3), OP_EQ, min);
+  tt_uint_op(buf_preferred_chunk_size(25), OP_EQ, min);
+  tt_uint_op(buf_preferred_chunk_size(0), OP_EQ, min);
+  tt_uint_op(buf_preferred_chunk_size(256), OP_EQ, 512);
+  tt_uint_op(buf_preferred_chunk_size(65400), OP_EQ, max);
   /* Here, we're implicitly saying that the chunk header overhead is
    * between 1 and 100 bytes. 24..48 would probably be more accurate. */
-  tt_uint_op(preferred_chunk_size(65536), OP_GT, 65536);
-  tt_uint_op(preferred_chunk_size(65536), OP_LT, 65536+100);
-  tt_uint_op(preferred_chunk_size(165536), OP_GT, 165536);
-  tt_uint_op(preferred_chunk_size(165536), OP_LT, 165536+100);
+  tt_uint_op(buf_preferred_chunk_size(65536), OP_GT, 65536);
+  tt_uint_op(buf_preferred_chunk_size(65536), OP_LT, 65536+100);
+  tt_uint_op(buf_preferred_chunk_size(165536), OP_GT, 165536);
+  tt_uint_op(buf_preferred_chunk_size(165536), OP_LT, 165536+100);
+ done:
+  ;
+}
+
+static void
+test_buffers_find_contentlen(void *arg)
+{
+  static const struct {
+    const char *headers;
+    int r;
+    int contentlen;
+  } results[] = {
+    { "Blah blah\r\nContent-Length: 1\r\n\r\n", 1, 1 },
+    { "Blah blah\r\n\r\n", 0, 0 }, /* no content-len */
+    { "Blah blah Content-Length: 1\r\n", 0, 0 }, /* no content-len. */
+    { "Blah blah\r\nContent-Length: 100000\r\n", 1, 100000},
+    { "Blah blah\r\nContent-Length: 1000000000000000000000000\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 0\r\n", 1, 0},
+    { "Blah blah\r\nContent-Length: -1\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1x\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1 x\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1 \r\n", 1, 1},
+    { "Blah blah\r\nContent-Length:  \r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: ", -1, 0},
+    { "Blah blah\r\nContent-Length: 5050", -1, 0},
+    { NULL, 0, 0 }
+  };
+  int i;
+
+  (void)arg;
+
+  for (i = 0; results[i].headers; ++i) {
+    int r;
+    size_t sz;
+    size_t headerlen = strlen(results[i].headers);
+    char * tmp = tor_memdup(results[i].headers, headerlen);/* ensure no eos */
+    sz = 999; /* to ensure it gets set */
+    r = buf_http_find_content_length(tmp, headerlen, &sz);
+    tor_free(tmp);
+    log_debug(LD_DIR, "%d: %s", i, escaped(results[i].headers));
+    tt_int_op(r, OP_EQ, results[i].r);
+    tt_int_op(sz, OP_EQ, results[i].contentlen);
+  }
  done:
   ;
 }
 
+static void
+test_buffer_peek_startswith(void *arg)
+{
+  (void)arg;
+  buf_t *buf;
+  buf = buf_new();
+  tt_ptr_op(buf, OP_NE, NULL);
+
+  tt_assert(buf_peek_startswith(buf, ""));
+  tt_assert(! buf_peek_startswith(buf, "X"));
+
+  buf_add(buf, "Tor", 3);
+
+  tt_assert(buf_peek_startswith(buf, ""));
+  tt_assert(buf_peek_startswith(buf, "T"));
+  tt_assert(buf_peek_startswith(buf, "To"));
+  tt_assert(buf_peek_startswith(buf, "Tor"));
+  tt_assert(! buf_peek_startswith(buf, "Top"));
+  tt_assert(! buf_peek_startswith(buf, "For"));
+  tt_assert(! buf_peek_startswith(buf, "Tork"));
+  tt_assert(! buf_peek_startswith(buf, "Torpor"));
+
+ done:
+  buf_free(buf);
+}
+
 struct testcase_t buffer_tests[] = {
   { "basic", test_buffers_basic, TT_FORK, NULL, NULL },
   { "copy", test_buffer_copy, TT_FORK, NULL, NULL },
   { "pullup", test_buffer_pullup, TT_FORK, NULL, NULL },
-  { "ext_or_cmd", test_buffer_ext_or_cmd, TT_FORK, NULL, NULL },
+  { "startswith", test_buffer_peek_startswith, 0, NULL, NULL },
   { "allocation_tracking", test_buffer_allocation_tracking, TT_FORK,
     NULL, NULL },
   { "time_tracking", test_buffer_time_tracking, TT_FORK, NULL, NULL },
-  { "zlib", test_buffers_zlib, TT_FORK, NULL, NULL },
-  { "zlib_fin_with_nil", test_buffers_zlib_fin_with_nil, TT_FORK, NULL, NULL },
-  { "zlib_fin_at_chunk_end", test_buffers_zlib_fin_at_chunk_end, TT_FORK,
-    NULL, NULL},
   { "tls_read_mocked", test_buffers_tls_read_mocked, 0,
     NULL, NULL },
   { "chunk_size", test_buffers_chunk_size, 0, NULL, NULL },
+  { "find_contentlen", test_buffers_find_contentlen, 0, NULL, NULL },
+
+  { "compress/zlib", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"deflate" },
+  { "compress/gzip", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"gzip" },
+  { "compress/zstd", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"x-zstd" },
+  { "compress/lzma", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"x-tor-lzma" },
+  { "compress/none", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"identity" },
+
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c
new file mode 100644
index 0000000000..463a36b24e
--- /dev/null
+++ b/src/test/test_bwmgt.c
@@ -0,0 +1,233 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_bwmgt.c
+ * \brief tests for bandwidth management / token bucket functions
+ */
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "or.h"
+#include "test.h"
+
+#include "token_bucket.h"
+
+// an imaginary time, in timestamp units. Chosen so it will roll over.
+static const uint32_t START_TS = UINT32_MAX-10;
+static const int32_t KB = 1024;
+static const uint32_t GB = (U64_LITERAL(1) << 30);
+
+static void
+test_bwmgt_token_buf_init(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+  // Burst is correct
+  tt_uint_op(b.cfg.burst, OP_EQ, 64*KB);
+  // Rate is correct, within 1 percent.
+  {
+    uint32_t ticks_per_sec =
+      (uint32_t) monotime_msec_to_approx_coarse_stamp_units(1000);
+    uint32_t rate_per_sec = (b.cfg.rate * ticks_per_sec / TICKS_PER_STEP);
+
+    tt_uint_op(rate_per_sec, OP_GT, 16*KB-160);
+    tt_uint_op(rate_per_sec, OP_LT, 16*KB+160);
+  }
+  // Bucket starts out full:
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS);
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 64*KB);
+
+ done:
+  ;
+}
+
+static void
+test_bwmgt_token_buf_adjust(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+
+  uint32_t rate_orig = b.cfg.rate;
+  // Increasing burst
+  token_bucket_rw_adjust(&b, 16*KB, 128*KB);
+  tt_uint_op(b.cfg.rate, OP_EQ, rate_orig);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 128*KB);
+
+  // Decreasing burst but staying above bucket
+  token_bucket_rw_adjust(&b, 16*KB, 96*KB);
+  tt_uint_op(b.cfg.rate, OP_EQ, rate_orig);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 96*KB);
+
+  // Decreasing burst below bucket,
+  token_bucket_rw_adjust(&b, 16*KB, 48*KB);
+  tt_uint_op(b.cfg.rate, OP_EQ, rate_orig);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 48*KB);
+
+  // Changing rate.
+  token_bucket_rw_adjust(&b, 32*KB, 48*KB);
+  tt_uint_op(b.cfg.rate, OP_GE, rate_orig*2 - 10);
+  tt_uint_op(b.cfg.rate, OP_LE, rate_orig*2 + 10);
+  tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB);
+  tt_uint_op(b.cfg.burst, OP_EQ, 48*KB);
+
+ done:
+  ;
+}
+
+static void
+test_bwmgt_token_buf_dec(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+
+  // full-to-not-full.
+  tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, KB));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 63*KB);
+
+  // Full to almost-not-full
+  tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 63*KB - 1));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 1);
+
+  // almost-not-full to empty.
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 1));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 0);
+
+  // reset bucket, try full-to-empty
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 0);
+
+  // reset bucket, try underflow.
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB + 1));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, -1);
+
+  // A second underflow does not make the bucket empty.
+  tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 1000));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, -1001);
+
+ done:
+  ;
+}
+
+static void
+test_bwmgt_token_buf_refill(void *arg)
+{
+  (void)arg;
+  token_bucket_rw_t b;
+  const uint32_t BW_SEC =
+    (uint32_t)monotime_msec_to_approx_coarse_stamp_units(1000);
+  token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS);
+
+  /* Make the buffer much emptier, then let one second elapse. */
+  token_bucket_rw_dec_read(&b, 48*KB);
+  tt_int_op(b.read_bucket.bucket, OP_EQ, 16*KB);
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB - 300);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB + 300);
+
+  /* Another half second. */
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 40*KB - 400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 40*KB + 400);
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + BW_SEC*3/2);
+
+  /* No time: nothing happens. */
+  {
+    const uint32_t bucket_orig = b.read_bucket.bucket;
+    tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2));
+    tt_int_op(b.read_bucket.bucket, OP_EQ, bucket_orig);
+  }
+
+  /* Another 30 seconds: fill the bucket. */
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*30));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst);
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ,
+             START_TS + BW_SEC*3/2 + BW_SEC*30);
+
+  /* Another 30 seconds: nothing happens. */
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*60));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst);
+  tt_uint_op(b.last_refilled_at_timestamp, OP_EQ,
+             START_TS + BW_SEC*3/2 + BW_SEC*60);
+
+  /* Empty the bucket, let two seconds pass, and make sure that a refill is
+   * noticed. */
+  tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.cfg.burst));
+  tt_int_op(0, OP_EQ, b.read_bucket.bucket);
+  tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*61));
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b,
+                                     START_TS + BW_SEC*3/2 + BW_SEC*62));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB-400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB+400);
+
+  /* Underflow the bucket, make sure we detect when it has tokens again. */
+  tt_int_op(1, OP_EQ,
+            token_bucket_rw_dec_read(&b, b.read_bucket.bucket+16*KB));
+  tt_int_op(-16*KB, OP_EQ, b.read_bucket.bucket);
+  // half a second passes...
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64));
+  tt_int_op(b.read_bucket.bucket, OP_GT, -8*KB-300);
+  tt_int_op(b.read_bucket.bucket, OP_LT, -8*KB+300);
+  // a second passes
+  tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*65));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400);
+
+  // We step a second backwards, and nothing happens.
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64));
+  tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400);
+  tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400);
+
+  // A ridiculous amount of time passes.
+  tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, INT32_MAX));
+  tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst);
+
+ done:
+  ;
+}
+
+/* Test some helper functions we use within the token bucket interface. */
+static void
+test_bwmgt_token_buf_helpers(void *arg)
+{
+  uint32_t ret;
+
+  (void) arg;
+
+  /* The returned value will be OS specific but in any case, it should be
+   * greater than 1 since we are passing 1GB/sec rate. */
+  ret = rate_per_sec_to_rate_per_step(1 * GB);
+  tt_u64_op(ret, OP_GT, 1);
+
+  /* We default to 1 in case rate is 0. */
+  ret = rate_per_sec_to_rate_per_step(0);
+  tt_u64_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+#define BWMGT(name)                                          \
+  { #name, test_bwmgt_ ## name , 0, NULL, NULL }
+
+struct testcase_t bwmgt_tests[] = {
+  BWMGT(token_buf_init),
+  BWMGT(token_buf_adjust),
+  BWMGT(token_buf_dec),
+  BWMGT(token_buf_refill),
+  BWMGT(token_buf_helpers),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c
index f429f4291d..54d9716780 100644
--- a/src/test/test_cell_formats.c
+++ b/src/test/test_cell_formats.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,6 +11,8 @@
 #include "channel.h"
 #include "connection_edge.h"
 #include "connection_or.h"
+#include "config.h"
+#include "crypto_rand.h"
 #include "onion.h"
 #include "onion_tap.h"
 #include "onion_fast.h"
@@ -476,7 +478,7 @@ test_cfmt_create_cells(void *arg)
   cell.command = CELL_CREATED;
   tt_int_op(-1, OP_EQ, create_cell_parse(&cc, &cell));
 
-  /* You can't acutally make an unparseable CREATE or CREATE_FAST cell. */
+  /* You can't actually make an unparseable CREATE or CREATE_FAST cell. */
 
   /* Try some CREATE2 cells.  First with a bad type. */
   cell.command = CELL_CREATE2;
@@ -698,6 +700,7 @@ test_cfmt_extend_cells(void *arg)
   tt_int_op(61681, OP_EQ, ec.orport_ipv4.port);
   tt_str_op("2002::f0:c51e", OP_EQ, fmt_addr(&ec.orport_ipv6.addr));
   tt_int_op(4370, OP_EQ, ec.orport_ipv6.port);
+  tt_assert(ed25519_public_key_is_zero(&ec.ed_pubkey));
   tt_mem_op(ec.node_id,OP_EQ, "anthropomorphization", 20);
   tt_int_op(cc->cell_type, OP_EQ, CELL_CREATE2);
   tt_int_op(cc->handshake_type, OP_EQ, 0x105);
@@ -717,6 +720,37 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op(p2+1+8+22+4,OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
 
+  /* Now let's add an ed25519 key to that extend2 cell. */
+  memcpy(ec.ed_pubkey.pubkey,
+         "brownshoesdontmakeit/brownshoesd", 32);
+
+  /* As before, since we aren't extending by ed25519. */
+  get_options_mutable()->ExtendByEd25519ID = 0;
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  test_memeq_hex(p2,
+                 "02000612F40001F0F1"
+                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 "01050063");
+
+  /* Now try with the ed25519 ID. */
+  get_options_mutable()->ExtendByEd25519ID = 1;
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_len, OP_EQ, 89+99-34-20 + 34);
+  test_memeq_hex(p2,
+                 "03000612F40001F0F1"
+                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 // ed digest follows:
+                 "0320" "62726f776e73686f6573646f6e746d616b656"
+                        "9742f62726f776e73686f657364"
+                 "01050063");
+  /* Can we parse that? Did the key come through right? */
+  memset(&ec, 0, sizeof(ec));
+  tt_int_op(0, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
+                                        p2, p2_len));
+  tt_mem_op("brownshoesdontmakeit/brownshoesd", OP_EQ,
+            ec.ed_pubkey.pubkey, 32);
+
   /* == Now try parsing some junk */
 
   /* Try a too-long handshake */
@@ -1257,7 +1291,7 @@ struct testcase_t cell_format_tests[] = {
   TEST(connected_cells, 0),
   TEST(create_cells, 0),
   TEST(created_cells, 0),
-  TEST(extend_cells, 0),
+  TEST(extend_cells, TT_FORK),
   TEST(extended_cells, 0),
   TEST(resolved_cells, 0),
   TEST(is_destroy, 0),
diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c
index 93ac9854d8..df987f82ce 100644
--- a/src/test/test_cell_queue.c
+++ b/src/test/test_cell_queue.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
@@ -130,8 +130,8 @@ test_circuit_n_cells(void *arg)
   tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(origin_c)), OP_EQ, 2);
 
  done:
-  circuit_free(TO_CIRCUIT(or_c));
-  circuit_free(TO_CIRCUIT(origin_c));
+  circuit_free_(TO_CIRCUIT(or_c));
+  circuit_free_(TO_CIRCUIT(origin_c));
 }
 
 struct testcase_t cell_queue_tests[] = {
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index a9e0634d9e..76124a6e75 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
@@ -6,17 +6,22 @@
 #include "or.h"
 #include "channel.h"
 /* For channel_note_destroy_not_pending */
+#define CIRCUITLIST_PRIVATE
 #include "circuitlist.h"
 #include "circuitmux.h"
+#include "circuitmux_ewma.h"
 /* For var_cell_free */
 #include "connection_or.h"
+#include "crypto_rand.h"
 /* For packed_cell stuff */
 #define RELAY_PRIVATE
 #include "relay.h"
 /* For init/free stuff */
 #include "scheduler.h"
+#include "networkstatus.h"
 
 /* Test suite stuff */
+#include "log_test_helpers.h"
 #include "test.h"
 #include "fakechans.h"
 
@@ -26,67 +31,23 @@ static cell_t * test_chan_last_seen_fixed_cell_ptr = NULL;
 static int test_chan_var_cells_recved = 0;
 static var_cell_t * test_chan_last_seen_var_cell_ptr = NULL;
 static int test_cells_written = 0;
-static int test_destroy_not_pending_calls = 0;
 static int test_doesnt_want_writes_count = 0;
 static int test_dumpstats_calls = 0;
 static int test_has_waiting_cells_count = 0;
-static double test_overhead_estimate = 1.0;
 static int test_releases_count = 0;
-static circuitmux_t *test_target_cmux = NULL;
-static unsigned int test_cmux_cells = 0;
 static channel_t *dump_statistics_mock_target = NULL;
 static int dump_statistics_mock_matches = 0;
-
-static void chan_test_channel_dump_statistics_mock(
-    channel_t *chan, int severity);
-static int chan_test_channel_flush_from_first_active_circuit_mock(
-    channel_t *chan, int max);
-static unsigned int chan_test_circuitmux_num_cells_mock(circuitmux_t *cmux);
-static void channel_note_destroy_not_pending_mock(channel_t *ch,
-                                                  circid_t circid);
-static void chan_test_cell_handler(channel_t *ch,
-                                   cell_t *cell);
-static const char * chan_test_describe_transport(channel_t *ch);
-static void chan_test_dumpstats(channel_t *ch, int severity);
-static void chan_test_var_cell_handler(channel_t *ch,
-                                       var_cell_t *var_cell);
-static void chan_test_close(channel_t *ch);
-static void chan_test_error(channel_t *ch);
-static void chan_test_finish_close(channel_t *ch);
-static const char * chan_test_get_remote_descr(channel_t *ch, int flags);
-static int chan_test_is_canonical(channel_t *ch, int req);
-static size_t chan_test_num_bytes_queued(channel_t *ch);
-static int chan_test_num_cells_writeable(channel_t *ch);
-static int chan_test_write_cell(channel_t *ch, cell_t *cell);
-static int chan_test_write_packed_cell(channel_t *ch,
-                                       packed_cell_t *packed_cell);
-static int chan_test_write_var_cell(channel_t *ch, var_cell_t *var_cell);
-static void scheduler_channel_doesnt_want_writes_mock(channel_t *ch);
-
-static void test_channel_dumpstats(void *arg);
-static void test_channel_flush(void *arg);
-static void test_channel_flushmux(void *arg);
-static void test_channel_incoming(void *arg);
-static void test_channel_lifecycle(void *arg);
-static void test_channel_multi(void *arg);
-static void test_channel_queue_incoming(void *arg);
-static void test_channel_queue_size(void *arg);
-static void test_channel_write(void *arg);
-
-static void
-channel_note_destroy_not_pending_mock(channel_t *ch,
-                                      circid_t circid)
-{
-  (void)ch;
-  (void)circid;
-
-  ++test_destroy_not_pending_calls;
-}
+static int test_close_called = 0;
+static int test_chan_should_be_canonical = 0;
+static int test_chan_should_match_target = 0;
+static int test_chan_canonical_should_be_reliable = 0;
+static int test_chan_listener_close_fn_called = 0;
+static int test_chan_listener_fn_called = 0;
 
 static const char *
 chan_test_describe_transport(channel_t *ch)
 {
-  tt_assert(ch != NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
  done:
   return "Fake channel for unit tests";
@@ -100,7 +61,7 @@ chan_test_describe_transport(channel_t *ch)
 static void
 chan_test_channel_dump_statistics_mock(channel_t *chan, int severity)
 {
-  tt_assert(chan != NULL);
+  tt_ptr_op(chan, OP_NE, NULL);
 
   (void)severity;
 
@@ -112,71 +73,14 @@ chan_test_channel_dump_statistics_mock(channel_t *chan, int severity)
   return;
 }
 
-/**
- * If the target cmux is the cmux for chan, make fake cells up to the
- * target number of cells and write them to chan.  Otherwise, invoke
- * the real channel_flush_from_first_active_circuit().
- */
-
-static int
-chan_test_channel_flush_from_first_active_circuit_mock(channel_t *chan,
-                                                       int max)
-{
-  int result = 0, c = 0;
-  packed_cell_t *cell = NULL;
-
-  tt_assert(chan != NULL);
-  if (test_target_cmux != NULL &&
-      test_target_cmux == chan->cmux) {
-    while (c <= max && test_cmux_cells > 0) {
-      cell = packed_cell_new();
-      channel_write_packed_cell(chan, cell);
-      ++c;
-      --test_cmux_cells;
-    }
-    result = c;
-  } else {
-    result = channel_flush_from_first_active_circuit__real(chan, max);
-  }
-
- done:
-  return result;
-}
-
-/**
- * If we have a target cmux set and this matches it, lie about how
- * many cells we have according to the number indicated; otherwise
- * pass to the real circuitmux_num_cells().
- */
-
-static unsigned int
-chan_test_circuitmux_num_cells_mock(circuitmux_t *cmux)
-{
-  unsigned int result = 0;
-
-  tt_assert(cmux != NULL);
-  if (cmux != NULL) {
-    if (cmux == test_target_cmux) {
-      result = test_cmux_cells;
-    } else {
-      result = circuitmux_num_cells__real(cmux);
-    }
-  }
-
- done:
-
-  return result;
-}
-
 /*
  * Handle an incoming fixed-size cell for unit tests
  */
 
 static void
-chan_test_cell_handler(channel_t *ch,
-                       cell_t *cell)
+chan_test_cell_handler(channel_t *chan, cell_t *cell)
 {
-  tt_assert(ch);
+  tt_assert(chan);
   tt_assert(cell);
 
   test_chan_last_seen_fixed_cell_ptr = cell;
@@ -193,7 +97,7 @@ chan_test_cell_handler(channel_t *ch,
 static void
 chan_test_dumpstats(channel_t *ch, int severity)
 {
-  tt_assert(ch != NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
   (void)severity;
 
@@ -226,6 +130,8 @@ chan_test_close(channel_t *ch)
 {
   tt_assert(ch);
 
+  ++test_close_called;
+
  done:
   return;
 }
@@ -268,41 +174,12 @@ static const char *
 chan_test_get_remote_descr(channel_t *ch, int flags)
 {
   tt_assert(ch);
-  tt_int_op(flags & ~(GRD_FLAG_ORIGINAL | GRD_FLAG_ADDR_ONLY), ==, 0);
+  tt_int_op(flags & ~(GRD_FLAG_ORIGINAL | GRD_FLAG_ADDR_ONLY), OP_EQ, 0);
 
  done:
   return "Fake channel for unit tests; no real endpoint";
 }
 
-static double
-chan_test_get_overhead_estimate(channel_t *ch)
-{
-  tt_assert(ch);
-
- done:
-  return test_overhead_estimate;
-}
-
-static int
-chan_test_is_canonical(channel_t *ch, int req)
-{
-  tt_assert(ch != NULL);
-  tt_assert(req == 0 || req == 1);
-
- done:
-  /* Fake channels are always canonical */
-  return 1;
-}
-
-static size_t
-chan_test_num_bytes_queued(channel_t *ch)
-{
-  tt_assert(ch);
-
- done:
-  return 0;
-}
-
 static int
 chan_test_num_cells_writeable(channel_t *ch)
 {
@@ -313,26 +190,6 @@ chan_test_num_cells_writeable(channel_t *ch)
 }
 
 static int
-chan_test_write_cell(channel_t *ch, cell_t *cell)
-{
-  int rv = 0;
-
-  tt_assert(ch);
-  tt_assert(cell);
-
-  if (test_chan_accept_cells) {
-    /* Free the cell and bump the counter */
-    tor_free(cell);
-    ++test_cells_written;
-    rv = 1;
-  }
-  /* else return 0, we didn't accept it */
-
- done:
-  return rv;
-}
-
-static int
 chan_test_write_packed_cell(channel_t *ch,
                             packed_cell_t *packed_cell)
 {
@@ -343,7 +200,6 @@ chan_test_write_packed_cell(channel_t *ch,
 
   if (test_chan_accept_cells) {
     /* Free the cell and bump the counter */
-    packed_cell_free(packed_cell);
     ++test_cells_written;
     rv = 1;
   }
@@ -380,7 +236,7 @@ chan_test_write_var_cell(channel_t *ch, var_cell_t *var_cell)
 void
 make_fake_cell(cell_t *c)
 {
-  tt_assert(c != NULL);
+  tt_ptr_op(c, OP_NE, NULL);
 
   c->circ_id = 1;
   c->command = CELL_RELAY;
@@ -397,7 +253,7 @@ make_fake_cell(cell_t *c)
 void
 make_fake_var_cell(var_cell_t *c)
 {
-  tt_assert(c != NULL);
+  tt_ptr_op(c, OP_NE, NULL);
 
   c->circ_id = 1;
   c->command = CELL_VERSIONS;
@@ -419,36 +275,27 @@ new_fake_channel(void)
   channel_init(chan);
 
   chan->close = chan_test_close;
-  chan->get_overhead_estimate = chan_test_get_overhead_estimate;
-  chan->get_remote_descr = chan_test_get_remote_descr;
-  chan->num_bytes_queued = chan_test_num_bytes_queued;
   chan->num_cells_writeable = chan_test_num_cells_writeable;
-  chan->write_cell = chan_test_write_cell;
+  chan->get_remote_descr = chan_test_get_remote_descr;
   chan->write_packed_cell = chan_test_write_packed_cell;
   chan->write_var_cell = chan_test_write_var_cell;
   chan->state = CHANNEL_STATE_OPEN;
 
+  chan->cmux = circuitmux_alloc();
+  circuitmux_set_policy(chan->cmux, &ewma_policy);
+
   return chan;
 }
 
 void
 free_fake_channel(channel_t *chan)
 {
-  cell_queue_entry_t *cell, *cell_tmp;
-
   if (! chan)
     return;
 
   if (chan->cmux)
     circuitmux_free(chan->cmux);
 
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->incoming_queue, next, cell_tmp) {
-      cell_queue_entry_free(cell, 0);
-  }
-  TOR_SIMPLEQ_FOREACH_SAFE(cell, &chan->outgoing_queue, next, cell_tmp) {
-      cell_queue_entry_free(cell, 0);
-  }
-
   tor_free(chan);
 }
 
@@ -489,16 +336,6 @@ scheduler_channel_doesnt_want_writes_mock(channel_t *ch)
 }
 
 /**
- * Counter query for scheduler_release_channel_mock()
- */
-
-int
-get_mock_scheduler_release_channel_count(void)
-{
-  return test_releases_count;
-}
-
-/**
  * Mock for scheduler_release_channel()
  */
 
@@ -513,6 +350,58 @@ scheduler_release_channel_mock(channel_t *ch)
   return;
 }
 
+static int
+test_chan_is_canonical(channel_t *chan, int req)
+{
+  tor_assert(chan);
+
+  if (req && test_chan_canonical_should_be_reliable) {
+    return 1;
+  }
+
+  if (test_chan_should_be_canonical) {
+    return 1;
+  }
+  return 0;
+}
+
+static int
+test_chan_matches_target(channel_t *chan, const tor_addr_t *target)
+{
+  (void) chan;
+  (void) target;
+
+  if (test_chan_should_match_target) {
+    return 1;
+  }
+  return 0;
+}
+
+static void
+test_chan_listener_close(channel_listener_t *chan)
+{
+  (void) chan;
+  ++test_chan_listener_close_fn_called;
+  return;
+}
+
+static void
+test_chan_listener_fn(channel_listener_t *listener, channel_t *chan)
+{
+  (void) listener;
+  (void) chan;
+
+  ++test_chan_listener_fn_called;
+  return;
+}
+
+static const char *
+test_chan_listener_describe_transport(channel_listener_t *chan)
+{
+  (void) chan;
+  return "Fake listener channel.";
+}
+
 /**
  * Test for channel_dumpstats() and limited test for
  * channel_dump_statistics()
@@ -523,6 +412,7 @@ test_channel_dumpstats(void *arg)
 {
   channel_t *ch = NULL;
   cell_t *cell = NULL;
+  packed_cell_t *p_cell = NULL;
   int old_count;
 
   (void)arg;
@@ -536,7 +426,6 @@ test_channel_dumpstats(void *arg)
   /* Set up a new fake channel */
   ch = new_fake_channel();
   tt_assert(ch);
-  ch->cmux = circuitmux_alloc();
 
   /* Try to register it */
   channel_register(ch);
@@ -552,24 +441,24 @@ test_channel_dumpstats(void *arg)
   channel_dumpstats(LOG_DEBUG);
 
   /* Assert that we hit the mock */
-  tt_int_op(dump_statistics_mock_matches, ==, 1);
+  tt_int_op(dump_statistics_mock_matches, OP_EQ, 1);
 
   /* Close the channel */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
 
   /* Try again and hit the finished channel */
   channel_dumpstats(LOG_DEBUG);
-  tt_int_op(dump_statistics_mock_matches, ==, 2);
+  tt_int_op(dump_statistics_mock_matches, OP_EQ, 2);
 
   channel_run_cleanup();
   ch = NULL;
 
   /* Now we should hit nothing */
   channel_dumpstats(LOG_DEBUG);
-  tt_int_op(dump_statistics_mock_matches, ==, 2);
+  tt_int_op(dump_statistics_mock_matches, OP_EQ, 2);
 
   /* Unmock */
   UNMOCK(channel_dump_statistics);
@@ -579,59 +468,56 @@ test_channel_dumpstats(void *arg)
   /* Now make another channel */
   ch = new_fake_channel();
   tt_assert(ch);
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
-  tt_assert(ch->registered);
+  tt_int_op(ch->registered, OP_EQ, 1);
   /* Lie about its age so dumpstats gets coverage for rate calculations */
   ch->timestamp_created = time(NULL) - 30;
-  tt_assert(ch->timestamp_created > 0);
-  tt_assert(time(NULL) > ch->timestamp_created);
+  tt_int_op(ch->timestamp_created, OP_GT, 0);
+  tt_int_op(time(NULL), OP_GT, ch->timestamp_created);
 
   /* Put cells through it both ways to make the counters non-zero */
-  cell = tor_malloc_zero(sizeof(*cell));
-  make_fake_cell(cell);
+  p_cell = packed_cell_new();
   test_chan_accept_cells = 1;
   old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  cell = NULL;
-  tt_int_op(test_cells_written, ==, old_count + 1);
-  tt_assert(ch->n_bytes_xmitted > 0);
-  tt_assert(ch->n_cells_xmitted > 0);
+  channel_write_packed_cell(ch, p_cell);
+  tt_int_op(test_cells_written, OP_EQ, old_count + 1);
+  tt_u64_op(ch->n_bytes_xmitted, OP_GT, 0);
+  tt_u64_op(ch->n_cells_xmitted, OP_GT, 0);
 
   /* Receive path */
   channel_set_cell_handlers(ch,
                             chan_test_cell_handler,
                             chan_test_var_cell_handler);
-  tt_ptr_op(channel_get_cell_handler(ch), ==, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, chan_test_var_cell_handler);
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
+  tt_ptr_op(channel_get_cell_handler(ch), OP_EQ, chan_test_cell_handler);
+  tt_ptr_op(channel_get_var_cell_handler(ch), OP_EQ,
+            chan_test_var_cell_handler);
+  cell = tor_malloc_zero(sizeof(*cell));
   old_count = test_chan_fixed_cells_recved;
-  channel_queue_cell(ch, cell);
-  tor_free(cell);
-  tt_int_op(test_chan_fixed_cells_recved, ==, old_count + 1);
-  tt_assert(ch->n_bytes_recved > 0);
-  tt_assert(ch->n_cells_recved > 0);
+  channel_process_cell(ch, cell);
+  tt_int_op(test_chan_fixed_cells_recved, OP_EQ, old_count + 1);
+  tt_u64_op(ch->n_bytes_recved, OP_GT, 0);
+  tt_u64_op(ch->n_cells_recved, OP_GT, 0);
 
   /* Test channel_dump_statistics */
   ch->describe_transport = chan_test_describe_transport;
   ch->dumpstats = chan_test_dumpstats;
-  ch->is_canonical = chan_test_is_canonical;
+  test_chan_should_be_canonical = 1;
+  ch->is_canonical = test_chan_is_canonical;
   old_count = test_dumpstats_calls;
   channel_dump_statistics(ch, LOG_DEBUG);
-  tt_int_op(test_dumpstats_calls, ==, old_count + 1);
+  tt_int_op(test_dumpstats_calls, OP_EQ, old_count + 1);
 
   /* Close the channel */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
  done:
-  tor_free(cell);
   free_fake_channel(ch);
+  tor_free(cell);
 
   UNMOCK(scheduler_channel_doesnt_want_writes);
   UNMOCK(scheduler_release_channel);
@@ -639,214 +525,235 @@ test_channel_dumpstats(void *arg)
   return;
 }
 
+/* Test outbound cell. The callstack is:
+ *  channel_flush_some_cells()
+ *   -> channel_flush_from_first_active_circuit()
+ *     -> channel_write_packed_cell()
+ *       -> write_packed_cell()
+ *         -> chan->write_packed_cell() fct ptr.
+ *
+ * This test goes from a cell in a circuit up to the channel write handler
+ * that should put them on the connection outbuf. */
 static void
-test_channel_flush(void *arg)
+test_channel_outbound_cell(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  packed_cell_t *p_cell = NULL;
-  var_cell_t *v_cell = NULL;
-  int init_count;
+  int old_count;
+  channel_t *chan = NULL;
+  packed_cell_t *p_cell = NULL, *p_cell2 = NULL;
+  origin_circuit_t *circ = NULL;
+  cell_queue_t *queue;
 
-  (void)arg;
+  (void) arg;
 
-  ch = new_fake_channel();
-  tt_assert(ch);
+  /* Set the test time to be mocked, since this test assumes that no
+   * time will pass, ewma values will not need to be re-scaled, and so on */
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(U64_LITERAL(1000000000) * 12345);
 
-  /* Cache the original count */
-  init_count = test_cells_written;
+  cmux_ewma_set_options(NULL,NULL);
 
-  /* Stop accepting so we can queue some */
-  test_chan_accept_cells = 0;
-
-  /* Queue a regular cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch, cell);
-  /* It should be queued, so assert that we didn't write it */
-  tt_int_op(test_cells_written, ==, init_count);
-
-  /* Queue a var cell */
-  v_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(v_cell);
-  channel_write_var_cell(ch, v_cell);
-  /* It should be queued, so assert that we didn't write it */
-  tt_int_op(test_cells_written, ==, init_count);
-
-  /* Try a packed cell now */
-  p_cell = packed_cell_new();
-  tt_assert(p_cell);
-  channel_write_packed_cell(ch, p_cell);
-  /* It should be queued, so assert that we didn't write it */
-  tt_int_op(test_cells_written, ==, init_count);
+  /* The channel will be freed so we need to hijack this so the scheduler
+   * doesn't get confused. */
+  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
 
-  /* Now allow writes through again */
+  /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
 
-  /* ...and flush */
-  channel_flush_cells(ch);
-
-  /* All three should have gone through */
-  tt_int_op(test_cells_written, ==, init_count + 3);
-
- done:
-  tor_free(ch);
-
-  return;
-}
-
-/**
- * Channel flush tests that require cmux mocking
- */
-
-static void
-test_channel_flushmux(void *arg)
-{
-  channel_t *ch = NULL;
-  int old_count, q_len_before, q_len_after;
-  ssize_t result;
-
-  (void)arg;
-
-  /* Install mocks we need for this test */
-  MOCK(channel_flush_from_first_active_circuit,
-       chan_test_channel_flush_from_first_active_circuit_mock);
-  MOCK(circuitmux_num_cells,
-       chan_test_circuitmux_num_cells_mock);
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-  ch->cmux = circuitmux_alloc();
-
+  /* Setup a valid circuit to queue a cell. */
+  circ = origin_circuit_new();
+  tt_assert(circ);
+  /* Circuit needs an origin purpose to be considered origin. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  TO_CIRCUIT(circ)->n_circ_id = 42;
+  /* This is the outbound test so use the next channel queue. */
+  queue = &TO_CIRCUIT(circ)->n_chan_cells;
+  /* Setup packed cell to queue on the circuit. */
+  p_cell = packed_cell_new();
+  tt_assert(p_cell);
+  p_cell2 = packed_cell_new();
+  tt_assert(p_cell2);
+  /* Setup a channel to put the circuit on. */
+  chan = new_fake_channel();
+  tt_assert(chan);
+  chan->state = CHANNEL_STATE_OPENING;
+  channel_change_state_open(chan);
+  /* Outbound channel. */
+  channel_mark_outgoing(chan);
+  /* Try to register it so we can clean it through the channel cleanup
+   * process. */
+  channel_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
+  /* Set EWMA policy so we can pick it when flushing. */
+  circuitmux_set_policy(chan->cmux, &ewma_policy);
+  tt_ptr_op(circuitmux_get_policy(chan->cmux), OP_EQ, &ewma_policy);
+
+  /* Register circuit to the channel circid map which will attach the circuit
+   * to the channel's cmux as well. */
+  circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan);
+  tt_int_op(channel_num_circuits(chan), OP_EQ, 1);
+  /* Test the cmux state. */
+  tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux);
+  tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+
+  /* Flush the channel without any cell on it. */
   old_count = test_cells_written;
-
-  test_target_cmux = ch->cmux;
-  test_cmux_cells = 1;
-
-  /* Enable cell acceptance */
-  test_chan_accept_cells = 1;
-
-  result = channel_flush_some_cells(ch, 1);
-
-  tt_int_op(result, ==, 1);
-  tt_int_op(test_cells_written, ==, old_count + 1);
-  tt_int_op(test_cmux_cells, ==, 0);
-
-  /* Now try it without accepting to force them into the queue */
-  test_chan_accept_cells = 0;
-  test_cmux_cells = 1;
-  q_len_before = chan_cell_queue_len(&(ch->outgoing_queue));
-
-  result = channel_flush_some_cells(ch, 1);
-
-  /* We should not have actually flushed any */
-  tt_int_op(result, ==, 0);
-  tt_int_op(test_cells_written, ==, old_count + 1);
-  /* But we should have gotten to the fake cellgen loop */
-  tt_int_op(test_cmux_cells, ==, 0);
-  /* ...and we should have a queued cell */
-  q_len_after = chan_cell_queue_len(&(ch->outgoing_queue));
-  tt_int_op(q_len_after, ==, q_len_before + 1);
-
-  /* Now accept cells again and drain the queue */
-  test_chan_accept_cells = 1;
-  channel_flush_cells(ch);
-  tt_int_op(test_cells_written, ==, old_count + 2);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  test_target_cmux = NULL;
-  test_cmux_cells = 0;
+  ssize_t flushed = channel_flush_some_cells(chan, 1);
+  tt_i64_op(flushed, OP_EQ, 0);
+  tt_int_op(test_cells_written, OP_EQ, old_count);
+  tt_int_op(channel_more_to_flush(chan), OP_EQ, 0);
+  tt_int_op(circuitmux_num_active_circuits(chan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 0);
+  tt_u64_op(chan->n_cells_xmitted, OP_EQ, 0);
+  tt_u64_op(chan->n_bytes_xmitted, OP_EQ, 0);
+
+  /* Queue cell onto the next queue that is the outbound direction. Than
+   * update its cmux so the circuit can be picked when flushing cells. */
+  cell_queue_append(queue, p_cell);
+  p_cell = NULL;
+  tt_int_op(queue->n, OP_EQ, 1);
+  cell_queue_append(queue, p_cell2);
+  p_cell2 = NULL;
+  tt_int_op(queue->n, OP_EQ, 2);
+
+  update_circuit_on_cmux(TO_CIRCUIT(circ), CELL_DIRECTION_OUT);
+  tt_int_op(circuitmux_num_active_circuits(chan->cmux), OP_EQ, 1);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 2);
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+
+  /* From this point on, we have a queued cell on an active circuit attached
+   * to the channel's cmux. */
+
+  /* Flush the first cell. This is going to go down the call stack. */
+  old_count = test_cells_written;
+  flushed = channel_flush_some_cells(chan, 1);
+  tt_i64_op(flushed, OP_EQ, 1);
+  tt_int_op(test_cells_written, OP_EQ, old_count + 1);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 1);
+  tt_int_op(channel_more_to_flush(chan), OP_EQ, 1);
+  /* Circuit should remain active because there is a second cell queued. */
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+  /* Should still be attached. */
+  tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+  tt_u64_op(chan->n_cells_xmitted, OP_EQ, 1);
+  tt_u64_op(chan->n_bytes_xmitted, OP_EQ, get_cell_network_size(0));
+
+  /* Flush second cell. This is going to go down the call stack. */
+  old_count = test_cells_written;
+  flushed = channel_flush_some_cells(chan, 1);
+  tt_i64_op(flushed, OP_EQ, 1);
+  tt_int_op(test_cells_written, OP_EQ, old_count + 1);
+  tt_int_op(circuitmux_num_cells(chan->cmux), OP_EQ, 0);
+  tt_int_op(channel_more_to_flush(chan), OP_EQ, 0);
+  /* No more cells should make the circuit inactive. */
+  tt_int_op(circuitmux_is_circuit_active(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 0);
+  /* Should still be attached. */
+  tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
+            OP_EQ, 1);
+  tt_u64_op(chan->n_cells_xmitted, OP_EQ, 2);
+  tt_u64_op(chan->n_bytes_xmitted, OP_EQ, get_cell_network_size(0) * 2);
 
  done:
-  if (ch)
-    circuitmux_free(ch->cmux);
-  tor_free(ch);
-
-  UNMOCK(channel_flush_from_first_active_circuit);
-  UNMOCK(circuitmux_num_cells);
-
-  test_chan_accept_cells = 0;
-
-  return;
+  if (circ) {
+    circuit_free_(TO_CIRCUIT(circ));
+  }
+  tor_free(p_cell);
+  channel_free_all();
+  UNMOCK(scheduler_release_channel);
+  monotime_disable_test_mocking();
 }
 
+/* Test inbound cell. The callstack is:
+ *  channel_process_cell()
+ *    -> chan->cell_handler()
+ *
+ * This test is about checking if we can process an inbound cell down to the
+ * channel handler. */
 static void
-test_channel_incoming(void *arg)
+test_channel_inbound_cell(void *arg)
 {
-  channel_t *ch = NULL;
+  channel_t *chan = NULL;
   cell_t *cell = NULL;
-  var_cell_t *var_cell = NULL;
   int old_count;
 
-  (void)arg;
+  (void) arg;
 
-  /* Mock these for duration of the test */
-  MOCK(scheduler_channel_doesnt_want_writes,
-       scheduler_channel_doesnt_want_writes_mock);
-  MOCK(scheduler_release_channel,
-       scheduler_release_channel_mock);
+  /* The channel will be freed so we need to hijack this so the scheduler
+   * doesn't get confused. */
+  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
 
   /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
 
-  ch = new_fake_channel();
-  tt_assert(ch);
+  chan = new_fake_channel();
+  tt_assert(chan);
   /* Start it off in OPENING */
-  ch->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch->cmux = circuitmux_alloc();
-
-  /* Install incoming cell handlers */
-  channel_set_cell_handlers(ch,
-                            chan_test_cell_handler,
-                            chan_test_var_cell_handler);
-  /* Test cell handler getters */
-  tt_ptr_op(channel_get_cell_handler(ch), ==, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, chan_test_var_cell_handler);
+  chan->state = CHANNEL_STATE_OPENING;
 
   /* Try to register it */
-  channel_register(ch);
-  tt_assert(ch->registered);
+  channel_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
 
   /* Open it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(chan);
+  tt_int_op(chan->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(chan->has_been_open, OP_EQ, 1);
 
-  /* Receive a fixed cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
+  /* Receive a cell now. */
+  cell = tor_malloc_zero(sizeof(*cell));
   make_fake_cell(cell);
   old_count = test_chan_fixed_cells_recved;
-  channel_queue_cell(ch, cell);
-  tor_free(cell);
-  tt_int_op(test_chan_fixed_cells_recved, ==, old_count + 1);
-
-  /* Receive a variable-size cell */
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  old_count = test_chan_var_cells_recved;
-  channel_queue_var_cell(ch, var_cell);
-  tor_free(cell);
-  tt_int_op(test_chan_var_cells_recved, ==, old_count + 1);
+  channel_process_cell(chan, cell);
+  tt_int_op(test_chan_fixed_cells_recved, OP_EQ, old_count);
+  tt_assert(monotime_coarse_is_zero(&chan->timestamp_xfer));
+  tt_u64_op(chan->timestamp_active, OP_EQ, 0);
+  tt_u64_op(chan->timestamp_recv, OP_EQ, 0);
+
+  /* Setup incoming cell handlers. We don't care about var cell, the channel
+   * layers is not handling those. */
+  channel_set_cell_handlers(chan, chan_test_cell_handler, NULL);
+  tt_ptr_op(chan->cell_handler, OP_EQ, chan_test_cell_handler);
+  /* Now process the cell, we should see it. */
+  old_count = test_chan_fixed_cells_recved;
+  channel_process_cell(chan, cell);
+  tt_int_op(test_chan_fixed_cells_recved, OP_EQ, old_count + 1);
+  /* We should have a series of timestamp set. */
+  tt_assert(!monotime_coarse_is_zero(&chan->timestamp_xfer));
+  tt_u64_op(chan->timestamp_active, OP_NE, 0);
+  tt_u64_op(chan->timestamp_recv, OP_NE, 0);
+  tt_assert(monotime_coarse_is_zero(&chan->next_padding_time));
+  tt_u64_op(chan->n_cells_recved, OP_EQ, 1);
+  tt_u64_op(chan->n_bytes_recved, OP_EQ, get_cell_network_size(0));
 
   /* Close it */
-  channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
-  chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  old_count = test_close_called;
+  channel_mark_for_close(chan);
+  tt_int_op(chan->state, OP_EQ, CHANNEL_STATE_CLOSING);
+  tt_int_op(chan->reason_for_closing, OP_EQ, CHANNEL_CLOSE_REQUESTED);
+  tt_int_op(test_close_called, OP_EQ, old_count + 1);
+
+  /* This closes the channe so it calls in the scheduler, make sure of it. */
+  old_count = test_releases_count;
+  chan_test_finish_close(chan);
+  tt_int_op(test_releases_count, OP_EQ, old_count + 1);
+  tt_int_op(chan->state, OP_EQ, CHANNEL_STATE_CLOSED);
+
+  /* The channel will be free, lets make sure it is not accessible. */
+  uint64_t chan_id = chan->global_identifier;
+  tt_ptr_op(channel_find_by_global_id(chan_id), OP_EQ, chan);
   channel_run_cleanup();
-  ch = NULL;
+  chan = channel_find_by_global_id(chan_id);
+  tt_assert(chan == NULL);
 
  done:
-  free_fake_channel(ch);
   tor_free(cell);
-  tor_free(var_cell);
-
-  UNMOCK(scheduler_channel_doesnt_want_writes);
   UNMOCK(scheduler_release_channel);
-
-  return;
 }
 
 /**
@@ -859,7 +766,7 @@ static void
 test_channel_lifecycle(void *arg)
 {
   channel_t *ch1 = NULL, *ch2 = NULL;
-  cell_t *cell = NULL;
+  packed_cell_t *p_cell = NULL;
   int old_count, init_doesnt_want_writes_count;
   int init_releases_count;
 
@@ -877,79 +784,71 @@ test_channel_lifecycle(void *arg)
 
   /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
 
   ch1 = new_fake_channel();
   tt_assert(ch1);
   /* Start it off in OPENING */
   ch1->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch1->cmux = circuitmux_alloc();
 
   /* Try to register it */
   channel_register(ch1);
   tt_assert(ch1->registered);
 
   /* Try to write a cell through (should queue) */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
+  p_cell = packed_cell_new();
   old_count = test_cells_written;
-  channel_write_cell(ch1, cell);
-  tt_int_op(old_count, ==, test_cells_written);
+  channel_write_packed_cell(ch1, p_cell);
+  tt_int_op(old_count, OP_EQ, test_cells_written);
 
   /* Move it to OPEN and flush */
-  channel_change_state(ch1, CHANNEL_STATE_OPEN);
-
-  /* Queue should drain */
-  tt_int_op(old_count + 1, ==, test_cells_written);
+  channel_change_state_open(ch1);
 
-  /* Get another one */
+/* Get another one */
   ch2 = new_fake_channel();
   tt_assert(ch2);
   ch2->state = CHANNEL_STATE_OPENING;
-  ch2->cmux = circuitmux_alloc();
 
   /* Register */
   channel_register(ch2);
   tt_assert(ch2->registered);
 
   /* Check counters */
-  tt_int_op(test_doesnt_want_writes_count, ==, init_doesnt_want_writes_count);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
+            init_doesnt_want_writes_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Move ch1 to MAINT */
   channel_change_state(ch1, CHANNEL_STATE_MAINT);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Move ch2 to OPEN */
-  channel_change_state(ch2, CHANNEL_STATE_OPEN);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  channel_change_state_open(ch2);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Move ch1 back to OPEN */
-  channel_change_state(ch1, CHANNEL_STATE_OPEN);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  channel_change_state_open(ch1);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count);
 
   /* Mark ch2 for close */
   channel_mark_for_close(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSING);
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSING);
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
-  tt_int_op(test_releases_count, ==, init_releases_count + 1);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count + 1);
 
   /* Shut down channels */
   channel_free_all();
   ch1 = ch2 = NULL;
-  tt_int_op(test_doesnt_want_writes_count, ==,
+  tt_int_op(test_doesnt_want_writes_count, OP_EQ,
             init_doesnt_want_writes_count + 1);
   /* channel_free() calls scheduler_release_channel() */
-  tt_int_op(test_releases_count, ==, init_releases_count + 4);
+  tt_int_op(test_releases_count, OP_EQ, init_releases_count + 4);
 
  done:
   free_fake_channel(ch1);
@@ -957,8 +856,6 @@ test_channel_lifecycle(void *arg)
 
   UNMOCK(scheduler_channel_doesnt_want_writes);
   UNMOCK(scheduler_release_channel);
-
-  return;
 }
 
 /**
@@ -985,15 +882,11 @@ test_channel_lifecycle_2(void *arg)
 
   /* Accept cells to lower layer */
   test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
 
   ch = new_fake_channel();
   tt_assert(ch);
   /* Start it off in OPENING */
   ch->state = CHANNEL_STATE_OPENING;
-  /* The full lifecycle test needs a cmux */
-  ch->cmux = circuitmux_alloc();
 
   /* Try to register it */
   channel_register(ch);
@@ -1001,11 +894,11 @@ test_channel_lifecycle_2(void *arg)
 
   /* Try to close it */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish closing it */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1013,18 +906,17 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
 
   /* Error exit from lower layer */
   chan_test_error(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_ERROR);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_ERROR);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1032,25 +924,24 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_OPEN);
 
   /* Go to maintenance state */
   channel_change_state(ch, CHANNEL_STATE_MAINT);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_MAINT);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_MAINT);
 
   /* Lower layer close */
   channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1061,25 +952,24 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_OPEN);
 
   /* Go to maintenance state */
   channel_change_state(ch, CHANNEL_STATE_MAINT);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_MAINT);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_MAINT);
 
   /* Lower layer close */
   channel_close_from_lower_layer(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSED);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1087,25 +977,24 @@ test_channel_lifecycle_2(void *arg)
   ch = new_fake_channel();
   tt_assert(ch);
   ch->state = CHANNEL_STATE_OPENING;
-  ch->cmux = circuitmux_alloc();
   channel_register(ch);
   tt_assert(ch->registered);
 
   /* Finish opening it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_OPEN);
 
   /* Go to maintenance state */
   channel_change_state(ch, CHANNEL_STATE_MAINT);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_MAINT);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_MAINT);
 
   /* Lower layer close */
   chan_test_error(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_CLOSING);
 
   /* Finish */
   chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_ERROR);
+  tt_int_op(ch->state, OP_EQ, CHANNEL_STATE_ERROR);
   channel_run_cleanup();
   ch = NULL;
 
@@ -1122,660 +1011,554 @@ test_channel_lifecycle_2(void *arg)
 }
 
 static void
-test_channel_multi(void *arg)
+test_channel_id_map(void *arg)
 {
-  channel_t *ch1 = NULL, *ch2 = NULL;
-  uint64_t global_queue_estimate;
-  cell_t *cell = NULL;
-
   (void)arg;
+#define N_CHAN 6
+  char rsa_id[N_CHAN][DIGEST_LEN];
+  ed25519_public_key_t *ed_id[N_CHAN];
+  channel_t *chan[N_CHAN];
+  int i;
+  ed25519_public_key_t ed_zero;
+  memset(&ed_zero, 0, sizeof(ed_zero));
+
+  tt_int_op(DIGEST_LEN, OP_EQ, sizeof(rsa_id[0])); // Do I remember C?
+
+  for (i = 0; i < N_CHAN; ++i) {
+    crypto_rand(rsa_id[i], DIGEST_LEN);
+    ed_id[i] = tor_malloc_zero(sizeof(*ed_id[i]));
+    crypto_rand((char*)ed_id[i]->pubkey, sizeof(ed_id[i]->pubkey));
+  }
 
-  /* Accept cells to lower layer */
-  test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
-
-  ch1 = new_fake_channel();
-  tt_assert(ch1);
-  ch2 = new_fake_channel();
-  tt_assert(ch2);
-
-  /* Initial queue size update */
-  channel_update_xmit_queue_size(ch1);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 0);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Queue some cells, check queue estimates */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch1, cell);
-
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch2, cell);
-
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 0);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Stop accepting cells at lower layer */
-  test_chan_accept_cells = 0;
-
-  /* Queue some cells and check queue estimates */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch1, cell);
-
-  channel_update_xmit_queue_size(ch1);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch2, cell);
-
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 1024);
-
-  /* Allow cells through again */
-  test_chan_accept_cells = 1;
-
-  /* Flush chan 2 */
-  channel_flush_cells(ch2);
-
-  /* Update and check queue sizes */
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 512);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  /* Flush chan 1 */
-  channel_flush_cells(ch1);
-
-  /* Update and check queue sizes */
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 0);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Now block again */
-  test_chan_accept_cells = 0;
-
-  /* Queue some cells */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch1, cell);
+  /* For channel 3, have no Ed identity. */
+  tor_free(ed_id[3]);
 
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch2, cell);
-  cell = NULL;
-
-  /* Check the estimates */
-  channel_update_xmit_queue_size(ch1);
-  channel_update_xmit_queue_size(ch2);
-  tt_u64_op(ch1->bytes_queued_for_xmit, ==, 512);
-  tt_u64_op(ch2->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 1024);
-
-  /* Now close channel 2; it should be subtracted from the global queue */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch2);
-  UNMOCK(scheduler_release_channel);
+  /* Channel 2 and 4 have same ROSA identity */
+  memcpy(rsa_id[4], rsa_id[2], DIGEST_LEN);
 
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
+  /* Channel 2 and 4 and 5 have same RSA identity */
+  memcpy(rsa_id[4], rsa_id[2], DIGEST_LEN);
+  memcpy(rsa_id[5], rsa_id[2], DIGEST_LEN);
 
-  /*
-   * Since the fake channels aren't registered, channel_free_all() can't
-   * see them properly.
-   */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch1);
-  UNMOCK(scheduler_release_channel);
+  /* Channels 2 and 5 have same Ed25519 identity */
+  memcpy(ed_id[5], ed_id[2], sizeof(*ed_id[2]));
 
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
+  for (i = 0; i < N_CHAN; ++i) {
+    chan[i] = new_fake_channel();
+    channel_register(chan[i]);
+    channel_set_identity_digest(chan[i], rsa_id[i], ed_id[i]);
+  }
 
-  /* Now free everything */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_free_all();
-  UNMOCK(scheduler_release_channel);
+  /* Lookup by RSA id only */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], NULL));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], NULL));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], NULL));
+  channel_t *ch;
+  ch = channel_find_by_remote_identity(rsa_id[2], NULL);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_ptr_op(ch, OP_EQ, NULL);
+
+  /* As above, but with zero Ed25519 ID (meaning "any ID") */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], &ed_zero));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], &ed_zero));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], &ed_zero));
+  ch = channel_find_by_remote_identity(rsa_id[2], &ed_zero);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_ptr_op(ch, OP_EQ, NULL);
+
+  /* Lookup nonexistent RSA identity */
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity("!!!!!!!!!!!!!!!!!!!!", NULL));
+
+  /* Look up by full identity pair */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], ed_id[0]));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], ed_id[1]));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], ed_id[3] /*NULL*/));
+  tt_ptr_op(chan[4], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[4], ed_id[4]));
+  ch = channel_find_by_remote_identity(rsa_id[2], ed_id[2]);
+  tt_assert(ch == chan[2] || ch == chan[5]);
+
+  /* Look up RSA identity with wrong ed25519 identity */
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[4], ed_id[0]));
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[2], ed_id[1]));
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], ed_id[1]));
 
  done:
-  free_fake_channel(ch1);
-  free_fake_channel(ch2);
-
-  return;
+  for (i = 0; i < N_CHAN; ++i) {
+    channel_clear_identity_digest(chan[i]);
+    channel_unregister(chan[i]);
+    free_fake_channel(chan[i]);
+    tor_free(ed_id[i]);
+  }
+#undef N_CHAN
 }
 
-/**
- * Check some hopefully-impossible edge cases in the channel queue we
- * can only trigger by doing evil things to the queue directly.
- */
-
 static void
-test_channel_queue_impossible(void *arg)
+test_channel_state(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  packed_cell_t *packed_cell = NULL;
-  var_cell_t *var_cell = NULL;
-  int old_count;
-  cell_queue_entry_t *q = NULL;
-  uint64_t global_queue_estimate;
-  uintptr_t cellintptr;
-
-  /* Cache the global queue size (see below) */
-  global_queue_estimate = channel_get_global_queue_estimate();
-
-  (void)arg;
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-
-  /* We test queueing here; tell it not to accept cells */
-  test_chan_accept_cells = 0;
-  /* ...and keep it from trying to flush the queue */
-  ch->state = CHANNEL_STATE_MAINT;
-
-  /* Cache the cell written count */
-  old_count = test_cells_written;
-
-  /* Assert that the queue is initially empty */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Get a fresh cell and write it to the channel*/
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  cellintptr = (uintptr_t)(void*)cell;
-  channel_write_cell(ch, cell);
-
-  /* Now it should be queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_FIXED);
-    tt_assert((uintptr_t)q->u.fixed.cell == cellintptr);
-  }
-  /* Do perverse things to it */
-  tor_free(q->u.fixed.cell);
-  q->u.fixed.cell = NULL;
-
-  /*
-   * Now change back to open with channel_change_state() and assert that it
-   * gets thrown away properly.
-   */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Same thing but for a var_cell */
-
-  test_chan_accept_cells = 0;
-  ch->state = CHANNEL_STATE_MAINT;
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  cellintptr = (uintptr_t)(void*)var_cell;
-  channel_write_var_cell(ch, var_cell);
-
-  /* Check that it's queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_VAR);
-    tt_assert((uintptr_t)q->u.var.var_cell == cellintptr);
-  }
-
-  /* Remove the cell from the queue entry */
-  tor_free(q->u.var.var_cell);
-  q->u.var.var_cell = NULL;
-
-  /* Let it drain and check that the bad entry is discarded */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Same thing with a packed_cell */
-
-  test_chan_accept_cells = 0;
-  ch->state = CHANNEL_STATE_MAINT;
-  packed_cell = packed_cell_new();
-  tt_assert(packed_cell);
-  cellintptr = (uintptr_t)(void*)packed_cell;
-  channel_write_packed_cell(ch, packed_cell);
-
-  /* Check that it's queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_PACKED);
-    tt_assert((uintptr_t)q->u.packed.packed_cell == cellintptr);
-  }
-
-  /* Remove the cell from the queue entry */
-  packed_cell_free(q->u.packed.packed_cell);
-  q->u.packed.packed_cell = NULL;
-
-  /* Let it drain and check that the bad entry is discarded */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
-
-  /* Unknown cell type case */
-  test_chan_accept_cells = 0;
-  ch->state = CHANNEL_STATE_MAINT;
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  cellintptr = (uintptr_t)(void*)cell;
-  channel_write_cell(ch, cell);
-
-  /* Check that it's queued */
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 1);
-  q = TOR_SIMPLEQ_FIRST(&(ch->outgoing_queue));
-  tt_assert(q);
-  if (q) {
-    tt_int_op(q->type, ==, CELL_QUEUE_FIXED);
-    tt_assert((uintptr_t)q->u.fixed.cell == cellintptr);
-  }
-  /* Clobber it, including the queue entry type */
-  tor_free(q->u.fixed.cell);
-  q->u.fixed.cell = NULL;
-  q->type = CELL_QUEUE_PACKED + 1;
-
-  /* Let it drain and check that the bad entry is discarded */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count);
-  tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
+  (void) arg;
+
+  /* Test state validity. */
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_CLOSED), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_OPEN), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_OPENING), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_MAINT), OP_EQ, 1);
+  tt_int_op(channel_state_is_valid(CHANNEL_STATE_LAST), OP_EQ, 0);
+  tt_int_op(channel_state_is_valid(INT_MAX), OP_EQ, 0);
+
+  /* Test listener state validity. */
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_CLOSED),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_LISTENING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_is_valid(CHANNEL_LISTENER_STATE_LAST),
+            OP_EQ, 0);
+  tt_int_op(channel_listener_state_is_valid(INT_MAX), OP_EQ, 0);
+
+  /* Test state transition. */
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSED,
+                                         CHANNEL_STATE_OPENING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSED,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSING,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSING,
+                                         CHANNEL_STATE_CLOSED), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_CLOSING,
+                                         CHANNEL_STATE_OPEN), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_OPEN), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_MAINT,
+                                         CHANNEL_STATE_OPENING), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPENING,
+                                         CHANNEL_STATE_OPEN), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPENING,
+                                         CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPENING,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_CLOSING), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_ERROR), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_OPEN,
+                                         CHANNEL_STATE_MAINT), OP_EQ, 1);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_LAST,
+                                         CHANNEL_STATE_MAINT), OP_EQ, 0);
+  tt_int_op(channel_state_can_transition(CHANNEL_STATE_LAST, INT_MAX),
+            OP_EQ, 0);
+
+  /* Test listener state transition. */
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSED,
+                                       CHANNEL_LISTENER_STATE_LISTENING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSED,
+                                       CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 0);
+
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSING,
+                                       CHANNEL_LISTENER_STATE_CLOSED),
+            OP_EQ, 1);
+
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_CLOSING,
+                                       CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_ERROR,
+                                       CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, 0);
+
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_LISTENING,
+                                       CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_LISTENING,
+                                       CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, 1);
+  tt_int_op(channel_listener_state_can_transition(
+                                       CHANNEL_LISTENER_STATE_LAST,
+                                       INT_MAX),
+            OP_EQ, 0);
+
+  /* Test state string. */
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_CLOSING), OP_EQ,
+            "closing");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_ERROR), OP_EQ,
+            "channel error");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_CLOSED), OP_EQ,
+            "closed");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_OPEN), OP_EQ,
+            "open");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_OPENING), OP_EQ,
+            "opening");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_MAINT), OP_EQ,
+            "temporarily suspended for maintenance");
+  tt_str_op(channel_state_to_string(CHANNEL_STATE_LAST), OP_EQ,
+            "unknown or invalid channel state");
+  tt_str_op(channel_state_to_string(INT_MAX), OP_EQ,
+            "unknown or invalid channel state");
+
+  /* Test listener state string. */
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_CLOSING),
+            OP_EQ, "closing");
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_ERROR),
+            OP_EQ, "channel listener error");
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_LISTENING),
+            OP_EQ, "listening");
+  tt_str_op(channel_listener_state_to_string(CHANNEL_LISTENER_STATE_LAST),
+            OP_EQ, "unknown or invalid channel listener state");
+  tt_str_op(channel_listener_state_to_string(INT_MAX),
+            OP_EQ, "unknown or invalid channel listener state");
 
  done:
-  free_fake_channel(ch);
+  ;
+}
 
-  /*
-   * Doing that meant that we couldn't correctly adjust the queue size
-   * for the var cell, so manually reset the global queue size estimate
-   * so the next test doesn't break if we run with --no-fork.
-   */
-  estimated_total_queue_size = global_queue_estimate;
+static networkstatus_t *mock_ns = NULL;
 
-  return;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return mock_ns;
 }
 
 static void
-test_channel_queue_incoming(void *arg)
+test_channel_duplicates(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  var_cell_t *var_cell = NULL;
-  int old_fixed_count, old_var_count;
-
-  (void)arg;
-
-  /* Mock these for duration of the test */
-  MOCK(scheduler_channel_doesnt_want_writes,
-       scheduler_channel_doesnt_want_writes_mock);
-  MOCK(scheduler_release_channel,
-       scheduler_release_channel_mock);
-
-  /* Accept cells to lower layer */
-  test_chan_accept_cells = 1;
-  /* Use default overhead factor */
-  test_overhead_estimate = 1.0;
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-  /* Start it off in OPENING */
-  ch->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch->cmux = circuitmux_alloc();
-
-  /* Test cell handler getters */
-  tt_ptr_op(channel_get_cell_handler(ch), ==, NULL);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, NULL);
-
-  /* Try to register it */
-  channel_register(ch);
-  tt_assert(ch->registered);
-
-  /* Open it */
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_OPEN);
-
-  /* Assert that the incoming queue is empty */
-  tt_assert(TOR_SIMPLEQ_EMPTY(&(ch->incoming_queue)));
-
-  /* Queue an incoming fixed-length cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_queue_cell(ch, cell);
-
-  /* Assert that the incoming queue has one entry */
-  tt_int_op(chan_cell_queue_len(&(ch->incoming_queue)), ==, 1);
-
-  /* Queue an incoming var cell */
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  channel_queue_var_cell(ch, var_cell);
-
-  /* Assert that the incoming queue has two entries */
-  tt_int_op(chan_cell_queue_len(&(ch->incoming_queue)), ==, 2);
-
-  /*
-   * Install cell handlers; this will drain the queue, so save the old
-   * cell counters first
-   */
-  old_fixed_count = test_chan_fixed_cells_recved;
-  old_var_count = test_chan_var_cells_recved;
-  channel_set_cell_handlers(ch,
-                            chan_test_cell_handler,
-                            chan_test_var_cell_handler);
-  tt_ptr_op(channel_get_cell_handler(ch), ==, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), ==, chan_test_var_cell_handler);
-
-  /* Assert cells were received */
-  tt_int_op(test_chan_fixed_cells_recved, ==, old_fixed_count + 1);
-  tt_int_op(test_chan_var_cells_recved, ==, old_var_count + 1);
-
-  /*
-   * Assert that the pointers are different from the cells we allocated;
-   * when queueing cells with no incoming cell handlers installed, the
-   * channel layer should copy them to a new buffer, and free them after
-   * delivery.  These pointers will have already been freed by the time
-   * we get here, so don't dereference them.
-   */
-  tt_ptr_op(test_chan_last_seen_fixed_cell_ptr, !=, cell);
-  tt_ptr_op(test_chan_last_seen_var_cell_ptr, !=, var_cell);
-
-  /* Assert queue is now empty */
-  tt_assert(TOR_SIMPLEQ_EMPTY(&(ch->incoming_queue)));
+  channel_t *chan = NULL;
+  routerstatus_t rs;
+
+  (void) arg;
+
+  setup_full_capture_of_logs(LOG_INFO);
+  /* Try a flat call with channel nor connections. */
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 0 connections to 0 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+
+  mock_ns = tor_malloc_zero(sizeof(*mock_ns));
+  mock_ns->routerstatus_list = smartlist_new();
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+
+  chan = new_fake_channel();
+  tt_assert(chan);
+  chan->is_canonical = test_chan_is_canonical;
+  memset(chan->identity_digest, 'A', sizeof(chan->identity_digest));
+  channel_add_to_digest_map(chan);
+  tt_ptr_op(channel_find_by_remote_identity(chan->identity_digest, NULL),
+            OP_EQ, chan);
+
+  /* No relay has been associated with this channel. */
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 0 connections to 0 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+
+  /* Associate relay to this connection in the consensus. */
+  memset(&rs, 0, sizeof(rs));
+  memset(rs.identity_digest, 'A', sizeof(rs.identity_digest));
+  smartlist_add(mock_ns->routerstatus_list, &rs);
+
+  /* Non opened channel. */
+  chan->state = CHANNEL_STATE_CLOSING;
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 0 connections to 0 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+  chan->state = CHANNEL_STATE_OPEN;
 
-  /* Close it; this contains an assertion that the incoming queue is empty */
-  channel_mark_for_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSING);
-  chan_test_finish_close(ch);
-  tt_int_op(ch->state, ==, CHANNEL_STATE_CLOSED);
-  channel_run_cleanup();
-  ch = NULL;
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 1 connections to 1 relays. Found 0 current canonical "
+    "connections, in 0 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+
+  test_chan_should_be_canonical = 1;
+  channel_check_for_duplicates();
+  expect_log_msg_containing(
+    "Found 1 connections to 1 relays. Found 1 current canonical "
+    "connections, in 1 of which we were a non-canonical peer. "
+    "0 relays had more than 1 connection, 0 had more than 2, and "
+    "0 had more than 4 connections.");
+  teardown_capture_of_logs();
 
  done:
-  free_fake_channel(ch);
-  tor_free(cell);
-  tor_free(var_cell);
-
-  UNMOCK(scheduler_channel_doesnt_want_writes);
-  UNMOCK(scheduler_release_channel);
-
-  return;
+  free_fake_channel(chan);
+  smartlist_clear(mock_ns->routerstatus_list);
+  networkstatus_vote_free(mock_ns);
+  UNMOCK(networkstatus_get_latest_consensus);
 }
 
 static void
-test_channel_queue_size(void *arg)
+test_channel_for_extend(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = NULL;
-  int n, old_count;
-  uint64_t global_queue_estimate;
-
-  (void)arg;
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-
-  /* Initial queue size update */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Test the call-through to our fake lower layer */
-  n = channel_num_cells_writeable(ch);
-  /* chan_test_num_cells_writeable() always returns 32 */
-  tt_int_op(n, ==, 32);
-
-  /*
-   * Now we queue some cells and check that channel_num_cells_writeable()
-   * adjusts properly
-   */
-
-  /* tell it not to accept cells */
-  test_chan_accept_cells = 0;
-  /* ...and keep it from trying to flush the queue */
-  ch->state = CHANNEL_STATE_MAINT;
-
-  /* Get a fresh cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-
-  old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  /* Assert that it got queued, not written through, correctly */
-  tt_int_op(test_cells_written, ==, old_count);
-
-  /* Now check chan_test_num_cells_writeable() again */
-  n = channel_num_cells_writeable(ch);
-  tt_int_op(n, ==, 0); /* Should return 0 since we're in CHANNEL_STATE_MAINT */
-
-  /* Update queue size estimates */
-  channel_update_xmit_queue_size(ch);
-  /* One cell, times an overhead factor of 1.0 */
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  /* Try a different overhead factor */
-  test_overhead_estimate = 0.5;
-  /* This one should be ignored since it's below 1.0 */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  /* Now try a larger one */
-  test_overhead_estimate = 2.0;
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 1024);
-  /* Go back to 1.0 */
-  test_overhead_estimate = 1.0;
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  /* Check the global estimate too */
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  /* Go to open */
-  old_count = test_cells_written;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-
-  /*
-   * It should try to write, but we aren't accepting cells right now, so
-   * it'll requeue
-   */
-  tt_int_op(test_cells_written, ==, old_count);
-
-  /* Check the queue size again */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 512);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 512);
-
-  /*
-   * Now the cell is in the queue, and we're open, so we should get 31
-   * writeable cells.
-   */
-  n = channel_num_cells_writeable(ch);
-  tt_int_op(n, ==, 31);
-
-  /* Accept cells again */
-  test_chan_accept_cells = 1;
-  /* ...and re-process the queue */
-  old_count = test_cells_written;
-  channel_flush_cells(ch);
-  tt_int_op(test_cells_written, ==, old_count + 1);
-
-  /* Should have 32 writeable now */
-  n = channel_num_cells_writeable(ch);
-  tt_int_op(n, ==, 32);
-
-  /* Should have queue size estimate of zero */
-  channel_update_xmit_queue_size(ch);
-  tt_u64_op(ch->bytes_queued_for_xmit, ==, 0);
-  global_queue_estimate = channel_get_global_queue_estimate();
-  tt_u64_op(global_queue_estimate, ==, 0);
-
-  /* Okay, now we're done with this one */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch);
-  UNMOCK(scheduler_release_channel);
+  channel_t *chan1 = NULL, *chan2 = NULL;
+  channel_t *ret_chan = NULL;
+  char digest[DIGEST_LEN];
+  ed25519_public_key_t ed_id;
+  tor_addr_t addr;
+  const char *msg;
+  int launch;
+  time_t now = time(NULL);
+
+  (void) arg;
+
+  memset(digest, 'A', sizeof(digest));
+  memset(&ed_id, 'B', sizeof(ed_id));
+
+  chan1 = new_fake_channel();
+  tt_assert(chan1);
+  /* Need to be registered to get added to the id map. */
+  channel_register(chan1);
+  tt_int_op(chan1->registered, OP_EQ, 1);
+  /* We need those for the test. */
+  chan1->is_canonical = test_chan_is_canonical;
+  chan1->matches_target = test_chan_matches_target;
+  chan1->timestamp_created = now - 9;
+
+  chan2 = new_fake_channel();
+  tt_assert(chan2);
+  /* Need to be registered to get added to the id map. */
+  channel_register(chan2);
+  tt_int_op(chan2->registered, OP_EQ, 1);
+  /* We need those for the test. */
+  chan2->is_canonical = test_chan_is_canonical;
+  chan2->matches_target = test_chan_matches_target;
+  /* Make it older than chan1. */
+  chan2->timestamp_created = chan1->timestamp_created - 1;
+
+  /* Set channel identities and add it to the channel map. The last one to be
+   * added is made the first one in the list so the lookup will always return
+   * that one first. */
+  channel_set_identity_digest(chan2, digest, &ed_id);
+  channel_set_identity_digest(chan1, digest, &ed_id);
+  tt_ptr_op(channel_find_by_remote_identity(digest, NULL), OP_EQ, chan1);
+  tt_ptr_op(channel_find_by_remote_identity(digest, &ed_id), OP_EQ, chan1);
+
+  /* The expected result is chan2 because it is older than chan1. */
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+
+  /* Switch that around from previous test. */
+  chan2->timestamp_created = chan1->timestamp_created + 1;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan1);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+
+  /* Same creation time, num circuits will be used and they both have 0 so the
+   * channel 2 should be picked due to how channel_is_better() work. */
+  chan2->timestamp_created = chan1->timestamp_created;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan1);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+
+  /* For the rest of the tests, we need channel 1 to be the older. */
+  chan2->timestamp_created = chan1->timestamp_created + 1;
+
+  /* Condemned the older channel. */
+  chan1->state = CHANNEL_STATE_CLOSING;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+  chan1->state = CHANNEL_STATE_OPEN;
+
+  /* Make the older channel a client one. */
+  channel_mark_client(chan1);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+  channel_clear_client(chan1);
+
+  /* Non matching ed identity with valid digest. */
+  ed25519_public_key_t dumb_ed_id;
+  memset(&dumb_ed_id, 0, sizeof(dumb_ed_id));
+  ret_chan = channel_get_for_extend(digest, &dumb_ed_id, &addr, &msg,
+                                    &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Not connected. Connecting.");
+  tt_int_op(launch, OP_EQ, 1);
+
+  /* Opening channel, we'll check if the target address matches. */
+  test_chan_should_match_target = 1;
+  chan1->state = CHANNEL_STATE_OPENING;
+  chan2->state = CHANNEL_STATE_OPENING;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Connection in progress; waiting.");
+  tt_int_op(launch, OP_EQ, 0);
+  chan1->state = CHANNEL_STATE_OPEN;
+  chan2->state = CHANNEL_STATE_OPEN;
+
+  /* Mark channel 1 as bad for circuits. */
+  channel_mark_bad_for_new_circs(chan1);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(ret_chan);
+  tt_ptr_op(ret_chan, OP_EQ, chan2);
+  tt_int_op(launch, OP_EQ, 0);
+  tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
+  chan1->is_bad_for_new_circs = 0;
+
+  /* Mark both channels as unusable. */
+  channel_mark_bad_for_new_circs(chan1);
+  channel_mark_bad_for_new_circs(chan2);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
+                        " Launching a new one.");
+  tt_int_op(launch, OP_EQ, 1);
+  chan1->is_bad_for_new_circs = 0;
+  chan2->is_bad_for_new_circs = 0;
+
+  /* Non canonical channels. */
+  test_chan_should_match_target = 0;
+  test_chan_canonical_should_be_reliable = 1;
+  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  tt_assert(!ret_chan);
+  tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
+                        " Launching a new one.");
+  tt_int_op(launch, OP_EQ, 1);
 
  done:
-  free_fake_channel(ch);
-
-  return;
+  free_fake_channel(chan1);
+  free_fake_channel(chan2);
 }
 
 static void
-test_channel_write(void *arg)
+test_channel_listener(void *arg)
 {
-  channel_t *ch = NULL;
-  cell_t *cell = tor_malloc_zero(sizeof(cell_t));
-  packed_cell_t *packed_cell = NULL;
-  var_cell_t *var_cell =
-    tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
   int old_count;
-
-  (void)arg;
-
-  packed_cell = packed_cell_new();
-  tt_assert(packed_cell);
-
-  ch = new_fake_channel();
-  tt_assert(ch);
-  make_fake_cell(cell);
-  make_fake_var_cell(var_cell);
-
-  /* Tell it to accept cells */
-  test_chan_accept_cells = 1;
-
-  old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  cell = NULL;
-  tt_assert(test_cells_written == old_count + 1);
-
-  channel_write_var_cell(ch, var_cell);
-  var_cell = NULL;
-  tt_assert(test_cells_written == old_count + 2);
-
-  channel_write_packed_cell(ch, packed_cell);
-  packed_cell = NULL;
-  tt_assert(test_cells_written == old_count + 3);
-
-  /* Now we test queueing; tell it not to accept cells */
-  test_chan_accept_cells = 0;
-  /* ...and keep it from trying to flush the queue */
-  ch->state = CHANNEL_STATE_MAINT;
-
-  /* Get a fresh cell */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-
-  old_count = test_cells_written;
-  channel_write_cell(ch, cell);
-  tt_assert(test_cells_written == old_count);
-
-  /*
-   * Now change back to open with channel_change_state() and assert that it
-   * gets drained from the queue.
-   */
-  test_chan_accept_cells = 1;
-  channel_change_state(ch, CHANNEL_STATE_OPEN);
-  tt_assert(test_cells_written == old_count + 1);
-
-  /*
-   * Check the note destroy case
-   */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  cell->command = CELL_DESTROY;
-
-  /* Set up the mock */
-  MOCK(channel_note_destroy_not_pending,
-       channel_note_destroy_not_pending_mock);
-
-  old_count = test_destroy_not_pending_calls;
-  channel_write_cell(ch, cell);
-  tt_assert(test_destroy_not_pending_calls == old_count + 1);
-
-  /* Now send a non-destroy and check we don't call it */
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch, cell);
-  tt_assert(test_destroy_not_pending_calls == old_count + 1);
-
-  UNMOCK(channel_note_destroy_not_pending);
-
-  /*
-   * Now switch it to CLOSING so we can test the discard-cells case
-   * in the channel_write_*() functions.
-   */
-  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
-  channel_mark_for_close(ch);
-  UNMOCK(scheduler_release_channel);
-
-  /* Send cells that will drop in the closing state */
-  old_count = test_cells_written;
-
-  cell = tor_malloc_zero(sizeof(cell_t));
-  make_fake_cell(cell);
-  channel_write_cell(ch, cell);
-  cell = NULL;
-  tt_assert(test_cells_written == old_count);
-
-  var_cell = tor_malloc_zero(sizeof(var_cell_t) + CELL_PAYLOAD_SIZE);
-  make_fake_var_cell(var_cell);
-  channel_write_var_cell(ch, var_cell);
-  var_cell = NULL;
-  tt_assert(test_cells_written == old_count);
-
-  packed_cell = packed_cell_new();
-  channel_write_packed_cell(ch, packed_cell);
-  packed_cell = NULL;
-  tt_assert(test_cells_written == old_count);
+  time_t now = time(NULL);
+  channel_listener_t *chan = NULL;
+
+  (void) arg;
+
+  chan = tor_malloc_zero(sizeof(*chan));
+  tt_assert(chan);
+  channel_init_listener(chan);
+  tt_u64_op(chan->global_identifier, OP_EQ, 1);
+  tt_int_op(chan->timestamp_created, OP_GE, now);
+  chan->close = test_chan_listener_close;
+
+  /* Register it. At this point, it is not open so it will be put in the
+   * finished list. */
+  channel_listener_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
+  channel_listener_unregister(chan);
+
+  /* Register it as listening now thus active. */
+  chan->state = CHANNEL_LISTENER_STATE_LISTENING;
+  channel_listener_register(chan);
+  tt_int_op(chan->registered, OP_EQ, 1);
+
+  /* Set the listener function. */
+  channel_listener_set_listener_fn(chan, test_chan_listener_fn);
+  tt_ptr_op(chan->listener, OP_EQ, test_chan_listener_fn);
+
+  /* Put a channel in the listener incoming list and queue it.
+   * function. By doing this, the listener() handler will be called. */
+  channel_t *in_chan = new_fake_channel();
+  old_count = test_chan_listener_fn_called;
+  channel_listener_queue_incoming(chan, in_chan);
+  free_fake_channel(in_chan);
+  tt_int_op(test_chan_listener_fn_called, OP_EQ, old_count + 1);
+
+  /* Put listener channel in CLOSING state. */
+  old_count = test_chan_listener_close_fn_called;
+  channel_listener_mark_for_close(chan);
+  tt_int_op(test_chan_listener_close_fn_called, OP_EQ, old_count + 1);
+  channel_listener_change_state(chan, CHANNEL_LISTENER_STATE_CLOSED);
+
+  /* Dump stats so we at least hit the code path. */
+  chan->describe_transport = test_chan_listener_describe_transport;
+  /* There is a check for "now > timestamp_created" when dumping the stats so
+   * make sure we go in. */
+  chan->timestamp_created = now - 10;
+  channel_listener_dump_statistics(chan, LOG_INFO);
 
  done:
-  free_fake_channel(ch);
-  tor_free(var_cell);
-  tor_free(cell);
-  packed_cell_free(packed_cell);
-  return;
+  channel_free_all();
 }
 
 struct testcase_t channel_tests[] = {
-  { "dumpstats", test_channel_dumpstats, TT_FORK, NULL, NULL },
-  { "flush", test_channel_flush, TT_FORK, NULL, NULL },
-  { "flushmux", test_channel_flushmux, TT_FORK, NULL, NULL },
-  { "incoming", test_channel_incoming, TT_FORK, NULL, NULL },
-  { "lifecycle", test_channel_lifecycle, TT_FORK, NULL, NULL },
-  { "lifecycle_2", test_channel_lifecycle_2, TT_FORK, NULL, NULL },
-  { "multi", test_channel_multi, TT_FORK, NULL, NULL },
-  { "queue_impossible", test_channel_queue_impossible, TT_FORK, NULL, NULL },
-  { "queue_incoming", test_channel_queue_incoming, TT_FORK, NULL, NULL },
-  { "queue_size", test_channel_queue_size, TT_FORK, NULL, NULL },
-  { "write", test_channel_write, TT_FORK, NULL, NULL },
+  { "inbound_cell", test_channel_inbound_cell, TT_FORK,
+    NULL, NULL },
+  { "outbound_cell", test_channel_outbound_cell, TT_FORK,
+    NULL, NULL },
+  { "id_map", test_channel_id_map, TT_FORK,
+    NULL, NULL },
+  { "lifecycle", test_channel_lifecycle, TT_FORK,
+    NULL, NULL },
+  { "lifecycle_2", test_channel_lifecycle_2, TT_FORK,
+    NULL, NULL },
+  { "dumpstats", test_channel_dumpstats, TT_FORK,
+    NULL, NULL },
+  { "state", test_channel_state, TT_FORK,
+    NULL, NULL },
+  { "duplicates", test_channel_duplicates, TT_FORK,
+    NULL, NULL },
+  { "get_channel_for_extend", test_channel_for_extend, TT_FORK,
+    NULL, NULL },
+  { "listener", test_channel_listener, TT_FORK,
+    NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
new file mode 100644
index 0000000000..4261bc1b67
--- /dev/null
+++ b/src/test/test_channelpadding.c
@@ -0,0 +1,1166 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define TOR_CHANNEL_INTERNAL_
+#define MAIN_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define TOR_TIMERS_PRIVATE
+#include "or.h"
+#include "test.h"
+#include "testsupport.h"
+#include "connection.h"
+#include "connection_or.h"
+#include "channel.h"
+#include "channeltls.h"
+#include "channelpadding.h"
+#include "compat_libevent.h"
+#include "config.h"
+#include "compat_time.h"
+#include "main.h"
+#include "networkstatus.h"
+#include "log_test_helpers.h"
+
+int channelpadding_get_netflow_inactive_timeout_ms(channel_t *chan);
+int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *chan);
+int channelpadding_send_disable_command(channel_t*);
+int channelpadding_find_timerslot(channel_t *chan);
+
+void test_channelpadding_timers(void *arg);
+void test_channelpadding_consensus(void *arg);
+void test_channelpadding_negotiation(void *arg);
+void test_channelpadding_decide_to_pad_channel(void *arg);
+void test_channelpadding_killonehop(void *arg);
+
+void dummy_nop_timer(void);
+
+#define NSEC_PER_MSEC (1000*1000)
+
+/* Thing to cast to fake tor_tls_t * to appease assert_connection_ok() */
+static int fake_tortls = 0; /* Bleh... */
+
+static int dont_stop_libevent = 0;
+
+// From test_channel.c
+channel_t * new_fake_channel(void);
+void free_fake_channel(channel_t*);
+
+static int
+mock_channel_has_queued_writes(channel_t *chan)
+{
+  (void)chan;
+  return 0;
+}
+
+static int tried_to_write_cell = 0;
+
+static channel_t *relay1_relay2;
+static channel_t *relay2_relay1;
+static channel_t *relay3_client;
+static channel_t *client_relay3;
+
+static int
+mock_channel_write_cell_relay2(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay1_relay2)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_relay1(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay2_relay1)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_relay3(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)client_relay3)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_client(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay3_client)->conn);
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell(channel_t *chan, cell_t *cell)
+{
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)chan)->conn);
+  if (!dont_stop_libevent)
+    tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return 0;
+}
+
+static void
+setup_fake_connection_for_channel(channel_tls_t *chan)
+{
+  or_connection_t *conn = (or_connection_t*)connection_new(CONN_TYPE_OR,
+                                                           AF_INET);
+
+  conn->base_.conn_array_index = smartlist_len(connection_array);
+  smartlist_add(connection_array, conn);
+
+  conn->chan = chan;
+  chan->conn = conn;
+
+  conn->base_.magic = OR_CONNECTION_MAGIC;
+  conn->base_.state = OR_CONN_STATE_OPEN;
+  conn->base_.type = CONN_TYPE_OR;
+  conn->base_.socket_family = AF_INET;
+  conn->base_.address = tor_strdup("<fake>");
+
+  conn->base_.port = 4242;
+
+  conn->tls = (tor_tls_t *)((void *)(&fake_tortls));
+
+  conn->link_proto = MIN_LINK_PROTO_FOR_CHANNEL_PADDING;
+
+  connection_or_set_canonical(conn, 1);
+}
+
+static channel_tls_t *
+new_fake_channeltls(uint8_t id)
+{
+  channel_tls_t *chan = tor_realloc(new_fake_channel(), sizeof(channel_tls_t));
+  chan->base_.magic = TLS_CHAN_MAGIC;
+  setup_fake_connection_for_channel(chan);
+  chan->base_.channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+  chan->base_.has_queued_writes = mock_channel_has_queued_writes;
+  chan->base_.write_cell = mock_channel_write_cell;
+  chan->base_.padding_enabled = 1;
+
+  chan->base_.identity_digest[0] = id;
+  channel_register(&chan->base_);
+
+  return chan;
+}
+
+static void
+free_fake_channeltls(channel_tls_t *chan)
+{
+  channel_unregister(&chan->base_);
+
+  tor_free(((channel_tls_t*)chan)->conn->base_.address);
+  buf_free(((channel_tls_t*)chan)->conn->base_.inbuf);
+  buf_free(((channel_tls_t*)chan)->conn->base_.outbuf);
+  tor_free(((channel_tls_t*)chan)->conn);
+
+  timer_free(chan->base_.padding_timer);
+  channel_handle_free(chan->base_.timer_handle);
+  channel_handles_clear(&chan->base_);
+
+  free_fake_channel(&chan->base_);
+
+  return;
+}
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus
+        = tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+  channelpadding_new_consensus_params(current_md_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+setup_mock_network(void)
+{
+  routerstatus_t *relay;
+  if (!connection_array)
+    connection_array = smartlist_new();
+
+  relay1_relay2 = (channel_t*)new_fake_channeltls(2);
+  relay1_relay2->write_cell = mock_channel_write_cell_relay1;
+  channel_timestamp_active(relay1_relay2);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 1;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  relay2_relay1 = (channel_t*)new_fake_channeltls(1);
+  relay2_relay1->write_cell = mock_channel_write_cell_relay2;
+  channel_timestamp_active(relay2_relay1);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 2;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  relay3_client = (channel_t*)new_fake_channeltls(0);
+  relay3_client->write_cell = mock_channel_write_cell_relay3;
+  relay3_client->is_client = 1;
+  channel_timestamp_active(relay3_client);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 3;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  client_relay3 = (channel_t*)new_fake_channeltls(3);
+  client_relay3->write_cell = mock_channel_write_cell_client;
+  channel_timestamp_active(client_relay3);
+
+  channel_do_open_actions(relay1_relay2);
+  channel_do_open_actions(relay2_relay1);
+  channel_do_open_actions(relay3_client);
+  channel_do_open_actions(client_relay3);
+}
+
+static void
+free_mock_network(void)
+{
+  free_fake_channeltls((channel_tls_t*)relay1_relay2);
+  free_fake_channeltls((channel_tls_t*)relay2_relay1);
+  free_fake_channeltls((channel_tls_t*)relay3_client);
+  free_fake_channeltls((channel_tls_t*)client_relay3);
+
+  smartlist_free(connection_array);
+}
+
+static void
+dummy_timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono)
+{
+  (void)t; (void)arg; (void)now_mono;
+  tor_libevent_exit_loop_after_callback(tor_libevent_get_base());
+  return;
+}
+
+// This hack adds a dummy timer so that the libevent base loop
+// actually returns when we don't expect any timers to fire. Otherwise,
+// the global_timer_event gets scheduled an hour from now, and the
+// base loop never returns.
+void
+dummy_nop_timer(void)
+{
+  tor_timer_t *dummy_timer = timer_new(dummy_timer_cb, NULL);
+  struct timeval timeout;
+  timeout.tv_sec = 1;
+  timeout.tv_usec = 0;
+
+  timer_schedule(dummy_timer, &timeout);
+
+  tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
+
+  timer_free(dummy_timer);
+}
+
+#define CHANNELPADDING_MAX_TIMERS 25
+#define CHANNELS_TO_TEST (CHANNELPADDING_MAX_TIMERS*4)
+/**
+ * Tests to ensure that we handle more than the max number of pending
+ * timers properly.
+ */
+void
+test_channelpadding_timers(void *arg)
+{
+  channelpadding_decision_t decision;
+  channel_t *chans[CHANNELS_TO_TEST];
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  if (!connection_array)
+    connection_array = smartlist_new();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  uint64_t nsec_mock = 1;
+  monotime_set_mock_time_nsec(nsec_mock);
+  monotime_coarse_set_mock_time_nsec(nsec_mock);
+
+  timers_initialize();
+  channelpadding_new_consensus_params(NULL);
+
+  for (int i = 0; i < CHANNELS_TO_TEST; i++) {
+    chans[i] = (channel_t*)new_fake_channeltls(0);
+    channel_timestamp_active(chans[i]);
+  }
+
+  for (int j = 0; j < 2; j++) {
+    tried_to_write_cell = 0;
+    int i = 0;
+
+    monotime_coarse_t now;
+    monotime_coarse_get(&now);
+
+    /* This loop fills our timerslot array with timers of increasing time
+     * until they fire */
+    for (; i < CHANNELPADDING_MAX_TIMERS; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                        &now, 10 + i*4);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to the first position in the timerslot
+     * array, since its timeout is before all other timers. */
+    for (; i < CHANNELS_TO_TEST/3; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                        &now, 1);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to our existing lists in a weak
+     * pseudorandom pattern.  It ensures that the lists can grow with multiple
+     * timers in them. */
+    for (; i < CHANNELS_TO_TEST/2; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                        &now, 10 + i*3 % CHANNELPADDING_MAX_TIMERS);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to the last position in the timerslot
+     * array, since its timeout is after all other timers. */
+    for (; i < CHANNELS_TO_TEST; i++) {
+      monotime_coarse_add_msec(&chans[i]->next_padding_time,
+                               &now, 500 + i % CHANNELPADDING_MAX_TIMERS);
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    // Wait for the timers and then kill the event loop.
+    nsec_mock += 1001 * NSEC_PER_MSEC;
+    monotime_coarse_set_mock_time_nsec(nsec_mock);
+    monotime_set_mock_time_nsec(nsec_mock);
+    timers_run_pending();
+
+    tt_int_op(tried_to_write_cell, OP_EQ, CHANNELS_TO_TEST);
+
+    // Test that we have no pending callbacks and all empty slots now
+    for (i = 0; i < CHANNELS_TO_TEST; i++) {
+      tt_assert(!chans[i]->pending_padding_callback);
+    }
+  }
+
+ done:
+  for (int i = 0; i < CHANNELS_TO_TEST; i++) {
+    free_fake_channeltls((channel_tls_t*)chans[i]);
+  }
+  smartlist_free(connection_array);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_killonehop(void *arg)
+{
+  channelpadding_decision_t decision;
+  int64_t new_time;
+  (void)arg;
+  tor_libevent_postfork();
+
+  routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  new_time = 1;
+
+  timers_initialize();
+  setup_mock_consensus();
+  setup_mock_network();
+
+  /* Do we disable padding if tor2webmode or rsos are enabled, and
+   * the consensus says don't pad?  */
+
+  /* Ensure we can kill tor2web and rsos padding if we want. */
+  // First, test that padding works if either is enabled
+  smartlist_clear(current_md_consensus->net_params);
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+
+  tried_to_write_cell = 0;
+  get_options_mutable()->Tor2webMode = 1;
+  monotime_coarse_add_msec(&client_relay3->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Then test disabling each via consensus param
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_tor2web=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  // Before the client tries to pad, the relay will still pad:
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&relay3_client->next_padding_time, &now, 100);
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->Tor2webMode = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(relay3_client->pending_padding_callback);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test client side (it should stop immediately, but send a negotiate)
+  tried_to_write_cell = 0;
+  tt_assert(relay3_client->padding_enabled);
+  tt_assert(client_relay3->padding_enabled);
+  get_options_mutable()->Tor2webMode = 1;
+  /* For the relay to receive the negotiate: */
+  get_options_mutable()->ORPort_set = 1;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+  tt_assert(!relay3_client->padding_enabled);
+
+  // Test relay side (it should have gotten the negotiation to disable)
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->Tor2webMode = 0;
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+
+  /* Repeat for SOS */
+  // First, test that padding works if either is enabled
+  smartlist_clear(current_md_consensus->net_params);
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  tried_to_write_cell = 0;
+  get_options_mutable()->ORPort_set = 0;
+  get_options_mutable()->HiddenServiceSingleHopMode = 1;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 1;
+
+  monotime_coarse_add_msec(&client_relay3->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101 * NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Then test disabling each via consensus param
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_single_onion=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  // Before the client tries to pad, the relay will still pad:
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&relay3_client->next_padding_time, &now, 100);
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->HiddenServiceSingleHopMode = 0;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(relay3_client->pending_padding_callback);
+
+  // Wait for the timer
+  new_time += 101 * NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test client side (it should stop immediately)
+  get_options_mutable()->HiddenServiceSingleHopMode = 1;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 1;
+  /* For the relay to receive the negotiate: */
+  get_options_mutable()->ORPort_set = 1;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test relay side (it should have gotten the negotiation to disable)
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->HiddenServiceSingleHopMode = 0;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 0;
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+
+ done:
+  free_mock_consensus();
+  free_mock_network();
+  tor_free(relay);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+}
+
+void
+test_channelpadding_consensus(void *arg)
+{
+  channelpadding_decision_t decision;
+  or_options_t *options = get_options_mutable();
+  int64_t val;
+  int64_t new_time;
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  /*
+   * Params tested:
+   *   nf_pad_before_usage
+   *   nf_pad_relays
+   *   nf_ito_low
+   *   nf_ito_high
+   *
+   * Plan:
+   * 1. Padding can be completely disabled via consensus
+   * 2. Negotiation can't re-enable consensus-disabled padding
+   * 3. Negotiation can't increase padding from relays beyond
+   *    consensus defaults
+   * 4. Relay-to-relay padding can be enabled/disabled in consensus
+   * 5. Can enable/disable padding before actually using a connection
+   * 6. Can we control circ and TLS conn lifetime from the consensus?
+   */
+  channel_t *chan;
+  routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  new_time = 1;
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  timers_initialize();
+
+  if (!connection_array)
+    connection_array = smartlist_new();
+  chan = (channel_t*)new_fake_channeltls(0);
+  channel_timestamp_active(chan);
+
+  setup_mock_consensus();
+
+  get_options_mutable()->ORPort_set = 1;
+
+  /* Test 1: Padding can be completely disabled via consensus */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=0");
+  get_options_mutable()->ConnectionPadding = 1;
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_EQ, 0);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_EQ, -2);
+
+  /* Test 2: Negotiation can't re-enable consensus-disabled padding */
+  channelpadding_send_enable_command(chan, 100, 200);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_EQ, 0);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_EQ, -2);
+  tt_assert(monotime_coarse_is_zero(&chan->next_padding_time));
+
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Test 3: Negotiation can't increase padding from relays beyond consensus
+   * values */
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=100");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=200");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 100);
+  tt_i64_op(val, OP_LE, 200);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 200);
+
+  // Wait for the timer
+  new_time += 201*NSEC_PER_MSEC;
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_clear(current_md_consensus->net_params);
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=1500");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=4500");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  channelpadding_send_enable_command(chan, 100, 200);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 1500);
+  tt_i64_op(val, OP_LE, 4500);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 4500);
+
+  /* Test 4: Relay-to-relay padding can be enabled/disabled in consensus */
+  /* Make this channel a relay's channel */
+  memcpy(relay->identity_digest,
+          ((channel_tls_t *)chan)->conn->identity_digest, DIGEST_LEN);
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+  relay = NULL; /* Prevent double-free */
+
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_relays=1");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 1500);
+  tt_i64_op(val, OP_LE, 4500);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 4500);
+
+  /* Test 5: If we disable padding before channel usage, does that work? */
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_before_usage=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test 6: Can we control circ and TLS conn lifetime from the consensus? */
+  val = channelpadding_get_channel_idle_timeout(NULL, 0);
+  tt_i64_op(val, OP_GE, 180);
+  tt_i64_op(val, OP_LE, 180+90);
+  val = channelpadding_get_channel_idle_timeout(chan, 0);
+  tt_i64_op(val, OP_GE, 180);
+  tt_i64_op(val, OP_LE, 180+90);
+  options->ReducedConnectionPadding = 1;
+  val = channelpadding_get_channel_idle_timeout(chan, 0);
+  tt_i64_op(val, OP_GE, 180/2);
+  tt_i64_op(val, OP_LE, (180+90)/2);
+
+  options->ReducedConnectionPadding = 0;
+  options->ORPort_set = 1;
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_conntimeout_relays=600");
+  channelpadding_new_consensus_params(current_md_consensus);
+  val = channelpadding_get_channel_idle_timeout(chan, 1);
+  tt_i64_op(val, OP_GE, 450);
+  tt_i64_op(val, OP_LE, 750);
+
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 30*60);
+  tt_i64_op(val, OP_LE, 30*60*2);
+
+  options->ReducedConnectionPadding = 1;
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_conntimeout_clients=600");
+  channelpadding_new_consensus_params(current_md_consensus);
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 600/2);
+  tt_i64_op(val, OP_LE, 600*2/2);
+
+  options->ReducedConnectionPadding = 0;
+  options->CircuitsAvailableTimeout = 24*60*60;
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 24*60*60);
+  tt_i64_op(val, OP_LE, 24*60*60*2);
+
+ done:
+  tor_free(relay);
+
+  free_mock_consensus();
+  free_fake_channeltls((channel_tls_t*)chan);
+  smartlist_free(connection_array);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_negotiation(void *arg)
+{
+  channelpadding_negotiate_t disable;
+  cell_t cell;
+  channelpadding_decision_t decision;
+  int val;
+  (void)arg;
+
+  /* Plan:
+   * 1. Clients reject negotiation, relays accept it.
+   *    * Bridges accept negotiation from their clients,
+   *      but not from relays.
+   * 2. Torrc options can override client-side negotiation
+   * 3. Test a version issue in channelpadidng cell
+   * 4. Test channelpadding_reduced_padding
+   */
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  timers_initialize();
+  setup_mock_consensus();
+  setup_mock_network();
+
+  /* Test case #1: Do the right things ignore negotiation? */
+  /* relay-to-client case: */
+  channelpadding_send_disable_command(relay3_client);
+  tt_assert(client_relay3->padding_enabled);
+
+  /* client-to-relay case: */
+  get_options_mutable()->ORPort_set = 1;
+  channelpadding_disable_padding_on_channel(client_relay3);
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  /* Bridge case from relay */
+  get_options_mutable()->BridgeRelay = 1;
+  channelpadding_disable_padding_on_channel(relay2_relay1);
+  tt_assert(relay1_relay2->padding_enabled);
+
+  /* Bridge case from client */
+  channelpadding_disable_padding_on_channel(client_relay3);
+  tt_assert(!relay3_client->padding_enabled);
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+  get_options_mutable()->BridgeRelay = 0;
+  get_options_mutable()->ORPort_set = 0;
+
+  /* Test case #2: Torrc options */
+  /* ConnectionPadding auto; Relay doesn't support us */
+  ((channel_tls_t*)relay3_client)->conn->link_proto = 4;
+  relay3_client->padding_enabled = 0;
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  ((channel_tls_t*)relay3_client)->conn->link_proto = 5;
+  relay3_client->padding_enabled = 1;
+
+  /* ConnectionPadding 1; Relay doesn't support us */
+  get_options_mutable()->ConnectionPadding = 1;
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  get_options_mutable()->ConnectionPadding = 0;
+
+  /* Test case #3: Test a version issue in channelpadding cell */
+  get_options_mutable()->ORPort_set = 1;
+  client_relay3->padding_enabled = 1;
+  relay3_client->padding_enabled = 1;
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&disable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP);
+  disable.version = 1;
+  channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, &disable);
+  client_relay3->write_cell(client_relay3, &cell);
+  tt_assert(relay3_client->padding_enabled);
+  tt_int_op(channelpadding_update_padding_for_channel(client_relay3, &disable),
+          OP_EQ, -1);
+  tt_assert(client_relay3->padding_enabled);
+
+  disable.version = 0;
+  channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, &disable);
+  client_relay3->write_cell(client_relay3, &cell);
+  tt_assert(!relay3_client->padding_enabled);
+
+  /* Test case 4: Reducing padding actually reduces it */
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+
+  channelpadding_reduce_padding_on_channel(client_relay3);
+
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+
+  get_options_mutable()->ORPort_set = 0;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+
+  tt_assert(!client_relay3->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(client_relay3);
+  tt_int_op(val, OP_GE, 9000);
+  tt_int_op(val, OP_LE, 14000);
+  int64_t val64 =
+    channelpadding_compute_time_until_pad_for_netflow(client_relay3);
+  tt_i64_op(val64, OP_LE, 14000);
+
+ done:
+  free_mock_network();
+  free_mock_consensus();
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_decide_to_pad_channel(void *arg)
+{
+  channelpadding_decision_t decision;
+  /**
+   * Test case plan:
+   *
+   * 1. Channel that has "sent a packet" before the timeout.
+   *    + We should decide to pad later
+   * 2. Channel that has not "sent a packet" before the timeout:
+   * 2a. Not within 1.1s of the timeout.
+   *    + We should decide to pad later
+   * 2b. Within 1.1s of the timemout.
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2c. Within 0.1s of the timeout.
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2d. Channel that asks to pad while timeout is scheduled
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2e. 0s of the timeout
+   *    + We should send padding immediately
+   *    + We should get feedback that we wrote a cell
+   * 2f. <0s of the timeout
+   *    + We should send padding immediately
+   *    + We should get feedback that we wrote a cell
+   * 3. Channel that sends a packet while timeout is scheduled
+   *    + We should not get feedback that we wrote a cell
+   * 4. Channel that closes while timeout is scheduled
+   *    + We should not get feedback that we wrote a cell
+   * 5. Make sure the channel still would work if repaired
+   *    + We should be able to schedule padding and resend
+   * 6. Channel is not used for full circuits
+   * 7. Channel that disappears while timeout is scheduled
+   *    + We should not send padding
+   */
+  channel_t *chan;
+  int64_t new_time;
+  if (!connection_array)
+    connection_array = smartlist_new();
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  new_time = 1;
+  monotime_coarse_t now;
+  monotime_coarse_get(&now);
+  timers_initialize();
+  setup_full_capture_of_logs(LOG_WARN);
+  channelpadding_new_consensus_params(NULL);
+
+  chan = (channel_t*)new_fake_channeltls(0);
+  channel_timestamp_active(chan);
+
+  /* Test case #1: Channel that has "sent a packet" before the timeout. */
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2a: > 1.1s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 1200);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2b: >= 1.0s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 1000);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  // Set up a timer for the <0 case below.
+  monotime_coarse_t now_minus_100s;
+  monotime_coarse_add_msec(&now_minus_100s, &now, 900);
+  // Wait for the timer from case #2b
+  new_time += 1000*NSEC_PER_MSEC;
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2c: > 0.1s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2d: Channel that asks to pad while timeout is scheduled */
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2e: 0s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 0);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SENT);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2f: <0s until timeout */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now_minus_100s, 0);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SENT);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #3: Channel that sends a packet while timeout is scheduled */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Pretend the channel sent a packet
+  channel_timestamp_active(chan);
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #4: Channel that closes while a timeout is scheduled */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Pretend the channel is temporarily down
+  chan->state = CHANNEL_STATE_MAINT;
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(!chan->pending_padding_callback);
+  chan->state = CHANNEL_STATE_OPEN;
+
+  /* Test case #5: Make sure previous test case didn't break everything */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  // Wait for the timer
+  new_time += 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #6. Channel is not used for full circuits */
+  chan->channel_usage = CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+
+  /* Test case #7. Channel is closed while timeout is scheduled.
+   *
+   * NOTE: This test deliberately breaks the channel callback mechanism.
+   * It must be last.
+   */
+  tried_to_write_cell = 0;
+  monotime_coarse_add_msec(&chan->next_padding_time, &now, 100);
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Close the connection while the timer is scheduled
+  free_fake_channeltls((channel_tls_t*)chan);
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  new_time = 101*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_get(&now);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+ done:
+  smartlist_free(connection_array);
+
+  teardown_capture_of_logs();
+  monotime_disable_test_mocking();
+  timers_shutdown();
+  channel_free_all();
+
+  return;
+}
+
+#define TEST_CHANNELPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t channelpadding_tests[] = {
+  //TEST_CHANNELPADDING(channelpadding_decide_to_pad_channel, 0),
+  TEST_CHANNELPADDING(channelpadding_decide_to_pad_channel, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_negotiation, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_consensus, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_killonehop, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_timers, TT_FORK),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 08442e01b6..94f1893cae 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -32,6 +32,7 @@ static or_connection_t * tlschan_connection_or_connect_mock(
     const tor_addr_t *addr,
     uint16_t port,
     const char *digest,
+    const ed25519_public_key_t *ed_id,
     channel_tls_t *tlschan);
 static int tlschan_is_local_addr_mock(const tor_addr_t *addr);
 
@@ -70,8 +71,8 @@ test_channeltls_create(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
-  tt_assert(ch != NULL);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
  done:
   if (ch) {
@@ -119,8 +120,8 @@ test_channeltls_num_bytes_queued(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
-  tt_assert(ch != NULL);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
   /*
    * Next, we have to test ch->num_bytes_queued, which is
@@ -131,7 +132,7 @@ test_channeltls_num_bytes_queued(void *arg)
 
   tt_assert(ch->num_bytes_queued != NULL);
   tlschan = BASE_CHAN_TO_TLS(ch);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
   if (TO_CONN(tlschan->conn)->outbuf == NULL) {
     /* We need an outbuf to make sure buf_datalen() gets called */
     fake_outbuf = 1;
@@ -141,7 +142,7 @@ test_channeltls_num_bytes_queued(void *arg)
   tlschan_buf_datalen_mock_size = 1024;
   MOCK(buf_datalen, tlschan_buf_datalen_mock);
   len = ch->num_bytes_queued(ch);
-  tt_int_op(len, ==, tlschan_buf_datalen_mock_size);
+  tt_int_op(len, OP_EQ, tlschan_buf_datalen_mock_size);
   /*
    * We also cover num_cells_writeable here; since wide_circ_ids = 0 on
    * the fake tlschans, cell_network_size returns 512, and so with
@@ -150,7 +151,7 @@ test_channeltls_num_bytes_queued(void *arg)
    * - 2 cells.
    */
   n = ch->num_cells_writeable(ch);
-  tt_int_op(n, ==, CEIL_DIV(OR_CONN_HIGHWATER, 512) - 2);
+  tt_int_op(n, OP_EQ, CEIL_DIV(OR_CONN_HIGHWATER, 512) - 2);
   UNMOCK(buf_datalen);
   tlschan_buf_datalen_mock_target = NULL;
   tlschan_buf_datalen_mock_size = 0;
@@ -204,12 +205,12 @@ test_channeltls_overhead_estimate(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
-  tt_assert(ch != NULL);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
+  tt_ptr_op(ch, OP_NE, NULL);
 
   /* First case: silly low ratios should get clamped to 1.0 */
   tlschan = BASE_CHAN_TO_TLS(ch);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
   tlschan->conn->bytes_xmitted = 128;
   tlschan->conn->bytes_xmitted_by_tls = 64;
   r = ch->get_overhead_estimate(ch);
@@ -266,14 +267,16 @@ static or_connection_t *
 tlschan_connection_or_connect_mock(const tor_addr_t *addr,
                                    uint16_t port,
                                    const char *digest,
+                                   const ed25519_public_key_t *ed_id,
                                    channel_tls_t *tlschan)
 {
   or_connection_t *result = NULL;
+  (void) ed_id; // XXXX Not yet used.
 
-  tt_assert(addr != NULL);
-  tt_assert(port != 0);
-  tt_assert(digest != NULL);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(addr, OP_NE, NULL);
+  tt_uint_op(port, OP_NE, 0);
+  tt_ptr_op(digest, OP_NE, NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
 
   /* Make a fake orconn */
   result = tor_malloc_zero(sizeof(*result));
@@ -298,11 +301,11 @@ tlschan_fake_close_method(channel_t *chan)
 {
   channel_tls_t *tlschan = NULL;
 
-  tt_assert(chan != NULL);
-  tt_int_op(chan->magic, ==, TLS_CHAN_MAGIC);
+  tt_ptr_op(chan, OP_NE, NULL);
+  tt_int_op(chan->magic, OP_EQ, TLS_CHAN_MAGIC);
 
   tlschan = BASE_CHAN_TO_TLS(chan);
-  tt_assert(tlschan != NULL);
+  tt_ptr_op(tlschan, OP_NE, NULL);
 
   /* Just free the fake orconn */
   tor_free(tlschan->conn->base_.address);
@@ -317,7 +320,7 @@ tlschan_fake_close_method(channel_t *chan)
 static int
 tlschan_is_local_addr_mock(const tor_addr_t *addr)
 {
-  tt_assert(addr != NULL);
+  tt_ptr_op(addr, OP_NE, NULL);
 
  done:
   return tlschan_local;
diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c
index fbb33f87f6..bf6a8376b3 100644
--- a/src/test/test_checkdir.c
+++ b/src/test/test_checkdir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -20,7 +20,7 @@
 #define umask(mask) ((void)0)
 #else
 #define tt_int_op_nowin(a,op,b) tt_int_op((a),op,(b))
-#endif
+#endif /* defined(_WIN32) */
 
 /** Run unit tests for private dir permission enforcement logic. */
 static void
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
new file mode 100644
index 0000000000..a5282df69d
--- /dev/null
+++ b/src/test/test_circuitbuild.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITBUILD_PRIVATE
+
+#include "or.h"
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+#include "config.h"
+#include "circuitbuild.h"
+
+/* Dummy nodes smartlist for testing */
+static smartlist_t dummy_nodes;
+/* Dummy exit extend_info for testing */
+static extend_info_t dummy_ei;
+
+static int
+mock_count_acceptable_nodes(smartlist_t *nodes)
+{
+  (void)nodes;
+
+  return DEFAULT_ROUTE_LEN + 1;
+}
+
+/* Test route lengths when the caller of new_route_len() doesn't
+ * specify exit_ei. */
+static void
+test_new_route_len_noexit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  r = new_route_len(CIRCUIT_PURPOSE_C_GENERAL, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  r = new_route_len(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  r = new_route_len(CIRCUIT_PURPOSE_S_CONNECT_REND, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Test route lengths where someone else chose the "exit" node, which
+ * require an extra hop for safety. */
+static void
+test_new_route_len_unsafe_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  /* connecting to hidden service directory */
+  r = new_route_len(CIRCUIT_PURPOSE_C_GENERAL, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+  /* client connecting to introduction point */
+  r = new_route_len(CIRCUIT_PURPOSE_C_INTRODUCING, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+  /* hidden service connecting to rendezvous point */
+  r = new_route_len(CIRCUIT_PURPOSE_S_CONNECT_REND, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Test route lengths where we chose the "exit" node, which don't
+ * require an extra hop for safety. */
+static void
+test_new_route_len_safe_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  /* hidden service connecting to introduction point */
+  r = new_route_len(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, &dummy_ei,
+                    &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  /* router testing its own reachability */
+  r = new_route_len(CIRCUIT_PURPOSE_TESTING, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Make sure a non-fatal assertion fails when new_route_len() gets an
+ * unexpected circuit purpose. */
+static void
+test_new_route_len_unhandled_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  tor_capture_bugs_(1);
+  setup_full_capture_of_logs(LOG_WARN);
+  r = new_route_len(CIRCUIT_PURPOSE_CONTROLLER, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(exit_ei && !known_purpose)");
+  expect_single_log_msg_containing("Unhandled purpose");
+  expect_single_log_msg_containing("with a chosen exit; assuming routelen");
+  teardown_capture_of_logs();
+  tor_end_capture_bugs_();
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+struct testcase_t circuitbuild_tests[] = {
+  { "noexit", test_new_route_len_noexit, 0, NULL, NULL },
+  { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL },
+  { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
+  { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c
index e996c42115..3794ffc2c6 100644
--- a/src/test/test_circuitlist.c
+++ b/src/test/test_circuitlist.c
@@ -1,13 +1,16 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
+#define HS_CIRCUITMAP_PRIVATE
 #include "or.h"
 #include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
+#include "circuitmux_ewma.h"
+#include "hs_circuitmap.h"
 #include "test.h"
 #include "log_test_helpers.h"
 
@@ -139,7 +142,7 @@ test_clist_maps(void *arg)
   /* Okay, now free ch2 and make sure that the circuit ID is STILL not
    * usable, because we haven't declared the destroy to be nonpending */
   tt_int_op(cdm.ncalls, OP_EQ, 0);
-  circuit_free(TO_CIRCUIT(or_c2));
+  circuit_free_(TO_CIRCUIT(or_c2));
   or_c2 = NULL; /* prevent free */
   tt_int_op(cdm.ncalls, OP_EQ, 2);
   memset(&cdm, 0, sizeof(cdm));
@@ -158,9 +161,9 @@ test_clist_maps(void *arg)
 
  done:
   if (or_c1)
-    circuit_free(TO_CIRCUIT(or_c1));
+    circuit_free_(TO_CIRCUIT(or_c1));
   if (or_c2)
-    circuit_free(TO_CIRCUIT(or_c2));
+    circuit_free_(TO_CIRCUIT(or_c2));
   if (ch1)
     tor_free(ch1->cmux);
   if (ch2)
@@ -178,6 +181,7 @@ static void
 test_rend_token_maps(void *arg)
 {
   or_circuit_t *c1, *c2, *c3, *c4;
+  origin_circuit_t *c5;
   const uint8_t tok1[REND_TOKEN_LEN] = "The cat can't tell y";
   const uint8_t tok2[REND_TOKEN_LEN] = "ou its name, and it ";
   const uint8_t tok3[REND_TOKEN_LEN] = "doesn't really care.";
@@ -185,10 +189,14 @@ test_rend_token_maps(void *arg)
 
   (void)arg;
   (void)tok1; //xxxx
+
+  hs_circuitmap_init();
+
   c1 = or_circuit_new(0, NULL);
   c2 = or_circuit_new(0, NULL);
   c3 = or_circuit_new(0, NULL);
   c4 = or_circuit_new(0, NULL);
+  c5 = origin_circuit_new();
 
   /* Make sure we really filled up the tok* variables */
   tt_int_op(tok1[REND_TOKEN_LEN-1], OP_EQ, 'y');
@@ -196,78 +204,87 @@ test_rend_token_maps(void *arg)
   tt_int_op(tok3[REND_TOKEN_LEN-1], OP_EQ, '.');
 
   /* No maps; nothing there. */
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok1));
 
-  circuit_set_rendezvous_cookie(c1, tok1);
-  circuit_set_intro_point_digest(c2, tok2);
+  hs_circuitmap_register_rend_circ_relay_side(c1, tok1);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c2, tok2);
 
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok3));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok3));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok2));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok1));
 
   /* Without purpose set, we don't get the circuits */
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   c1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
   c2->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
 
   /* Okay, make sure they show up now. */
-  tt_ptr_op(c1, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(c2, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(c1, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(c2, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   /* Two items at the same place with the same token. */
   c3->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
-  circuit_set_rendezvous_cookie(c3, tok2);
-  tt_ptr_op(c2, OP_EQ, circuit_get_intro_point(tok2));
-  tt_ptr_op(c3, OP_EQ, circuit_get_rendezvous(tok2));
+  hs_circuitmap_register_rend_circ_relay_side(c3, tok2);
+  tt_ptr_op(c2, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
 
   /* Marking a circuit makes it not get returned any more */
   circuit_mark_for_close(TO_CIRCUIT(c1), END_CIRC_REASON_FINISHED);
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  circuit_free(TO_CIRCUIT(c1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  circuit_free_(TO_CIRCUIT(c1));
   c1 = NULL;
 
   /* Freeing a circuit makes it not get returned any more. */
-  circuit_free(TO_CIRCUIT(c2));
+  circuit_free_(TO_CIRCUIT(c2));
   c2 = NULL;
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   /* c3 -- are you still there? */
-  tt_ptr_op(c3, OP_EQ, circuit_get_rendezvous(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
   /* Change its cookie.  This never happens in Tor per se, but hey. */
   c3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
-  circuit_set_intro_point_digest(c3, tok3);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c3, tok3);
 
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok2));
-  tt_ptr_op(c3, OP_EQ, circuit_get_intro_point(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
   /* Now replace c3 with c4. */
   c4->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
-  circuit_set_intro_point_digest(c4, tok3);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c4, tok3);
 
-  tt_ptr_op(c4, OP_EQ, circuit_get_intro_point(tok3));
+  tt_ptr_op(c4, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
-  tt_ptr_op(c3->rendinfo, OP_EQ, NULL);
-  tt_ptr_op(c4->rendinfo, OP_NE, NULL);
-  tt_mem_op(c4->rendinfo, OP_EQ, tok3, REND_TOKEN_LEN);
+  tt_ptr_op(TO_CIRCUIT(c3)->hs_token, OP_EQ, NULL);
+  tt_ptr_op(TO_CIRCUIT(c4)->hs_token, OP_NE, NULL);
+  tt_mem_op(TO_CIRCUIT(c4)->hs_token->token, OP_EQ, tok3, REND_TOKEN_LEN);
 
   /* Now clear c4's cookie. */
-  circuit_set_intro_point_digest(c4, NULL);
-  tt_ptr_op(c4->rendinfo, OP_EQ, NULL);
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok3));
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(c4));
+  tt_ptr_op(TO_CIRCUIT(c4)->hs_token, OP_EQ, NULL);
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
+
+  /* Now let's do a check for the client-side rend circuitmap */
+  c5->base_.purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
+  hs_circuitmap_register_rend_circ_client_side(c5, tok1);
+
+  tt_ptr_op(c5, OP_EQ, hs_circuitmap_get_rend_circ_client_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_client_side(tok2));
 
  done:
   if (c1)
-    circuit_free(TO_CIRCUIT(c1));
+    circuit_free_(TO_CIRCUIT(c1));
   if (c2)
-    circuit_free(TO_CIRCUIT(c2));
+    circuit_free_(TO_CIRCUIT(c2));
   if (c3)
-    circuit_free(TO_CIRCUIT(c3));
+    circuit_free_(TO_CIRCUIT(c3));
   if (c4)
-    circuit_free(TO_CIRCUIT(c4));
+    circuit_free_(TO_CIRCUIT(c4));
+  if (c5)
+    circuit_free_(TO_CIRCUIT(c5));
 }
 
 static void
@@ -365,10 +382,89 @@ test_pick_circid(void *arg)
   UNMOCK(channel_dump_statistics);
 }
 
+/** Test that the circuit pools of our HS circuitmap are isolated based on
+ *  their token type. */
+static void
+test_hs_circuitmap_isolation(void *arg)
+{
+  or_circuit_t *circ1 = NULL;
+  origin_circuit_t *circ2 = NULL;
+  or_circuit_t *circ3 = NULL;
+  origin_circuit_t *circ4 = NULL;
+
+  (void)arg;
+
+  hs_circuitmap_init();
+
+  {
+    const uint8_t tok1[REND_TOKEN_LEN] = "bet i got some of th";
+
+    circ1 = or_circuit_new(0, NULL);
+    tt_assert(circ1);
+    circ1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
+
+    /* check that circuitmap is empty right? */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+
+    /* Register circ1 with tok1 as relay-side rend circ */
+    hs_circuitmap_register_rend_circ_relay_side(circ1, tok1);
+
+    /* check that service-side getters don't work */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_service_side(tok1));
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_service_side(tok1));
+
+    /* Check that the right getter works. */
+    tt_ptr_op(circ1, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  }
+
+  {
+    const uint8_t tok2[REND_TOKEN_LEN] = "you dont know anythi";
+
+    circ2 = origin_circuit_new();
+    tt_assert(circ2);
+    circ2->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+    circ3 = or_circuit_new(0, NULL);
+    tt_assert(circ3);
+    circ3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
+    circ4 = origin_circuit_new();
+    tt_assert(circ4);
+    circ4->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+
+    /* Register circ2 with tok2 as service-side intro v2 circ */
+    hs_circuitmap_register_intro_circ_v2_service_side(circ2, tok2);
+    /* Register circ3 with tok2 again but for different purpose */
+    hs_circuitmap_register_intro_circ_v2_relay_side(circ3, tok2);
+
+    /* Check that the getters work */
+    tt_ptr_op(circ2, OP_EQ,
+              hs_circuitmap_get_intro_circ_v2_service_side(tok2));
+    tt_ptr_op(circ3, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
+
+    /* Register circ4 with tok2: it should override circ2 */
+    hs_circuitmap_register_intro_circ_v2_service_side(circ4, tok2);
+
+    /* check that relay-side getters don't work */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+
+    /* Check that the getter returns circ4; the last circuit registered with
+     * that token. */
+    tt_ptr_op(circ4, OP_EQ,
+              hs_circuitmap_get_intro_circ_v2_service_side(tok2));
+  }
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ1));
+  circuit_free_(TO_CIRCUIT(circ2));
+  circuit_free_(TO_CIRCUIT(circ3));
+  circuit_free_(TO_CIRCUIT(circ4));
+}
+
 struct testcase_t circuitlist_tests[] = {
   { "maps", test_clist_maps, TT_FORK, NULL, NULL },
   { "rend_token_maps", test_rend_token_maps, TT_FORK, NULL, NULL },
   { "pick_circid", test_pick_circid, TT_FORK, NULL, NULL },
+  { "hs_circuitmap_isolation", test_hs_circuitmap_isolation,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c
index 1ffa17247d..c81d53ae51 100644
--- a/src/test/test_circuitmux.c
+++ b/src/test/test_circuitmux.c
@@ -1,16 +1,20 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
 #define CIRCUITMUX_PRIVATE
+#define CIRCUITMUX_EWMA_PRIVATE
 #define RELAY_PRIVATE
 #include "or.h"
 #include "channel.h"
 #include "circuitmux.h"
+#include "circuitmux_ewma.h"
 #include "relay.h"
 #include "scheduler.h"
 #include "test.h"
 
+#include <math.h>
+
 /* XXXX duplicated function from test_circuitlist.c */
 static channel_t *
 new_fake_channel(void)
@@ -45,12 +49,13 @@ test_cmux_destroy_cell_queue(void *arg)
   cmux = circuitmux_alloc();
   tt_assert(cmux);
   ch = new_fake_channel();
+  circuitmux_set_policy(cmux, &ewma_policy);
   ch->has_queued_writes = has_queued_writes;
   ch->wide_circ_ids = 1;
 
   circ = circuitmux_get_first_active_circuit(cmux, &cq);
-  tt_assert(!circ);
-  tt_assert(!cq);
+  tt_ptr_op(circ, OP_EQ, NULL);
+  tt_ptr_op(cq, OP_EQ, NULL);
 
   circuitmux_append_destroy_cell(ch, cmux, 100, 10);
   circuitmux_append_destroy_cell(ch, cmux, 190, 6);
@@ -59,7 +64,7 @@ test_cmux_destroy_cell_queue(void *arg)
   tt_int_op(circuitmux_num_cells(cmux), OP_EQ, 3);
 
   circ = circuitmux_get_first_active_circuit(cmux, &cq);
-  tt_assert(!circ);
+  tt_ptr_op(circ, OP_EQ, NULL);
   tt_assert(cq);
 
   tt_int_op(cq->n, OP_EQ, 3);
@@ -77,8 +82,50 @@ test_cmux_destroy_cell_queue(void *arg)
   tor_free(dc);
 }
 
+static void
+test_cmux_compute_ticks(void *arg)
+{
+  const int64_t NS_PER_S = 1000 * 1000 * 1000;
+  const int64_t START_NS = U64_LITERAL(1217709000)*NS_PER_S;
+  int64_t now;
+  double rem;
+  unsigned tick;
+  (void)arg;
+  circuitmux_ewma_free_all();
+  monotime_enable_test_mocking();
+
+  monotime_coarse_set_mock_time_nsec(START_NS);
+  cell_ewma_initialize_ticks();
+  const unsigned tick_zero = cell_ewma_get_current_tick_and_fraction(&rem);
+  tt_double_op(rem, OP_GT, -1e-9);
+  tt_double_op(rem, OP_LT, 1e-9);
+
+  /* 1.5 second later and we should still be in the same tick. */
+  now = START_NS + NS_PER_S + NS_PER_S/2;
+  monotime_coarse_set_mock_time_nsec(now);
+  tick = cell_ewma_get_current_tick_and_fraction(&rem);
+  tt_uint_op(tick, OP_EQ, tick_zero);
+#ifdef USING_32BIT_MSEC_HACK
+  const double tolerance = .0005;
+#else
+  const double tolerance = .00000001;
+#endif
+  tt_double_op(fabs(rem - .15), OP_LT, tolerance);
+
+  /* 25 second later and we should be in another tick. */
+  now = START_NS + NS_PER_S * 25;
+  monotime_coarse_set_mock_time_nsec(now);
+  tick = cell_ewma_get_current_tick_and_fraction(&rem);
+  tt_uint_op(tick, OP_EQ, tick_zero + 2);
+  tt_double_op(fabs(rem - .5), OP_LT, tolerance);
+
+ done:
+  ;
+}
+
 struct testcase_t circuitmux_tests[] = {
   { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL },
+  { "compute_ticks", test_cmux_compute_ticks, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c
new file mode 100644
index 0000000000..8ebef659ca
--- /dev/null
+++ b/src/test/test_circuitstats.c
@@ -0,0 +1,201 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITSTATS_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CHANNEL_PRIVATE_
+
+#include "or.h"
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "circuitbuild.h"
+#include "circuitstats.h"
+#include "circuituse.h"
+#include "channel.h"
+
+void test_circuitstats_timeout(void *arg);
+void test_circuitstats_hoplen(void *arg);
+origin_circuit_t *subtest_fourhop_circuit(struct timeval, int);
+origin_circuit_t *add_opened_threehop(void);
+origin_circuit_t *build_unopened_fourhop(struct timeval);
+
+int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+
+static int marked_for_close;
+/* Mock function because we are not trying to test the close circuit that does
+ * an awful lot of checks on the circuit object. */
+static void
+mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
+                            const char *file)
+{
+  (void) circ;
+  (void) reason;
+  (void) line;
+  (void) file;
+  marked_for_close = 1;
+  return;
+}
+
+origin_circuit_t *
+add_opened_threehop(void)
+{
+  origin_circuit_t *or_circ = origin_circuit_new();
+  extend_info_t fakehop;
+  memset(&fakehop, 0, sizeof(fakehop));
+
+  TO_CIRCUIT(or_circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN;
+
+  onion_append_hop(&or_circ->cpath, &fakehop);
+  onion_append_hop(&or_circ->cpath, &fakehop);
+  onion_append_hop(&or_circ->cpath, &fakehop);
+
+  or_circ->has_opened = 1;
+  TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
+  TO_CIRCUIT(or_circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  return or_circ;
+}
+
+origin_circuit_t *
+build_unopened_fourhop(struct timeval circ_start_time)
+{
+  origin_circuit_t *or_circ = origin_circuit_new();
+  extend_info_t *fakehop = tor_malloc_zero(sizeof(extend_info_t));
+  memset(fakehop, 0, sizeof(extend_info_t));
+
+  TO_CIRCUIT(or_circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  TO_CIRCUIT(or_circ)->timestamp_began = circ_start_time;
+  TO_CIRCUIT(or_circ)->timestamp_created = circ_start_time;
+
+  or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  or_circ->build_state->desired_path_len = 4;
+
+  onion_append_hop(&or_circ->cpath, fakehop);
+  onion_append_hop(&or_circ->cpath, fakehop);
+  onion_append_hop(&or_circ->cpath, fakehop);
+  onion_append_hop(&or_circ->cpath, fakehop);
+
+  tor_free(fakehop);
+
+  return or_circ;
+}
+
+origin_circuit_t *
+subtest_fourhop_circuit(struct timeval circ_start_time, int should_timeout)
+{
+  origin_circuit_t *or_circ = build_unopened_fourhop(circ_start_time);
+
+  // Now make them open one at a time and call
+  // circuit_build_times_handle_completed_hop();
+  or_circ->cpath->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 0);
+
+  or_circ->cpath->next->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 0);
+
+  // Third hop: We should count it now.
+  or_circ->cpath->next->next->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ,
+            !should_timeout); // 1 if counted, 0 otherwise
+
+  // Fourth hop: Don't double count
+  or_circ->cpath->next->next->next->state = CPATH_STATE_OPEN;
+  circuit_build_times_handle_completed_hop(or_circ);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ,
+            !should_timeout);
+
+ done:
+  return or_circ;
+}
+
+void
+test_circuitstats_hoplen(void *arg)
+{
+  /* Plan:
+   *   0. Test no other opened circs (relaxed timeout)
+   *   1. Check >3 hop circ building w/o timeout
+   *   2. Check >3 hop circs w/ timeouts..
+   */
+  struct timeval circ_start_time;
+  origin_circuit_t *threehop = NULL;
+  origin_circuit_t *fourhop = NULL;
+  (void)arg;
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+
+  circuit_build_times_init(get_circuit_build_times_mutable());
+
+  // Let's set a close_ms to 2X the initial timeout, so we can
+  // test relaxed functionality (which uses the close_ms timeout)
+  get_circuit_build_times_mutable()->close_ms *= 2;
+
+  tor_gettimeofday(&circ_start_time);
+  circ_start_time.tv_sec -= 119; // make us hit "relaxed" cutoff
+
+  // Test 1: Build a fourhop circuit that should get marked
+  // as relaxed and eventually counted by circuit_expire_building
+  // (but not before)
+  fourhop = subtest_fourhop_circuit(circ_start_time, 0);
+  tt_int_op(fourhop->relaxed_timeout, OP_EQ, 0);
+  tt_int_op(marked_for_close, OP_EQ, 0);
+  circuit_expire_building();
+  tt_int_op(marked_for_close, OP_EQ, 0);
+  tt_int_op(fourhop->relaxed_timeout, OP_EQ, 1);
+  TO_CIRCUIT(fourhop)->timestamp_began.tv_sec -= 119;
+  circuit_expire_building();
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+  tt_int_op(marked_for_close, OP_EQ, 1);
+
+  circuit_free_(TO_CIRCUIT(fourhop));
+  circuit_build_times_reset(get_circuit_build_times_mutable());
+
+  // Test 2: Add a threehop circuit for non-relaxed timeouts
+  threehop = add_opened_threehop();
+
+  /* This circuit should not timeout */
+  tor_gettimeofday(&circ_start_time);
+  circ_start_time.tv_sec -= 59;
+  fourhop = subtest_fourhop_circuit(circ_start_time, 0);
+  circuit_expire_building();
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+  tt_int_op(TO_CIRCUIT(fourhop)->purpose, OP_NE,
+            CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+
+  circuit_free_((circuit_t *)fourhop);
+  circuit_build_times_reset(get_circuit_build_times_mutable());
+
+  /* Test 3: This circuit should now time out and get marked as a
+   * measurement circuit, but still get counted (and counted only once)
+   */
+  circ_start_time.tv_sec -= 2;
+  fourhop = subtest_fourhop_circuit(circ_start_time, 0);
+  tt_int_op(TO_CIRCUIT(fourhop)->purpose, OP_EQ,
+            CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+  circuit_expire_building();
+  tt_int_op(get_circuit_build_times()->total_build_times, OP_EQ, 1);
+
+ done:
+  UNMOCK(circuit_mark_for_close_);
+  circuit_free_(TO_CIRCUIT(threehop));
+  circuit_free_(TO_CIRCUIT(fourhop));
+  circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
+}
+
+#define TEST_CIRCUITSTATS(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t circuitstats_tests[] = {
+  TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c
new file mode 100644
index 0000000000..df1b43807f
--- /dev/null
+++ b/src/test/test_circuituse.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITLIST_PRIVATE
+
+#include "or.h"
+#include "test.h"
+#include "test_helpers.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "circuitbuild.h"
+#include "nodelist.h"
+
+static void
+test_circuit_is_available_for_use_ret_false_when_marked_for_close(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->marked_for_close = 1;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_when_timestamp_dirty(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->timestamp_dirty = 1;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_general_purpose(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_general_origin(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_origin_purpose(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_unusable_for_new_conns(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ));
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_returns_false_for_onehop_tunnel(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->onehop_tunnel = 1;
+
+  tt_int_op(0, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_returns_true_for_clean_circuit(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->onehop_tunnel = 0;
+
+  tt_int_op(1, OP_EQ, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static int
+mock_circuit_all_predicted_ports_handled(time_t now,
+                                         int *need_uptime,
+                                         int *need_capacity)
+{
+  (void)now;
+
+  if (need_uptime && need_capacity)
+    return 0;
+  return 1;
+}
+
+static consensus_path_type_t
+mock_router_have_unknown_consensus_path(void)
+{
+  return CONSENSUS_PATH_UNKNOWN;
+}
+
+static consensus_path_type_t
+mock_router_have_exit_consensus_path(void)
+{
+  return CONSENSUS_PATH_EXIT;
+}
+
+static void
+test_needs_exit_circuits_ret_false_for_predicted_ports_and_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 0;
+
+  time_t now = time(NULL);
+  tt_int_op(0, OP_EQ,
+            needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+}
+
+static void
+test_needs_exit_circuits_ret_false_for_non_exit_consensus_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 1;
+  MOCK(router_have_consensus_path, mock_router_have_unknown_consensus_path);
+
+  time_t now = time(NULL);
+  tt_int_op(0, OP_EQ,
+            needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_exit_circuits_ret_true_for_predicted_ports_and_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 1;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+
+  time_t now = time(NULL);
+  tt_int_op(1, OP_EQ,
+            needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_circuits_for_build_ret_false_consensus_path_unknown(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_unknown_consensus_path);
+  tt_int_op(0, OP_EQ, needs_circuits_for_build(0));
+  done: ;
+}
+
+static void
+test_needs_circuits_for_build_ret_false_if_num_less_than_max(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+  tt_int_op(0, OP_EQ, needs_circuits_for_build(13));
+  done:
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_circuits_for_build_returns_true_when_more_are_needed(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+  tt_int_op(1, OP_EQ, needs_circuits_for_build(0));
+  done:
+    UNMOCK(router_have_consensus_path);
+}
+
+struct testcase_t circuituse_tests[] = {
+ { "marked",
+   test_circuit_is_available_for_use_ret_false_when_marked_for_close,
+   TT_FORK, NULL, NULL
+ },
+ { "timestamp",
+   test_circuit_is_available_for_use_ret_false_when_timestamp_dirty,
+   TT_FORK, NULL, NULL
+ },
+ { "non_general",
+   test_circuit_is_available_for_use_ret_false_for_non_general_purpose,
+   TT_FORK, NULL, NULL
+ },
+ { "non_general",
+  test_circuit_is_available_for_use_ret_false_for_non_general_origin,
+   TT_FORK, NULL, NULL
+ },
+ { "origin",
+   test_circuit_is_available_for_use_ret_false_for_non_origin_purpose,
+   TT_FORK, NULL, NULL
+ },
+ { "clean",
+   test_circuit_is_available_for_use_ret_false_unusable_for_new_conns,
+   TT_FORK, NULL, NULL
+ },
+ { "onehop",
+   test_circuit_is_available_for_use_returns_false_for_onehop_tunnel,
+   TT_FORK, NULL, NULL
+ },
+ { "clean_circ",
+   test_circuit_is_available_for_use_returns_true_for_clean_circuit,
+   TT_FORK, NULL, NULL
+ },
+ { "exit_f",
+   test_needs_exit_circuits_ret_false_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "exit_t",
+   test_needs_exit_circuits_ret_true_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "non_exit",
+   test_needs_exit_circuits_ret_false_for_non_exit_consensus_path,
+   TT_FORK, NULL, NULL
+ },
+ { "true",
+   test_needs_exit_circuits_ret_true_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "consensus_path_unknown",
+   test_needs_circuits_for_build_ret_false_consensus_path_unknown,
+   TT_FORK, NULL, NULL
+ },
+ { "less_than_max",
+   test_needs_circuits_for_build_ret_false_if_num_less_than_max,
+   TT_FORK, NULL, NULL
+ },
+ { "more_needed",
+   test_needs_circuits_for_build_returns_true_when_more_are_needed,
+   TT_FORK, NULL, NULL
+ },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 0443cc0b1c..85f69bd626 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define COMPAT_LIBEVENT_PRIVATE
@@ -10,7 +10,6 @@
 #include "compat_libevent.h"
 
 #include <event2/event.h>
-#include <event2/thread.h>
 
 #include "log_test_helpers.h"
 
@@ -122,10 +121,70 @@ test_compat_libevent_header_version(void *ignored)
   (void)0;
 }
 
+/* Test for postloop events */
+
+/* Event callback to increment a counter. */
+static void
+increment_int_counter_cb(periodic_timer_t *timer, void *arg)
+{
+  (void)timer;
+  int *ctr = arg;
+  ++*ctr;
+}
+
+static int activated_counter = 0;
+
+/* Mainloop event callback to activate another mainloop event */
+static void
+activate_event_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  mainloop_event_t **other_event = arg;
+  mainloop_event_activate(*other_event);
+  ++activated_counter;
+}
+
+static void
+test_compat_libevent_postloop_events(void *arg)
+{
+  (void)arg;
+  mainloop_event_t *a = NULL, *b = NULL;
+  periodic_timer_t *timed = NULL;
+
+  tor_libevent_postfork();
+
+  /* If postloop events don't work, then these events will activate one
+   * another ad infinitum and, and the periodic event will never occur. */
+  b = mainloop_event_postloop_new(activate_event_cb, &a);
+  a = mainloop_event_postloop_new(activate_event_cb, &b);
+
+  int counter = 0;
+  struct timeval fifty_ms = { 0, 10 * 1000 };
+  timed = periodic_timer_new(tor_libevent_get_base(), &fifty_ms,
+                             increment_int_counter_cb, &counter);
+
+  mainloop_event_activate(a);
+  int r;
+  do {
+    r = tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
+    if (r == -1)
+      break;
+  } while (counter < 5);
+
+  tt_int_op(activated_counter, OP_GE, 2);
+
+ done:
+  mainloop_event_free(a);
+  mainloop_event_free(b);
+  periodic_timer_free(timed);
+}
+
 struct testcase_t compat_libevent_tests[] = {
   { "logging_callback", test_compat_libevent_logging_callback,
     TT_FORK, NULL, NULL },
   { "header_version", test_compat_libevent_header_version, 0, NULL, NULL },
+  { "postloop_events", test_compat_libevent_postloop_events,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_config.c b/src/test/test_config.c
index 89f9b3e2aa..8662b832b8 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,6 +11,7 @@
 #include "or.h"
 #include "address.h"
 #include "addressmap.h"
+#include "bridges.h"
 #include "circuitmux_ewma.h"
 #include "circuitbuild.h"
 #include "config.h"
@@ -19,12 +20,11 @@
 #include "connection_edge.h"
 #include "test.h"
 #include "util.h"
-#include "address.h"
 #include "connection_or.h"
 #include "control.h"
 #include "cpuworker.h"
 #include "dirserv.h"
-#include "dirvote.h"
+#include "dirauth/dirvote.h"
 #include "dns.h"
 #include "entrynodes.h"
 #include "transports.h"
@@ -41,9 +41,8 @@
 #include "routerlist.h"
 #include "routerset.h"
 #include "statefile.h"
-#include "test.h"
-#include "transports.h"
-#include "util.h"
+
+#include "test_helpers.h"
 
 static void
 test_config_addressmap(void *arg)
@@ -276,7 +275,7 @@ test_config_check_or_create_data_subdir(void *arg)
   tt_assert(!is_private_dir(subpath));
   tt_assert(!check_or_create_data_subdir(subdir));
   tt_assert(is_private_dir(subpath));
-#endif
+#endif /* !defined (_WIN32) */
 
  done:
   rmdir(subpath);
@@ -320,7 +319,7 @@ test_config_write_to_data_subdir(void *arg)
   tt_int_op(mkdir(options->DataDirectory, 0700), OP_EQ, 0);
 #endif
 
-  // Write attempt shoudl fail, if subdirectory doesn't exist.
+  // Write attempt should fail, if subdirectory doesn't exist.
   tt_assert(write_to_data_subdir(subdir, fname, str, NULL));
   tt_assert(! check_or_create_data_subdir(subdir));
 
@@ -365,12 +364,12 @@ good_bridge_line_test(const char *string, const char *test_addrport,
   /* If we were asked to validate a digest, but we did not get a
      digest after parsing, we failed. */
   if (test_digest && tor_digest_is_zero(bridge_line->digest))
-    tt_assert(0);
+    tt_abort();
 
   /* If we were not asked to validate a digest, and we got a digest
      after parsing, we failed again. */
   if (!test_digest && !tor_digest_is_zero(bridge_line->digest))
-    tt_assert(0);
+    tt_abort();
 
   /* If we were asked to validate a digest, and we got a digest after
      parsing, make sure it's correct. */
@@ -384,17 +383,17 @@ good_bridge_line_test(const char *string, const char *test_addrport,
   /* If we were asked to validate a transport name, make sure tha it
      matches with the transport name that was parsed. */
   if (test_transport && !bridge_line->transport_name)
-    tt_assert(0);
+    tt_abort();
   if (!test_transport && bridge_line->transport_name)
-    tt_assert(0);
+    tt_abort();
   if (test_transport)
     tt_str_op(test_transport,OP_EQ, bridge_line->transport_name);
 
   /* Validate the SOCKS argument smartlist. */
   if (test_socks_args && !bridge_line->socks_args)
-    tt_assert(0);
+    tt_abort();
   if (!test_socks_args && bridge_line->socks_args)
-    tt_assert(0);
+    tt_abort();
   if (test_socks_args)
     tt_assert(smartlist_strings_eq(test_socks_args,
                                      bridge_line->socks_args));
@@ -412,7 +411,7 @@ bad_bridge_line_test(const char *string)
   bridge_line_t *bridge_line = parse_bridge_line(string);
   if (bridge_line)
     TT_FAIL(("%s was supposed to fail, but it didn't.", string));
-  tt_assert(!bridge_line);
+  tt_ptr_op(bridge_line, OP_EQ, NULL);
 
  done:
   bridge_line_free(bridge_line);
@@ -520,18 +519,18 @@ test_config_parse_transport_options_line(void *arg)
 
   { /* too small line */
     options_sl = get_options_from_transport_options_line("valley", NULL);
-    tt_assert(!options_sl);
+    tt_ptr_op(options_sl, OP_EQ, NULL);
   }
 
   { /* no k=v values */
     options_sl = get_options_from_transport_options_line("hit it!", NULL);
-    tt_assert(!options_sl);
+    tt_ptr_op(options_sl, OP_EQ, NULL);
   }
 
   { /* correct line, but wrong transport specified */
     options_sl =
       get_options_from_transport_options_line("trebuchet k=v", "rook");
-    tt_assert(!options_sl);
+    tt_ptr_op(options_sl, OP_EQ, NULL);
   }
 
   { /* correct -- no transport specified */
@@ -582,6 +581,22 @@ test_config_parse_transport_options_line(void *arg)
   }
 }
 
+/* Mocks needed for the compute_max_mem_in_queues test */
+static int get_total_system_memory_mock(size_t *mem_out);
+
+static size_t total_system_memory_output = 0;
+static int total_system_memory_return = 0;
+
+static int
+get_total_system_memory_mock(size_t *mem_out)
+{
+  if (! mem_out)
+    return -1;
+
+  *mem_out = total_system_memory_output;
+  return total_system_memory_return;
+}
+
 /* Mocks needed for the transport plugin line test */
 
 static void pt_kickstart_proxy_mock(const smartlist_t *transport_list,
@@ -655,84 +670,85 @@ test_config_parse_transport_plugin_line(void *arg)
 
   /* Bad transport lines - too short */
   r = parse_transport_line(options, "bad", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options, "bad", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options, "bad bad", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options, "bad bad", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* Test transport list parsing */
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
    "transport_1 exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* Bad transport identifiers */
   r = parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* Check SOCKS cases for client transport */
   r = parse_transport_line(options,
       "transport_1 socks4 1.2.3.4:567", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 1, 0);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* Proxy case for server transport */
   r = parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 1, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* Multiple-transport error exit */
   r = parse_transport_line(options,
       "transport_1,transport_2 socks5 1.2.3.4:567", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1,transport_2 proxy 1.2.3.4:567", 1, 1);
+  tt_int_op(r, OP_LT, 0);
   /* No port error exit */
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3.4", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
      "transport_1 proxy 1.2.3.4", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   /* Unparsable address error exit */
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3:6x7", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1 proxy 1.2.3:6x7", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* "Strange {Client|Server}TransportPlugin field" error exit */
   r = parse_transport_line(options,
       "transport_1 foo bar", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1 foo bar", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
 
   /* No sandbox mode error exit */
   tmp = options->Sandbox;
   options->Sandbox = 1;
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 1);
-  tt_assert(r < 0);
+  tt_int_op(r, OP_LT, 0);
   options->Sandbox = tmp;
 
   /*
@@ -744,7 +760,7 @@ test_config_parse_transport_plugin_line(void *arg)
     pt_kickstart_proxy_mock_call_count;
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   tt_assert(pt_kickstart_proxy_mock_call_count ==
       old_pt_kickstart_proxy_mock_call_count + 1);
   UNMOCK(pt_kickstart_proxy);
@@ -752,7 +768,7 @@ test_config_parse_transport_plugin_line(void *arg)
   /* This one hits a log line in the !validate_only case only */
   r = parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 0, 1);
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
 
   /* Check mocked client transport cases */
   MOCK(pt_kickstart_proxy, pt_kickstart_proxy_mock);
@@ -770,7 +786,7 @@ test_config_parse_transport_plugin_line(void *arg)
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /* transport_is_needed() should have been called */
   tt_assert(transport_is_needed_mock_call_count ==
       old_transport_is_needed_mock_call_count + 1);
@@ -794,7 +810,7 @@ test_config_parse_transport_plugin_line(void *arg)
   r = parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /*
    * transport_is_needed() and pt_kickstart_proxy() should have been
    * called.
@@ -818,7 +834,7 @@ test_config_parse_transport_plugin_line(void *arg)
   r = parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 0, 0);
   /* Should have succeeded */
-  tt_assert(r == 0);
+  tt_int_op(r, OP_EQ, 0);
   /*
    * transport_is_needed() and transport_add_from_config() should have
    * been called.
@@ -851,9 +867,23 @@ static void
 test_config_fix_my_family(void *arg)
 {
   char *err = NULL;
-  const char *family = "$1111111111111111111111111111111111111111, "
-                       "1111111111111111111111111111111111111112, "
-                       "$1111111111111111111111111111111111111113";
+  config_line_t *family = tor_malloc_zero(sizeof(config_line_t));
+  family->key = tor_strdup("MyFamily");
+  family->value = tor_strdup("$1111111111111111111111111111111111111111, "
+                             "1111111111111111111111111111111111111112, "
+                             "$1111111111111111111111111111111111111113");
+
+  config_line_t *family2 = tor_malloc_zero(sizeof(config_line_t));
+  family2->key = tor_strdup("MyFamily");
+  family2->value = tor_strdup("1111111111111111111111111111111111111114");
+
+  config_line_t *family3 = tor_malloc_zero(sizeof(config_line_t));
+  family3->key = tor_strdup("MyFamily");
+  family3->value = tor_strdup("$1111111111111111111111111111111111111115");
+
+  family->next = family2;
+  family2->next = family3;
+  family3->next = NULL;
 
   or_options_t* options = options_new();
   or_options_t* defaults = options_new();
@@ -861,7 +891,7 @@ test_config_fix_my_family(void *arg)
 
   options_init(options);
   options_init(defaults);
-  options->MyFamily = tor_strdup(family);
+  options->MyFamily_lines = family;
 
   options_validate(NULL, options, defaults, 0, &err) ;
 
@@ -869,18 +899,23 @@ test_config_fix_my_family(void *arg)
     TT_FAIL(("options_validate failed: %s", err));
   }
 
-  tt_str_op(options->MyFamily,OP_EQ,
-                                "$1111111111111111111111111111111111111111, "
-                                "$1111111111111111111111111111111111111112, "
-                                "$1111111111111111111111111111111111111113");
-
-  done:
-    if (err != NULL) {
-      tor_free(err);
-    }
+  const char *valid[] = { "$1111111111111111111111111111111111111111",
+                          "$1111111111111111111111111111111111111112",
+                          "$1111111111111111111111111111111111111113",
+                          "$1111111111111111111111111111111111111114",
+                          "$1111111111111111111111111111111111111115" };
+  int ret_size = 0;
+  config_line_t *ret;
+  for (ret = options->MyFamily; ret && ret_size < 5; ret = ret->next) {
+    tt_str_op(ret->value, OP_EQ, valid[ret_size]);
+    ret_size++;
+  }
+  tt_int_op(ret_size, OP_EQ, 5);
 
-    or_options_free(options);
-    or_options_free(defaults);
+ done:
+  tor_free(err);
+  or_options_free(options);
+  or_options_free(defaults);
 }
 
 static int n_hostname_01010101 = 0;
@@ -1117,7 +1152,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(retval == 0);
-  tt_want_str_op(method_used,==,"CONFIGURED");
+  tt_want_str_op(method_used,OP_EQ,"CONFIGURED");
   tt_want(hostname_out == NULL);
   tt_assert(resolved_addr == 0x80348069);
 
@@ -1142,8 +1177,8 @@ test_config_resolve_my_address(void *arg)
 
   tt_want(retval == 0);
   tt_want(n_hostname_01010101 == prev_n_hostname_01010101 + 1);
-  tt_want_str_op(method_used,==,"RESOLVED");
-  tt_want_str_op(hostname_out,==,"www.torproject.org");
+  tt_want_str_op(method_used,OP_EQ,"RESOLVED");
+  tt_want_str_op(hostname_out,OP_EQ,"www.torproject.org");
   tt_assert(resolved_addr == 0x01010101);
 
   UNMOCK(tor_lookup_hostname);
@@ -1174,8 +1209,8 @@ test_config_resolve_my_address(void *arg)
   tt_want(retval == 0);
   tt_want(n_gethostname_replacement == prev_n_gethostname_replacement + 1);
   tt_want(n_hostname_01010101 == prev_n_hostname_01010101 + 1);
-  tt_want_str_op(method_used,==,"GETHOSTNAME");
-  tt_want_str_op(hostname_out,==,"onionrouter!");
+  tt_want_str_op(method_used,OP_EQ,"GETHOSTNAME");
+  tt_want_str_op(hostname_out,OP_EQ,"onionrouter!");
   tt_assert(resolved_addr == 0x01010101);
 
   UNMOCK(tor_gethostname);
@@ -1197,7 +1232,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(resolved_addr == 0);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   tor_free(options->Address);
   tor_free(hostname_out);
@@ -1219,7 +1254,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_hostname_failure == prev_n_hostname_failure + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_lookup_hostname);
 
@@ -1240,7 +1275,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_gethostname_failure == prev_n_gethostname_failure + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
   tor_free(hostname_out);
@@ -1263,11 +1298,11 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(retval == 0);
-  tt_want_int_op(n_gethostname_replacement, ==,
+  tt_want_int_op(n_gethostname_replacement, OP_EQ,
                  prev_n_gethostname_replacement + 1);
-  tt_want_int_op(n_get_interface_address, ==,
+  tt_want_int_op(n_get_interface_address, OP_EQ,
                  prev_n_get_interface_address + 1);
-  tt_want_str_op(method_used,==,"INTERFACE");
+  tt_want_str_op(method_used,OP_EQ,"INTERFACE");
   tt_want(hostname_out == NULL);
   tt_assert(resolved_addr == 0x08080808);
 
@@ -1293,7 +1328,7 @@ test_config_resolve_my_address(void *arg)
           prev_n_get_interface_address_failure + 1);
   tt_want(n_gethostname_replacement ==
           prev_n_gethostname_replacement + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(get_interface_address);
   tor_free(hostname_out);
@@ -1323,7 +1358,7 @@ test_config_resolve_my_address(void *arg)
   tt_want(n_hostname_failure == prev_n_hostname_failure + 1);
   tt_want(n_gethostname_replacement == prev_n_gethostname_replacement + 1);
   tt_want(retval == 0);
-  tt_want_str_op(method_used,==,"INTERFACE");
+  tt_want_str_op(method_used,OP_EQ,"INTERFACE");
   tt_assert(resolved_addr == 0x09090909);
 
   UNMOCK(tor_lookup_hostname);
@@ -1353,7 +1388,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_hostname_failure == prev_n_hostname_failure + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
   UNMOCK(tor_lookup_hostname);
@@ -1368,7 +1403,7 @@ test_config_resolve_my_address(void *arg)
    *      if running on.
    *   3. Hostname from previous step cannot be converted to
    *      address by using tor_inet_aton() function.
-   *   4. However, tor_lookup_hostname() succeds in resolving the
+   *   4. However, tor_lookup_hostname() succeeds in resolving the
    *      hostname from step 2.
    *   5. Unfortunately, tor_addr_is_internal() deems this address
    *      to be internal.
@@ -1397,9 +1432,9 @@ test_config_resolve_my_address(void *arg)
   tt_want(n_hostname_localhost == prev_n_hostname_localhost + 1);
   tt_want(n_get_interface_address6 == prev_n_get_interface_address6 + 1);
 
-  tt_str_op(method_used,==,"INTERFACE");
-  tt_assert(!hostname_out);
-  tt_assert(retval == 0);
+  tt_str_op(method_used,OP_EQ,"INTERFACE");
+  tt_ptr_op(hostname_out, OP_EQ, NULL);
+  tt_int_op(retval, OP_EQ, 0);
 
   /*
    * CASE 11b:
@@ -1424,7 +1459,7 @@ test_config_resolve_my_address(void *arg)
   tt_want(n_get_interface_address6_failure ==
           prev_n_get_interface_address6_failure + 1);
 
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
   UNMOCK(tor_lookup_hostname);
@@ -1453,7 +1488,7 @@ test_config_resolve_my_address(void *arg)
                               &method_used,&hostname_out);
 
   tt_want(n_gethostname_localhost == prev_n_gethostname_localhost + 1);
-  tt_assert(retval == -1);
+  tt_int_op(retval, OP_EQ, -1);
 
   UNMOCK(tor_gethostname);
 
@@ -1488,18 +1523,18 @@ test_config_adding_trusted_dir_server(void *arg)
                               NULL, V3_DIRINFO, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(get_n_authorities(V3_DIRINFO) == 1);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 1);
+  tt_int_op(get_n_authorities(V3_DIRINFO), OP_EQ, 1);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 1);
 
   /* create a trusted ds with an IPv6 address and port */
   rv = tor_addr_port_parse(LOG_WARN, "[::1]:9061", &ipv6.addr, &ipv6.port, -1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, &ipv6, digest,
                               NULL, V3_DIRINFO, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(get_n_authorities(V3_DIRINFO) == 2);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 2);
+  tt_int_op(get_n_authorities(V3_DIRINFO), OP_EQ, 2);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 2);
 
  done:
   clear_dir_servers();
@@ -1521,21 +1556,21 @@ test_config_adding_fallback_dir_server(void *arg)
   routerlist_free_all();
 
   rv = tor_addr_parse(&ipv4, "127.0.0.1");
-  tt_assert(rv == AF_INET);
+  tt_int_op(rv, OP_EQ, AF_INET);
 
   /* create a trusted ds without an IPv6 address and port */
   ds = fallback_dir_server_new(&ipv4, 9059, 9060, NULL, digest, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 1);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 1);
 
   /* create a trusted ds with an IPv6 address and port */
   rv = tor_addr_port_parse(LOG_WARN, "[::1]:9061", &ipv6.addr, &ipv6.port, -1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   ds = fallback_dir_server_new(&ipv4, 9059, 9060, &ipv6, digest, 1.0);
   tt_assert(ds);
   dir_server_add(ds);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 2);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 2);
 
  done:
   clear_dir_servers();
@@ -1566,14 +1601,14 @@ test_config_parsing_trusted_dir_server(void *arg)
   rv = parse_dir_authority_line(TEST_DIR_AUTH_LINE_START
                                 TEST_DIR_AUTH_LINE_END,
                                 V3_DIRINFO, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* parse a trusted dir server with an IPv6 address and port */
   rv = parse_dir_authority_line(TEST_DIR_AUTH_LINE_START
                                 TEST_DIR_AUTH_IPV6_FLAG
                                 TEST_DIR_AUTH_LINE_END,
                                 V3_DIRINFO, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* Since we are only validating, there is no cleanup. */
  done:
@@ -1601,13 +1636,13 @@ test_config_parsing_fallback_dir_server(void *arg)
 
   /* parse a trusted dir server without an IPv6 address and port */
   rv = parse_dir_fallback_line(TEST_DIR_FALLBACK_LINE, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* parse a trusted dir server with an IPv6 address and port */
   rv = parse_dir_fallback_line(TEST_DIR_FALLBACK_LINE
                                TEST_DIR_FALLBACK_IPV6_FLAG,
                                1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* Since we are only validating, there is no cleanup. */
  done:
@@ -1627,8 +1662,8 @@ test_config_adding_default_trusted_dir_servers(void *arg)
 
   /* Assume we only have one bridge authority */
   add_default_trusted_dir_authorities(BRIDGE_DIRINFO);
-  tt_assert(get_n_authorities(BRIDGE_DIRINFO) == 1);
-  tt_assert(smartlist_len(router_get_fallback_dir_servers()) == 1);
+  tt_int_op(get_n_authorities(BRIDGE_DIRINFO), OP_EQ, 1);
+  tt_int_op(smartlist_len(router_get_fallback_dir_servers()), OP_EQ, 1);
 
   /* Assume we have eight V3 authorities */
   add_default_trusted_dir_authorities(V3_DIRINFO);
@@ -1816,7 +1851,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 1);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 1);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -1835,7 +1870,7 @@ test_config_adding_dir_servers(void *arg)
                          1 : 0)
                         );
       /* If we have no default bridge authority, something has gone wrong */
-      tt_assert(n_default_alt_bridge_authority >= 1);
+      tt_int_op(n_default_alt_bridge_authority, OP_GE, 1);
 
       /* Count v3 Authorities */
       SMARTLIST_FOREACH(fallback_servers,
@@ -1847,7 +1882,7 @@ test_config_adding_dir_servers(void *arg)
                          1 : 0)
                         );
       /* If we have no default authorities, something has gone really wrong */
-      tt_assert(n_default_alt_dir_authority >= 1);
+      tt_int_op(n_default_alt_dir_authority, OP_GE, 1);
 
       /* Calculate Fallback Directory Count */
       n_default_fallback_dir = (smartlist_len(fallback_servers) -
@@ -1857,7 +1892,7 @@ test_config_adding_dir_servers(void *arg)
        * or some authorities aren't being added as fallback directories.
        * (networkstatus_consensus_can_use_extra_fallbacks depends on all
        * authorities being fallback directories.) */
-      tt_assert(n_default_fallback_dir >= 0);
+      tt_int_op(n_default_fallback_dir, OP_GE, 0);
     }
   }
 
@@ -1898,7 +1933,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -1907,7 +1942,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* D0, (No B1), (No A2) */
-      tt_assert(smartlist_len(dir_servers) == 1);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 1);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -1919,7 +1954,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -1931,7 +1966,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -1943,14 +1978,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* D0, (No B1), (No A2), Custom Fallback */
-      tt_assert(smartlist_len(fallback_servers) == 2);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 2);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -1962,7 +1997,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -1974,7 +2009,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -1986,7 +2021,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -1998,7 +2033,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2010,7 +2045,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2039,7 +2074,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we just have the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 0);
@@ -2048,7 +2083,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* D0, (No B1), (No A2) */
-      tt_assert(smartlist_len(dir_servers) == 1);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 1);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2060,7 +2095,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2072,7 +2107,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2084,14 +2119,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* D0, (No B1), (No A2), (No Fallback) */
-      tt_assert(smartlist_len(fallback_servers) == 1);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 1);
 
       /* DirAuthority - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2103,7 +2138,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 1);
+      tt_int_op(found_D0, OP_EQ, 1);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2115,7 +2150,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2127,7 +2162,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2139,7 +2174,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2151,7 +2186,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2180,7 +2215,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2189,7 +2224,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* (No D0), B1, A2 */
-      tt_assert(smartlist_len(dir_servers) == 2);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 2);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2201,7 +2236,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2213,7 +2248,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2225,14 +2260,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* (No D0), B1, A2, Custom Fallback */
-      tt_assert(smartlist_len(fallback_servers) == 3);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 3);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2244,7 +2279,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2256,7 +2291,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2268,7 +2303,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2280,7 +2315,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2292,7 +2327,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2322,7 +2357,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 0);
@@ -2331,7 +2366,7 @@ test_config_adding_dir_servers(void *arg)
       /* trusted_dir_servers */
       const smartlist_t *dir_servers = router_get_trusted_dir_servers();
       /* (No D0), B1, A2 */
-      tt_assert(smartlist_len(dir_servers) == 2);
+      tt_int_op(smartlist_len(dir_servers), OP_EQ, 2);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2343,7 +2378,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2355,7 +2390,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2367,14 +2402,14 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
     }
 
     {
       /* fallback_dir_servers */
       const smartlist_t *fallback_servers = router_get_fallback_dir_servers();
       /* (No D0), B1, A2, (No Fallback) */
-      tt_assert(smartlist_len(fallback_servers) == 2);
+      tt_int_op(smartlist_len(fallback_servers), OP_EQ, 2);
 
       /* (No DirAuthority) - D0 - dir_port: 60090 */
       int found_D0 = 0;
@@ -2386,7 +2421,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2398,7 +2433,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2410,7 +2445,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2422,7 +2457,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2434,7 +2469,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
     }
   }
 
@@ -2474,7 +2509,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2495,7 +2530,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2507,7 +2542,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2519,7 +2554,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2545,7 +2580,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2557,7 +2592,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2569,7 +2604,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2581,7 +2616,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2593,7 +2628,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2628,7 +2663,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 1);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 1);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2649,7 +2684,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2661,7 +2696,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2673,7 +2708,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2699,7 +2734,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* AlternateBridgeAuthority - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2711,7 +2746,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 1);
+      tt_int_op(found_B1, OP_EQ, 1);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2723,7 +2758,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2735,7 +2770,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* Default FallbackDir - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2747,7 +2782,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 1);
+      tt_int_op(found_default_fallback, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default v3 non-Bridge directory authorities, so let's assume that
@@ -2791,7 +2826,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -2813,7 +2848,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2825,7 +2860,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2837,7 +2872,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -2864,7 +2899,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2876,7 +2911,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2888,7 +2923,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -2900,7 +2935,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -2912,7 +2947,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -2948,7 +2983,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we just have the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 0);
@@ -2971,7 +3006,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -2983,7 +3018,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -2995,7 +3030,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -3022,7 +3057,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3034,7 +3069,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* AlternateDirAuthority - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3046,7 +3081,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 1);
+      tt_int_op(found_A2, OP_EQ, 1);
 
       /* (No Custom FallbackDir) - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -3058,7 +3093,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -3070,7 +3105,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge authorities (except for hard-coding tonga's details),
@@ -3114,7 +3149,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must not have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 0);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 0);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -3136,7 +3171,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3148,7 +3183,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3160,7 +3195,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, so let's assume that
@@ -3187,7 +3222,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3199,7 +3234,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3211,7 +3246,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -3223,7 +3258,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 1);
+      tt_int_op(found_non_default_fallback, OP_EQ, 1);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -3235,7 +3270,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 0);
+      tt_int_op(found_default_fallback, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, so let's assume that
@@ -3277,7 +3312,7 @@ test_config_adding_dir_servers(void *arg)
     /* check outcome */
 
     /* we must have added the default fallback dirs */
-    tt_assert(n_add_default_fallback_dir_servers_known_default == 1);
+    tt_int_op(n_add_default_fallback_dir_servers_known_default, OP_EQ, 1);
 
     /* we have more fallbacks than just the authorities */
     tt_assert(networkstatus_consensus_can_use_extra_fallbacks(options) == 1);
@@ -3299,7 +3334,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3311,7 +3346,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3323,7 +3358,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, so let's assume that
@@ -3350,7 +3385,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60090 ?
                          1 : 0)
                         );
-      tt_assert(found_D0 == 0);
+      tt_int_op(found_D0, OP_EQ, 0);
 
       /* (No AlternateBridgeAuthority) - B1 - dir_port: 60091 */
       int found_B1 = 0;
@@ -3362,7 +3397,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60091 ?
                          1 : 0)
                         );
-      tt_assert(found_B1 == 0);
+      tt_int_op(found_B1, OP_EQ, 0);
 
       /* (No AlternateDirAuthority) - A2 - dir_port: 60092 */
       int found_A2 = 0;
@@ -3374,7 +3409,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60092 ?
                          1 : 0)
                         );
-      tt_assert(found_A2 == 0);
+      tt_int_op(found_A2, OP_EQ, 0);
 
       /* Custom FallbackDir - No Nickname - dir_port: 60093 */
       int found_non_default_fallback = 0;
@@ -3386,7 +3421,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60093 ?
                          1 : 0)
                         );
-      tt_assert(found_non_default_fallback == 0);
+      tt_int_op(found_non_default_fallback, OP_EQ, 0);
 
       /* (No Default FallbackDir) - No Nickname - dir_port: 60099 */
       int found_default_fallback = 0;
@@ -3398,7 +3433,7 @@ test_config_adding_dir_servers(void *arg)
                         (ds->dir_port == 60099 ?
                          1 : 0)
                         );
-      tt_assert(found_default_fallback == 1);
+      tt_int_op(found_default_fallback, OP_EQ, 1);
 
       /* There's no easy way of checking that we have included all the
        * default Bridge & V3 Directory authorities, and the default
@@ -3455,7 +3490,7 @@ test_config_default_dir_servers(void *arg)
   opts = NULL;
 
   /* assume a release will never go out with less than 7 authorities */
-  tt_assert(trusted_count >= 7);
+  tt_int_op(trusted_count, OP_GE, 7);
   /* if we disable the default fallbacks, there must not be any extra */
   tt_assert(fallback_count == trusted_count);
 
@@ -3468,7 +3503,7 @@ test_config_default_dir_servers(void *arg)
   opts = NULL;
 
   /* assume a release will never go out with less than 7 authorities */
-  tt_assert(trusted_count >= 7);
+  tt_int_op(trusted_count, OP_GE, 7);
   /* XX/teor - allow for default fallbacks to be added without breaking
    * the unit tests. Set a minimum fallback count once the list is stable. */
   tt_assert(fallback_count >= trusted_count);
@@ -3537,18 +3572,18 @@ test_config_directory_fetch(void *arg)
   options->ClientOnly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* Bridge Clients can use multiple directory mirrors for bootstrap */
   memset(options, 0, sizeof(or_options_t));
   options->UseBridges = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* Bridge Relays (Bridges) must act like clients, and use multiple
    * directory mirrors for bootstrap */
@@ -3557,9 +3592,9 @@ test_config_directory_fetch(void *arg)
   options->ORPort_set = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* Clients set to FetchDirInfoEarly must fetch it from the authorities,
    * but can use multiple authorities for bootstrap */
@@ -3567,9 +3602,9 @@ test_config_directory_fetch(void *arg)
   options->FetchDirInfoEarly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 1);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 1);
 
   /* OR servers only fetch the consensus from the authorities when they don't
    * know their own address, but never use multiple directories for bootstrap
@@ -3580,16 +3615,16 @@ test_config_directory_fetch(void *arg)
   mock_router_pick_published_address_result = -1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   /* Exit OR servers only fetch the consensus from the authorities when they
    * refuse unknown exits, but never use multiple directories for bootstrap
@@ -3607,17 +3642,17 @@ test_config_directory_fetch(void *arg)
   options->RefuseUnknownExits = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   options->RefuseUnknownExits = 0;
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   /* Dir servers fetch the consensus from the authorities, unless they are not
    * advertising themselves (hibernating) or have no routerinfo or are not
@@ -3636,26 +3671,26 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 0;
   routerinfo.dir_port = 1;
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 1;
   mock_router_get_my_routerinfo_result = NULL;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 1;
   routerinfo.dir_port = 0;
@@ -3663,9 +3698,9 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 0);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
   mock_advertised_server_mode_result = 1;
   routerinfo.dir_port = 1;
@@ -3673,9 +3708,9 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_assert(directory_fetches_from_authorities(options) == 1);
-  tt_assert(networkstatus_consensus_can_use_multiple_directories(options)
-            == 0);
+  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
+            OP_EQ, 0);
 
  done:
   tor_free(options);
@@ -3722,119 +3757,119 @@ test_config_port_cfg_line_extract_addrport(void *arg)
 
   tt_int_op(port_cfg_line_extract_addrport("", &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "");;
+  tt_str_op(a, OP_EQ, "");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("hello", &a, &unixy, &rest),
             OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "hello");;
+  tt_str_op(a, OP_EQ, "hello");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(" flipperwalt gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(" flipperwalt \t gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("flipperwalt \t gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:flipperwalt \t gersplut",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "flipperwalt");;
+  tt_str_op(a, OP_EQ, "flipperwalt");
   tt_str_op(rest, OP_EQ, "gersplut");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:lolol ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(" unix:lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("foobar:lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, "foobar:lolol");;
+  tt_str_op(a, OP_EQ, "foobar:lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport(":lolol",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 0);
-  tt_str_op(a, OP_EQ, ":lolol");;
+  tt_str_op(a, OP_EQ, ":lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lolol\"",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lolol\" ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lolol\" foo ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lolol");;
+  tt_str_op(a, OP_EQ, "lolol");
   tt_str_op(rest, OP_EQ, "foo ");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lol ol\" foo ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lol ol");;
+  tt_str_op(a, OP_EQ, "lol ol");
   tt_str_op(rest, OP_EQ, "foo ");
   tor_free(a);
 
   tt_int_op(port_cfg_line_extract_addrport("unix:\"lol\\\" ol\" foo ",
                                            &a, &unixy, &rest), OP_EQ, 0);
   tt_int_op(unixy, OP_EQ, 1);
-  tt_str_op(a, OP_EQ, "lol\" ol");;
+  tt_str_op(a, OP_EQ, "lol\" ol");
   tt_str_op(rest, OP_EQ, "foo ");
   tor_free(a);
 
@@ -3861,144 +3896,6 @@ mock_config_line(const char *key, const char *val)
 }
 
 static void
-test_config_parse_port_config__listenaddress(void *data)
-{
-  (void)data;
-  int ret;
-  config_line_t *config_listen_address = NULL, *config_listen_address2 = NULL,
-    *config_listen_address3 = NULL;
-  config_line_t *config_port1 = NULL, *config_port2 = NULL,
-    *config_port3 = NULL, *config_port4 = NULL, *config_port5 = NULL;
-  smartlist_t *slout = NULL;
-  port_cfg_t *port_cfg = NULL;
-
-  // Test basic invocation with no arguments
-  ret = parse_port_config(NULL, NULL, NULL, NULL, 0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Setup some test data
-  config_listen_address = mock_config_line("DNSListenAddress", "127.0.0.1");
-  config_listen_address2 = mock_config_line("DNSListenAddress", "x$$$:::345");
-  config_listen_address3 = mock_config_line("DNSListenAddress",
-                                            "127.0.0.1:1442");
-  config_port1 = mock_config_line("DNSPort", "42");
-  config_port2 = mock_config_line("DNSPort", "43");
-  config_port1->next = config_port2;
-  config_port3 = mock_config_line("DNSPort", "auto");
-  config_port4 = mock_config_line("DNSPort", "55542");
-  config_port5 = mock_config_line("DNSPort", "666777");
-
-  // Test failure when we have a ListenAddress line and several
-  // Port lines for the same portname
-  ret = parse_port_config(NULL, config_port1, config_listen_address, "DNS", 0,
-                          NULL, 0, 0);
-
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test case when we have a listen address, no default port and allow
-  // spurious listen address lines
-  ret = parse_port_config(NULL, NULL, config_listen_address, "DNS", 0, NULL,
-                          0, CL_PORT_ALLOW_EXTRA_LISTENADDR);
-  tt_int_op(ret, OP_EQ, 1);
-
-  // Test case when we have a listen address, no default port but doesn't
-  // allow spurious listen address lines
-  ret = parse_port_config(NULL, NULL, config_listen_address, "DNS", 0, NULL,
-                          0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test case when we have a listen address, and a port that points to auto,
-  // should use the AUTO port
-  slout = smartlist_new();
-  ret = parse_port_config(slout, config_port3, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 1);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
-  tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
-
-  // Test when we have a listen address and a custom port
-  ret = parse_port_config(slout, config_port4, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 2);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 1);
-  tt_int_op(port_cfg->port, OP_EQ, 55542);
-
-  // Test when we have a listen address and an invalid custom port
-  ret = parse_port_config(slout, config_port5, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test we get a server port configuration when asked for it
-  ret = parse_port_config(slout, NULL, config_listen_address, "DNS", 0, NULL,
-                          123, CL_PORT_SERVER_OPTIONS);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 4);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 2);
-  tt_int_op(port_cfg->port, OP_EQ, 123);
-  tt_int_op(port_cfg->server_cfg.no_listen, OP_EQ, 1);
-  tt_int_op(port_cfg->server_cfg.bind_ipv4_only, OP_EQ, 1);
-
-  // Test an invalid ListenAddress configuration
-  ret = parse_port_config(NULL, NULL, config_listen_address2, "DNS", 0, NULL,
-                          222, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test default to the port in the listen address if available
-  ret = parse_port_config(slout, config_port2, config_listen_address3, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 5);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 4);
-  tt_int_op(port_cfg->port, OP_EQ, 1442);
-
-  // Test we work correctly without an out, but with a listen address
-  // and a port
-  ret = parse_port_config(NULL, config_port2, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal control
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_CONTROL_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal ext or listener
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_EXT_OR_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal other
-  SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
-  smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          0, NULL, 0, CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal control without an out
-  ret = parse_port_config(NULL, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_CONTROL_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
- done:
-  config_free_lines(config_listen_address);
-  config_free_lines(config_listen_address2);
-  config_free_lines(config_listen_address3);
-  config_free_lines(config_port1);
-  /* 2 was linked from 1. */
-  config_free_lines(config_port3);
-  config_free_lines(config_port4);
-  config_free_lines(config_port5);
-  if (slout)
-    SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
-  smartlist_free(slout);
-}
-
-static void
 test_config_parse_port_config__ports__no_ports_given(void *data)
 {
   (void)data;
@@ -4009,40 +3906,40 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   slout = smartlist_new();
 
   // Test no defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and out, adds a new port cfg
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
@@ -4053,7 +3950,7 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   // for a unix address
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "/foo/bar/unixdomain",
+  ret = parse_port_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
                           42, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4082,28 +3979,28 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters an empty unix domain specification
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "unix:");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters a unix domain specification but the listener
   // doesn't support domain sockets
   config_port_valid = mock_config_line("DNSPort", "unix:/tmp/foo/bar");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS",
+  ret = parse_port_config(NULL, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test valid unix domain
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0, 0);
 #ifdef _WIN32
   tt_int_op(ret, OP_EQ, -1);
@@ -4118,16 +4015,17 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.dns_request, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 1);
+  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.prefer_ipv6_virtaddr, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test failure if we have no ipv4 and no ipv6 and no onion (DNS only)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("SOCKSPort",
                                          "unix:/tmp/foo/bar NoIPv4Traffic "
+                                         "NoIPv6Traffic "
                                          "NoOnionTraffic");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "SOCKS",
+  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4136,7 +4034,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.1:80 NoDNSRequest");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS",
+  ret = parse_port_config(NULL, config_port_invalid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4147,8 +4045,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.1:80 "
+                                       "NoIPv6Traffic "
                                        "NoIPv4Traffic NoOnionTraffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS",
+  ret = parse_port_config(slout, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4162,8 +4061,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure if we have DNS but no ipv4 and no ipv6
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("SOCKSPort",
+                                         "NoIPv6Traffic "
                                          "unix:/tmp/foo/bar NoIPv4Traffic");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "SOCKS",
+  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4174,8 +4074,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4188,15 +4089,16 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test success with quoted unix: address.
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar\" "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4209,15 +4111,16 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test failure with broken quoted unix: address.
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4227,8 +4130,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"\" "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4239,7 +4143,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "OnionTrafficOnly");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4252,7 +4156,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test success with no ipv4 but take ipv6
   config_free_lines(config_port_valid); config_port_valid = NULL;
@@ -4260,7 +4164,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4271,7 +4175,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test success with both ipv4 and ipv6
   config_free_lines(config_port_valid); config_port_valid = NULL;
@@ -4279,7 +4183,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "IPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4290,33 +4194,33 @@ test_config_parse_port_config__ports__ports_given(void *data)
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
   // Test failure if we specify world writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 WorldWritable");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 GroupWritable");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 RelaxDirModeCheck");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with only a port (this will fail without a default address)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4325,7 +4229,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestPort");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4338,7 +4242,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateDestPorts");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4351,7 +4255,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestAddr");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4364,7 +4268,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4377,7 +4281,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientProtocol");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4390,7 +4294,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientAddr");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4401,7 +4305,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with ignored unknown options
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42 ThisOptionDoesntExist");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4410,7 +4314,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4423,7 +4327,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort",
                                        "42 IPv6Traffic PreferIPv6");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.42", 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4436,7 +4340,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4449,12 +4353,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv6DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
-  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 1);
+  tt_int_op(port_cfg->entry_cfg.cache_ipv4_answers, OP_EQ, 0);
   tt_int_op(port_cfg->entry_cfg.cache_ipv6_answers, OP_EQ, 1);
 
   // Test success with no cache ipv4 DNS
@@ -4462,7 +4366,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoCacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4475,7 +4379,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheDNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4488,7 +4392,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv4Cache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4501,7 +4405,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv6Cache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4514,7 +4418,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseDNSCache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4527,7 +4431,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoPreferIPv6Automap");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4539,7 +4443,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 PreferSOCKSNoAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4554,14 +4458,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("DNSPort", "0");
   config_port_valid = mock_config_line("DNSPort", "42");
   config_port_invalid->next = config_port_valid;
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with warn non-local control
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "Control",
+  ret = parse_port_config(slout, config_port_valid, "Control",
                           CONN_TYPE_CONTROL_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4569,7 +4473,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local listener
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "ExtOR",
+  ret = parse_port_config(slout, config_port_valid, "ExtOR",
                           CONN_TYPE_EXT_OR_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4577,12 +4481,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local other
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test success with warn non-local other without out
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4593,7 +4497,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IPv4Traffic "
                                        "IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0,
                           CL_PORT_TAKES_HOSTNAMES |
                           CL_PORT_NO_STREAM_OPTIONS);
@@ -4601,14 +4505,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
-  tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
+  tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
 
   // Test failure for a SessionGroup argument with invalid value
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=invalid");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4620,7 +4524,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4630,7 +4534,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123 "
                                          "SessionGroup=321");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4639,7 +4543,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 SessionGroup=1111122");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4651,7 +4555,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "0");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
@@ -4661,7 +4565,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "something");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4674,7 +4578,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "auto");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4688,7 +4592,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.122:auto");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4700,8 +4604,10 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure when asked to parse an invalid address followed by auto
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "invalidstuff!!:auto");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
+  UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with parsing both an address and a real port
@@ -4709,7 +4615,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.123:656");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4723,7 +4629,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "something wrong");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4732,7 +4638,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.1.0:123:auto");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4742,7 +4648,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/somewhere");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.46", 0,
                           CL_PORT_DFLT_GROUP_WRITABLE);
 #ifdef _WIN32
@@ -4752,7 +4658,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->is_group_writable, OP_EQ, 1);
-#endif
+#endif /* defined(_WIN32) */
 
  done:
   if (slout)
@@ -4777,7 +4683,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort",
                                        "127.0.0.124:656 NoAdvertise");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4790,7 +4696,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4804,7 +4710,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen "
                                          "NoAdvertise");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4813,7 +4719,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4826,7 +4732,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "[::1]:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4840,7 +4746,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 IPv6Only "
                                          "IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4849,7 +4755,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 unknown");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4860,7 +4766,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.124:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4869,7 +4775,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "[::1]:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4878,7 +4784,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("ORPort", "unix:\"\"");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "ORPort", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "ORPort", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4890,6 +4796,891 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
 }
 
+static void
+test_config_parse_log_severity(void *data)
+{
+  int ret;
+  const char *severity_log_lines[] = {
+    "debug file /tmp/debug.log",
+    "debug\tfile /tmp/debug.log",
+    "[handshake]debug [~net,~mm]info notice stdout",
+    "[handshake]debug\t[~net,~mm]info\tnotice\tstdout",
+    NULL
+  };
+  int i;
+  log_severity_list_t *severity;
+
+  (void) data;
+
+  severity = tor_malloc(sizeof(log_severity_list_t));
+  for (i = 0; severity_log_lines[i]; i++) {
+    memset(severity, 0, sizeof(log_severity_list_t));
+    ret = parse_log_severity_config(&severity_log_lines[i], severity);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  tor_free(severity);
+}
+
+static void
+test_config_include_limit(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrc_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_limit"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrc_path, "%s"PATH_SEPARATOR"torrc", dir);
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               torrc_path);
+  tt_int_op(write_str_to_file(torrc_path, torrc_contents, 0), OP_EQ, 0);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrc_path);
+  tor_free(dir);
+}
+
+static void
+test_config_include_does_not_exist(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_does_not_exist"));
+  char *missing_path = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&missing_path, "%s"PATH_SEPARATOR"missing", dir);
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               missing_path);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(missing_path);
+}
+
+static void
+test_config_include_error_in_included_file(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *dir = tor_strdup(get_fname("test_error_in_included_file"));
+  char *invalid_path = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&invalid_path, "%s"PATH_SEPARATOR"invalid", dir);
+  tt_int_op(write_str_to_file(invalid_path, "unclosed \"", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               invalid_path);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(invalid_path);
+}
+
+static void
+test_config_include_empty_file_folder(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *folder_path = NULL;
+  char *file_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_empty_file_folder"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&folder_path, "%s"PATH_SEPARATOR"empty_dir", dir);
+#ifdef _WIN32
+  tt_int_op(mkdir(folder_path), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(folder_path, 0700), OP_EQ, 0);
+#endif
+  tor_asprintf(&file_path, "%s"PATH_SEPARATOR"empty_file", dir);
+  tt_int_op(write_str_to_file(file_path, "", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n"
+               "%%include %s\n",
+               folder_path, file_path);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(folder_path);
+  tor_free(file_path);
+  tor_free(dir);
+}
+
+#ifndef _WIN32
+static void
+test_config_include_no_permission(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *folder_path = NULL;
+  char *dir = NULL;
+  if (geteuid() == 0)
+    tt_skip();
+
+  dir = tor_strdup(get_fname("test_include_forbidden_folder"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+
+  tor_asprintf(&folder_path, "%s"PATH_SEPARATOR"forbidden_dir", dir);
+  tt_int_op(mkdir(folder_path, 0100), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               folder_path);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,
+                                     &include_used, NULL),
+            OP_EQ, -1);
+  tt_ptr_op(result, OP_EQ, NULL);
+
+ done:
+  config_free_lines(result);
+  tor_free(folder_path);
+  if (dir)
+    chmod(dir, 0700);
+  tor_free(dir);
+}
+#endif
+
+static void
+test_config_include_recursion_before_after(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrc_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_recursion_before_after"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrc_path, "%s"PATH_SEPARATOR"torrc", dir);
+
+  char file_contents[1000];
+  const int limit = MAX_INCLUDE_RECURSION_LEVEL;
+  int i;
+  // Loop backwards so file_contents has the contents of the first file by the
+  // end of the loop
+  for (i = limit; i > 0; i--) {
+    if (i < limit) {
+      tor_snprintf(file_contents, sizeof(file_contents),
+                   "Test %d\n"
+                   "%%include %s%d\n"
+                   "Test %d\n",
+                   i, torrc_path, i + 1, 2 * limit - i);
+    } else {
+      tor_snprintf(file_contents, sizeof(file_contents), "Test %d\n", i);
+    }
+
+    if (i > 1) {
+      char *file_path = NULL;
+      tor_asprintf(&file_path, "%s%d", torrc_path, i);
+      tt_int_op(write_str_to_file(file_path, file_contents, 0), OP_EQ, 0);
+      tor_free(file_path);
+    }
+  }
+
+  int include_used;
+  tt_int_op(config_get_lines_include(file_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 2 * limit - 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_path);
+}
+
+static void
+test_config_include_recursion_after_only(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrc_path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_recursion_after_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrc_path, "%s"PATH_SEPARATOR"torrc", dir);
+
+  char file_contents[1000];
+  const int limit = MAX_INCLUDE_RECURSION_LEVEL;
+  int i;
+  // Loop backwards so file_contents has the contents of the first file by the
+  // end of the loop
+  for (i = limit; i > 0; i--) {
+    int n = (i - limit - 1) * -1;
+    if (i < limit) {
+      tor_snprintf(file_contents, sizeof(file_contents),
+                   "%%include %s%d\n"
+                   "Test %d\n",
+                   torrc_path, i + 1, n);
+    } else {
+      tor_snprintf(file_contents, sizeof(file_contents), "Test %d\n", n);
+    }
+
+    if (i > 1) {
+      char *file_path = NULL;
+      tor_asprintf(&file_path, "%s%d", torrc_path, i);
+      tt_int_op(write_str_to_file(file_path, file_contents, 0), OP_EQ, 0);
+      tor_free(file_path);
+    }
+  }
+
+  int include_used;
+  tt_int_op(config_get_lines_include(file_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, limit);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_path);
+}
+
+static void
+test_config_include_folder_order(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrcd = NULL;
+  char *path = NULL;
+  char *path2 = NULL;
+  char *dir = tor_strdup(get_fname("test_include_folder_order"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  // test that files in subfolders are ignored
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "subfolder");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(path), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(path, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path2, "%s"PATH_SEPARATOR"%s", path, "01_ignore");
+  tt_int_op(write_str_to_file(path2, "ShouldNotSee 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  // test that files starting with . are ignored
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, ".dot");
+  tt_int_op(write_str_to_file(path, "ShouldNotSee 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  // test file order
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "01_1st");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "02_2nd");
+  tt_int_op(write_str_to_file(path, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "aa_3rd");
+  tt_int_op(write_str_to_file(path, "Test 3\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "ab_4th");
+  tt_int_op(write_str_to_file(path, "Test 4\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 4);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(path);
+  tor_free(path2);
+  tor_free(dir);
+}
+
+static void
+test_config_include_path_syntax(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_path_syntax"));
+  char *esc_dir = NULL, *dir_with_pathsep = NULL,
+    *esc_dir_with_pathsep = NULL, *torrc_contents = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  esc_dir = esc_for_log(dir);
+  tor_asprintf(&dir_with_pathsep, "%s%s", dir, PATH_SEPARATOR);
+  esc_dir_with_pathsep = esc_for_log(dir_with_pathsep);
+
+  tor_asprintf(&torrc_contents,
+               "%%include %s\n"
+               "%%include %s%s \n" // space to avoid suppressing newline
+               "%%include %s\n",
+               esc_dir,
+               dir, PATH_SEPARATOR,
+               esc_dir_with_pathsep);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_contents);
+  tor_free(esc_dir);
+  tor_free(dir_with_pathsep);
+  tor_free(esc_dir_with_pathsep);
+}
+
+static void
+test_config_include_not_processed(void *data)
+{
+  (void)data;
+
+  char torrc_contents[1000] = "%include does_not_exist\n";
+  config_line_t *result = NULL;
+  tt_int_op(config_get_lines(torrc_contents, &result, 0),OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    tt_str_op(next->key, OP_EQ, "%include");
+    tt_str_op(next->value, OP_EQ, "does_not_exist");
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+}
+
+static void
+test_config_include_has_include(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_has_include"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrc_contents[1000] = "Test 1\n";
+  int include_used;
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 0);
+  config_free_lines(result);
+
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s\n", dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used,
+            NULL), OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_flag_both_without(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  // test with defaults-torrc and torrc without include
+  int ret = options_init_from_string(conf_empty, conf_empty, CMD_RUN_UNITTESTS,
+                                     NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 0);
+
+ done:
+  tor_free(errmsg);
+}
+
+static void
+test_config_include_flag_torrc_only(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_flag_torrc_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", dir, "dummy");
+  tt_int_op(write_str_to_file(path, "\n", 0), OP_EQ, 0);
+
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  char conf_include[1000];
+  tor_snprintf(conf_include, sizeof(conf_include), "%%include %s", path);
+
+  // test with defaults-torrc without include and torrc with include
+  int ret = options_init_from_string(conf_empty, conf_include,
+                                     CMD_RUN_UNITTESTS, NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 1);
+
+ done:
+  tor_free(errmsg);
+  tor_free(path);
+  tor_free(dir);
+}
+
+static void
+test_config_include_flag_defaults_only(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_flag_defaults_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", dir, "dummy");
+  tt_int_op(write_str_to_file(path, "\n", 0), OP_EQ, 0);
+
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  char conf_include[1000];
+  tor_snprintf(conf_include, sizeof(conf_include), "%%include %s", path);
+
+  // test with defaults-torrc with include and torrc without include
+  int ret = options_init_from_string(conf_include, conf_empty,
+                                     CMD_RUN_UNITTESTS, NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 0);
+
+ done:
+  tor_free(errmsg);
+  tor_free(path);
+  tor_free(dir);
+}
+
+static void
+test_config_dup_and_filter(void *arg)
+{
+  (void)arg;
+  /* Test normal input. */
+  config_line_t *line = NULL;
+  config_line_append(&line, "abc", "def");
+  config_line_append(&line, "ghi", "jkl");
+  config_line_append(&line, "ABCD", "mno");
+
+  config_line_t *line_dup = config_lines_dup_and_filter(line, "aBc");
+  tt_ptr_op(line_dup, OP_NE, NULL);
+  tt_ptr_op(line_dup->next, OP_NE, NULL);
+  tt_ptr_op(line_dup->next->next, OP_EQ, NULL);
+
+  tt_str_op(line_dup->key, OP_EQ, "abc");
+  tt_str_op(line_dup->value, OP_EQ, "def");
+  tt_str_op(line_dup->next->key, OP_EQ, "ABCD");
+  tt_str_op(line_dup->next->value, OP_EQ, "mno");
+
+  /* empty output */
+  config_free_lines(line_dup);
+  line_dup = config_lines_dup_and_filter(line, "skdjfsdkljf");
+  tt_ptr_op(line_dup, OP_EQ, NULL);
+
+  /* empty input */
+  config_free_lines(line_dup);
+  line_dup = config_lines_dup_and_filter(NULL, "abc");
+  tt_ptr_op(line_dup, OP_EQ, NULL);
+
+ done:
+  config_free_lines(line);
+  config_free_lines(line_dup);
+}
+
+/* If we're not configured to be a bridge, but we set
+ * BridgeDistribution, then options_validate () should return -1. */
+static void
+test_config_check_bridge_distribution_setting_not_a_bridge(void *arg)
+{
+  or_options_t* options = get_options_mutable();
+  or_options_t* old_options = options;
+  or_options_t* default_options = options;
+  char* message = NULL;
+  int ret;
+
+  (void)arg;
+
+  options->BridgeRelay = 0;
+  options->BridgeDistribution = (char*)("https");
+
+  ret = options_validate(old_options, options, default_options, 0, &message);
+
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(message, OP_EQ, "You set BridgeDistribution, but you "
+            "didn't set BridgeRelay!");
+ done:
+  tor_free(message);
+  options->BridgeDistribution = NULL;
+}
+
+/* If the BridgeDistribution setting was valid, 0 should be returned. */
+static void
+test_config_check_bridge_distribution_setting_valid(void *arg)
+{
+  int ret = check_bridge_distribution_setting("https");
+
+  (void)arg;
+
+  tt_int_op(ret, OP_EQ, 0);
+ done:
+  return;
+}
+
+/* If the BridgeDistribution setting was invalid, -1 should be returned. */
+static void
+test_config_check_bridge_distribution_setting_invalid(void *arg)
+{
+  int ret = check_bridge_distribution_setting("hyphens-are-allowed");
+
+  (void)arg;
+
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = check_bridge_distribution_setting("asterisks*are*forbidden");
+
+  tt_int_op(ret, OP_EQ, -1);
+ done:
+  return;
+}
+
+/* If the BridgeDistribution setting was unrecognised, a warning should be
+ * logged and 0 should be returned. */
+static void
+test_config_check_bridge_distribution_setting_unrecognised(void *arg)
+{
+  int ret = check_bridge_distribution_setting("unicorn");
+
+  (void)arg;
+
+  tt_int_op(ret, OP_EQ, 0);
+ done:
+  return;
+}
+
+static void
+test_config_include_opened_file_list(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  smartlist_t *opened_files = smartlist_new();
+  char *torrcd = NULL;
+  char *subfolder = NULL;
+  char *path = NULL;
+  char *empty = NULL;
+  char *file = NULL;
+  char *dot = NULL;
+  char *dir = tor_strdup(get_fname("test_include_opened_file_list"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&subfolder, "%s"PATH_SEPARATOR"%s", torrcd, "subfolder");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(subfolder), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(subfolder, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", subfolder,
+               "01_file_in_subfolder");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+
+  tor_asprintf(&empty, "%s"PATH_SEPARATOR"%s", torrcd, "empty");
+  tt_int_op(write_str_to_file(empty, "", 0), OP_EQ, 0);
+
+  tor_asprintf(&file, "%s"PATH_SEPARATOR"%s", torrcd, "file");
+  tt_int_op(write_str_to_file(file, "Test 2\n", 0), OP_EQ, 0);
+
+  tor_asprintf(&dot, "%s"PATH_SEPARATOR"%s", torrcd, ".dot");
+  tt_int_op(write_str_to_file(dot, "Test 3\n", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            opened_files), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  tt_int_op(smartlist_len(opened_files), OP_EQ, 4);
+  tt_int_op(smartlist_contains_string(opened_files, torrcd), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, subfolder), OP_EQ, 1);
+  // files inside subfolders are not opended, only the subfolder is opened
+  tt_int_op(smartlist_contains_string(opened_files, empty), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, file), OP_EQ, 1);
+  // dot files are not opened as we ignore them when we get their name from
+  // their parent folder
+
+ done:
+  SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
+  smartlist_free(opened_files);
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(subfolder);
+  tor_free(path);
+  tor_free(empty);
+  tor_free(file);
+  tor_free(dot);
+  tor_free(dir);
+}
+
+static void
+test_config_compute_max_mem_in_queues(void *data)
+{
+#define GIGABYTE(x) (U64_LITERAL(x) << 30)
+#define MEGABYTE(x) (U64_LITERAL(x) << 20)
+  (void)data;
+  MOCK(get_total_system_memory, get_total_system_memory_mock);
+
+  /* We are unable to detect the amount of memory on the system. Tor will try
+   * to use some sensible default values for 64-bit and 32-bit systems. */
+  total_system_memory_return = -1;
+
+#if SIZEOF_VOID_P >= 8
+  /* We are on a 64-bit system. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(8));
+#else
+  /* We are on a 32-bit system. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
+#endif
+
+  /* We are able to detect the amount of RAM on the system. */
+  total_system_memory_return = 0;
+
+  /* We are running on a system with one gigabyte of RAM. */
+  total_system_memory_output = GIGABYTE(1);
+
+  /* We have 0.75 * RAM available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            3 * (GIGABYTE(1) / 4));
+
+  /* We are running on a tiny machine with 256 MB of RAM. */
+  total_system_memory_output = MEGABYTE(256);
+
+  /* We will now enforce a minimum of 256 MB of RAM available for the
+   * MaxMemInQueues here, even though we should only have had 0.75 * 256 = 192
+   * MB available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, MEGABYTE(256));
+
+#if SIZEOF_SIZE_T > 4
+  /* We are running on a machine with 8 GB of RAM. */
+  total_system_memory_output = GIGABYTE(8);
+
+  /* We will have 0.4 * RAM available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            2 * (GIGABYTE(8) / 5));
+
+  /* We are running on a machine with 16 GB of RAM. */
+  total_system_memory_output = GIGABYTE(16);
+
+  /* We will have 0.4 * RAM available. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            2 * (GIGABYTE(16) / 5));
+
+  /* We are running on a machine with 32 GB of RAM. */
+  total_system_memory_output = GIGABYTE(32);
+
+  /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
+  tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
+            MAX_DEFAULT_MEMORY_QUEUE_SIZE);
+#endif
+
+ done:
+  UNMOCK(get_total_system_memory);
+
+#undef GIGABYTE
+#undef MEGABYTE
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -4912,10 +5703,33 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(fix_my_family, 0),
   CONFIG_TEST(directory_fetch, 0),
   CONFIG_TEST(port_cfg_line_extract_addrport, 0),
-  CONFIG_TEST(parse_port_config__listenaddress, 0),
   CONFIG_TEST(parse_port_config__ports__no_ports_given, 0),
   CONFIG_TEST(parse_port_config__ports__server_options, 0),
   CONFIG_TEST(parse_port_config__ports__ports_given, 0),
+  CONFIG_TEST(parse_log_severity, 0),
+  CONFIG_TEST(include_limit, 0),
+  CONFIG_TEST(include_does_not_exist, 0),
+  CONFIG_TEST(include_error_in_included_file, 0),
+  CONFIG_TEST(include_empty_file_folder, 0),
+#ifndef _WIN32
+  CONFIG_TEST(include_no_permission, 0),
+#endif
+  CONFIG_TEST(include_recursion_before_after, 0),
+  CONFIG_TEST(include_recursion_after_only, 0),
+  CONFIG_TEST(include_folder_order, 0),
+  CONFIG_TEST(include_path_syntax, 0),
+  CONFIG_TEST(include_not_processed, 0),
+  CONFIG_TEST(include_has_include, 0),
+  CONFIG_TEST(include_flag_both_without, TT_FORK),
+  CONFIG_TEST(include_flag_torrc_only, TT_FORK),
+  CONFIG_TEST(include_flag_defaults_only, TT_FORK),
+  CONFIG_TEST(dup_and_filter, 0),
+  CONFIG_TEST(check_bridge_distribution_setting_not_a_bridge, TT_FORK),
+  CONFIG_TEST(check_bridge_distribution_setting_valid, 0),
+  CONFIG_TEST(check_bridge_distribution_setting_invalid, 0),
+  CONFIG_TEST(check_bridge_distribution_setting_unrecognised, 0),
+  CONFIG_TEST(include_opened_file_list, 0),
+  CONFIG_TEST(compute_max_mem_in_queues, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_connection.c b/src/test/test_connection.c
index d394fc9852..dc0f6860d9 100644
--- a/src/test/test_connection.c
+++ b/src/test/test_connection.c
@@ -1,24 +1,27 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CONNECTION_PRIVATE
 #define MAIN_PRIVATE
+#define CONNECTION_OR_PRIVATE
 
 #include "or.h"
 #include "test.h"
 
 #include "connection.h"
+#include "hs_common.h"
 #include "main.h"
 #include "microdesc.h"
+#include "nodelist.h"
 #include "networkstatus.h"
 #include "rendcache.h"
 #include "directory.h"
+#include "connection_or.h"
 
-static void test_conn_lookup_addr_helper(const char *address,
-                                         int family,
-                                         tor_addr_t *addr);
+#include "test_connection.h"
+#include "test_helpers.h"
 
 static void * test_conn_get_basic_setup(const struct testcase_t *tc);
 static int test_conn_get_basic_teardown(const struct testcase_t *tc,
@@ -61,48 +64,7 @@ static int test_conn_get_rsrc_teardown(const struct testcase_t *tc,
 #define TEST_CONN_UNATTACHED_STATE (AP_CONN_STATE_CIRCUIT_WAIT)
 #define TEST_CONN_ATTACHED_STATE   (AP_CONN_STATE_CONNECT_WAIT)
 
-#define TEST_CONN_FD_INIT 50
-static int mock_connection_connect_sockaddr_called = 0;
-static int fake_socket_number = TEST_CONN_FD_INIT;
-
-static int
-mock_connection_connect_sockaddr(connection_t *conn,
-                                 const struct sockaddr *sa,
-                                 socklen_t sa_len,
-                                 const struct sockaddr *bindaddr,
-                                 socklen_t bindaddr_len,
-                                 int *socket_error)
-{
-  (void)sa_len;
-  (void)bindaddr;
-  (void)bindaddr_len;
-
-  tor_assert(conn);
-  tor_assert(sa);
-  tor_assert(socket_error);
-
-  mock_connection_connect_sockaddr_called++;
-
-  conn->s = fake_socket_number++;
-  tt_assert(SOCKET_OK(conn->s));
-  /* We really should call tor_libevent_initialize() here. Because we don't,
-   * we are relying on other parts of the code not checking if the_event_base
-   * (and therefore event->ev_base) is NULL.  */
-  tt_assert(connection_add_connecting(conn) == 0);
-
- done:
-  /* Fake "connected" status */
-  return 1;
-}
-
-static int
-fake_close_socket(evutil_socket_t sock)
-{
-  (void)sock;
-  return 0;
-}
-
-static void
+void
 test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr)
 {
   int rv = 0;
@@ -111,7 +73,7 @@ test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr)
 
   rv = tor_addr_lookup(address, family, addr);
   /* XXXX - should we retry on transient failure? */
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   tt_assert(tor_addr_is_loopback(addr));
   tt_assert(tor_addr_is_v4(addr));
 
@@ -121,51 +83,6 @@ test_conn_lookup_addr_helper(const char *address, int family, tor_addr_t *addr)
   tor_addr_make_null(addr, TEST_CONN_FAMILY);
 }
 
-static connection_t *
-test_conn_get_connection(uint8_t state, uint8_t type, uint8_t purpose)
-{
-  connection_t *conn = NULL;
-  tor_addr_t addr;
-  int socket_err = 0;
-  int in_progress = 0;
-
-  MOCK(connection_connect_sockaddr,
-       mock_connection_connect_sockaddr);
-  MOCK(tor_close_socket, fake_close_socket);
-
-  init_connection_lists();
-
-  conn = connection_new(type, TEST_CONN_FAMILY);
-  tt_assert(conn);
-
-  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS, TEST_CONN_FAMILY, &addr);
-  tt_assert(!tor_addr_is_null(&addr));
-
-  tor_addr_copy_tight(&conn->addr, &addr);
-  conn->port = TEST_CONN_PORT;
-  mock_connection_connect_sockaddr_called = 0;
-  in_progress = connection_connect(conn, TEST_CONN_ADDRESS_PORT, &addr,
-                                   TEST_CONN_PORT, &socket_err);
-  tt_assert(mock_connection_connect_sockaddr_called == 1);
-  tt_assert(!socket_err);
-  tt_assert(in_progress == 0 || in_progress == 1);
-
-  /* fake some of the attributes so the connection looks OK */
-  conn->state = state;
-  conn->purpose = purpose;
-  assert_connection_ok(conn, time(NULL));
-
-  UNMOCK(connection_connect_sockaddr);
-  UNMOCK(tor_close_socket);
-  return conn;
-
-  /* On failure */
- done:
-  UNMOCK(connection_connect_sockaddr);
-  UNMOCK(tor_close_socket);
-  return NULL;
-}
-
 static void *
 test_conn_get_basic_setup(const struct testcase_t *tc)
 {
@@ -264,14 +181,10 @@ test_conn_get_rend_setup(const struct testcase_t *tc)
 
   rend_cache_init();
 
-  /* TODO: use directory_initiate_command_rend() to do this - maybe? */
-  conn->rend_data = tor_malloc_zero(sizeof(rend_data_t));
+  /* TODO: use directory_initiate_request() to do this - maybe? */
   tor_assert(strlen(TEST_CONN_REND_ADDR) == REND_SERVICE_ID_LEN_BASE32);
-  memcpy(conn->rend_data->onion_address,
-         TEST_CONN_REND_ADDR,
-         REND_SERVICE_ID_LEN_BASE32+1);
-  conn->rend_data->hsdirs_fp = smartlist_new();
-
+  conn->rend_data = rend_data_client_create(TEST_CONN_REND_ADDR, NULL, NULL,
+                                            REND_NO_AUTH);
   assert_connection_ok(&conn->base_, time(NULL));
   return conn;
 
@@ -382,7 +295,7 @@ test_conn_download_status_teardown(const struct testcase_t *tc, void *arg)
 
       /* connection_free_() cleans up requested_resource */
       rv = test_conn_get_rsrc_teardown(tc, conn);
-      tt_assert(rv == 1);
+      tt_int_op(rv, OP_EQ, 1);
     }
   } SMARTLIST_FOREACH_END(conn);
 
@@ -456,12 +369,13 @@ test_conn_get_basic(void *arg)
    * its attributes, but get NULL when we supply a different value. */
 
   tt_assert(connection_get_by_global_id(conn->global_identifier) == conn);
-  tt_assert(connection_get_by_global_id(!conn->global_identifier) == NULL);
+  tt_ptr_op(connection_get_by_global_id(!conn->global_identifier), OP_EQ,
+            NULL);
 
   tt_assert(connection_get_by_type(conn->type) == conn);
   tt_assert(connection_get_by_type(TEST_CONN_TYPE) == conn);
-  tt_assert(connection_get_by_type(!conn->type) == NULL);
-  tt_assert(connection_get_by_type(!TEST_CONN_TYPE) == NULL);
+  tt_ptr_op(connection_get_by_type(!conn->type), OP_EQ, NULL);
+  tt_ptr_op(connection_get_by_type(!TEST_CONN_TYPE), OP_EQ, NULL);
 
   tt_assert(connection_get_by_type_state(conn->type, conn->state)
             == conn);
@@ -551,7 +465,8 @@ test_conn_get_rend(void *arg)
   tt_assert(connection_get_by_type_state_rendquery(
                                             conn->base_.type,
                                             conn->base_.state,
-                                            conn->rend_data->onion_address)
+                                            rend_data_get_address(
+                                                      conn->rend_data))
             == TO_CONN(conn));
   tt_assert(connection_get_by_type_state_rendquery(
                                             TEST_CONN_TYPE,
@@ -574,7 +489,7 @@ test_conn_get_rend(void *arg)
 #define sl_is_conn_assert(sl_input, conn) \
   do {                                               \
     the_sl = (sl_input);                             \
-    tt_assert(smartlist_len((the_sl)) == 1);         \
+    tt_int_op(smartlist_len((the_sl)), OP_EQ, 1);         \
     tt_assert(smartlist_get((the_sl), 0) == (conn)); \
     smartlist_free(the_sl); the_sl = NULL;           \
   } while (0)
@@ -582,7 +497,7 @@ test_conn_get_rend(void *arg)
 #define sl_no_conn_assert(sl_input)          \
   do {                                       \
     the_sl = (sl_input);                     \
-    tt_assert(smartlist_len((the_sl)) == 0); \
+    tt_int_op(smartlist_len((the_sl)), OP_EQ, 0); \
     smartlist_free(the_sl); the_sl = NULL;   \
   } while (0)
 
@@ -628,43 +543,32 @@ test_conn_get_rsrc(void *arg)
                                                     TEST_CONN_RSRC_2,
                                                     !TEST_CONN_STATE));
 
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    conn->base_.purpose,
-                                                    conn->requested_resource)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    !conn->base_.purpose,
-                                                    "")
-            == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                    !TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC_2)
-            == 0);
-
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    conn->base_.purpose,
-                                                    conn->requested_resource,
-                                                    conn->base_.state)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC,
-                                                    TEST_CONN_STATE)
-            == 1);
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    !conn->base_.purpose,
-                                                    "",
-                                                    !conn->base_.state)
-            == 0);
-  tt_assert(connection_dir_count_by_purpose_resource_and_state(
-                                                    !TEST_CONN_RSRC_PURPOSE,
-                                                    TEST_CONN_RSRC_2,
-                                                    !TEST_CONN_STATE)
-            == 0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 conn->base_.purpose, conn->requested_resource),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 !conn->base_.purpose, ""),
+            OP_EQ, 0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                 !TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC_2),
+            OP_EQ, 0);
+
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 conn->base_.purpose, conn->requested_resource,
+                 conn->base_.state),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC, TEST_CONN_STATE),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 !conn->base_.purpose, "", !conn->base_.state),
+            OP_EQ, 0);
+  tt_int_op(connection_dir_count_by_purpose_resource_and_state(
+                 !TEST_CONN_RSRC_PURPOSE, TEST_CONN_RSRC_2, !TEST_CONN_STATE),
+            OP_EQ, 0);
 
  done:
   smartlist_free(the_sl);
@@ -690,117 +594,127 @@ test_conn_download_status(void *arg)
   const char *other_res = networkstatus_get_flavor_name(other_flavor);
 
   /* no connections */
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, not downloading */
   conn = test_conn_download_status_add_a_connection(res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, downloading but not linked (not possible on a client,
    * but possible on a relay) */
   conn->base_.state = TEST_CONN_DL_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, downloading and linked, but not yet attached */
   ap_conn = test_conn_get_linked_connection(TO_CONN(conn),
                                             TEST_CONN_UNATTACHED_STATE);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
     /* one connection, downloading and linked and attached */
   ap_conn->state = TEST_CONN_ATTACHED_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* one connection, linked and attached but not downloading */
   conn->base_.state = TEST_CONN_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* two connections, both not downloading */
   conn2 = test_conn_download_status_add_a_connection(res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 2);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 2);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* two connections, one downloading */
   conn->base_.state = TEST_CONN_DL_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 2);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 2);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
   conn->base_.state = TEST_CONN_STATE;
 
   /* more connections, all not downloading */
   /* ignore the return value, it's free'd using the connection list */
   (void)test_conn_download_status_add_a_connection(res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 0);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* more connections, one downloading */
   conn->base_.state = TEST_CONN_DL_STATE;
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* more connections, two downloading (should never happen, but needs
    * to be tested for completeness) */
@@ -808,38 +722,41 @@ test_conn_download_status(void *arg)
   /* ignore the return value, it's free'd using the connection list */
   (void)test_conn_get_linked_connection(TO_CONN(conn2),
                                         TEST_CONN_ATTACHED_STATE);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
   conn->base_.state = TEST_CONN_STATE;
 
   /* more connections, a different one downloading */
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                         other_res) == 0);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                           TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 0);
 
   /* a connection for the other flavor (could happen if a client is set to
    * cache directory documents), one preferred flavor downloading
    */
   conn4 = test_conn_download_status_add_a_connection(other_res);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 0);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        other_res) == 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            0);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 1);
 
   /* a connection for the other flavor (could happen if a client is set to
    * cache directory documents), both flavors downloading
@@ -848,23 +765,117 @@ test_conn_download_status(void *arg)
   /* ignore the return value, it's free'd using the connection list */
   (void)test_conn_get_linked_connection(TO_CONN(conn4),
                                         TEST_CONN_ATTACHED_STATE);
-  tt_assert(networkstatus_consensus_is_already_downloading(res) == 1);
-  tt_assert(networkstatus_consensus_is_already_downloading(other_res) == 1);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        res) == 3);
-  tt_assert(connection_dir_count_by_purpose_and_resource(
-                                                        TEST_CONN_RSRC_PURPOSE,
-                                                        other_res) == 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(res), OP_EQ, 1);
+  tt_int_op(networkstatus_consensus_is_already_downloading(other_res), OP_EQ,
+            1);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, res),
+            OP_EQ, 3);
+  tt_int_op(connection_dir_count_by_purpose_and_resource(
+                                   TEST_CONN_RSRC_PURPOSE, other_res),
+            OP_EQ, 1);
 
  done:
   /* the teardown function removes all the connections in the global list*/;
 }
 
+static node_t test_node;
+
+static node_t *
+mock_node_get_mutable_by_id(const char *digest)
+{
+  (void) digest;
+  static routerinfo_t node_ri;
+  memset(&node_ri, 0, sizeof(node_ri));
+
+  test_node.ri = &node_ri;
+  memset(test_node.identity, 'c', sizeof(test_node.identity));
+
+  tor_addr_t ipv4_addr;
+  tor_addr_parse(&ipv4_addr, "18.0.0.1");
+  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+  node_ri.or_port = 1;
+
+  return &test_node;
+}
+
+static const node_t *
+mock_node_get_by_id(const char *digest)
+{
+  (void) digest;
+  memset(test_node.identity, 'c', sizeof(test_node.identity));
+  return &test_node;
+}
+
+/* Test whether we correctly track failed connections between relays. */
+static void
+test_failed_orconn_tracker(void *arg)
+{
+  (void) arg;
+
+  int can_connect;
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  (void) now;
+
+  update_approx_time(now);
+
+  /* Prepare the OR connection that will be used in this test */
+  or_connection_t or_conn;
+  tt_int_op(AF_INET,OP_EQ, tor_addr_parse(&or_conn.real_addr, "18.0.0.1"));
+  tt_int_op(AF_INET,OP_EQ, tor_addr_parse(&or_conn.base_.addr, "18.0.0.1"));
+  or_conn.base_.port = 1;
+  memset(or_conn.identity_digest, 'c', sizeof(or_conn.identity_digest));
+
+  /* Check whether we can connect with an empty failure cache:
+   * this should succeed */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+  /* Now add the destination to the failure cache */
+  note_or_connect_failed(&or_conn);
+
+  /* Check again: now it shouldn't connect */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 0);
+
+  /* Move time forward and check again: the cache should have been cleared and
+   * now it should connect */
+  now += 3600;
+  update_approx_time(now);
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+  /* Now mock the node_get_*by_id() functions to start using the node subsystem
+   * optimization. */
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  MOCK(node_get_mutable_by_id, mock_node_get_mutable_by_id);
+
+  /* Since we just started using the node subsystem it will allow connections
+   * now */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+  /* Mark it as failed */
+  note_or_connect_failed(&or_conn);
+
+  /* Check that it shouldn't connect now */
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 0);
+
+  /* Move time forward and check again: now it should connect  */
+  now += 3600;
+  update_approx_time(now);
+  can_connect = should_connect_to_relay(&or_conn);
+  tt_int_op(can_connect, OP_EQ, 1);
+
+ done:
+  ;
+}
+
 #define CONNECTION_TESTCASE(name, fork, setup)                           \
   { #name, test_conn_##name, fork, &setup, NULL }
 
-/* where arg is an expression (constant, varaible, compound expression) */
+/* where arg is an expression (constant, variable, compound expression) */
 #define CONNECTION_TESTCASE_ARG(name, fork, setup, arg)                  \
   { #name "_" #arg, test_conn_##name, fork, &setup, (void *)arg }
 
@@ -877,6 +888,7 @@ struct testcase_t connection_tests[] = {
   CONNECTION_TESTCASE_ARG(download_status,  TT_FORK,
                           test_conn_download_status_st, FLAV_NS),
 //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair),
+  { "failed_orconn_tracker", test_failed_orconn_tracker, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_connection.h b/src/test/test_connection.h
new file mode 100644
index 0000000000..392783b53b
--- /dev/null
+++ b/src/test/test_connection.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** Some constants used by test_connection and helpers */
+#define TEST_CONN_FAMILY        (AF_INET)
+#define TEST_CONN_ADDRESS       "127.0.0.1"
+#define TEST_CONN_PORT          (12345)
+#define TEST_CONN_ADDRESS_PORT  "127.0.0.1:12345"
+#define TEST_CONN_FD_INIT 50
+
+void test_conn_lookup_addr_helper(const char *address,
+                                  int family, tor_addr_t *addr);
+
diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c
new file mode 100644
index 0000000000..ffec3149b0
--- /dev/null
+++ b/src/test/test_conscache.c
@@ -0,0 +1,340 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_conscache_open_failure(void *arg)
+{
+  (void) arg;
+  /* Try opening a directory that doesn't exist and which we shouldn't be
+   * able to create. */
+  consensus_cache_t *cache = consensus_cache_open("a/b/c/d/e/f/g", 128);
+  tt_ptr_op(cache, OP_EQ, NULL);
+
+ done:
+  ;
+}
+
+static void
+test_conscache_simple_usage(void *arg)
+{
+  (void)arg;
+  consensus_cache_entry_t *ent = NULL, *ent2 = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create object; make sure it exists. */
+  config_line_t *labels = NULL;
+  config_line_append(&labels, "Hello", "world");
+  config_line_append(&labels, "Adios", "planetas");
+  ent = consensus_cache_add(cache,
+                            labels, (const uint8_t *)"A\0B\0C", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent);
+
+  /* Make a second object */
+  config_line_append(&labels, "Hello", "mundo");
+  config_line_append(&labels, "Adios", "planets");
+  ent2 = consensus_cache_add(cache,
+                             labels, (const uint8_t *)"xyzzy", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent2);
+  tt_assert(! consensus_cache_entry_is_mapped(ent2));
+  consensus_cache_entry_decref(ent2);
+  ent2 = NULL;
+
+  /* Check get_value */
+  tt_ptr_op(NULL, OP_EQ, consensus_cache_entry_get_value(ent, "hebbo"));
+  tt_str_op("world", OP_EQ, consensus_cache_entry_get_value(ent, "Hello"));
+
+  /* Check find_first */
+  ent2 = consensus_cache_find_first(cache, "Hello", "world!");
+  tt_ptr_op(ent2, OP_EQ, NULL);
+  ent2 = consensus_cache_find_first(cache, "Hello", "world");
+  tt_ptr_op(ent2, OP_EQ, ent);
+  ent2 = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_ptr_op(ent2, OP_NE, ent);
+
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+
+  /* Check get_body */
+  const uint8_t *bp = NULL;
+  size_t sz = 0;
+  int r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "A\0B\0C", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* Free and re-create the cache, to rescan the directory. */
+  consensus_cache_free(cache);
+  consensus_cache_entry_decref(ent);
+  cache = consensus_cache_open("cons", 128);
+
+  /* Make sure the entry is still there */
+  ent = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_assert(ent);
+  ent2 = consensus_cache_find_first(cache, "Adios", "planets");
+  tt_ptr_op(ent, OP_EQ, ent2);
+  consensus_cache_entry_incref(ent);
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+  r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "xyzzy", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* There should be two entries total. */
+  smartlist_t *entries = smartlist_new();
+  consensus_cache_find_all(entries, cache, NULL, NULL);
+  int n = smartlist_len(entries);
+  smartlist_free(entries);
+  tt_int_op(n, OP_EQ, 2);
+
+ done:
+  consensus_cache_entry_decref(ent);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_cleanup(void *arg)
+{
+  (void)arg;
+  const int N = 20;
+  consensus_cache_entry_t **ents =
+    tor_calloc(N, sizeof(consensus_cache_entry_t*));
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries. */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "cleanup");
+    config_line_append(&labels, "index", num);
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    ents[i] = consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ents[i]);
+    /* We're still holding a reference to each entry at this point. */
+  }
+
+  /* Page all of the entries into RAM */
+  for (i = 0; i < N; ++i) {
+    const uint8_t *bp;
+    size_t sz;
+    tt_assert(! consensus_cache_entry_is_mapped(ents[i]));
+    consensus_cache_entry_get_body(ents[i], &bp, &sz);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as deletable. */
+  for (i = 7; i < N; i += 7) {
+    consensus_cache_entry_mark_for_removal(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as aggressively unpaged. */
+  for (i = 3; i < N; i += 3) {
+    consensus_cache_entry_mark_for_aggressive_release(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Incref some of the entries again */
+  for (i = 0; i < N; i += 2) {
+    consensus_cache_entry_incref(ents[i]);
+  }
+
+  /* Now we're going to decref everything. We do so at a specific time.  I'm
+   * picking the moment when I was writing this test, at 2017-04-05 12:16:48
+   * UTC. */
+  const time_t example_time = 1491394608;
+  update_approx_time(example_time);
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    if (i % 2) {
+      ents[i] = NULL; /* We're no longer holding any reference here. */
+    }
+  }
+
+  /* At this point, the aggressively-released items with refcount 1 should
+   * be unmapped. Nothing should be deleted. */
+  consensus_cache_entry_t *e_tmp;
+  e_tmp = consensus_cache_find_first(cache, "index", "3");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "5");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "6");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "7");
+  tt_ptr_op(e_tmp, OP_EQ, NULL); // not found because pending deletion.
+
+  /* Delete the pending-deletion items. */
+  consensus_cache_delete_pending(cache, 0);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 20 - 2); /* 1 entry was deleted; 1 is not-found. */
+  }
+  e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1...
+  tt_ptr_op(e_tmp, OP_EQ, NULL); // so deleted.
+  e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2
+  tt_ptr_op(e_tmp, OP_EQ, NULL); // not deleted; but not found.
+
+  /* Now do lazy unmapping. */
+  // should do nothing.
+  consensus_cache_unmap_lazy(cache, example_time - 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  // should actually unmap
+  consensus_cache_unmap_lazy(cache, example_time + 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  // This one will still be mapped, since it has a reference.
+  e_tmp = consensus_cache_find_first(cache, "index", "16");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    ents[i] = NULL;
+  }
+
+  /* Free and re-create the cache, to rescan the directory. Make sure the
+   * deleted thing is still deleted, along with the other deleted thing. */
+  consensus_cache_free(cache);
+  cache = consensus_cache_open("cons", 128);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 18);
+  }
+
+ done:
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+  }
+  tor_free(ents);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_filter(void *arg)
+{
+  (void)arg;
+  const int N = 30;
+  smartlist_t *lst = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries with different labels */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "filter");
+    config_line_append(&labels, "index", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 3);
+    config_line_append(&labels, "mod3", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 5);
+    config_line_append(&labels, "mod5", num);
+
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    consensus_cache_entry_t *ent =
+      consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ent);
+    consensus_cache_entry_decref(ent);
+  }
+
+  lst = smartlist_new();
+  /* Find nothing. */
+  consensus_cache_find_all(lst, cache, "mod5", "5");
+  tt_int_op(smartlist_len(lst), OP_EQ, 0);
+  /* Find everything. */
+  consensus_cache_find_all(lst, cache, "test-id", "filter");
+  tt_int_op(smartlist_len(lst), OP_EQ, N);
+
+  /* Now filter to find the entries that have i%3 == 1 */
+  consensus_cache_filter_list(lst, "mod3", "1");
+  tt_int_op(smartlist_len(lst), OP_EQ, 10);
+  /* Now filter to find the entries that also have i%5 == 3 */
+  consensus_cache_filter_list(lst, "mod5", "3");
+  tt_int_op(smartlist_len(lst), OP_EQ, 2);
+  /* So now we have those entries for which i%15 == 13. */
+
+  consensus_cache_entry_t *ent1 = smartlist_get(lst, 0);
+  consensus_cache_entry_t *ent2 = smartlist_get(lst, 1);
+  const char *idx1 = consensus_cache_entry_get_value(ent1, "index");
+  const char *idx2 = consensus_cache_entry_get_value(ent2, "index");
+  tt_assert( (!strcmp(idx1, "28") && !strcmp(idx2, "13")) ||
+             (!strcmp(idx1, "13") && !strcmp(idx2, "28")) );
+
+ done:
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+  smartlist_free(lst);
+}
+
+#define ENT(name)                                               \
+  { #name, test_conscache_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t conscache_tests[] = {
+  ENT(open_failure),
+  ENT(simple_usage),
+  ENT(cleanup),
+  ENT(filter),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
new file mode 100644
index 0000000000..fda3a7f186
--- /dev/null
+++ b/src/test/test_consdiff.c
@@ -0,0 +1,1185 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "or.h"
+#include "test.h"
+
+#include "consdiff.h"
+#include "memarea.h"
+#include "log_test_helpers.h"
+
+#define tt_str_eq_line(a,b) \
+  tt_assert(line_str_eq((b),(a)))
+
+static void
+test_consdiff_smartlist_slice(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_slice_t *sls;
+  int items[6] = {0,0,0,0,0,0};
+
+  /* Create a regular smartlist. */
+  (void)arg;
+  smartlist_add(sl, &items[1]);
+  smartlist_add(sl, &items[2]);
+  smartlist_add(sl, &items[3]);
+  smartlist_add(sl, &items[4]);
+  smartlist_add(sl, &items[5]);
+
+  /* See if the slice was done correctly. */
+  sls = smartlist_slice(sl, 2, 5);
+  tt_ptr_op(sl, OP_EQ, sls->list);
+  tt_ptr_op(&items[3], OP_EQ, smartlist_get(sls->list, sls->offset));
+  tt_ptr_op(&items[5], OP_EQ,
+      smartlist_get(sls->list, sls->offset + (sls->len-1)));
+  tor_free(sls);
+
+  /* See that using -1 as the end does get to the last element. */
+  sls = smartlist_slice(sl, 2, -1);
+  tt_ptr_op(sl, OP_EQ, sls->list);
+  tt_ptr_op(&items[3], OP_EQ, smartlist_get(sls->list, sls->offset));
+  tt_ptr_op(&items[5], OP_EQ,
+      smartlist_get(sls->list, sls->offset + (sls->len-1)));
+
+ done:
+  tor_free(sls);
+  smartlist_free(sl);
+}
+
+static void
+test_consdiff_smartlist_slice_string_pos(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_slice_t *sls;
+  memarea_t *area = memarea_new();
+
+  /* Create a regular smartlist. */
+  (void)arg;
+  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+
+  /* See that smartlist_slice_string_pos respects the bounds of the slice. */
+  sls = smartlist_slice(sl, 2, 5);
+  cdline_t a_line = { "a", 1 };
+  tt_int_op(3, OP_EQ, smartlist_slice_string_pos(sls, &a_line));
+  cdline_t d_line = { "d", 1 };
+  tt_int_op(-1, OP_EQ, smartlist_slice_string_pos(sls, &d_line));
+
+ done:
+  tor_free(sls);
+  smartlist_free(sl);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_lcs_lengths(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2;
+  int *lengths1, *lengths2;
+  memarea_t *area = memarea_new();
+
+  /* Expected lcs lengths in regular and reverse order. */
+  int e_lengths1[] = { 0, 1, 2, 3, 3, 4 };
+  int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+
+  lengths1 = lcs_lengths(sls1, sls2, 1);
+  lengths2 = lcs_lengths(sls1, sls2, -1);
+  tt_mem_op(e_lengths1, OP_EQ, lengths1, sizeof(int) * 6);
+  tt_mem_op(e_lengths2, OP_EQ, lengths2, sizeof(int) * 6);
+
+ done:
+  tor_free(lengths1);
+  tor_free(lengths2);
+  tor_free(sls1);
+  tor_free(sls2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_trim_slices(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_t *sl3 = smartlist_new();
+  smartlist_t *sl4 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2, *sls3, *sls4;
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  sls3 = smartlist_slice(sl3, 0, -1);
+  sls4 = smartlist_slice(sl4, 0, -1);
+
+  /* They should be trimmed by one line at each end. */
+  tt_int_op(5, OP_EQ, sls1->len);
+  tt_int_op(5, OP_EQ, sls2->len);
+  trim_slices(sls1, sls2);
+  tt_int_op(3, OP_EQ, sls1->len);
+  tt_int_op(3, OP_EQ, sls2->len);
+
+  /* They should not be trimmed at all. */
+  tt_int_op(5, OP_EQ, sls3->len);
+  tt_int_op(5, OP_EQ, sls4->len);
+  trim_slices(sls3, sls4);
+  tt_int_op(5, OP_EQ, sls3->len);
+  tt_int_op(5, OP_EQ, sls4->len);
+
+ done:
+  tor_free(sls1);
+  tor_free(sls2);
+  tor_free(sls3);
+  tor_free(sls4);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  smartlist_free(sl3);
+  smartlist_free(sl4);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_set_changed(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  bitarray_t *changed1 = bitarray_init_zero(4);
+  bitarray_t *changed2 = bitarray_init_zero(4);
+  smartlist_slice_t *sls1, *sls2;
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines(sl2, "a\na\na\na\n", area);
+
+  /* Length of sls1 is 0. */
+  sls1 = smartlist_slice(sl1, 0, 0);
+  sls2 = smartlist_slice(sl2, 1, 3);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The former is not changed, the latter changes all of its elements. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+  bitarray_clear(changed2, 1);
+  bitarray_clear(changed2, 2);
+
+  /* Length of sls1 is 1 and its element is in sls2. */
+  tor_free(sls1);
+  sls1 = smartlist_slice(sl1, 0, 1);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The latter changes all elements but the (first) common one. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(!bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+  bitarray_clear(changed2, 2);
+
+  /* Length of sls1 is 1 and its element is not in sls2. */
+  tor_free(sls1);
+  sls1 = smartlist_slice(sl1, 1, 2);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The former changes its element, the latter changes all elements. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+
+ done:
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  tor_free(sls1);
+  tor_free(sls2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_calc_changes(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2;
+  bitarray_t *changed1 = bitarray_init_zero(4);
+  bitarray_t *changed2 = bitarray_init_zero(4);
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\na\na\na\n", area);
+  consensus_split_lines(sl2, "a\na\na\na\n", area);
+
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* Nothing should be set to changed. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(!bitarray_is_set(changed2, 1));
+  tt_assert(!bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+
+  smartlist_clear(sl2);
+  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  tor_free(sls1);
+  tor_free(sls2);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* Two elements are changed. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+  bitarray_clear(changed1, 1);
+  bitarray_clear(changed1, 2);
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(!bitarray_is_set(changed2, 2));
+  tt_assert(bitarray_is_set(changed2, 3));
+  bitarray_clear(changed1, 1);
+  bitarray_clear(changed1, 3);
+
+  smartlist_clear(sl2);
+  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  tor_free(sls1);
+  tor_free(sls2);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* All elements are changed. */
+  tt_assert(bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(bitarray_is_set(changed1, 2));
+  tt_assert(bitarray_is_set(changed1, 3));
+
+  tt_assert(bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(bitarray_is_set(changed2, 3));
+
+ done:
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  tor_free(sls1);
+  tor_free(sls2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_get_id_hash(void *arg)
+{
+  (void)arg;
+
+  cdline_t line1 = { "r name", 6 };
+  cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 };
+  cdline_t line3 = { "r name hash+valid+base64 etc", 28 };
+  cdline_t tmp;
+
+  /* No hash. */
+  tt_int_op(-1, OP_EQ, get_id_hash(&line1, &tmp));
+  /* The hash contains characters that are not base64. */
+  tt_int_op(-1, OP_EQ, get_id_hash(&line2, &tmp));
+
+  /* valid hash. */
+  tt_int_op(0, OP_EQ, get_id_hash(&line3, &tmp));
+  tt_ptr_op(tmp.s, OP_EQ, line3.s + 7);
+  tt_uint_op(tmp.len, OP_EQ, line3.len - 11);
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_is_valid_router_entry(void *arg)
+{
+  /* Doesn't start with "r ". */
+  (void)arg;
+  cdline_t line0 = { "foo", 3 };
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line0));
+
+  /* These are already tested with get_id_hash, but make sure it's run
+   * properly. */
+
+  cdline_t line1 = { "r name", 6 };
+  cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 };
+  cdline_t line3 = { "r name hash+valid+base64 etc", 28 };
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line1));
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line2));
+  tt_int_op(1, OP_EQ, is_valid_router_entry(&line3));
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_next_router(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  memarea_t *area = memarea_new();
+  (void)arg;
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area,
+      "r name hash+longer+than+27+chars+and+valid+base64 etc");
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area,
+      "r name hash+longer+than+27+chars+and+valid+base64 etc");
+  smartlist_add_linecpy(sl, area, "foo");
+
+  /* Not currently on a router entry line, finding the next one. */
+  tt_int_op(1, OP_EQ, next_router(sl, 0));
+  tt_int_op(4, OP_EQ, next_router(sl, 2));
+
+  /* Already at the beginning of a router entry line, ignore it. */
+  tt_int_op(4, OP_EQ, next_router(sl, 1));
+
+  /* There are no more router entries, so return the line after the last. */
+  tt_int_op(6, OP_EQ, next_router(sl, 4));
+  tt_int_op(6, OP_EQ, next_router(sl, 5));
+
+ done:
+  smartlist_free(sl);
+  memarea_drop_all(area);
+}
+
+static int
+base64cmp_wrapper(const char *a, const char *b)
+{
+  cdline_t aa = { a, a ? (uint32_t) strlen(a) : 0 };
+  cdline_t bb = { b, b ? (uint32_t) strlen(b) : 0 };
+  return base64cmp(&aa, &bb);
+}
+
+static void
+test_consdiff_base64cmp(void *arg)
+{
+  /* NULL arguments. */
+  (void)arg;
+  tt_int_op(0, OP_EQ, base64cmp_wrapper(NULL, NULL));
+  tt_int_op(-1, OP_EQ, base64cmp_wrapper(NULL, "foo"));
+  tt_int_op(1, OP_EQ, base64cmp_wrapper("bar", NULL));
+
+  /* Nil base64 values. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("", ""));
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("_", "&"));
+
+  /* Exact same valid strings. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+", "abcABC/+"));
+  /* Both end with an invalid base64 char other than '\0'. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+ "));
+  /* Only one ends with an invalid base64 char other than '\0'. */
+  tt_int_op(-1, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+a"));
+
+  /* Comparisons that would return differently with strcmp(). */
+  tt_int_op(strcmp("/foo", "Afoo"), OP_LT, 0);
+  tt_int_op(base64cmp_wrapper("/foo", "Afoo"), OP_GT, 0);
+  tt_int_op(strcmp("Afoo", "0foo"), OP_GT, 0);
+  tt_int_op(base64cmp_wrapper("Afoo", "0foo"), OP_LT, 0);
+
+  /* Comparisons that would return the same as with strcmp(). */
+  tt_int_op(strcmp("afoo", "Afoo"), OP_GT, 0);
+  tt_int_op(base64cmp_wrapper("afoo", "Afoo"), OP_GT, 0);
+
+  /* Different lengths */
+  tt_int_op(base64cmp_wrapper("afoo", "afooo"), OP_LT, 0);
+  tt_int_op(base64cmp_wrapper("afooo", "afoo"), OP_GT, 0);
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_gen_ed_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  int i;
+  memarea_t *area = memarea_new();
+  setup_capture_of_logs(LOG_WARN);
+
+  (void)arg;
+  cons1 = smartlist_new();
+  cons2 = smartlist_new();
+
+  /* Identity hashes are not sorted properly, return NULL. */
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  smartlist_add_linecpy(cons2, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+  smartlist_add_linecpy(cons2, area, "foo");
+  smartlist_add_linecpy(cons2, area, "r name ccccccccccccccccccccccccccc etc");
+  smartlist_add_linecpy(cons2, area, "bar");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Same, but now with the second consensus. */
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the target consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Same as the two above, but with the reversed thing immediately after a
+     match. (The code handles this differently) */
+  smartlist_del(cons1, 0);
+  smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the target consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Identity hashes are repeated, return NULL. */
+  smartlist_clear(cons1);
+
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* We have to add a line that is just a dot, return NULL. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  smartlist_add_linecpy(cons1, area, "foo1");
+  smartlist_add_linecpy(cons1, area, "foo2");
+
+  smartlist_add_linecpy(cons2, area, "foo1");
+  smartlist_add_linecpy(cons2, area, ".");
+  smartlist_add_linecpy(cons2, area, "foo2");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Cannot generate consensus diff "
+         "because one of the lines to be added is \".\".");
+
+#define MAX_LINE_COUNT (10000)
+  /* Too many lines to be fed to the quadratic-time function. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "a");
+  for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "b");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because we found too few common router ids.");
+
+  /* We have dot lines, but they don't interfere with the script format. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  smartlist_add_linecpy(cons1, area, "foo1");
+  smartlist_add_linecpy(cons1, area, ".");
+  smartlist_add_linecpy(cons1, area, ".");
+  smartlist_add_linecpy(cons1, area, "foo2");
+
+  smartlist_add_linecpy(cons2, area, "foo1");
+  smartlist_add_linecpy(cons2, area, ".");
+  smartlist_add_linecpy(cons2, area, "foo2");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  smartlist_free(diff);
+
+  /* Empty diff tests. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(0, OP_EQ, smartlist_len(diff));
+  smartlist_free(diff);
+
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  smartlist_add_linecpy(cons2, area, "foo");
+  smartlist_add_linecpy(cons2, area, "bar");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(0, OP_EQ, smartlist_len(diff));
+  smartlist_free(diff);
+
+  /* Everything is deleted. */
+  smartlist_clear(cons2);
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(1, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("1,2d", smartlist_get(diff, 0));
+
+  smartlist_free(diff);
+
+  /* Everything is added. */
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(4, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("0a", smartlist_get(diff, 0));
+  tt_str_eq_line("foo", smartlist_get(diff, 1));
+  tt_str_eq_line("bar", smartlist_get(diff, 2));
+  tt_str_eq_line(".", smartlist_get(diff, 3));
+
+  smartlist_free(diff);
+
+  /* Everything is changed. */
+  smartlist_add_linecpy(cons2, area, "foo2");
+  smartlist_add_linecpy(cons2, area, "bar2");
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(4, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("1,2c", smartlist_get(diff, 0));
+  tt_str_eq_line("foo2", smartlist_get(diff, 1));
+  tt_str_eq_line("bar2", smartlist_get(diff, 2));
+  tt_str_eq_line(".", smartlist_get(diff, 3));
+
+  smartlist_free(diff);
+
+  /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(7, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("5a", smartlist_get(diff, 0));
+  tt_str_eq_line("U", smartlist_get(diff, 1));
+  tt_str_eq_line(".", smartlist_get(diff, 2));
+  tt_str_eq_line("4c", smartlist_get(diff, 3));
+  tt_str_eq_line("O", smartlist_get(diff, 4));
+  tt_str_eq_line(".", smartlist_get(diff, 5));
+  tt_str_eq_line("2d", smartlist_get(diff, 6));
+
+  smartlist_free(diff);
+
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+  consensus_split_lines(cons1, "B\n", area);
+  consensus_split_lines(cons2, "A\nB\n", area);
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(3, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("0a", smartlist_get(diff, 0));
+  tt_str_eq_line("A", smartlist_get(diff, 1));
+  tt_str_eq_line(".", smartlist_get(diff, 2));
+
+  /* TODO: small real use-cases, i.e. consensuses. */
+
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_apply_ed_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  memarea_t *area = memarea_new();
+  (void)arg;
+  cons1 = smartlist_new();
+  diff = smartlist_new();
+  setup_capture_of_logs(LOG_WARN);
+
+  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+
+  /* Command without range. */
+  smartlist_add_linecpy(diff, area, "a");
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  smartlist_clear(diff);
+  expect_single_log_msg_containing("an ed command was missing a line number");
+
+  /* Range without command. */
+  smartlist_add_linecpy(diff, area, "1");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("a line with no ed command was found");
+
+  smartlist_clear(diff);
+
+  /* Range without end. */
+  smartlist_add_linecpy(diff, area, "1,");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a range "
+                                   "end line number.");
+
+  smartlist_clear(diff);
+
+  /* Incoherent ranges. */
+  smartlist_add_linecpy(diff, area, "1,1");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an invalid range was found");
+
+  smartlist_clear(diff);
+
+  smartlist_add_linecpy(diff, area, "3,2");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an invalid range was found");
+
+  smartlist_clear(diff);
+
+  /* Unexpected range for add command. */
+  smartlist_add_linecpy(diff, area, "1,2a");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("add lines after a range");
+
+  smartlist_clear(diff);
+
+  /* $ for a non-delete command. */
+  smartlist_add_linecpy(diff, area, "1,$c");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it wanted to use $ with a command "
+                                   "other than delete");
+
+  smartlist_clear(diff);
+
+  /* Script is not in reverse order. */
+  smartlist_add_linecpy(diff, area, "1d");
+  smartlist_add_linecpy(diff, area, "3d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("its commands are not properly sorted");
+
+  smartlist_clear(diff);
+
+  /* Script contains unrecognised commands longer than one char. */
+  smartlist_add_linecpy(diff, area, "1foo");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* Script contains unrecognised commands. */
+  smartlist_add_linecpy(diff, area, "1e");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an unrecognised ed command was found");
+
+  smartlist_clear(diff);
+
+  /* Command that should be followed by at least one line and a ".", but
+   * isn't. */
+  smartlist_add_linecpy(diff, area, "0a");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it has an ed command that tries to "
+                                   "insert zero lines.");
+
+  /* Now it is followed by a ".", but it inserts zero lines. */
+  smartlist_add_linecpy(diff, area, ".");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it has an ed command that tries to "
+                                   "insert zero lines.");
+
+  smartlist_clear(diff);
+
+  /* Now it it inserts something, but has no terminator. */
+  smartlist_add_linecpy(diff, area, "0a");
+  smartlist_add_linecpy(diff, area, "hello");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("lines to be inserted that don't end with "
+                                   "a \".\".");
+
+  smartlist_clear(diff);
+
+  /* Ranges must be numeric only and cannot contain spaces. */
+  smartlist_add_linecpy(diff, area, "0, 4d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a range "
+                                   "end line number.");
+
+  smartlist_clear(diff);
+
+  /* '+' is not a number. */
+  smartlist_add_linecpy(diff, area, "+0,4d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a line number");
+
+  smartlist_clear(diff);
+
+  /* range duplication */
+  smartlist_add_linecpy(diff, area, "0,4d,5d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* space before command */
+  smartlist_add_linecpy(diff, area, "0,4 d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* space inside number */
+  smartlist_add_linecpy(diff, area, "0,4 5d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* Test appending text, 'a'. */
+  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(8, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("V", smartlist_get(cons2, 0));
+  tt_str_eq_line("A", smartlist_get(cons2, 1));
+  tt_str_eq_line("B", smartlist_get(cons2, 2));
+  tt_str_eq_line("C", smartlist_get(cons2, 3));
+  tt_str_eq_line("U", smartlist_get(cons2, 4));
+  tt_str_eq_line("O", smartlist_get(cons2, 5));
+  tt_str_eq_line("D", smartlist_get(cons2, 6));
+  tt_str_eq_line("E", smartlist_get(cons2, 7));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test deleting text, 'd'. */
+  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(2, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("C", smartlist_get(cons2, 0));
+  tt_str_eq_line("E", smartlist_get(cons2, 1));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test changing text, 'c'. */
+  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(5, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("M", smartlist_get(cons2, 0));
+  tt_str_eq_line("C", smartlist_get(cons2, 1));
+  tt_str_eq_line("T", smartlist_get(cons2, 2));
+  tt_str_eq_line("X", smartlist_get(cons2, 3));
+  tt_str_eq_line("E", smartlist_get(cons2, 4));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test 'a', 'd' and 'c' together. */
+  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(6, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("M", smartlist_get(cons2, 0));
+  tt_str_eq_line("A", smartlist_get(cons2, 1));
+  tt_str_eq_line("C", smartlist_get(cons2, 2));
+  tt_str_eq_line("T", smartlist_get(cons2, 3));
+  tt_str_eq_line("X", smartlist_get(cons2, 4));
+  tt_str_eq_line("E", smartlist_get(cons2, 5));
+
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_gen_diff(void *arg)
+{
+  char *cons1_str=NULL, *cons2_str=NULL;
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  consensus_digest_t digests1, digests2;
+  memarea_t *area = memarea_new();
+  (void)arg;
+  cons1 = smartlist_new();
+  cons2 = smartlist_new();
+
+  /* Identity hashes are not sorted properly, return NULL.
+   * Already tested in gen_ed_diff, but see that a NULL ed diff also makes
+   * gen_diff return NULL. */
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name bbbbbbbbbbbbbbbbb etc\nfoo\n"
+      "r name aaaaaaaaaaaaaaaaa etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  cons2_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name aaaaaaaaaaaaaaaaa etc\nfoo\n"
+      "r name ccccccccccccccccc etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest_as_signed(cons1_str, &digests1));
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest(cons2_str, &digests2));
+
+  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines(cons2, cons2_str, area);
+
+  diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+
+  /* Check that the headers are done properly. */
+  tor_free(cons1_str);
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nfoo\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest_as_signed(cons1_str, &digests1));
+  smartlist_clear(cons1);
+  consensus_split_lines(cons1, cons1_str, area);
+  diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(11, OP_EQ, smartlist_len(diff));
+  tt_assert(line_str_eq(smartlist_get(diff, 0),
+                        "network-status-diff-version 1"));
+  tt_assert(line_str_eq(smartlist_get(diff, 1), "hash "
+      "95D70F5A3CC65F920AA8B44C4563D7781A082674329661884E19E94B79D539C2 "
+      "7AFECEFA4599BA33D603653E3D2368F648DF4AC4723929B0F7CF39281596B0C1"));
+  tt_assert(line_str_eq(smartlist_get(diff, 2), "6,$d"));
+  tt_assert(line_str_eq(smartlist_get(diff, 3), "3,4c"));
+  tt_assert(line_str_eq(smartlist_get(diff, 4), "bar"));
+  tt_assert(line_str_eq(smartlist_get(diff, 5),
+                        "directory-signature foo bar"));
+  tt_assert(line_str_eq(smartlist_get(diff, 6),
+                        "."));
+  tt_assert(line_str_eq(smartlist_get(diff, 7), "1a"));
+  tt_assert(line_str_eq(smartlist_get(diff, 8),
+                        "r name aaaaaaaaaaaaaaaaa etc"));
+  tt_assert(line_str_eq(smartlist_get(diff, 9), "foo"));
+  tt_assert(line_str_eq(smartlist_get(diff, 10), "."));
+
+  /* TODO: small real use-cases, i.e. consensuses. */
+
+ done:
+  tor_free(cons1_str);
+  tor_free(cons2_str);
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_apply_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *diff=NULL;
+  char *cons1_str=NULL, *cons2 = NULL;
+  consensus_digest_t digests1;
+  (void)arg;
+  memarea_t *area = memarea_new();
+  cons1 = smartlist_new();
+  diff = smartlist_new();
+  setup_capture_of_logs(LOG_INFO);
+
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nfoo\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest(cons1_str, &digests1));
+  consensus_split_lines(cons1, cons1_str, area);
+
+  /* diff doesn't have enough lines. */
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("too short")
+
+  /* first line doesn't match format-version string. */
+  smartlist_add_linecpy(diff, area, "foo-bar");
+  smartlist_add_linecpy(diff, area, "header-line");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("format is not known")
+
+  /* The first word of the second header line is not "hash". */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "word a b");
+  smartlist_add_linecpy(diff, area, "x");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("does not include the necessary digests")
+
+  /* Wrong number of words after "hash". */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash a b c");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("does not include the necessary digests")
+
+  /* base16 digests do not have the expected length. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash aaa bbb");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("includes base16-encoded digests of "
+                                   "incorrect size")
+
+  /* base16 digests contain non-base16 characters. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      " ????????????????????????????????????????????????????????????????"
+      " ----------------------------------------------------------------");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("includes malformed digests")
+
+  /* Invalid ed diff.
+   * As tested in apply_ed_diff, but check that apply_diff does return NULL if
+   * the ed diff can't be applied. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* sha256 of cons2. */
+      " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA");
+  smartlist_add_linecpy(diff, area, "foobar");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("because an ed command was missing a line "
+                                   "number")
+
+  /* Base consensus doesn't match its digest as found in the diff. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* bogus sha256. */
+      " 3333333333333333333333333333333333333333333333333333333333333333"
+      /* sha256 of cons2. */
+      " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("base consensus doesn't match the digest "
+                            "as found");
+
+  /* Resulting consensus doesn't match its digest as found in the diff. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* bogus sha3. */
+      " 3333333333333333333333333333333333333333333333333333333333333333");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("resulting consensus doesn't match the "
+                            "digest as found");
+
+#if 0
+  /* XXXX No longer possible, since we aren't using the other algorithm. */
+  /* Resulting consensus digest cannot be computed */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* bogus sha3. */
+      " 3333333333333333333333333333333333333333333333333333333333333333");
+  smartlist_add_linecpy(diff, area, "1,2d"); // remove starting line
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("Could not compute digests of the consensus "
+                            "resulting from applying a consensus diff.");
+#endif /* 0 */
+
+  /* Very simple test, only to see that nothing errors. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* sha3 of cons2. */
+      " 90A418881B2FCAB3D9E60EE02E4D666D56CFA38F8A3B7AA3E0ADBA530DDA9353");
+  smartlist_add_linecpy(diff, area, "3c");
+  smartlist_add_linecpy(diff, area, "sample");
+  smartlist_add_linecpy(diff, area, ".");
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_str_op(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nsample\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n", OP_EQ,
+      cons2);
+  tor_free(cons2);
+
+  /* Check that lowercase letters in base16-encoded digests work too. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646d6cf563a41869d3b02e73254372ae3140046c5e7d83c9f71e54976af9b4"
+      /* sha3 of cons2. */
+      " 90a418881b2fcab3d9e60ee02e4d666d56cfa38f8a3b7aa3e0adba530dda9353");
+  smartlist_add_linecpy(diff, area, "3c");
+  smartlist_add_linecpy(diff, area, "sample");
+  smartlist_add_linecpy(diff, area, ".");
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_str_op(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nsample\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n", OP_EQ,
+      cons2);
+  tor_free(cons2);
+
+  smartlist_clear(diff);
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(cons1_str);
+  smartlist_free(cons1);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+#define CONSDIFF_LEGACY(name)                                          \
+  { #name, test_consdiff_ ## name , 0, NULL, NULL }
+
+struct testcase_t consdiff_tests[] = {
+  CONSDIFF_LEGACY(smartlist_slice),
+  CONSDIFF_LEGACY(smartlist_slice_string_pos),
+  CONSDIFF_LEGACY(lcs_lengths),
+  CONSDIFF_LEGACY(trim_slices),
+  CONSDIFF_LEGACY(set_changed),
+  CONSDIFF_LEGACY(calc_changes),
+  CONSDIFF_LEGACY(get_id_hash),
+  CONSDIFF_LEGACY(is_valid_router_entry),
+  CONSDIFF_LEGACY(next_router),
+  CONSDIFF_LEGACY(base64cmp),
+  CONSDIFF_LEGACY(gen_ed_diff),
+  CONSDIFF_LEGACY(apply_ed_diff),
+  CONSDIFF_LEGACY(gen_diff),
+  CONSDIFF_LEGACY(apply_diff),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
new file mode 100644
index 0000000000..3b91baca39
--- /dev/null
+++ b/src/test/test_consdiffmgr.c
@@ -0,0 +1,897 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "consdiff.h"
+#include "consdiffmgr.h"
+#include "cpuworker.h"
+#include "crypto_rand.h"
+#include "networkstatus.h"
+#include "routerparse.h"
+#include "workqueue.h"
+
+#include "test.h"
+#include "log_test_helpers.h"
+
+// ============================== Setup/teardown the consdiffmgr
+// These functions get run before/after each test in this module
+
+static void *
+consdiffmgr_test_setup(const struct testcase_t *arg)
+{
+  (void)arg;
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+
+  consdiff_cfg_t consdiff_cfg = { 300 };
+  consdiffmgr_configure(&consdiff_cfg);
+  return (void *)1; // must return something non-null.
+}
+static int
+consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
+{
+  (void)arg;
+  (void)ignore;
+  consdiffmgr_free_all();
+  return 1;
+}
+static struct testcase_setup_t setup_diffmgr = {
+  consdiffmgr_test_setup,
+  consdiffmgr_test_teardown
+};
+
+// ============================== NS faking functions
+// These functions are for making quick fake consensus objects and
+// strings that are just good enough for consdiff and consdiffmgr.
+
+static networkstatus_t *
+fake_ns_new(consensus_flavor_t flav, time_t valid_after)
+{
+  networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->type = NS_TYPE_CONSENSUS;
+  ns->flavor = flav;
+  ns->valid_after = valid_after;
+  return ns;
+}
+
+static char *
+fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
+{
+  const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
+  char valid_after_string[ISO_TIME_LEN+1];
+
+  format_iso_time(valid_after_string, valid_after);
+  char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
+  char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
+
+  char *consensus;
+  tor_asprintf(&consensus,
+               "network-status-version 3%s\n"
+               "vote-status consensus\n"
+               "valid-after %s\n"
+               "r name ccccccccccccccccc etc\nsample\n"
+               "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+               "%s\n"
+               "directory-signature hello-there\n"
+               "directory-signature %s\n",
+               flavor_string,
+               valid_after_string,
+               random_stuff,
+               random_stuff2);
+  tor_free(random_stuff);
+  tor_free(random_stuff2);
+  return consensus;
+}
+
+// ============================== Cpuworker mocking code
+// These mocking functions and types capture the cpuworker calls
+// so we can inspect them and run them in the main thread.
+static smartlist_t *fake_cpuworker_queue = NULL;
+typedef struct fake_work_queue_ent_t {
+  enum workqueue_reply_t (*fn)(void *, void *);
+  void (*reply_fn)(void *);
+  void *arg;
+} fake_work_queue_ent_t;
+static struct workqueue_entry_s *
+mock_cpuworker_queue_work(workqueue_priority_t prio,
+                          enum workqueue_reply_t (*fn)(void *, void *),
+                          void (*reply_fn)(void *),
+                          void *arg)
+{
+  (void) prio;
+
+  if (! fake_cpuworker_queue)
+    fake_cpuworker_queue = smartlist_new();
+
+  fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
+  ent->fn = fn;
+  ent->reply_fn = reply_fn;
+  ent->arg = arg;
+  smartlist_add(fake_cpuworker_queue, ent);
+  return (struct workqueue_entry_s *)ent;
+}
+static int
+mock_cpuworker_run_work(void)
+{
+  if (! fake_cpuworker_queue)
+    return 0;
+  SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
+      enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
+      if (r != WQ_RPL_REPLY)
+        return -1;
+  });
+  return 0;
+}
+static void
+mock_cpuworker_handle_replies(void)
+{
+  if (! fake_cpuworker_queue)
+    return;
+  SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
+      ent->reply_fn(ent->arg);
+      tor_free(ent);
+  });
+  smartlist_free(fake_cpuworker_queue);
+  fake_cpuworker_queue = NULL;
+}
+
+// ==============================  Other helpers
+
+static consdiff_status_t
+lookup_diff_from(consensus_cache_entry_t **out,
+                 consensus_flavor_t flav,
+                 const char *str1)
+{
+  uint8_t digest[DIGEST256_LEN];
+  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+    TT_FAIL(("Unable to compute sha3-as-signed"));
+    return CONSDIFF_NOT_FOUND;
+  }
+  return consdiffmgr_find_diff_from(out, flav,
+                                    DIGEST_SHA3_256, digest, sizeof(digest),
+                                    NO_METHOD);
+}
+
+static int
+lookup_apply_and_verify_diff(consensus_flavor_t flav,
+                             const char *str1,
+                             const char *str2)
+{
+  consensus_cache_entry_t *ent = NULL;
+  consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
+  if (ent == NULL || status != CONSDIFF_AVAILABLE) {
+    return -1;
+  }
+
+  consensus_cache_entry_incref(ent);
+  size_t size;
+  char *diff_string = NULL;
+  int r = uncompress_or_copy(&diff_string, &size, ent);
+  consensus_cache_entry_decref(ent);
+  if (diff_string == NULL || r < 0)
+    return -1;
+
+  char *applied = consensus_diff_apply(str1, diff_string);
+  tor_free(diff_string);
+  if (applied == NULL)
+    return -1;
+
+  int match = !strcmp(applied, str2);
+  tor_free(applied);
+  return match ? 0 : -1;
+}
+
+static void
+cdm_reload(void)
+{
+  consdiffmgr_free_all();
+  cdm_cache_get();
+  consdiffmgr_rescan();
+}
+
+// ==============================  Beginning of tests
+
+#if 0
+static int got_failure = 0;
+static void
+got_assertion_failure(void)
+{
+  ++got_failure;
+}
+
+/* XXXX This test won't work, because there is currently no way to actually
+ * XXXX capture a real assertion failure. */
+static void
+test_consdiffmgr_init_failure(void *arg)
+{
+  (void)arg;
+  // Capture assertions and bugs.
+
+  /* As in ...test_setup, but do not create the datadir. The missing directory
+   * will cause a failure. */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
+  tor_free(get_options_mutable()->CacheDirectory);
+  get_options_mutable()->CacheDirectory = ddir_fname; // now owns the pointer.
+
+  consdiff_cfg_t consdiff_cfg = { 7200, 300 };
+
+  tor_set_failed_assertion_callback(got_assertion_failure);
+  tor_capture_bugs_(1);
+  consdiffmgr_configure(&consdiff_cfg); // This should fail.
+  tt_int_op(got_failure, OP_EQ, 1);
+  const smartlist_t *bugs = tor_get_captured_bug_log_();
+  tt_int_op(smartlist_len(bugs), OP_EQ, 1);
+
+ done:
+  tor_end_capture_bugs_();
+}
+#endif /* 0 */
+
+static void
+test_consdiffmgr_sha3_helper(void *arg)
+{
+  (void) arg;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+  config_line_t *lines = NULL;
+  char *mem_op_hex_tmp = NULL;
+  config_line_prepend(&lines, "good-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+  config_line_prepend(&lines, "short-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF0");
+  config_line_prepend(&lines, "long-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
+  config_line_prepend(&lines, "not-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DXXXX");
+  consensus_cache_entry_t *ent =
+    consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+
+  uint8_t buf[DIGEST256_LEN];
+  tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
+  tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
+  test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+
+  tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
+
+ done:
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_consdiffmgr_add(void *arg)
+{
+  (void) arg;
+  time_t now = approx_time();
+
+  char *body = NULL;
+
+  consensus_cache_entry_t *ent = NULL;
+  networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
+  const char *dummy = "foo";
+  int r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* If we add it again, it won't work */
+  setup_capture_of_logs(LOG_INFO);
+  dummy = "bar";
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, -1);
+  expect_single_log_msg_containing("We already have a copy of that "
+                                   "consensus");
+  mock_clean_saved_logs();
+
+  /* But it will work fine if the flavor is different */
+  dummy = "baz";
+  ns_tmp->flavor = FLAV_MICRODESC;
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* And it will work fine if the time is different */
+  dummy = "quux";
+  ns_tmp->flavor = FLAV_NS;
+  ns_tmp->valid_after = now - 60;
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* If we add one a long long time ago, it will fail. */
+  dummy = "xyzzy";
+  ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("it's too old.");
+
+  /* Try looking up a consensuses. */
+  ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
+  tt_assert(ent);
+  consensus_cache_entry_incref(ent);
+  size_t s;
+  r = uncompress_or_copy(&body, &s, ent);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(s, OP_EQ, 4);
+  tt_mem_op(body, OP_EQ, "quux", 4);
+
+  /* Try looking up another entry, but fail */
+  tt_ptr_op(cdm_cache_lookup_consensus(FLAV_MICRODESC, now - 60), OP_EQ, NULL);
+  tt_ptr_op(cdm_cache_lookup_consensus(FLAV_NS, now - 61), OP_EQ, NULL);
+
+ done:
+  networkstatus_vote_free(ns_tmp);
+  teardown_capture_of_logs();
+  consensus_cache_entry_decref(ent);
+  tor_free(body);
+}
+
+static void
+test_consdiffmgr_make_diffs(void *arg)
+{
+  (void)arg;
+  networkstatus_t *ns = NULL;
+  char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
+  char *applied = NULL, *diff_text = NULL;
+  time_t now = approx_time();
+  int r;
+  consensus_cache_entry_t *diff = NULL;
+  uint8_t md_ns_sha3[DIGEST256_LEN];
+  consdiff_status_t diff_status;
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  // Try rescan with no consensuses: shouldn't crash or queue work.
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  // Make two consensuses, 1 hour sec ago.
+  ns = fake_ns_new(FLAV_NS, now-3600);
+  ns_body = fake_ns_body_new(FLAV_NS, now-3600);
+  r = consdiffmgr_add_consensus(ns_body, ns);
+  networkstatus_vote_free(ns);
+  tor_free(ns_body);
+  tt_int_op(r, OP_EQ, 0);
+
+  ns = fake_ns_new(FLAV_MICRODESC, now-3600);
+  md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
+  r = consdiffmgr_add_consensus(md_ns_body, ns);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  networkstatus_vote_free(ns);
+  tt_int_op(r, OP_EQ, 0);
+
+  // No diffs will be generated.
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  // Add a MD consensus from 45 minutes ago. This should cause one diff
+  // worth of work to get queued.
+  ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
+  md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
+  r = consdiffmgr_add_consensus(md_ns_body_2, ns);
+  networkstatus_vote_free(ns);
+  tt_int_op(r, OP_EQ, 0);
+
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
+                                           DIGEST_SHA3_256,
+                                           md_ns_sha3, DIGEST256_LEN,
+                                           NO_METHOD);
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
+
+  // Now run that process and get the diff.
+  r = mock_cpuworker_run_work();
+  tt_int_op(r, OP_EQ, 0);
+  mock_cpuworker_handle_replies();
+
+  // At this point we should be able to get that diff.
+  diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
+                                           DIGEST_SHA3_256,
+                                           md_ns_sha3, DIGEST256_LEN,
+                                           NO_METHOD);
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
+  tt_assert(diff);
+
+  /* Make sure applying the diff actually works */
+  const uint8_t *diff_body;
+  size_t diff_size;
+  r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
+  tt_int_op(r, OP_EQ, 0);
+  diff_text = tor_memdup_nulterm(diff_body, diff_size);
+  applied = consensus_diff_apply(md_ns_body, diff_text);
+  tt_assert(applied);
+  tt_str_op(applied, OP_EQ, md_ns_body_2);
+
+  /* Rescan again: no more work to do. */
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+ done:
+  tor_free(md_ns_body);
+  tor_free(md_ns_body_2);
+  tor_free(diff_text);
+  tor_free(applied);
+}
+
+static void
+test_consdiffmgr_diff_rules(void *arg)
+{
+  (void)arg;
+#define N 6
+  char *md_body[N], *ns_body[N];
+  networkstatus_t *md_ns[N], *ns_ns[N];
+  int i;
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* Create a bunch of consensus things at 15-second intervals. */
+  time_t start = approx_time() - 120;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 15;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    ns_body[i] = fake_ns_body_new(FLAV_NS, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+    ns_ns[i] = fake_ns_new(FLAV_NS, when);
+  }
+
+  /* For the MD consensuses: add 4 of them, and make sure that
+   * diffs are created to one consensus (the most recent) only. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  /* For the NS consensuses: add 3, generate, and add one older one and
+   * make sure that older one is the only one whose diff is generated */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  /* At this point, we should actually have working diffs! */
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
+
+  /* Self-to-self diff won't be present */
+  consensus_cache_entry_t *ent;
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+       lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
+  /* No diff from 2 has been added yet */
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+       lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
+  /* No diff arriving at old things. */
+  tt_int_op(-1, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+  /* No backwards diff */
+  tt_int_op(-1, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
+
+  /* Now, an update: add number 2 and make sure it's the only one whose diff
+   * is regenerated. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  tt_int_op(0, OP_EQ,
+            lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
+
+  /* Finally: reload, and make sure that the information is still indexed */
+  cdm_reload();
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
+
+ done:
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    tor_free(ns_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+    networkstatus_vote_free(ns_ns[i]);
+  }
+  UNMOCK(cpuworker_queue_work);
+#undef N
+}
+
+static void
+test_consdiffmgr_diff_failure(void *arg)
+{
+  (void)arg;
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* We're going to make sure that if we have a bogus request where
+   * we can't actually compute a diff, the world must not end. */
+  networkstatus_t *ns1 = NULL;
+  networkstatus_t *ns2 = NULL;
+  int r;
+
+  ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
+  ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
+  r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
+  tt_int_op(r, OP_EQ, 0);
+  // We refuse to compute a diff to or from a line holding only a single dot.
+  // We can add it here, though.
+  r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
+  tt_int_op(r, OP_EQ, 0);
+
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  setup_capture_of_logs(LOG_WARN);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  expect_single_log_msg_containing("one of the lines to be added is \".\".");
+  mock_clean_saved_logs();
+  mock_cpuworker_handle_replies();
+  expect_single_log_msg_containing("Worker was unable to compute consensus "
+                                   "diff from ");
+
+  /* Make sure the diff is not present */
+  consensus_cache_entry_t *ent;
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
+
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(cpuworker_queue_work);
+  networkstatus_vote_free(ns1);
+  networkstatus_vote_free(ns2);
+}
+
+static void
+test_consdiffmgr_diff_pending(void *arg)
+{
+#define N 3
+  (void)arg;
+  char *md_body[N];
+  networkstatus_t *md_ns[N];
+  time_t start = approx_time() - 120;
+  int i;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 30;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+  }
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  /* Make a diff */
+  consdiffmgr_rescan();
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+
+  /* Look it up.  Is it pending? */
+  consensus_cache_entry_t *ent = NULL;
+  consdiff_status_t diff_status;
+  diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
+  tt_ptr_op(ent, OP_EQ, NULL);
+
+  /* Add another old consensus.  only one new diff should launch! */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
+  consdiffmgr_rescan();
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+
+ done:
+  UNMOCK(cpuworker_queue_work);
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+  }
+#undef N
+}
+
+static void
+test_consdiffmgr_cleanup_old(void *arg)
+{
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  /* This item will be will be cleanable because it has a valid-after
+   * time far in the past. */
+  config_line_prepend(&labels, "document-type", "confribble-blarg");
+  config_line_prepend(&labels, "consensus-valid-after",
+                      "1980-10-10T10:10:10");
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Deleting entry because its consensus-valid-"
+                            "after value (1980-10-10T10:10:10) was too old");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_bad_valid_after(void *arg)
+{
+  /* This will seem cleanable, but isn't, because its valid-after time is
+   * misformed. */
+
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  config_line_prepend(&labels, "document-type", "consensus");
+  config_line_prepend(&labels, "consensus-valid-after",
+                      "whan that aprille with his shoures soote"); // (~1385?)
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Ignoring entry because its consensus-valid-"
+                            "after value (\"whan that aprille with his "
+                            "shoures soote\") was unparseable");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_no_valid_after(void *arg)
+{
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  /* This item will be will be uncleanable because it has no recognized
+   * valid-after. */
+  config_line_prepend(&labels, "document-type", "consensus");
+  config_line_prepend(&labels, "confrooble-voolid-oofter",
+                      "2010-10-10T09:08:07");
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Ignoring entry because it had no consensus-"
+                            "valid-after label");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_old_diffs(void *arg)
+{
+  (void)arg;
+#define N 4
+  char *md_body[N];
+  networkstatus_t *md_ns[N];
+  int i;
+  consensus_cache_entry_t *hold_ent = NULL, *ent;
+
+  /* Make sure that the cleanup function removes diffs to the not-most-recent
+   * consensus. */
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* Create a bunch of consensus things at 15-second intervals. */
+  time_t start = approx_time() - 120;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 15;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+  }
+
+  /* add the first 3. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  /* Make diffs. */
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  /* Nothing is deletable now */
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
+            lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
+  consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
+
+  /* Now add an even-more-recent consensus; this should make all previous
+   * diffs deletable, and make delete */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
+  tt_int_op(2 * n_diff_compression_methods() +
+            (n_consensus_compression_methods() - 1) , OP_EQ,
+            consdiffmgr_cleanup());
+
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
+  /* This one is marked deletable but still in the hashtable */
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
+
+  /* Everything should be valid at this point */
+  tt_int_op(0, OP_EQ, consdiffmgr_validate());
+
+  /* And if we recan NOW, we'll purge the hashtable of the entries,
+   * and launch attempts to generate new ones */
+  consdiffmgr_rescan();
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
+
+  /* We're still holding on to this, though, so we can still map it! */
+  const uint8_t *t1 = NULL;
+  size_t s;
+  int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(t1);
+
+ done:
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+  }
+  consensus_cache_entry_decref(hold_ent);
+  UNMOCK(cpuworker_queue_work);
+#undef N
+}
+
+static void
+test_consdiffmgr_validate(void *arg)
+{
+  (void)arg;
+  config_line_t *lines = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+  smartlist_t *vals = smartlist_new();
+
+  /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
+   * one with a wrong sha3, and one with no sha3. */
+  config_line_prepend(&lines, "id", "wrong sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "bad sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "now is the winter of our dicotheque");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "no sha3");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "good sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "8d8b1998616cd6b4c4055da8d38728dc"
+                      "93c758d4131a53c7d81aa6337dee1c05");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  cdm_reload();
+  cache = cdm_cache_get();
+  tt_int_op(1, OP_EQ, consdiffmgr_validate());
+
+  consensus_cache_find_all(vals, cache, "id", "good sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 1);
+  smartlist_clear(vals);
+
+  consensus_cache_find_all(vals, cache, "id", "no sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 1);
+  smartlist_clear(vals);
+
+  consensus_cache_find_all(vals, cache, "id", "wrong sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 0);
+  consensus_cache_find_all(vals, cache, "id", "bad sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 0);
+
+ done:
+  smartlist_free(vals);
+}
+
+#define TEST(name)                                      \
+  { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
+
+struct testcase_t consdiffmgr_tests[] = {
+#if 0
+  { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
+#endif
+  TEST(sha3_helper),
+  TEST(add),
+  TEST(make_diffs),
+  TEST(diff_rules),
+  TEST(diff_failure),
+  TEST(diff_pending),
+  TEST(cleanup_old),
+  TEST(cleanup_bad_valid_after),
+  TEST(cleanup_no_valid_after),
+  TEST(cleanup_old_diffs),
+  TEST(validate),
+
+  // XXXX Test: non-cacheing cases of replyfn().
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index d8b82e0661..3fc3523af4 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -1,10 +1,11 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include "or.h"
+#include "crypto_rand.h"
 #include "fp_pair.h"
 #include "test.h"
 
@@ -501,31 +502,31 @@ test_container_smartlist_pos(void *arg)
   (void) arg;
   smartlist_t *sl = smartlist_new();
 
-  smartlist_add(sl, tor_strdup("This"));
-  smartlist_add(sl, tor_strdup("is"));
-  smartlist_add(sl, tor_strdup("a"));
-  smartlist_add(sl, tor_strdup("test"));
-  smartlist_add(sl, tor_strdup("for"));
-  smartlist_add(sl, tor_strdup("a"));
-  smartlist_add(sl, tor_strdup("function"));
+  smartlist_add_strdup(sl, "This");
+  smartlist_add_strdup(sl, "is");
+  smartlist_add_strdup(sl, "a");
+  smartlist_add_strdup(sl, "test");
+  smartlist_add_strdup(sl, "for");
+  smartlist_add_strdup(sl, "a");
+  smartlist_add_strdup(sl, "function");
 
   /* Test string_pos */
-  tt_int_op(smartlist_string_pos(NULL, "Fred"), ==, -1);
-  tt_int_op(smartlist_string_pos(sl, "Fred"), ==, -1);
-  tt_int_op(smartlist_string_pos(sl, "This"), ==, 0);
-  tt_int_op(smartlist_string_pos(sl, "a"), ==, 2);
-  tt_int_op(smartlist_string_pos(sl, "function"), ==, 6);
+  tt_int_op(smartlist_string_pos(NULL, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_string_pos(sl, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_string_pos(sl, "This"), OP_EQ, 0);
+  tt_int_op(smartlist_string_pos(sl, "a"), OP_EQ, 2);
+  tt_int_op(smartlist_string_pos(sl, "function"), OP_EQ, 6);
 
   /* Test pos */
-  tt_int_op(smartlist_pos(NULL, "Fred"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "Fred"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "This"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "a"), ==, -1);
-  tt_int_op(smartlist_pos(sl, "function"), ==, -1);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,0)), ==, 0);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,2)), ==, 2);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,5)), ==, 5);
-  tt_int_op(smartlist_pos(sl, smartlist_get(sl,6)), ==, 6);
+  tt_int_op(smartlist_pos(NULL, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "Fred"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "This"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "a"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, "function"), OP_EQ, -1);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,0)), OP_EQ, 0);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,2)), OP_EQ, 2);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,5)), OP_EQ, 5);
+  tt_int_op(smartlist_pos(sl, smartlist_get(sl,6)), OP_EQ, 6);
 
  done:
   SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
@@ -681,7 +682,7 @@ test_container_pqueue(void *arg)
 {
   smartlist_t *sl = smartlist_new();
   int (*cmp)(const void *, const void*);
-  const int offset = STRUCT_OFFSET(pq_entry_t, idx);
+  const int offset = offsetof(pq_entry_t, idx);
 #define ENTRY(s) pq_entry_t s = { #s, -1 }
   ENTRY(cows);
   ENTRY(zebras);
@@ -830,7 +831,7 @@ test_container_strmap(void *arg)
   found_keys = smartlist_new();
   while (!strmap_iter_done(iter)) {
     strmap_iter_get(iter,&k,&v);
-    smartlist_add(found_keys, tor_strdup(k));
+    smartlist_add_strdup(found_keys, k);
     tt_ptr_op(v,OP_EQ, strmap_get(map, k));
 
     if (!strcmp(k, "K2")) {
@@ -882,6 +883,46 @@ test_container_strmap(void *arg)
   tor_free(v105);
 }
 
+static void
+test_container_smartlist_remove(void *arg)
+{
+  (void) arg;
+  int array[5];
+  smartlist_t *sl = smartlist_new();
+  int i,j;
+
+  for (j=0; j < 2; ++j)
+    for (i=0; i < 5; ++i)
+      smartlist_add(sl, &array[i]);
+
+  smartlist_remove(sl, &array[0]);
+  smartlist_remove(sl, &array[3]);
+  smartlist_remove(sl, &array[4]);
+  tt_assert(! smartlist_contains(sl, &array[0]));
+  tt_assert(smartlist_contains(sl, &array[1]));
+  tt_assert(smartlist_contains(sl, &array[2]));
+  tt_assert(! smartlist_contains(sl, &array[3]));
+  tt_assert(! smartlist_contains(sl, &array[4]));
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+
+  smartlist_clear(sl);
+  for (j=0; j < 2; ++j)
+    for (i=0; i < 5; ++i)
+      smartlist_add(sl, &array[i]);
+
+  smartlist_remove_keeporder(sl, &array[0]);
+  smartlist_remove_keeporder(sl, &array[3]);
+  smartlist_remove_keeporder(sl, &array[4]);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  tt_ptr_op(smartlist_get(sl, 0), OP_EQ, &array[1]);
+  tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
+  tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for getting the median of a list. */
 static void
 test_container_order_functions(void *arg)
@@ -950,13 +991,13 @@ test_container_order_functions(void *arg)
   tt_assert(15 == median_time(times, 5));
 
   int32_t int32s[] = { -5, -10, -50, 100 };
-  tt_int_op(-5, ==, median_int32(int32s, 1));
-  tt_int_op(-10, ==, median_int32(int32s, 2));
-  tt_int_op(-10, ==, median_int32(int32s, 3));
-  tt_int_op(-10, ==, median_int32(int32s, 4));
+  tt_int_op(-5, OP_EQ, median_int32(int32s, 1));
+  tt_int_op(-10, OP_EQ, median_int32(int32s, 2));
+  tt_int_op(-10, OP_EQ, median_int32(int32s, 3));
+  tt_int_op(-10, OP_EQ, median_int32(int32s, 4));
 
   long longs[] = { -30, 30, 100, -100, 7 };
-  tt_int_op(7, ==, find_nth_long(longs, 5, 2));
+  tt_int_op(7, OP_EQ, find_nth_long(longs, 5, 2));
 
  done:
   ;
@@ -1066,7 +1107,7 @@ test_container_fp_pair_map(void *arg)
   tt_int_op(fp_pair_map_size(map),OP_EQ, 4);
   fp_pair_map_assert_ok(map);
   fp_pair_map_set(map, &fp5, v104);
-  fp_pair_map_set(map, &fp6, v105);
+  fp_pair_map_set_by_digests(map, fp6.first, fp6.second, v105);
   fp_pair_map_assert_ok(map);
 
   /* Test iterator. */
@@ -1084,7 +1125,8 @@ test_container_fp_pair_map(void *arg)
 
   /* Make sure we removed fp2, but not the others. */
   tt_ptr_op(fp_pair_map_get(map, &fp2),OP_EQ, NULL);
-  tt_ptr_op(fp_pair_map_get(map, &fp5),OP_EQ, v104);
+  tt_ptr_op(fp_pair_map_get_by_digests(map, fp5.first, fp5.second),
+            OP_EQ, v104);
 
   fp_pair_map_assert_ok(map);
   /* Clean up after ourselves. */
@@ -1113,31 +1155,31 @@ test_container_smartlist_most_frequent(void *arg)
   const char *cp;
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 0);
-  tt_ptr_op(cp, ==, NULL);
+  tt_int_op(count, OP_EQ, 0);
+  tt_ptr_op(cp, OP_EQ, NULL);
 
   /* String must be sorted before we call get_most_frequent */
   smartlist_split_string(sl, "abc:def:ghi", ":", 0, 0);
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 1);
-  tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */
+  tt_int_op(count, OP_EQ, 1);
+  tt_str_op(cp, OP_EQ, "ghi"); /* Ties broken in favor of later element */
 
   smartlist_split_string(sl, "def:ghi", ":", 0, 0);
   smartlist_sort_strings(sl);
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 2);
-  tt_ptr_op(cp, !=, NULL);
-  tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */
+  tt_int_op(count, OP_EQ, 2);
+  tt_ptr_op(cp, OP_NE, NULL);
+  tt_str_op(cp, OP_EQ, "ghi"); /* Ties broken in favor of later element */
 
   smartlist_split_string(sl, "def:abc:qwop", ":", 0, 0);
   smartlist_sort_strings(sl);
 
   cp = smartlist_get_most_frequent_string_(sl, &count);
-  tt_int_op(count, ==, 3);
-  tt_ptr_op(cp, !=, NULL);
-  tt_str_op(cp, ==, "def"); /* No tie */
+  tt_int_op(count, OP_EQ, 3);
+  tt_ptr_op(cp, OP_NE, NULL);
+  tt_str_op(cp, OP_EQ, "def"); /* No tie */
 
  done:
   SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
@@ -1166,7 +1208,7 @@ test_container_smartlist_sort_ptrs(void *arg)
     smartlist_shuffle(sl);
     smartlist_sort_pointers(sl);
     for (j = 0; j < ARRAY_LENGTH(arrayptrs); ++j) {
-      tt_ptr_op(smartlist_get(sl, j), ==, arrayptrs[j]);
+      tt_ptr_op(smartlist_get(sl, j), OP_EQ, arrayptrs[j]);
     }
   }
 
@@ -1192,11 +1234,11 @@ test_container_smartlist_strings_eq(void *arg)
   } while (0)
 
   /* Both NULL, so equal */
-  tt_int_op(1, ==, smartlist_strings_eq(NULL, NULL));
+  tt_int_op(1, OP_EQ, smartlist_strings_eq(NULL, NULL));
 
   /* One NULL, not equal. */
-  tt_int_op(0, ==, smartlist_strings_eq(NULL, sl1));
-  tt_int_op(0, ==, smartlist_strings_eq(sl1, NULL));
+  tt_int_op(0, OP_EQ, smartlist_strings_eq(NULL, sl1));
+  tt_int_op(0, OP_EQ, smartlist_strings_eq(sl1, NULL));
 
   /* Both empty, both equal. */
   EQ_SHOULD_SAY("", "", 1);
@@ -1239,6 +1281,7 @@ struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
+  CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index f19c846144..1a350f66c0 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,20 +1,100 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONTROL_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "control.h"
 #include "entrynodes.h"
+#include "hs_common.h"
 #include "networkstatus.h"
 #include "rendservice.h"
 #include "routerlist.h"
 #include "test.h"
+#include "test_helpers.h"
 
 static void
-test_add_onion_helper_keyarg(void *arg)
+test_add_onion_helper_keyarg_v3(void *arg)
 {
-  crypto_pk_t *pk = NULL;
-  crypto_pk_t *pk2 = NULL;
+  int ret, hs_version;
+  add_onion_secret_key_t pk;
+  char *key_new_blob = NULL;
+  char *err_msg = NULL;
+  const char *key_new_alg = NULL;
+
+  (void) arg;
+
+  memset(&pk, 0, sizeof(pk));
+
+  /* Test explicit ED25519-V3 key generation. */
+  ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg,
+                                &key_new_blob, &pk, &hs_version,
+                                &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
+  tt_assert(key_new_blob);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+
+  /* Test discarding the private key. */
+  ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg,
+                                &key_new_blob, &pk, &hs_version,
+                                &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+
+  /* Test passing a key blob. */
+  {
+    /* The base64 key and hex key are the same. Hex key is 64 bytes long. The
+     * sk has been generated randomly using python3. */
+    const char *base64_sk =
+      "a9bT19PqGC9Y+BmOo1IQvCGjjwxMiaaxEXZ+FKMxpEQW"
+      "6AmSV5roThUGMRCaqQSCnR2jI1vL2QxHORzI4RxMmw==";
+    const char *hex_sk =
+      "\x6b\xd6\xd3\xd7\xd3\xea\x18\x2f\x58\xf8\x19\x8e\xa3\x52\x10\xbc"
+      "\x21\xa3\x8f\x0c\x4c\x89\xa6\xb1\x11\x76\x7e\x14\xa3\x31\xa4\x44"
+      "\x16\xe8\x09\x92\x57\x9a\xe8\x4e\x15\x06\x31\x10\x9a\xa9\x04\x82"
+      "\x9d\x1d\xa3\x23\x5b\xcb\xd9\x0c\x47\x39\x1c\xc8\xe1\x1c\x4c\x9b";
+    char *key_blob = NULL;
+
+    tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk);
+    tt_assert(key_blob);
+    ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg,
+                                  &key_new_blob, &pk, &hs_version,
+                                  &err_msg);
+    tor_free(key_blob);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+    tt_assert(pk.v3);
+    tt_mem_op(pk.v3, OP_EQ, hex_sk, 64);
+    tt_ptr_op(key_new_alg, OP_EQ, NULL);
+    tt_ptr_op(key_new_blob, OP_EQ, NULL);
+    tt_ptr_op(err_msg, OP_EQ, NULL);
+    tor_free(pk.v3); pk.v3 = NULL;
+    tor_free(key_new_blob);
+  }
+
+ done:
+  tor_free(pk.v3);
+  tor_free(key_new_blob);
+  tor_free(err_msg);
+}
+
+static void
+test_add_onion_helper_keyarg_v2(void *arg)
+{
+  int ret, hs_version;
+  add_onion_secret_key_t pk;
+  crypto_pk_t *pk1 = NULL;
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
   char *err_msg = NULL;
@@ -23,83 +103,100 @@ test_add_onion_helper_keyarg(void *arg)
 
   (void) arg;
 
+  memset(&pk, 0, sizeof(pk));
+
   /* Test explicit RSA1024 key generation. */
-  pk = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(pk);
+  ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "BEST" key generation (Assumes BEST = RSA1024). */
-  crypto_pk_free(pk);
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  pk = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(pk);
+  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test discarding the private key. */
-  crypto_pk_free(pk);
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  pk = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
-  tt_assert(!err_msg);
+  ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test generating a invalid key type. */
-  crypto_pk_free(pk);
-  pk = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(!pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
+  ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(!pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
   tt_assert(err_msg);
 
   /* Test loading a RSA1024 key. */
   tor_free(err_msg);
-  pk = pk_generate(0);
-  tt_int_op(0, OP_EQ, crypto_pk_base64_encode(pk, &encoded));
+  pk1 = pk_generate(0);
+  tt_int_op(0, OP_EQ, crypto_pk_base64_encode(pk1, &encoded));
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
-  pk2 = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &err_msg);
-  tt_assert(pk2);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
-  tt_assert(!err_msg);
-  tt_assert(crypto_pk_cmp_keys(pk, pk2) == 0);
+  ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0);
 
   /* Test loading a invalid key type. */
   tor_free(arg_str);
-  crypto_pk_free(pk); pk = NULL;
+  crypto_pk_free(pk1); pk1 = NULL;
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_asprintf(&arg_str, "RSA512:%s", encoded);
-  pk = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(!pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
+  ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(!pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
   tt_assert(err_msg);
 
   /* Test loading a invalid key. */
   tor_free(arg_str);
-  crypto_pk_free(pk); pk = NULL;
+  crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(err_msg);
   encoded[strlen(encoded)/2] = '\0';
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
-  pk = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &err_msg);
-  tt_assert(!pk);
-  tt_assert(!key_new_alg);
-  tt_assert(!key_new_blob);
+  ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, &err_msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
+  tt_assert(!pk.v2);
+  tt_ptr_op(key_new_alg, OP_EQ, NULL);
+  tt_ptr_op(key_new_blob, OP_EQ, NULL);
   tt_assert(err_msg);
 
  done:
-  crypto_pk_free(pk);
-  crypto_pk_free(pk2);
+  crypto_pk_free(pk1);
+  crypto_pk_free(pk.v2);
   tor_free(key_new_blob);
   tor_free(err_msg);
   tor_free(encoded);
@@ -107,6 +204,45 @@ test_add_onion_helper_keyarg(void *arg)
 }
 
 static void
+test_getinfo_helper_onion(void *arg)
+{
+  (void)arg;
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+  char *service_id = NULL;
+  int rt = 0;
+
+  dummy.ephemeral_onion_services = NULL;
+
+  /* successfully get an empty answer */
+  rt = getinfo_helper_onions(&dummy, "onions/current", &answer, &errmsg);
+  tt_int_op(rt, OP_EQ, 0);
+  tt_str_op(answer, OP_EQ, "");
+  tor_free(answer);
+
+  /* successfully get an empty answer */
+  rt = getinfo_helper_onions(&dummy, "onions/detached", &answer, &errmsg);
+  tt_int_op(rt, OP_EQ, 0);
+  tt_str_op(answer, OP_EQ, "");
+  tor_free(answer);
+
+  /* get an answer for one onion service */
+  service_id = tor_strdup("dummy_onion_id");
+  dummy.ephemeral_onion_services = smartlist_new();
+  smartlist_add(dummy.ephemeral_onion_services, service_id);
+  rt = getinfo_helper_onions(&dummy, "onions/current", &answer, &errmsg);
+  tt_int_op(rt, OP_EQ, 0);
+  tt_str_op(answer, OP_EQ, "dummy_onion_id");
+
+ done:
+  tor_free(answer);
+  tor_free(service_id);
+  smartlist_free(dummy.ephemeral_onion_services);
+}
+
+static void
 test_rend_service_parse_port_config(void *arg)
 {
   const char *sep = ",";
@@ -118,25 +254,25 @@ test_rend_service_parse_port_config(void *arg)
   /* Test "VIRTPORT" only. */
   cfg = rend_service_parse_port_config("80", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "VIRTPORT,TARGET" (Target is port). */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("80,8080", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "VIRTPORT,TARGET" (Target is IPv4:port). */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("80,192.0.2.1:8080", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
 
   /* Test "VIRTPORT,TARGET" (Target is IPv6:port). */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("80,[2001:db8::1]:8080", sep, &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_service_port_config_free(cfg);
   cfg = NULL;
 
@@ -145,13 +281,13 @@ test_rend_service_parse_port_config(void *arg)
   /* Test empty config. */
   rend_service_port_config_free(cfg);
   cfg = rend_service_parse_port_config("", sep, &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_assert(err_msg);
 
   /* Test invalid port. */
   tor_free(err_msg);
   cfg = rend_service_parse_port_config("90001", sep, &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
@@ -163,7 +299,7 @@ test_rend_service_parse_port_config(void *arg)
   cfg = rend_service_parse_port_config("100 unix:\"/tmp/foo bar\"",
                                        " ", &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_service_port_config_free(cfg);
   cfg = NULL;
 
@@ -172,22 +308,24 @@ test_rend_service_parse_port_config(void *arg)
   cfg = rend_service_parse_port_config("100 unix:\"/tmp/foo bar\"",
                                        " ", &err_msg);
   tt_assert(cfg);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_service_port_config_free(cfg);
   cfg = NULL;
 
   /* quoted unix port, missing end quote */
   cfg = rend_service_parse_port_config("100 unix:\"/tmp/foo bar",
                                        " ", &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_str_op(err_msg, OP_EQ, "Couldn't process address <unix:\"/tmp/foo bar> "
             "from hidden service configuration");
   tor_free(err_msg);
 
   /* bogus IP address */
-  cfg = rend_service_parse_port_config("100 1.2.3.4.5:9000",
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  cfg = rend_service_parse_port_config("100 foo!!.example.com:9000",
                                        " ", &err_msg);
-  tt_assert(!cfg);
+  UNMOCK(tor_addr_lookup);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_str_op(err_msg, OP_EQ, "Unparseable address in hidden service port "
             "configuration.");
   tor_free(err_msg);
@@ -195,7 +333,7 @@ test_rend_service_parse_port_config(void *arg)
   /* bogus port port */
   cfg = rend_service_parse_port_config("100 99999",
                                        " ", &err_msg);
-  tt_assert(!cfg);
+  tt_ptr_op(cfg, OP_EQ, NULL);
   tt_str_op(err_msg, OP_EQ, "Unparseable or out-of-range port \"99999\" "
             "in hidden service port configuration.");
   tor_free(err_msg);
@@ -218,7 +356,7 @@ test_add_onion_helper_clientauth(void *arg)
   client = add_onion_helper_clientauth("alice", &created, &err_msg);
   tt_assert(client);
   tt_assert(created);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test "ClientName:Blob" */
@@ -226,26 +364,26 @@ test_add_onion_helper_clientauth(void *arg)
                                        &created, &err_msg);
   tt_assert(client);
   tt_assert(!created);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test invalid client names */
   client = add_onion_helper_clientauth("no*asterisks*allowed", &created,
                                        &err_msg);
-  tt_assert(!client);
+  tt_ptr_op(client, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
   /* Test invalid auth cookie */
   client = add_onion_helper_clientauth("alice:12345", &created, &err_msg);
-  tt_assert(!client);
+  tt_ptr_op(client, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
   /* Test invalid syntax */
   client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created,
                                        &err_msg);
-  tt_assert(!client);
+  tt_ptr_op(client, OP_EQ, NULL);
   tt_assert(err_msg);
   tor_free(err_msg);
 
@@ -258,7 +396,7 @@ test_add_onion_helper_clientauth(void *arg)
 
 static const download_status_t dl_status_default =
   { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static download_status_t ns_dl_status[N_CONSENSUS_FLAVORS];
 static download_status_t ns_dl_status_bootstrap[N_CONSENSUS_FLAVORS];
 static download_status_t ns_dl_status_running[N_CONSENSUS_FLAVORS];
@@ -269,7 +407,7 @@ static download_status_t ns_dl_status_running[N_CONSENSUS_FLAVORS];
  */
 static const download_status_t dls_sample_1 =
   { 1467163900, 0, 0, DL_SCHED_GENERIC, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_DETERMINISTIC, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static const char * dls_sample_1_str =
     "next-attempt-at 2016-06-29 01:31:40\n"
     "n-download-failures 0\n"
@@ -277,10 +415,12 @@ static const char * dls_sample_1_str =
     "schedule DL_SCHED_GENERIC\n"
     "want-authority DL_WANT_ANY_DIRSERVER\n"
     "increment-on DL_SCHED_INCREMENT_FAILURE\n"
-    "backoff DL_SCHED_DETERMINISTIC\n";
+    "backoff DL_SCHED_RANDOM_EXPONENTIAL\n"
+    "last-backoff-position 0\n"
+    "last-delay-used 0\n";
 static const download_status_t dls_sample_2 =
   { 1467164400, 1, 2, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_DETERMINISTIC, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static const char * dls_sample_2_str =
     "next-attempt-at 2016-06-29 01:40:00\n"
     "n-download-failures 1\n"
@@ -288,10 +428,12 @@ static const char * dls_sample_2_str =
     "schedule DL_SCHED_CONSENSUS\n"
     "want-authority DL_WANT_AUTHORITY\n"
     "increment-on DL_SCHED_INCREMENT_FAILURE\n"
-    "backoff DL_SCHED_DETERMINISTIC\n";
+    "backoff DL_SCHED_RANDOM_EXPONENTIAL\n"
+    "last-backoff-position 0\n"
+    "last-delay-used 0\n";
 static const download_status_t dls_sample_3 =
   { 1467154400, 12, 25, DL_SCHED_BRIDGE, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_DETERMINISTIC, 0, 0 };
+    DL_SCHED_INCREMENT_ATTEMPT, 0, 0 };
 static const char * dls_sample_3_str =
     "next-attempt-at 2016-06-28 22:53:20\n"
     "n-download-failures 12\n"
@@ -299,10 +441,12 @@ static const char * dls_sample_3_str =
     "schedule DL_SCHED_BRIDGE\n"
     "want-authority DL_WANT_ANY_DIRSERVER\n"
     "increment-on DL_SCHED_INCREMENT_ATTEMPT\n"
-    "backoff DL_SCHED_DETERMINISTIC\n";
+    "backoff DL_SCHED_RANDOM_EXPONENTIAL\n"
+    "last-backoff-position 0\n"
+    "last-delay-used 0\n";
 static const download_status_t dls_sample_4 =
   { 1467166600, 3, 0, DL_SCHED_GENERIC, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 };
+    DL_SCHED_INCREMENT_FAILURE, 0, 0 };
 static const char * dls_sample_4_str =
     "next-attempt-at 2016-06-29 02:16:40\n"
     "n-download-failures 3\n"
@@ -315,7 +459,7 @@ static const char * dls_sample_4_str =
     "last-delay-used 0\n";
 static const download_status_t dls_sample_5 =
   { 1467164600, 3, 7, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER,
-    DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 1, 2112, };
+    DL_SCHED_INCREMENT_FAILURE, 1, 2112, };
 static const char * dls_sample_5_str =
     "next-attempt-at 2016-06-29 01:43:20\n"
     "n-download-failures 3\n"
@@ -328,7 +472,7 @@ static const char * dls_sample_5_str =
     "last-delay-used 2112\n";
 static const download_status_t dls_sample_6 =
   { 1467164200, 4, 9, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY,
-    DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 3, 432 };
+    DL_SCHED_INCREMENT_ATTEMPT, 3, 432 };
 static const char * dls_sample_6_str =
     "next-attempt-at 2016-06-29 01:36:40\n"
     "n-download-failures 4\n"
@@ -501,7 +645,7 @@ cert_dl_status_def_for_auth_mock(const char *digest)
   download_status_t *dl = NULL;
   char digest_str[HEX_DIGEST_LEN+1];
 
-  tt_assert(digest != NULL);
+  tt_ptr_op(digest, OP_NE, NULL);
   base16_encode(digest_str, HEX_DIGEST_LEN + 1,
                 digest, DIGEST_LEN);
   digest_str[HEX_DIGEST_LEN] = '\0';
@@ -525,7 +669,7 @@ cert_dl_status_sks_for_auth_id_mock(const char *digest)
   char *tmp;
   int len;
 
-  tt_assert(digest != NULL);
+  tt_ptr_op(digest, OP_NE, NULL);
   base16_encode(digest_str, HEX_DIGEST_LEN + 1,
                 digest, DIGEST_LEN);
   digest_str[HEX_DIGEST_LEN] = '\0';
@@ -579,11 +723,11 @@ cert_dl_status_fp_sk_mock(const char *fp_digest, const char *sk_digest)
    * dl status we want.
    */
 
-  tt_assert(fp_digest != NULL);
+  tt_ptr_op(fp_digest, OP_NE, NULL);
   base16_encode(fp_digest_str, HEX_DIGEST_LEN + 1,
                 fp_digest, DIGEST_LEN);
   fp_digest_str[HEX_DIGEST_LEN] = '\0';
-  tt_assert(sk_digest != NULL);
+  tt_ptr_op(sk_digest, OP_NE, NULL);
   base16_encode(sk_digest_str, HEX_DIGEST_LEN + 1,
                 sk_digest, DIGEST_LEN);
   sk_digest_str[HEX_DIGEST_LEN] = '\0';
@@ -663,7 +807,7 @@ descbr_get_dl_by_digest_mock(const char *digest)
   char digest_str[HEX_DIGEST_LEN+1];
 
   if (!disable_descbr) {
-    tt_assert(digest != NULL);
+    tt_ptr_op(digest, OP_NE, NULL);
     base16_encode(digest_str, HEX_DIGEST_LEN + 1,
                   digest, DIGEST_LEN);
     digest_str[HEX_DIGEST_LEN] = '\0';
@@ -730,7 +874,7 @@ test_download_status_consensus(void *arg)
 
   /* Check that the unknown prefix case works; no mocks needed yet */
   getinfo_helper_downloads(&dummy, "downloads/foo", &answer, &errmsg);
-  tt_assert(answer == NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown download status query");
 
   setup_ns_mocks();
@@ -745,8 +889,8 @@ test_download_status_consensus(void *arg)
          sizeof(download_status_t));
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/ns",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_1_str);
   tor_free(answer);
   errmsg = NULL;
@@ -756,8 +900,8 @@ test_download_status_consensus(void *arg)
          sizeof(download_status_t));
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/microdesc",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_2_str);
   tor_free(answer);
   errmsg = NULL;
@@ -767,8 +911,8 @@ test_download_status_consensus(void *arg)
          sizeof(download_status_t));
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/ns/bootstrap",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_3_str);
   tor_free(answer);
   errmsg = NULL;
@@ -779,8 +923,8 @@ test_download_status_consensus(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/networkstatus/microdesc/bootstrap",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_4_str);
   tor_free(answer);
   errmsg = NULL;
@@ -791,8 +935,8 @@ test_download_status_consensus(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/networkstatus/ns/running",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_5_str);
   tor_free(answer);
   errmsg = NULL;
@@ -803,8 +947,8 @@ test_download_status_consensus(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/networkstatus/microdesc/running",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_6_str);
   tor_free(answer);
   errmsg = NULL;
@@ -812,8 +956,8 @@ test_download_status_consensus(void *arg)
   /* Now check the error case */
   getinfo_helper_downloads(&dummy, "downloads/networkstatus/foo",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown flavor");
   errmsg = NULL;
 
@@ -847,8 +991,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/cert/fps",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, auth_id_digest_expected_list);
   tor_free(answer);
   errmsg = NULL;
@@ -857,10 +1001,10 @@ test_download_status_cert(void *arg)
   memcpy(&auth_def_cert_download_status_1, &dls_sample_1,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s", auth_id_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_1_str);
   tor_free(question);
   tor_free(answer);
@@ -870,10 +1014,10 @@ test_download_status_cert(void *arg)
   memcpy(&auth_def_cert_download_status_2, &dls_sample_2,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s", auth_id_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_2_str);
   tor_free(question);
   tor_free(answer);
@@ -881,10 +1025,10 @@ test_download_status_cert(void *arg)
 
   /* Case 4 - list of signing key digests for 1st auth id */
   tor_asprintf(&question, "downloads/cert/fp/%s/sks", auth_id_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, auth_1_sk_digest_expected_list);
   tor_free(question);
   tor_free(answer);
@@ -892,10 +1036,10 @@ test_download_status_cert(void *arg)
 
   /* Case 5 - list of signing key digests for 2nd auth id */
   tor_asprintf(&question, "downloads/cert/fp/%s/sks", auth_id_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, auth_2_sk_digest_expected_list);
   tor_free(question);
   tor_free(answer);
@@ -906,10 +1050,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_1_str, auth_1_sk_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_3_str);
   tor_free(question);
   tor_free(answer);
@@ -920,10 +1064,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_1_str, auth_1_sk_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_4_str);
   tor_free(question);
   tor_free(answer);
@@ -934,10 +1078,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_2_str, auth_2_sk_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_5_str);
   tor_free(question);
   tor_free(answer);
@@ -948,10 +1092,10 @@ test_download_status_cert(void *arg)
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/cert/fp/%s/%s",
                auth_id_digest_2_str, auth_2_sk_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_6_str);
   tor_free(question);
   tor_free(answer);
@@ -962,8 +1106,8 @@ test_download_status_cert(void *arg)
   /* Case 1 - query is garbage after downloads/cert/ part */
   getinfo_helper_downloads(&dummy, "downloads/cert/blahdeblah",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown certificate download status query");
   errmsg = NULL;
 
@@ -973,8 +1117,8 @@ test_download_status_cert(void *arg)
    */
   getinfo_helper_downloads(&dummy, "downloads/cert/fp/2B1D36D32B2942406",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -985,8 +1129,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/82F52AF55D250115FE44D3GC81D49643241D56A1",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -997,8 +1141,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this authority identity digest");
   errmsg = NULL;
@@ -1010,8 +1154,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/82F52AF55D250115FE44D3GC81D49643241D56A1/blah",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like an identity digest");
   errmsg = NULL;
 
@@ -1022,8 +1166,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/82F52AF55D25/blah",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like an identity digest");
   errmsg = NULL;
 
@@ -1034,8 +1178,8 @@ test_download_status_cert(void *arg)
   getinfo_helper_downloads(&dummy,
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/sks",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get list of signing key digests for this authority "
       "identity digest");
@@ -1049,8 +1193,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/"
       "82F52AF55D250115FE44D3GC81D49643241D56A1",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a signing key digest");
   errmsg = NULL;
 
@@ -1062,8 +1206,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/"
       "82F52AF55D250115FE44D",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a signing key digest");
   errmsg = NULL;
 
@@ -1075,8 +1219,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/C6B05DF332F74DB9A13498EE3BBC7AA2F69FCB45/"
       "3A214FC21AE25B012C2ECCB5F4EC8A3602D0545D",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this identity/"
       "signing key digest pair");
@@ -1090,8 +1234,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/63CDD326DFEF0CA020BDD3FEB45A3286FE13A061/"
       "3A214FC21AE25B012C2ECCB5F4EC8A3602D0545D",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this identity/"
       "signing key digest pair");
@@ -1105,8 +1249,8 @@ test_download_status_cert(void *arg)
       "downloads/cert/fp/63CDD326DFEF0CA020BDD3FEB45A3286FE13A061/"
       "9451B8F1B10952384EB58B5F230C0BB701626C9B",
       &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
       "Failed to get download status for this identity/"
       "signing key digest pair");
@@ -1142,8 +1286,8 @@ test_download_status_desc(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/desc/descs",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, descbr_expected_list);
   tor_free(answer);
   errmsg = NULL;
@@ -1152,10 +1296,10 @@ test_download_status_desc(void *arg)
   memcpy(&descbr_digest_1_dl, &dls_sample_1,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/desc/%s", descbr_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_1_str);
   tor_free(question);
   tor_free(answer);
@@ -1165,10 +1309,10 @@ test_download_status_desc(void *arg)
   memcpy(&descbr_digest_2_dl, &dls_sample_2,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/desc/%s", descbr_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_2_str);
   tor_free(question);
   tor_free(answer);
@@ -1179,8 +1323,8 @@ test_download_status_desc(void *arg)
   /* Case 1 - non-digest-length garbage after downloads/desc */
   getinfo_helper_downloads(&dummy, "downloads/desc/blahdeblah",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown router descriptor download status query");
   errmsg = NULL;
 
@@ -1189,8 +1333,8 @@ test_download_status_desc(void *arg)
     &dummy,
     "downloads/desc/774EC52FD9A5B80A6FACZE536616E8022E3470AG",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -1199,8 +1343,8 @@ test_download_status_desc(void *arg)
     &dummy,
     "downloads/desc/B05B46135B0B2C04EBE1DD6A6AE4B12D7CD2226A",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "No such descriptor digest found");
   errmsg = NULL;
 
@@ -1209,8 +1353,8 @@ test_download_status_desc(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/desc/descs",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ,
             "We don't seem to have a networkstatus-flavored consensus");
   errmsg = NULL;
@@ -1246,8 +1390,8 @@ test_download_status_bridge(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/bridge/bridges",
                            &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, descbr_expected_list);
   tor_free(answer);
   errmsg = NULL;
@@ -1256,10 +1400,10 @@ test_download_status_bridge(void *arg)
   memcpy(&descbr_digest_1_dl, &dls_sample_3,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/bridge/%s", descbr_digest_1_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_3_str);
   tor_free(question);
   tor_free(answer);
@@ -1269,10 +1413,10 @@ test_download_status_bridge(void *arg)
   memcpy(&descbr_digest_2_dl, &dls_sample_4,
          sizeof(download_status_t));
   tor_asprintf(&question, "downloads/bridge/%s", descbr_digest_2_str);
-  tt_assert(question != NULL);
+  tt_ptr_op(question, OP_NE, NULL);
   getinfo_helper_downloads(&dummy, question, &answer, &errmsg);
-  tt_assert(answer != NULL);
-  tt_assert(errmsg == NULL);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
   tt_str_op(answer, OP_EQ, dls_sample_4_str);
   tor_free(question);
   tor_free(answer);
@@ -1283,8 +1427,8 @@ test_download_status_bridge(void *arg)
   /* Case 1 - non-digest-length garbage after downloads/bridge */
   getinfo_helper_downloads(&dummy, "downloads/bridge/blahdeblah",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "Unknown bridge descriptor download status query");
   errmsg = NULL;
 
@@ -1293,8 +1437,8 @@ test_download_status_bridge(void *arg)
     &dummy,
     "downloads/bridge/774EC52FD9A5B80A6FACZE536616E8022E3470AG",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "That didn't look like a digest");
   errmsg = NULL;
 
@@ -1303,8 +1447,8 @@ test_download_status_bridge(void *arg)
     &dummy,
     "downloads/bridge/B05B46135B0B2C04EBE1DD6A6AE4B12D7CD2226A",
     &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "No such bridge identity digest found");
   errmsg = NULL;
 
@@ -1313,8 +1457,8 @@ test_download_status_bridge(void *arg)
   getinfo_helper_downloads(&dummy,
                            "downloads/bridge/bridges",
                            &answer, &errmsg);
-  tt_assert(answer == NULL);
-  tt_assert(errmsg != NULL);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
   tt_str_op(errmsg, OP_EQ, "We don't seem to be using bridges");
   errmsg = NULL;
   disable_descbr = 0;
@@ -1326,8 +1470,67 @@ test_download_status_bridge(void *arg)
   return;
 }
 
+/** Set timeval to a mock date and time. This is necessary
+ * to make tor_gettimeofday() mockable. */
+static void
+mock_tor_gettimeofday(struct timeval *timeval)
+{
+  timeval->tv_sec = 1523405073;
+  timeval->tv_usec = 271645;
+}
+
+static void
+test_current_time(void *arg)
+{
+  /* We just need one of these to pass, it doesn't matter what's in it */
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+
+  /* We need these for storing the (mock) time. */
+  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  /* Case 1 - local time */
+  format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  getinfo_helper_current_time(&dummy,
+                              "current-time/local",
+                              &answer, &errmsg);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tt_str_op(answer, OP_EQ, timebuf);
+  tor_free(answer);
+  errmsg = NULL;
+
+  /* Case 2 - UTC time */
+  format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  getinfo_helper_current_time(&dummy,
+                              "current-time/utc",
+                              &answer, &errmsg);
+  tt_ptr_op(answer, OP_NE, NULL);
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tt_str_op(answer, OP_EQ, timebuf);
+  tor_free(answer);
+  errmsg = NULL;
+
+ done:
+  UNMOCK(tor_gettimeofday);
+  tor_free(answer);
+
+  return;
+}
+
 struct testcase_t controller_tests[] = {
-  { "add_onion_helper_keyarg", test_add_onion_helper_keyarg, 0, NULL, NULL },
+  { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0,
+    NULL, NULL },
+  { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0,
+    NULL, NULL },
+  { "getinfo_helper_onion", test_getinfo_helper_onion, 0, NULL, NULL },
   { "rend_service_parse_port_config", test_rend_service_parse_port_config, 0,
     NULL, NULL },
   { "add_onion_helper_clientauth", test_add_onion_helper_clientauth, 0, NULL,
@@ -1338,6 +1541,7 @@ struct testcase_t controller_tests[] = {
     NULL },
   { "download_status_desc", test_download_status_desc, 0, NULL, NULL },
   { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL },
+  { "current_time", test_current_time, 0, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 11e1e3dc8f..e81aea8d66 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
@@ -12,79 +12,6 @@
 #include "test.h"
 
 static void
-help_test_bucket_note_empty(uint32_t expected_msec_since_midnight,
-                            int tokens_before, size_t tokens_removed,
-                            uint32_t msec_since_epoch)
-{
-  uint32_t timestamp_var = 0;
-  struct timeval tvnow;
-  tvnow.tv_sec = msec_since_epoch / 1000;
-  tvnow.tv_usec = (msec_since_epoch % 1000) * 1000;
-  connection_buckets_note_empty_ts(&timestamp_var, tokens_before,
-                                   tokens_removed, &tvnow);
-  tt_int_op(expected_msec_since_midnight, OP_EQ, timestamp_var);
-
- done:
-  ;
-}
-
-static void
-test_cntev_bucket_note_empty(void *arg)
-{
-  (void)arg;
-
-  /* Two cases with nothing to note, because bucket was empty before;
-   * 86442200 == 1970-01-02 00:00:42.200000 */
-  help_test_bucket_note_empty(0, 0, 0, 86442200);
-  help_test_bucket_note_empty(0, -100, 100, 86442200);
-
-  /* Nothing to note, because bucket has not been emptied. */
-  help_test_bucket_note_empty(0, 101, 100, 86442200);
-
-  /* Bucket was emptied, note 42200 msec since midnight. */
-  help_test_bucket_note_empty(42200, 101, 101, 86442200);
-  help_test_bucket_note_empty(42200, 101, 102, 86442200);
-}
-
-static void
-test_cntev_bucket_millis_empty(void *arg)
-{
-  struct timeval tvnow;
-  (void)arg;
-
-  /* 1970-01-02 00:00:42.200000 */
-  tvnow.tv_sec = 86400 + 42;
-  tvnow.tv_usec = 200000;
-
-  /* Bucket has not been refilled. */
-  tt_int_op(0, OP_EQ, bucket_millis_empty(0, 42120, 0, 100, &tvnow));
-  tt_int_op(0, OP_EQ, bucket_millis_empty(-10, 42120, -10, 100, &tvnow));
-
-  /* Bucket was not empty. */
-  tt_int_op(0, OP_EQ, bucket_millis_empty(10, 42120, 20, 100, &tvnow));
-
-  /* Bucket has been emptied 80 msec ago and has just been refilled. */
-  tt_int_op(80, OP_EQ, bucket_millis_empty(-20, 42120, -10, 100, &tvnow));
-  tt_int_op(80, OP_EQ, bucket_millis_empty(-10, 42120, 0, 100, &tvnow));
-  tt_int_op(80, OP_EQ, bucket_millis_empty(0, 42120, 10, 100, &tvnow));
-
-  /* Bucket has been emptied 180 msec ago, last refill was 100 msec ago
-   * which was insufficient to make it positive, so cap msec at 100. */
-  tt_int_op(100, OP_EQ, bucket_millis_empty(0, 42020, 1, 100, &tvnow));
-
-  /* 1970-01-02 00:00:00:050000 */
-  tvnow.tv_sec = 86400;
-  tvnow.tv_usec = 50000;
-
-  /* Last emptied 30 msec before midnight, tvnow is 50 msec after
-   * midnight, that's 80 msec in total. */
-  tt_int_op(80, OP_EQ, bucket_millis_empty(0, 86400000 - 30, 1, 100, &tvnow));
-
- done:
-  ;
-}
-
-static void
 add_testing_cell_stats_entry(circuit_t *circ, uint8_t command,
                              unsigned int waiting_time,
                              unsigned int removed, unsigned int exitward)
@@ -395,8 +322,6 @@ test_cntev_event_mask(void *arg)
   { #name, test_cntev_ ## name, flags, 0, NULL }
 
 struct testcase_t controller_event_tests[] = {
-  TEST(bucket_note_empty, TT_FORK),
-  TEST(bucket_millis_empty, TT_FORK),
   TEST(sum_up_cell_stats, TT_FORK),
   TEST(append_cell_stats, TT_FORK),
   TEST(format_cell_stats, TT_FORK),
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 8fd9ca7671..bb2e340dd2 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define CRYPTO_CURVE25519_PRIVATE
-#define CRYPTO_PRIVATE
+#define CRYPTO_RAND_PRIVATE
 #include "or.h"
 #include "test.h"
 #include "aes.h"
@@ -13,11 +13,9 @@
 #include "siphash.h"
 #include "crypto_curve25519.h"
 #include "crypto_ed25519.h"
+#include "crypto_rand.h"
 #include "ed25519_vectors.inc"
 
-#include <openssl/evp.h>
-#include <openssl/rand.h>
-
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
 test_crypto_dh(void *arg)
@@ -331,38 +329,6 @@ test_crypto_rng_strongest(void *arg)
 #undef N
 }
 
-/* Test for rectifying openssl RAND engine. */
-static void
-test_crypto_rng_engine(void *arg)
-{
-  (void)arg;
-  RAND_METHOD dummy_method;
-  memset(&dummy_method, 0, sizeof(dummy_method));
-
-  /* We should be a no-op if we're already on RAND_OpenSSL */
-  tt_int_op(0, ==, crypto_force_rand_ssleay());
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-
-  /* We should correct the method if it's a dummy. */
-  RAND_set_rand_method(&dummy_method);
-#ifdef LIBRESSL_VERSION_NUMBER
-  /* On libressl, you can't override the RNG. */
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-  tt_int_op(0, ==, crypto_force_rand_ssleay());
-#else
-  tt_assert(RAND_get_rand_method() == &dummy_method);
-  tt_int_op(1, ==, crypto_force_rand_ssleay());
-#endif
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-
-  /* Make sure we aren't calling dummy_method */
-  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
-  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
-
- done:
-  ;
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -1135,6 +1101,54 @@ test_crypto_sha3_xof(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+/* Test our MAC-SHA3 function. There are not actually any MAC-SHA3 test
+ * vectors out there for our H(len(k) || k || m) construction. Hence what we
+ * are gonna do is test our crypto_mac_sha3_256() function against manually
+ * doing H(len(k) || k||m).  If in the future the Keccak group decides to
+ * standarize an MAC construction and make test vectors, we should
+ * incorporate them here. */
+static void
+test_crypto_mac_sha3(void *arg)
+{
+  const char msg[] = "i am in a library somewhere using my computer";
+  const char key[] = "i'm from the past talking to the future.";
+
+  uint8_t hmac_test[DIGEST256_LEN];
+  char hmac_manual[DIGEST256_LEN];
+
+  (void) arg;
+
+  /* First let's use our nice HMAC-SHA3 function */
+  crypto_mac_sha3_256(hmac_test, sizeof(hmac_test),
+                      (uint8_t *) key, strlen(key),
+                      (uint8_t *) msg, strlen(msg));
+
+  /* Now let's try a manual H(len(k) || k || m) construction */
+  {
+    char *key_msg_concat = NULL, *all = NULL;
+    int result;
+    const uint64_t key_len_netorder = tor_htonll(strlen(key));
+    size_t all_len;
+
+    tor_asprintf(&key_msg_concat, "%s%s", key, msg);
+    all_len = sizeof(key_len_netorder) + strlen(key_msg_concat);
+    all = tor_malloc_zero(all_len);
+    memcpy(all, &key_len_netorder, sizeof(key_len_netorder));
+    memcpy(all + sizeof(key_len_netorder), key_msg_concat,
+           strlen(key_msg_concat));
+
+    result = crypto_digest256(hmac_manual, all, all_len, DIGEST_SHA3_256);
+    tor_free(key_msg_concat);
+    tor_free(all);
+    tt_int_op(result, OP_EQ, 0);
+  }
+
+  /* Now compare the two results */
+  tt_mem_op(hmac_test, OP_EQ, hmac_manual, DIGEST256_LEN);
+
+ done: ;
+}
+
 /** Run unit tests for our public key crypto functions */
 static void
 test_crypto_pk(void *arg)
@@ -1195,12 +1209,12 @@ test_crypto_pk(void *arg)
   tt_assert(! crypto_pk_write_private_key_to_filename(pk1,
                                                         get_fname("pkey1")));
   /* failing case for read: can't read. */
-  tt_assert(crypto_pk_read_private_key_from_filename(pk2,
-                                                   get_fname("xyzzy")) < 0);
+  tt_int_op(crypto_pk_read_private_key_from_filename(pk2, get_fname("xyzzy")),
+            OP_LT, 0);
   write_str_to_file(get_fname("xyzzy"), "foobar", 6);
   /* Failing case for read: no key. */
-  tt_assert(crypto_pk_read_private_key_from_filename(pk2,
-                                                   get_fname("xyzzy")) < 0);
+  tt_int_op(crypto_pk_read_private_key_from_filename(pk2, get_fname("xyzzy")),
+            OP_LT, 0);
   tt_assert(! crypto_pk_read_private_key_from_filename(pk2,
                                                          get_fname("pkey1")));
   tt_int_op(15,OP_EQ,
@@ -1232,17 +1246,17 @@ test_crypto_pk(void *arg)
   i = crypto_pk_asn1_encode(pk1, data1, 1024);
   tt_int_op(i, OP_GT, 0);
   pk2 = crypto_pk_asn1_decode(data1, i);
-  tt_assert(crypto_pk_cmp_keys(pk1,pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk2), OP_EQ, 0);
 
   /* Try with hybrid encryption wrappers. */
   crypto_rand(data1, 1024);
   for (i = 85; i < 140; ++i) {
     memset(data2,0,1024);
     memset(data3,0,1024);
-    len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2),
+    len = crypto_pk_obsolete_public_hybrid_encrypt(pk1,data2,sizeof(data2),
                                           data1,i,PK_PKCS1_OAEP_PADDING,0);
     tt_int_op(len, OP_GE, 0);
-    len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3),
+    len = crypto_pk_obsolete_private_hybrid_decrypt(pk1,data3,sizeof(data3),
                                            data2,len,PK_PKCS1_OAEP_PADDING,1);
     tt_int_op(len,OP_EQ, i);
     tt_mem_op(data1,OP_EQ, data3,i);
@@ -1251,9 +1265,9 @@ test_crypto_pk(void *arg)
   /* Try copy_full */
   crypto_pk_free(pk2);
   pk2 = crypto_pk_copy_full(pk1);
-  tt_assert(pk2 != NULL);
+  tt_ptr_op(pk2, OP_NE, NULL);
   tt_ptr_op(pk1, OP_NE, pk2);
-  tt_assert(crypto_pk_cmp_keys(pk1,pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk2), OP_EQ, 0);
 
  done:
   if (pk1)
@@ -1331,17 +1345,17 @@ test_crypto_pk_base64(void *arg)
   /* Test decoding a valid key. */
   pk2 = crypto_pk_base64_decode(encoded, strlen(encoded));
   tt_assert(pk2);
-  tt_assert(crypto_pk_cmp_keys(pk1,pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(pk1, pk2), OP_EQ, 0);
   crypto_pk_free(pk2);
 
   /* Test decoding a invalid key (not Base64). */
   static const char *invalid_b64 = "The key is in another castle!";
   pk2 = crypto_pk_base64_decode(invalid_b64, strlen(invalid_b64));
-  tt_assert(!pk2);
+  tt_ptr_op(pk2, OP_EQ, NULL);
 
   /* Test decoding a truncated Base64 blob. */
   pk2 = crypto_pk_base64_decode(encoded, strlen(encoded)/2);
-  tt_assert(!pk2);
+  tt_ptr_op(pk2, OP_EQ, NULL);
 
  done:
   crypto_pk_free(pk1);
@@ -1410,7 +1424,7 @@ do_truncate(const char *fname, size_t len)
   tor_free(bytes);
   return r;
 }
-#endif
+#endif /* defined(HAVE_TRUNCATE) */
 
 /** Sanity check for crypto pk digests  */
 static void
@@ -1433,6 +1447,7 @@ test_crypto_digests(void *arg)
              AUTHORITY_SIGNKEY_A_DIGEST, HEX_DIGEST_LEN);
 
   r = crypto_pk_get_common_digests(k, &pkey_digests);
+  tt_int_op(r, OP_EQ, 0);
 
   tt_mem_op(hex_str(pkey_digests.d[DIGEST_SHA1], DIGEST_LEN),OP_EQ,
              AUTHORITY_SIGNKEY_A_DIGEST, HEX_DIGEST_LEN);
@@ -1469,28 +1484,6 @@ test_crypto_digest_names(void *arg)
   ;
 }
 
-#ifndef OPENSSL_1_1_API
-#define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX))
-#define EVP_ENCODE_CTX_free(ctx) tor_free(ctx)
-#endif
-
-/** Encode src into dest with OpenSSL's EVP Encode interface, returning the
- * length of the encoded data in bytes.
- */
-static int
-base64_encode_evp(char *dest, char *src, size_t srclen)
-{
-  const unsigned char *s = (unsigned char*)src;
-  EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
-  int len, ret;
-
-  EVP_EncodeInit(ctx);
-  EVP_EncodeUpdate(ctx, (unsigned char *)dest, &len, s, (int)srclen);
-  EVP_EncodeFinal(ctx, (unsigned char *)(dest + len), &ret);
-  EVP_ENCODE_CTX_free(ctx);
-  return ret+ len;
-}
-
 /** Run unit tests for misc crypto formatting functionality (base64, base32,
  * fingerprints, etc) */
 static void
@@ -1519,7 +1512,7 @@ test_crypto_formats(void *arg)
     tt_int_op(i, OP_GE, 0);
     tt_int_op(i, OP_EQ, strlen(data2));
     tt_assert(! strchr(data2, '='));
-    j = base64_decode_nopad((uint8_t*)data3, 1024, data2, i);
+    j = base64_decode(data3, 1024, data2, i);
     tt_int_op(j, OP_EQ, idx);
     tt_mem_op(data3,OP_EQ, data1, idx);
   }
@@ -1544,21 +1537,7 @@ test_crypto_formats(void *arg)
   tt_mem_op(data1,OP_EQ, data3, DIGEST_LEN);
   tt_int_op(99,OP_EQ, data3[DIGEST_LEN+1]);
 
-  tt_assert(digest_from_base64(data3, "###") < 0);
-
-  for (i = 0; i < 256; i++) {
-    /* Test the multiline format Base64 encoder with 0 .. 256 bytes of
-     * output against OpenSSL.
-     */
-    const size_t enclen = base64_encode_size(i, BASE64_ENCODE_MULTILINE);
-    data1[i] = i;
-    j = base64_encode(data2, 1024, data1, i, BASE64_ENCODE_MULTILINE);
-    tt_int_op(j, OP_EQ, enclen);
-    j = base64_encode_evp(data3, data1, i);
-    tt_int_op(j, OP_EQ, enclen);
-    tt_mem_op(data2, OP_EQ, data3, enclen);
-    tt_int_op(j, OP_EQ, strlen(data2));
-  }
+  tt_int_op(digest_from_base64(data3, "###"), OP_LT, 0);
 
   /* Encoding SHA256 */
   crypto_rand(data2, DIGEST256_LEN);
@@ -1969,7 +1948,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "e0544770bc7de853b38f9100489e3e79";
   const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88"
                                 "bd13861475516bc2cd2b6e005e805064";
-#else
+#else /* !(defined(SLOW_CURVE25519_TEST)) */
   const int loop_max=200;
   const char e1_expected[]    = "bc7112cde03f97ef7008cad1bdc56be3"
                                 "c6a1037d74cceb3712e9206871dcf654";
@@ -1977,7 +1956,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "8e3ee1a63c7d14274ea5d4c67f065467";
   const char e1e2k_expected[] = "7ddb98bd89025d2347776b33901b3e7e"
                                 "c0ee98cb2257a4545c0cfb2ca3e1812b";
-#endif
+#endif /* defined(SLOW_CURVE25519_TEST) */
 
   unsigned char e1k[32];
   unsigned char e2k[32];
@@ -2233,6 +2212,9 @@ test_crypto_ed25519_simple(void *arg)
   tt_int_op(0, OP_EQ, ed25519_public_key_generate(&pub1, &sec1));
   tt_int_op(0, OP_EQ, ed25519_public_key_generate(&pub2, &sec1));
 
+  tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, 0);
+  tt_int_op(ed25519_validate_pubkey(&pub2), OP_EQ, 0);
+
   tt_mem_op(pub1.pubkey, OP_EQ, pub2.pubkey, sizeof(pub1.pubkey));
   tt_assert(ed25519_pubkey_eq(&pub1, &pub2));
   tt_assert(ed25519_pubkey_eq(&pub1, &pub1));
@@ -2604,6 +2586,39 @@ test_crypto_ed25519_blinding(void *arg)
   ;
 }
 
+/** Test that our blinding functions will fail if we pass them bad pubkeys */
+static void
+test_crypto_ed25519_blinding_fail(void *arg)
+{
+  int retval;
+  uint8_t param[32] = {2};
+  ed25519_public_key_t pub;
+  ed25519_public_key_t pub_blinded;
+
+  (void)arg;
+
+  /* This point is not on the curve: the blind routines should fail */
+  const char badkey[] =
+    "e19c65de75c68cf3b7643ea732ba9eb1a3d20d6d57ba223c2ece1df66feb5af0";
+  retval = base16_decode((char*)pub.pubkey, sizeof(pub.pubkey),
+                         badkey, strlen(badkey));
+  tt_int_op(retval, OP_EQ, sizeof(pub.pubkey));
+  retval = ed25519_public_blind(&pub_blinded, &pub, param);
+  tt_int_op(retval, OP_EQ, -1);
+
+  /* This point is legit: blind routines should be happy */
+  const char goodkey[] =
+    "4ba2e44760dff4c559ef3c38768c1c14a8a54740c782c8d70803e9d6e3ad8794";
+  retval = base16_decode((char*)pub.pubkey, sizeof(pub.pubkey),
+                         goodkey, strlen(goodkey));
+  tt_int_op(retval, OP_EQ, sizeof(pub.pubkey));
+  retval = ed25519_public_blind(&pub_blinded, &pub, param);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  ;
+}
+
 static void
 test_crypto_ed25519_testvectors(void *arg)
 {
@@ -2621,6 +2636,8 @@ test_crypto_ed25519_testvectors(void *arg)
     ed25519_signature_t sig;
     int sign;
 
+    memset(&curvekp, 0xd0, sizeof(curvekp));
+
 #define DECODE(p,s) base16_decode((char*)(p),sizeof(p),(s),strlen(s))
 #define EQ(a,h) test_memeq_hex((const char*)(a), (h))
 
@@ -2702,8 +2719,8 @@ test_crypto_ed25519_storage(void *arg)
   tor_free(tag);
 
   /* whitebox test: truncated keys. */
-  tt_int_op(0, ==, do_truncate(fname_1, 40));
-  tt_int_op(0, ==, do_truncate(fname_2, 40));
+  tt_int_op(0, OP_EQ, do_truncate(fname_1, 40));
+  tt_int_op(0, OP_EQ, do_truncate(fname_2, 40));
   tt_int_op(-1, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_2));
   tt_ptr_op(tag, OP_EQ, NULL);
   tor_free(tag);
@@ -2895,6 +2912,67 @@ crypto_rand_check_failure_mode_predict(void)
 
 #undef FAILURE_MODE_BUFFER_SIZE
 
+/** Test that our ed25519 validation function rejects evil public keys and
+ *  accepts good ones. */
+static void
+test_crypto_ed25519_validation(void *arg)
+{
+  (void) arg;
+
+  int retval;
+  ed25519_public_key_t pub1;
+
+  /* See https://lists.torproject.org/pipermail/tor-dev/2017-April/012230.html
+     for a list of points with torsion components in ed25519. */
+
+  { /* Point with torsion component (order 8l) */
+    const char badkey[] =
+      "300ef2e64e588e1df55b48e4da0416ffb64cc85d5b00af6463d5cc6c2b1c185e";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* Point with torsion component (order 4l) */
+    const char badkey[] =
+      "f43e3a046db8749164c6e69b193f1e942c7452e7d888736f40b98093d814d5e7";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* Point with torsion component (order 2l) */
+    const char badkey[] =
+      "c9fff3af0471c28e33e98c2043e44f779d0427b1e37c521a6bddc011ed1869af";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* This point is not even on the curve */
+    const char badkey[] =
+      "e19c65de75c68cf3b7643ea732ba9eb1a3d20d6d57ba223c2ece1df66feb5af0";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           badkey, strlen(badkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1);
+  }
+
+  { /* This one is a good key */
+    const char goodkey[] =
+      "4ba2e44760dff4c559ef3c38768c1c14a8a54740c782c8d70803e9d6e3ad8794";
+    retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey),
+                           goodkey, strlen(goodkey));
+    tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey));
+    tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, 0);
+  }
+
+ done: ;
+}
+
 static void
 test_crypto_failure_modes(void *arg)
 {
@@ -2902,17 +2980,17 @@ test_crypto_failure_modes(void *arg)
   (void)arg;
 
   rv = crypto_early_init();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   /* Check random works */
   rv = crypto_rand_check_failure_mode_zero();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   rv = crypto_rand_check_failure_mode_identical();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
   rv = crypto_rand_check_failure_mode_predict();
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
  done:
   ;
@@ -2933,7 +3011,6 @@ struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
   CRYPTO_LEGACY(rng),
   { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL },
   { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
   { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
     &passthrough_setup, (void*)"nosyscall" },
@@ -2959,6 +3036,7 @@ struct testcase_t crypto_tests[] = {
   { "digest_names", test_crypto_digest_names, 0, NULL, NULL },
   { "sha3", test_crypto_sha3, TT_FORK, NULL, NULL},
   { "sha3_xof", test_crypto_sha3_xof, TT_FORK, NULL, NULL},
+  { "mac_sha3", test_crypto_mac_sha3, TT_FORK, NULL, NULL},
   CRYPTO_LEGACY(dh),
   { "aes_iv_AES", test_crypto_aes_iv, TT_FORK, &passthrough_setup,
     (void*)"aes" },
@@ -2981,7 +3059,9 @@ struct testcase_t crypto_tests[] = {
   ED25519_TEST(encode, 0),
   ED25519_TEST(convert, 0),
   ED25519_TEST(blinding, 0),
+  ED25519_TEST(blinding_fail, 0),
   ED25519_TEST(testvectors, 0),
+  ED25519_TEST(validation, 0),
   { "ed25519_storage", test_crypto_ed25519_storage, 0, NULL, NULL },
   { "siphash", test_crypto_siphash, 0, NULL, NULL },
   { "failure_modes", test_crypto_failure_modes, TT_FORK, NULL, NULL },
diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c
new file mode 100644
index 0000000000..a016277508
--- /dev/null
+++ b/src/test/test_crypto_openssl.c
@@ -0,0 +1,107 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define CRYPTO_RAND_PRIVATE
+
+#include "crypto_rand.h"
+#include "util.h"
+#include "util_format.h"
+#include "compat.h"
+#include "test.h"
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+#include "compat_openssl.h"
+
+/* Test for rectifying openssl RAND engine. */
+static void
+test_crypto_rng_engine(void *arg)
+{
+  (void)arg;
+  RAND_METHOD dummy_method;
+  memset(&dummy_method, 0, sizeof(dummy_method));
+
+  /* We should be a no-op if we're already on RAND_OpenSSL */
+  tt_int_op(0, OP_EQ, crypto_force_rand_ssleay());
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+
+  /* We should correct the method if it's a dummy. */
+  RAND_set_rand_method(&dummy_method);
+#ifdef LIBRESSL_VERSION_NUMBER
+  /* On libressl, you can't override the RNG. */
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+  tt_int_op(0, OP_EQ, crypto_force_rand_ssleay());
+#else
+  tt_assert(RAND_get_rand_method() == &dummy_method);
+  tt_int_op(1, OP_EQ, crypto_force_rand_ssleay());
+#endif /* defined(LIBRESSL_VERSION_NUMBER) */
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+
+  /* Make sure we aren't calling dummy_method */
+  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
+  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
+
+ done:
+  ;
+}
+
+#ifndef OPENSSL_1_1_API
+#define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX))
+#define EVP_ENCODE_CTX_free(ctx) tor_free(ctx)
+#endif
+
+/** Encode src into dest with OpenSSL's EVP Encode interface, returning the
+ * length of the encoded data in bytes.
+ */
+static int
+base64_encode_evp(char *dest, char *src, size_t srclen)
+{
+  const unsigned char *s = (unsigned char*)src;
+  EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
+  int len, ret;
+
+  EVP_EncodeInit(ctx);
+  EVP_EncodeUpdate(ctx, (unsigned char *)dest, &len, s, (int)srclen);
+  EVP_EncodeFinal(ctx, (unsigned char *)(dest + len), &ret);
+  EVP_ENCODE_CTX_free(ctx);
+  return ret+ len;
+}
+
+static void
+test_crypto_base64_encode_matches(void *arg)
+{
+  (void)arg;
+  int i, j;
+  char data1[1024];
+  char data2[1024];
+  char data3[1024];
+
+  for (i = 0; i < 256; i++) {
+    /* Test the multiline format Base64 encoder with 0 .. 256 bytes of
+     * output against OpenSSL.
+     */
+    const size_t enclen = base64_encode_size(i, BASE64_ENCODE_MULTILINE);
+    data1[i] = i;
+    j = base64_encode(data2, 1024, data1, i, BASE64_ENCODE_MULTILINE);
+    tt_int_op(j, OP_EQ, enclen);
+    j = base64_encode_evp(data3, data1, i);
+    tt_int_op(j, OP_EQ, enclen);
+    tt_mem_op(data2, OP_EQ, data3, enclen);
+    tt_int_op(j, OP_EQ, strlen(data2));
+  }
+
+ done:
+  ;
+}
+
+struct testcase_t crypto_openssl_tests[] = {
+  { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL },
+  { "base64_encode_match", test_crypto_base64_encode_matches,
+    TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index 0be58c9389..0e1f5bd227 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -9,6 +9,7 @@
 #include "test.h"
 #include "crypto_s2k.h"
 #include "crypto_pwbox.h"
+#include "crypto_rand.h"
 
 #if defined(HAVE_LIBSCRYPT_H) && defined(HAVE_LIBSCRYPT_SCRYPT)
 #define HAVE_LIBSCRYPT
@@ -164,10 +165,10 @@ test_libscrypt_eq_openssl(void *arg)
   EVP_PBE_scrypt((const char *)"", 0, (const unsigned char *)"", 0,
                   N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   memset(buf1,0,64);
   memset(buf2,0,64);
@@ -185,10 +186,10 @@ test_libscrypt_eq_openssl(void *arg)
                  (const unsigned char *)"NaCl", strlen("NaCl"),
                  N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   memset(buf1,0,64);
   memset(buf2,0,64);
@@ -210,10 +211,10 @@ test_libscrypt_eq_openssl(void *arg)
                  strlen("SodiumChloride"),
                  N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   memset(buf1,0,64);
   memset(buf2,0,64);
@@ -234,15 +235,15 @@ test_libscrypt_eq_openssl(void *arg)
                  strlen("SodiumChloride"),
                  N, r, p, maxmem, buf2, dk_len);
 
-  tt_int_op(libscrypt_retval, ==, 0);
-  tt_int_op(openssl_retval, ==, 1);
+  tt_int_op(libscrypt_retval, OP_EQ, 0);
+  tt_int_op(openssl_retval, OP_EQ, 1);
 
-  tt_mem_op(buf1, ==, buf2, 64);
+  tt_mem_op(buf1, OP_EQ, buf2, 64);
 
   done:
   return;
 }
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) && defined(HAVE_EVP_PBE_SCRYPT) */
 
 static void
 test_crypto_s2k_errors(void *arg)
@@ -283,7 +284,7 @@ test_crypto_s2k_errors(void *arg)
                                                  "ABC", 3, 0));
   tt_int_op(S2K_TRUNCATED, OP_EQ, secret_to_key_new(buf, 50, &sz,
                                                  "ABC", 3, S2K_FLAG_LOW_MEM));
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) */
   tt_int_op(S2K_TRUNCATED, OP_EQ, secret_to_key_new(buf, 37, &sz,
                                               "ABC", 3, S2K_FLAG_USE_PBKDF2));
   tt_int_op(S2K_TRUNCATED, OP_EQ, secret_to_key_new(buf, 29, &sz,
@@ -318,7 +319,7 @@ test_crypto_s2k_errors(void *arg)
   tt_int_op(S2K_BAD_PARAMS, OP_EQ,
             secret_to_key_derivekey(buf2, sizeof(buf2),
                                     buf, 19, "ABC", 3));
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) */
 
  done:
   ;
@@ -516,7 +517,7 @@ test_crypto_ed25519_fuzz_donna(void *arg)
   unsigned i;
   (void)arg;
 
-  tt_assert(sizeof(msg) == iters);
+  tt_uint_op(iters, OP_EQ, sizeof(msg));
   crypto_rand((char*) msg, sizeof(msg));
 
   /* Fuzz Ed25519-donna vs ref10, alternating the implementation used to
@@ -600,7 +601,7 @@ struct testcase_t slow_crypto_tests[] = {
 #ifdef HAVE_EVP_PBE_SCRYPT
   { "libscrypt_eq_openssl", test_libscrypt_eq_openssl, 0, NULL, NULL },
 #endif
-#endif
+#endif /* defined(HAVE_LIBSCRYPT) */
   { "s2k_pbkdf2", test_crypto_s2k_general, 0, &passthrough_setup,
     (void*)"pbkdf2" },
   { "s2k_rfc2440_general", test_crypto_s2k_general, 0, &passthrough_setup,
diff --git a/src/test/test_data.c b/src/test/test_data.c
index 788489a097..ce6c3394f6 100644
--- a/src/test/test_data.c
+++ b/src/test/test_data.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "test.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 4cdbfb4f84..52d3ef159c 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -1,12 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <math.h>
 
 #define CONFIG_PRIVATE
+#define CONTROL_PRIVATE
 #define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
 #define ROUTER_PRIVATE
@@ -17,12 +18,16 @@
 #define RELAY_PRIVATE
 
 #include "or.h"
+#include "bridges.h"
 #include "confparse.h"
 #include "config.h"
+#include "control.h"
 #include "crypto_ed25519.h"
+#include "crypto_rand.h"
 #include "directory.h"
 #include "dirserv.h"
-#include "dirvote.h"
+#include "dirauth/dirvote.h"
+#include "entrynodes.h"
 #include "hibernate.h"
 #include "memarea.h"
 #include "networkstatus.h"
@@ -31,12 +36,13 @@
 #include "routerlist.h"
 #include "routerparse.h"
 #include "routerset.h"
-#include "shared_random_state.h"
+#include "dirauth/shared_random_state.h"
 #include "test.h"
 #include "test_dir_common.h"
 #include "torcert.h"
 #include "relay.h"
 #include "log_test_helpers.h"
+#include "voting_schedule.h"
 
 #define NS_MODULE dir
 
@@ -272,8 +278,8 @@ test_dir_formats(void *arg)
   tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
   tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
   tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
-  tt_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0);
-  tt_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0);
+  tt_int_op(crypto_pk_cmp_keys(rp1->onion_pkey, pk1), OP_EQ, 0);
+  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0);
   tt_assert(rp1->supports_tunnelled_dir_requests);
   //tt_assert(rp1->exit_policy == NULL);
   tor_free(buf);
@@ -291,9 +297,9 @@ test_dir_formats(void *arg)
   strlcat(buf2, "master-key-ed25519 ", sizeof(buf2));
   {
     char k[ED25519_BASE64_LEN+1];
-    tt_assert(ed25519_public_to_base64(k,
-                                &r2->cache_info.signing_key_cert->signing_key)
-              >= 0);
+    tt_int_op(ed25519_public_to_base64(k,
+                          &r2->cache_info.signing_key_cert->signing_key),
+              OP_GE, 0);
     strlcat(buf2, k, sizeof(buf2));
     strlcat(buf2, "\n", sizeof(buf2));
   }
@@ -328,7 +334,7 @@ test_dir_formats(void *arg)
     ntor_cc = make_ntor_onion_key_crosscert(&r2_onion_keypair,
                                           &kp1.pubkey,
                                           r2->cache_info.published_on,
-                                          MIN_ONION_KEY_LIFETIME,
+                                          get_onion_key_lifetime(),
                                           &ntor_cc_sign);
     tt_assert(ntor_cc);
     base64_encode(cert_buf, sizeof(cert_buf),
@@ -389,8 +395,8 @@ test_dir_formats(void *arg)
   tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
              r2->onion_curve25519_pkey->public_key,
              CURVE25519_PUBKEY_LEN);
-  tt_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0);
-  tt_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0);
+  tt_int_op(crypto_pk_cmp_keys(rp2->onion_pkey, pk2), OP_EQ, 0);
+  tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0);
   tt_assert(rp2->supports_tunnelled_dir_requests);
 
   tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
@@ -419,7 +425,7 @@ test_dir_formats(void *arg)
     add_fingerprint_to_dir(buf, fingerprint_list, 0);
   }
 
-#endif
+#endif /* 0 */
   dirserv_free_fingerprint_list();
 
  done:
@@ -475,34 +481,34 @@ test_dir_routerinfo_parsing(void *arg)
   routerinfo_free(ri);
   ri = router_parse_entry_from_string(EX_RI_MINIMAL, NULL, 0, 0,
                                       "@purpose bridge\n", NULL);
-  tt_assert(ri != NULL);
+  tt_ptr_op(ri, OP_NE, NULL);
   tt_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
   routerinfo_free(ri);
 
   /* bad annotations prepended. */
   ri = router_parse_entry_from_string(EX_RI_MINIMAL,
                                       NULL, 0, 0, "@purpose\n", NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* bad annotations on router. */
   ri = router_parse_entry_from_string("@purpose\nrouter x\n", NULL, 0, 1,
                                       NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* unwanted annotations on router. */
   ri = router_parse_entry_from_string("@purpose foo\nrouter x\n", NULL, 0, 0,
                                       NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* No signature. */
   ri = router_parse_entry_from_string("router x\n", NULL, 0, 0,
                                       NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   /* Not a router */
   routerinfo_free(ri);
   ri = router_parse_entry_from_string("hello\n", NULL, 0, 0, NULL, NULL);
-  tt_assert(ri == NULL);
+  tt_ptr_op(ri, OP_EQ, NULL);
 
   CHECK_FAIL(EX_RI_BAD_SIG1, 1);
   CHECK_FAIL(EX_RI_BAD_SIG2, 1);
@@ -563,7 +569,7 @@ test_dir_routerinfo_parsing(void *arg)
 static void
 routerinfo_free_wrapper_(void *arg)
 {
-  routerinfo_free(arg);
+  routerinfo_free_(arg);
 }
 
 static void
@@ -629,11 +635,11 @@ test_dir_extrainfo_parsing(void *arg)
   ADD(EX_EI_ED_MISPLACED_SIG);
 
   CHECK_OK(EX_EI_MINIMAL);
-  tt_assert(!ei->pending_sig);
+  tt_ptr_op(ei->pending_sig, OP_EQ, NULL);
   CHECK_OK(EX_EI_MAXIMAL);
-  tt_assert(!ei->pending_sig);
+  tt_ptr_op(ei->pending_sig, OP_EQ, NULL);
   CHECK_OK(EX_EI_GOOD_ED_EI);
-  tt_assert(!ei->pending_sig);
+  tt_ptr_op(ei->pending_sig, OP_EQ, NULL);
 
   CHECK_FAIL(EX_EI_BAD_SIG1,1);
   CHECK_FAIL(EX_EI_BAD_SIG2,1);
@@ -660,7 +666,7 @@ test_dir_extrainfo_parsing(void *arg)
   escaped(NULL);
   extrainfo_free(ei);
   routerinfo_free(ri);
-  digestmap_free((digestmap_t*)map, routerinfo_free_wrapper_);
+  digestmap_free_((digestmap_t*)map, routerinfo_free_wrapper_);
 }
 
 static void
@@ -678,16 +684,16 @@ test_dir_parse_router_list(void *arg)
   routerinfo_t *ri = NULL;
   char d[DIGEST_LEN];
 
-  smartlist_add(chunks, tor_strdup(EX_RI_MINIMAL));     // ri 0
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_PORTS));   // bad ri 0
-  smartlist_add(chunks, tor_strdup(EX_EI_MAXIMAL));     // ei 0
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_SIG2));    // bad ei --
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_NICKNAME));// bad ei 0
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_SIG1));    // bad ri --
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_PUBLISHED));  // bad ei 1
-  smartlist_add(chunks, tor_strdup(EX_RI_MAXIMAL));     // ri 1
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_FAMILY));  // bad ri 1
-  smartlist_add(chunks, tor_strdup(EX_EI_MINIMAL));     // ei 1
+  smartlist_add_strdup(chunks, EX_RI_MINIMAL);     // ri 0
+  smartlist_add_strdup(chunks, EX_RI_BAD_PORTS);   // bad ri 0
+  smartlist_add_strdup(chunks, EX_EI_MAXIMAL);     // ei 0
+  smartlist_add_strdup(chunks, EX_EI_BAD_SIG2);    // bad ei --
+  smartlist_add_strdup(chunks, EX_EI_BAD_NICKNAME);// bad ei 0
+  smartlist_add_strdup(chunks, EX_RI_BAD_SIG1);    // bad ri --
+  smartlist_add_strdup(chunks, EX_EI_BAD_PUBLISHED);  // bad ei 1
+  smartlist_add_strdup(chunks, EX_RI_MAXIMAL);     // ri 1
+  smartlist_add_strdup(chunks, EX_RI_BAD_FAMILY);  // bad ri 1
+  smartlist_add_strdup(chunks, EX_EI_MINIMAL);     // ei 1
 
   list = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -756,7 +762,7 @@ test_dir_parse_router_list(void *arg)
   smartlist_free(chunks);
   routerinfo_free(ri);
   if (map) {
-    digestmap_free((digestmap_t*)map, routerinfo_free_wrapper_);
+    digestmap_free_((digestmap_t*)map, routerinfo_free_wrapper_);
     router_get_routerlist()->identity_map =
       (struct digest_ri_map_t*)digestmap_new();
   }
@@ -812,19 +818,19 @@ test_dir_load_routers(void *arg)
 #define ADD(str)                                                        \
   do {                                                                  \
     tt_int_op(0,OP_EQ,router_get_router_hash(str, strlen(str), buf));      \
-    smartlist_add(wanted, tor_strdup(hex_str(buf, DIGEST_LEN)));        \
+    smartlist_add_strdup(wanted, hex_str(buf, DIGEST_LEN));        \
   } while (0)
 
   MOCK(router_get_dl_status_by_descriptor_digest, mock_router_get_dl_status);
 
   update_approx_time(1412510400);
 
-  smartlist_add(chunks, tor_strdup(EX_RI_MINIMAL));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_FINGERPRINT));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_SIG2));
-  smartlist_add(chunks, tor_strdup(EX_RI_MAXIMAL));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_PORTS));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_TOKENS));
+  smartlist_add_strdup(chunks, EX_RI_MINIMAL);
+  smartlist_add_strdup(chunks, EX_RI_BAD_FINGERPRINT);
+  smartlist_add_strdup(chunks, EX_RI_BAD_SIG2);
+  smartlist_add_strdup(chunks, EX_RI_MAXIMAL);
+  smartlist_add_strdup(chunks, EX_RI_BAD_PORTS);
+  smartlist_add_strdup(chunks, EX_RI_BAD_TOKENS);
 
   /* not ADDing MINIMIAL */
   ADD(EX_RI_MAXIMAL);
@@ -909,6 +915,23 @@ mock_get_by_ei_desc_digest(const char *d)
   }
 }
 
+static signed_descriptor_t *
+mock_ei_get_by_ei_digest(const char *d)
+{
+  char hex[HEX_DIGEST_LEN+1];
+  base16_encode(hex, sizeof(hex), d, DIGEST_LEN);
+  signed_descriptor_t *sd = &sd_ei_minimal;
+
+  if (!strcmp(hex, "11E0EDF526950739F7769810FCACAB8C882FAEEE")) {
+    sd->signed_descriptor_body = (char *)EX_EI_MINIMAL;
+    sd->signed_descriptor_len = sizeof(EX_EI_MINIMAL);
+    sd->annotations_len = 0;
+    sd->saved_location = SAVED_NOWHERE;
+    return sd;
+  }
+  return NULL;
+}
+
 static smartlist_t *mock_ei_insert_list = NULL;
 static was_router_added_t
 mock_ei_insert(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)
@@ -932,18 +955,18 @@ test_dir_load_extrainfo(void *arg)
 #define ADD(str)                                                        \
   do {                                                                  \
     tt_int_op(0,OP_EQ,router_get_extrainfo_hash(str, strlen(str), buf));   \
-    smartlist_add(wanted, tor_strdup(hex_str(buf, DIGEST_LEN)));        \
+    smartlist_add_strdup(wanted, hex_str(buf, DIGEST_LEN));        \
   } while (0)
 
   mock_ei_insert_list = smartlist_new();
   MOCK(router_get_by_extrainfo_digest, mock_get_by_ei_desc_digest);
   MOCK(extrainfo_insert, mock_ei_insert);
 
-  smartlist_add(chunks, tor_strdup(EX_EI_MINIMAL));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_NICKNAME));
-  smartlist_add(chunks, tor_strdup(EX_EI_MAXIMAL));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_PUBLISHED));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_TOKENS));
+  smartlist_add_strdup(chunks, EX_EI_MINIMAL);
+  smartlist_add_strdup(chunks, EX_EI_BAD_NICKNAME);
+  smartlist_add_strdup(chunks, EX_EI_MAXIMAL);
+  smartlist_add_strdup(chunks, EX_EI_BAD_PUBLISHED);
+  smartlist_add_strdup(chunks, EX_EI_BAD_TOKENS);
 
   /* not ADDing MINIMIAL */
   ADD(EX_EI_MAXIMAL);
@@ -998,6 +1021,37 @@ test_dir_load_extrainfo(void *arg)
 }
 
 static void
+test_dir_getinfo_extra(void *arg)
+{
+  int r;
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+  MOCK(extrainfo_get_by_descriptor_digest, mock_ei_get_by_ei_digest);
+  r = getinfo_helper_dir(NULL, "extra-info/digest/"
+                         "11E0EDF526950739F7769810FCACAB8C882FAEEE", &answer,
+                         &errmsg);
+  tt_int_op(0, OP_EQ, r);
+  tt_ptr_op(NULL, OP_EQ, errmsg);
+  tt_str_op(answer, OP_EQ, EX_EI_MINIMAL);
+  tor_free(answer);
+
+  answer = NULL;
+  r = getinfo_helper_dir(NULL, "extra-info/digest/"
+                         "NOTAVALIDHEXSTRINGNOTAVALIDHEXSTRINGNOTA", &answer,
+                         &errmsg);
+  tt_int_op(0, OP_EQ, r);
+  /* getinfo_helper_dir() should maybe return an error here but doesn't */
+  tt_ptr_op(NULL, OP_EQ, errmsg);
+  /* In any case, there should be no answer for an invalid hex string. */
+  tt_ptr_op(NULL, OP_EQ, answer);
+
+ done:
+  UNMOCK(extrainfo_get_by_descriptor_digest);
+}
+
+static void
 test_dir_versions(void *arg)
 {
   tor_version_t ver1;
@@ -1064,6 +1118,7 @@ test_dir_versions(void *arg)
   tt_int_op(0, OP_EQ, ver1.patchlevel);
   tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
   tt_str_op("alpha", OP_EQ, ver1.status_tag);
+  /* Go through the full set of status tags */
   tt_int_op(0, OP_EQ, tor_version_parse("2.1.700-alpha", &ver1));
   tt_int_op(2, OP_EQ, ver1.major);
   tt_int_op(1, OP_EQ, ver1.minor);
@@ -1078,6 +1133,60 @@ test_dir_versions(void *arg)
   tt_int_op(0, OP_EQ, ver1.patchlevel);
   tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
   tt_str_op("alpha-dev", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.5-rc", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(5, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("rc", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.6-rc-dev", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(6, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("rc-dev", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.8", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(8, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.9-dev", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(9, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("dev", OP_EQ, ver1.status_tag);
+  /* In #21450, we fixed an inconsistency in parsing versions > INT32_MAX
+   * between i386 and x86_64, as we used tor_parse_long, and then cast to int
+   */
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2147483647.0", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2147483647, OP_EQ, ver1.minor);
+  tt_int_op(0, OP_EQ, ver1.micro);
+  tt_int_op(0, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("", OP_EQ, ver1.status_tag);
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.2147483648.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.4294967295.0", &ver1));
+  /* In #21278, we reject negative version components */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-1.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-2147483648.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-4294967295.0", &ver1));
+  /* In #21507, we reject version components with non-numeric prefixes */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("+1.0.0", &ver1));
+  /* use the list in isspace() */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\t0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\n0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\v0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\f0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\r0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0. 0.0", &ver1));
 
 #define tt_versionstatus_op(vs1, op, vs2)                               \
   tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t,       \
@@ -1097,6 +1206,7 @@ test_dir_versions(void *arg)
   test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8");
   test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs");
   test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6");
+  test_v_i_o(VS_NEW, "0.2.9.9-dev", "0.2.9.9");
   /* Not on list, but newer than any in same series. */
   test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3",
              "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0");
@@ -1135,6 +1245,70 @@ test_dir_versions(void *arg)
                                    "Tor 0.2.1.0-dev (r99)"));
   tt_int_op(1,OP_EQ, tor_version_as_new_as("Tor 0.2.1.1",
                                    "Tor 0.2.1.0-dev (r99)"));
+  /* And git revisions */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-56788a2489127072)",
+                                        "Tor 0.2.9.9 (git-56788a2489127072)"));
+  /* a git revision is newer than no git revision */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-56788a2489127072)",
+                                        "Tor 0.2.9.9"));
+  /* a longer git revision is newer than a shorter git revision
+   * this should be true if they prefix-match, but if they don't, they are
+   * incomparable, because hashes aren't ordered (but we compare their bytes
+   * anyway) */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                  "Tor 0.2.9.9 (git-56788a2489127072d513cf4baf35a8ff475f3c7b)",
+                  "Tor 0.2.9.9 (git-56788a2489127072)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-0102)",
+                                        "Tor 0.2.9.9 (git-03)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-0102)",
+                                        "Tor 0.2.9.9 (git-00)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2.9.9 (git-01)",
+                                           "Tor 0.2.9.9 (git-00)"));
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2.9.9 (git-00)",
+                                           "Tor 0.2.9.9 (git-01)"));
+  /* In #21278, we compare without integer overflows.
+   * But since #21450 limits version components to [0, INT32_MAX], it is no
+   * longer possible to cause an integer overflow in tor_version_compare() */
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.0.0.0",
+                                           "Tor 2147483647.0.0.0"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 2147483647.0.0.0",
+                                           "Tor 0.0.0.0"));
+  /* These versions used to cause an overflow, now they don't parse
+   * (and authorities reject their descriptors), and log a BUG message */
+  setup_full_capture_of_logs(LOG_WARN);
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.0.0.0",
+                                           "Tor 0.-2147483648.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2147483647.0.0",
+                                           "Tor 0.-1.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2147483647.0.0",
+                                           "Tor 0.-2147483648.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 4294967295.0.0.0",
+                                           "Tor 0.0.0.0"));
+  expect_no_log_entry();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.4294967295.0.0",
+                                           "Tor 0.-4294967295.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  teardown_capture_of_logs();
 
   /* Now try git revisions */
   tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1));
@@ -1144,11 +1318,24 @@ test_dir_versions(void *arg)
   tt_int_op(7,OP_EQ, ver1.patchlevel);
   tt_int_op(3,OP_EQ, ver1.git_tag_len);
   tt_mem_op(ver1.git_tag,OP_EQ, "\xff\x00\xff", 3);
+  /* reject bad hex digits */
   tt_int_op(-1,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1));
+  /* reject odd hex digit count */
   tt_int_op(-1,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1));
+  /* ignore "git " */
   tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1));
+  /* standard length is 16 hex digits */
+  tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git-0010203040506070)",
+                                       &ver1));
+  /* length limit is 40 hex digits */
+  tt_int_op(0,OP_EQ, tor_version_parse(
+                     "0.5.6.7 (git-000102030405060708090a0b0c0d0e0f10111213)",
+                     &ver1));
+  tt_int_op(-1,OP_EQ, tor_version_parse(
+                    "0.5.6.7 (git-000102030405060708090a0b0c0d0e0f1011121314)",
+                    &ver1));
  done:
-  ;
+  teardown_capture_of_logs();
 }
 
 /** Run unit tests for directory fp_pair functions. */
@@ -1314,6 +1501,26 @@ test_dir_measured_bw_kb(void *arg)
                 "bw=1024 junk=007\n",
     "misc=junk node_id=$557365204145532d32353620696e73746561642e  "
                 "bw=1024 junk=007\n",
+    /* check whether node_id can be at the end */
+    "bw=1024 node_id=$557365204145532d32353620696e73746561642e\n",
+    /* check whether node_id can be at the end and bw has something in front*/
+    "foo=bar bw=1024 node_id=$557365204145532d32353620696e73746561642e\n",
+    /* check whether node_id can be at the end and something in the
+     * in the middle of bw and node_id */
+    "bw=1024 foo=bar node_id=$557365204145532d32353620696e73746561642e\n",
+
+    /* Test that a line with vote=1 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=1\n",
+    /* Test that a line with unmeasured=1 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 unmeasured=1\n",
+    /* Test that a line with vote=1 and unmeasured=1 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=1"
+    "unmeasured=1\n",
+    /* Test that a line with unmeasured=0 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 unmeasured=0\n",
+    /* Test that a line with vote=1 and unmeasured=0 will pass. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=1"
+    "unmeasured=0\n",
     "end"
   };
   const char *lines_fail[] = {
@@ -1347,18 +1554,30 @@ test_dir_measured_bw_kb(void *arg)
     "node_id=$55736520414552d32353620696e73746561642e bw=1024\n",
     "node_id=557365204145532d32353620696e73746561642e bw=1024\n",
     "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n",
+
+    /* Test that a line with vote=0 will fail too, so that it is ignored. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=0\n",
+    /* Test that a line with vote=0 will fail even if unmeasured=0. */
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 vote=0 "
+    "unmeasured=0\n",
     "end"
   };
 
   (void)arg;
   for (i = 0; strcmp(lines_fail[i], "end"); i++) {
     //fprintf(stderr, "Testing: %s\n", lines_fail[i]);
-    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1);
+    /* Testing only with line_is_after_headers = 1. Tests with
+     * line_is_after_headers = 0 in
+     * test_dir_measured_bw_kb_line_is_after_headers */
+    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1);
   }
 
   for (i = 0; strcmp(lines_pass[i], "end"); i++) {
     //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n'));
-    tt_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0);
+    /* Testing only with line_is_after_headers = 1. Tests with
+     * line_is_after_headers = 0 in
+     * test_dir_measured_bw_kb_line_is_after_headers */
+    tt_assert(measured_bw_line_parse(&mbwl, lines_pass[i], 1) == 0);
     tt_assert(mbwl.bw_kb == 1024);
     tt_assert(strcmp(mbwl.node_hex,
                 "557365204145532d32353620696e73746561642e") == 0);
@@ -1370,7 +1589,7 @@ test_dir_measured_bw_kb(void *arg)
 
 /* Test dirserv_read_measured_bandwidths */
 static void
-test_dir_dirserv_read_measured_bandwidths(void *arg)
+test_dir_dirserv_read_measured_bandwidths_empty(void *arg)
 {
   char *fname=NULL;
   (void)arg;
@@ -1387,6 +1606,167 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   teardown_capture_of_logs();
 }
 
+/* Unit tests for measured_bw_line_parse using line_is_after_headers flag.
+ * When the end of the header is detected (a first complete bw line is parsed),
+ * incomplete lines fail and give warnings, but do not give warnings if
+ * the header is not ended, allowing to ignore additional header lines. */
+static void
+test_dir_measured_bw_kb_line_is_after_headers(void *arg)
+{
+  (void)arg;
+  measured_bw_line_t mbwl;
+  const char *line_pass = \
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024\n";
+  int i;
+  const char *lines_fail[] = {
+    "node_id=$557365204145532d32353620696e73746561642e \n",
+    "bw=1024\n",
+    "rtt=300\n",
+    "end"
+  };
+
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Test bw lines when header has ended */
+  for (i = 0; strcmp(lines_fail[i], "end"); i++) {
+    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1);
+    expect_log_msg_containing("Incomplete line in bandwidth file:");
+    mock_clean_saved_logs();
+  }
+
+  tt_assert(measured_bw_line_parse(&mbwl, line_pass, 1) == 0);
+
+  /* Test bw lines when header has not ended */
+  for (i = 0; strcmp(lines_fail[i], "end"); i++) {
+    tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 0) == -1);
+    expect_log_msg_containing("Missing bw or node_id in bandwidth file line:");
+    mock_clean_saved_logs();
+  }
+
+  tt_assert(measured_bw_line_parse(&mbwl, line_pass, 0) == 0);
+
+ done:
+  teardown_capture_of_logs();
+}
+
+/* Test dirserv_read_measured_bandwidths with whole files. */
+static void
+test_dir_dirserv_read_measured_bandwidths(void *arg)
+{
+  (void)arg;
+  char *content = NULL;
+  time_t timestamp = time(NULL);
+  char *fname = tor_strdup(get_fname("V3BandwidthsFile"));
+
+  /* Test Torflow file only with timestamp*/
+  tor_asprintf(&content, "%ld", (long)timestamp);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test Torflow file with timestamp followed by '\n' */
+  tor_asprintf(&content, "%ld\n", (long)timestamp);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test Torflow complete file*/
+  const char *torflow_relay_lines=
+    "node_id=$557365204145532d32353620696e73746561642e bw=1024 "
+    "nick=Test measured_at=1523911725 updated_at=1523911725 "
+    "pid_error=4.11374090719 pid_error_sum=4.11374090719 "
+    "pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 "
+    "scanner=/filepath\n";
+
+  tor_asprintf(&content, "%ld\n%s", (long)timestamp, torflow_relay_lines);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test Torflow complete file including v1.1.0 headers */
+  const char *v110_header_lines=
+    "version=1.1.0\n"
+    "software=sbws\n"
+    "software_version=0.1.0\n"
+    "generator_started=2018-05-08T16:13:25\n"
+    "earliest_bandwidth=2018-05-08T16:13:26\n"
+    "====\n";
+
+  tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, v110_header_lines,
+               torflow_relay_lines);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test Torflow with additional headers afer a correct bw line */
+  tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, torflow_relay_lines,
+               v110_header_lines);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test Torflow with additional headers afer a correct bw line and more
+   * bw lines after the headers. */
+  tor_asprintf(&content, "%ld\n%s%s%s", (long)timestamp, torflow_relay_lines,
+               v110_header_lines, torflow_relay_lines);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test sbws file */
+  const char *sbws_relay_lines=
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test rtt=380 time=2018-05-08T16:13:26\n";
+
+  tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, v110_header_lines,
+               sbws_relay_lines);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+
+  /* Test v1.x.x bandwidth line with vote=0.
+   * It will be ignored it and logged it at debug level. */
+  const char *relay_lines_ignore =
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=0\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=0"
+    "unmeasured=1\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=0"
+    "unmeasured=0\n";
+
+  /* Create the bandwidth file */
+  tor_asprintf(&content, "%ld\n%s", (long)timestamp, relay_lines_ignore);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+
+  /* Read the bandwidth file */
+  setup_full_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+  expect_log_msg_containing("Ignoring bandwidth file line");
+  teardown_capture_of_logs();
+
+  /* Test v1.x.x bandwidth line with "vote=1" or "unmeasured=1" or
+   * "unmeasured=0".
+   * They will not be ignored. */
+  /* Create the bandwidth file */
+  const char *relay_lines_vote =
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 vote=1\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 unmeasured=0\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 bw=1024 unmeasured=1\n";
+  tor_asprintf(&content, "%ld\n%s", (long)timestamp, relay_lines_vote);
+  write_str_to_file(fname, content, 0);
+  tor_free(content);
+
+  /* Read the bandwidth file */
+  setup_full_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL));
+  expect_log_msg_not_containing("Ignoring bandwidth file line");
+  teardown_capture_of_logs();
+
+ done:
+  tor_free(fname);
+}
+
 #define MBWC_INIT_TIME 1000
 
 /** Do the measured bandwidth cache unit test */
@@ -1512,6 +1892,15 @@ test_dir_param_voting(void *arg)
   tt_int_op(-8,OP_EQ, networkstatus_get_param(&vote4, "ab", -12, -100, -8));
   tt_int_op(0,OP_EQ, networkstatus_get_param(&vote4, "foobar", 0, -100, 8));
 
+  tt_int_op(100,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -1, "x-yz", 50, 0, 300));
+  tt_int_op(30,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, 30, "x-yz", 50, 0, 300));
+  tt_int_op(0,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -101, "foobar", 0, -100, 8));
+  tt_int_op(-99,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -99, "foobar", 0, -100, 8));
+
   smartlist_add(votes, &vote1);
 
   /* Do the first tests without adding all the other votes, for
@@ -1611,7 +2000,7 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "moomin", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(n_found > 1)");
+            "n_found == 0");
   tor_end_capture_bugs_();
   /* There is no 'fred=', so that is treated as not existing. */
   tt_int_op(-100, OP_EQ,
@@ -1699,8 +2088,7 @@ vote_tweaks_for_v3ns(networkstatus_t *v, int voter, time_t now)
     measured_bw_line_t mbw;
     memset(mbw.node_id, 33, sizeof(mbw.node_id));
     mbw.bw_kb = 1024;
-    tt_assert(measured_bw_line_apply(&mbw,
-                v->routerstatus_list) == 1);
+    tt_int_op(measured_bw_line_apply(&mbw, v->routerstatus_list), OP_EQ, 1);
   } else if (voter == 2 || voter == 3) {
     /* Monkey around with the list a bit */
     vrs = smartlist_get(v->routerstatus_list, 2);
@@ -1816,7 +2204,7 @@ test_consensus_for_v3ns(networkstatus_t *con, time_t now)
   (void)now;
 
   tt_assert(con);
-  tt_assert(!con->cert);
+  tt_ptr_op(con->cert, OP_EQ, NULL);
   tt_int_op(2,OP_EQ, smartlist_len(con->routerstatus_list));
   /* There should be two listed routers: one with identity 3, one with
    * identity 5. */
@@ -1886,13 +2274,260 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now)
     /* XXXX check version */
   } else {
     /* Weren't expecting this... */
-    tt_assert(0);
+    tt_abort();
   }
 
  done:
   return;
 }
 
+static void
+test_dir_networkstatus_compute_bw_weights_v10(void *arg)
+{
+  (void) arg;
+  smartlist_t *chunks = smartlist_new();
+  int64_t G, M, E, D, T, weight_scale;
+  int ret;
+  weight_scale = 10000;
+
+  /* no case. one or more of the values is 0 */
+  G = M = E = D = 0;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(smartlist_len(chunks), OP_EQ, 0);
+
+  /* case 1 */
+  /* XXX dir-spec not followed? See #20272. If it isn't closed, then this is
+   * testing current behavior, not spec. */
+  G = E = 10;
+  M = D = 1;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(smartlist_len(chunks), OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=3333 "
+    "Wbe=3000 Wbg=3000 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=7000 "
+    "Weg=3333 Wem=7000 Wgb=10000 Wgd=3333 Wgg=7000 Wgm=7000 Wmb=10000 "
+    "Wmd=3333 Wme=3000 Wmg=3000 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2a E scarce */
+  M = 100;
+  G = 20;
+  E = D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2a G scarce */
+  M = 100;
+  E = 20;
+  G = D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=10000 Weg=0 Wem=10000 "
+    "Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 Wme=0 Wmg=0 "
+    "Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b1 (Wgg=1, Wmd=Wgd) */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 100;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=4000 "
+    "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=2000 Wee=10000 Weg=2000 "
+    "Wem=10000 Wgb=10000 Wgd=4000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=4000 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b2 */
+  M = 60;
+  E = 30;
+  G = 10;
+  D = 100;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=666 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3666 Wee=10000 Weg=3666 "
+    "Wem=10000 Wgb=10000 Wgd=5668 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=666 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b3 */
+  /* XXX I can't get a combination of values that hits this case without error,
+   * so this just tests that it fails. See #20285. Also see #20284 as 2b3 does
+   * not follow dir-spec. */
+  /* (E < T/3 && G < T/3) && (E+D>=G || G+D>=E) && (M > T/3) */
+  M = 80;
+  E = 30;
+  G = 30;
+  D = 30;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 0);
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3a G scarce */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 "
+    "Wbe=3333 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=6667 Weg=0 "
+    "Wem=6667 Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 "
+    "Wme=3333 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3a E scarce */
+  M = 10;
+  E = 10;
+  G = 30;
+  D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=3333 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=6667 Wgm=6667 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=3333 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3bg */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 10;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 "
+    "Wbe=3334 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=6666 Weg=0 "
+    "Wem=6666 Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 "
+    "Wme=3334 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3be */
+  M = 10;
+  E = 10;
+  G = 30;
+  D = 10;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=3334 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=6666 Wgm=6666 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=3334 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 21 Jul 2013 (3be) */
+  G = 5483409;
+  M = 1455379;
+  E = 980834;
+  D = 3385803;
+  T = 11305425;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_assert(ret);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=883 Wbe=0 "
+    "Wbg=3673 Wbm=10000 Wdb=10000 Web=10000 Wed=8233 Wee=10000 Weg=8233 "
+    "Wem=10000 Wgb=10000 Wgd=883 Wgg=6327 Wgm=6327 Wmb=10000 Wmd=883 Wme=0 "
+    "Wmg=3673 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 04 Oct 2016 (3a E scarce) */
+  G=29322240;
+  M=4721546;
+  E=1522058;
+  D=9273571;
+  T=44839415;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_assert(ret);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=4194 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=5806 Wgm=5806 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=4194 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 04 Sep 2013 (2b1) */
+  G=3091352;
+  M=1838837;
+  E=2109300;
+  D=2469369;
+  T=9508858;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_assert(ret);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=317 "
+    "Wbe=5938 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=9366 Wee=4061 "
+    "Weg=9366 Wem=4061 Wgb=10000 Wgd=317 Wgg=10000 Wgm=10000 Wmb=10000 "
+    "Wmd=317 Wme=5938 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* explicitly test initializing weights to 1*/
+  G=1;
+  M=1;
+  E=1;
+  D=1;
+  T=4;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=3333 "
+    "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 "
+    "Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  tt_assert(ret);
+
+ done:
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+}
+
 static authority_cert_t *mock_cert;
 
 static authority_cert_t *
@@ -1958,6 +2593,7 @@ test_a_networkstatus(
   sign_skey_2 = crypto_pk_new();
   sign_skey_3 = crypto_pk_new();
   sign_skey_leg1 = pk_generate(4);
+  voting_schedule_recalculate_timing(get_options(), now);
   sr_state_init(0, 0);
 
   tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1,
@@ -2495,8 +3131,9 @@ gen_routerstatus_for_umbw(int idx, time_t now)
       rs->addr = 0x99008801;
       rs->or_port = 443;
       rs->dir_port = 8000;
-      /* all flags but running cleared */
+      /* all flags but running and valid cleared */
       rs->is_flagged_running = 1;
+      rs->is_valid = 1;
       /*
        * This one has measured bandwidth below the clip cutoff, and
        * so shouldn't be clipped; we'll have to test that it isn't
@@ -2569,8 +3206,9 @@ gen_routerstatus_for_umbw(int idx, time_t now)
       rs->addr = 0xC0000203;
       rs->or_port = 500;
       rs->dir_port = 1999;
-      /* all flags but running cleared */
+      /* all flags but running and valid cleared */
       rs->is_flagged_running = 1;
+      rs->is_valid = 1;
       /*
        * This one has unmeasured bandwidth below the clip cutoff, and
        * so shouldn't be clipped; we'll have to test that it isn't
@@ -2587,12 +3225,12 @@ gen_routerstatus_for_umbw(int idx, time_t now)
       break;
     default:
       /* Shouldn't happen */
-      tt_assert(0);
+      tt_abort();
   }
   if (vrs) {
     vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
     tor_asprintf(&vrs->microdesc->microdesc_hash_line,
-                 "m 9,10,11,12,13,14,15,16,17 "
+                 "m 25,26,27,28 "
                  "sha256=xyzajkldsdsajdadlsdjaslsdksdjlsdjsdaskdaaa%d\n",
                  idx);
   }
@@ -2618,7 +3256,7 @@ vote_tweaks_for_umbw(networkstatus_t *v, int voter, time_t now)
   smartlist_clear(v->supported_methods);
   /* Method 17 is MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB */
   smartlist_split_string(v->supported_methods,
-                         "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17",
+                         "25 26 27 28",
                          NULL, 0, -1);
   /* If we're using a non-default clip bandwidth, add it to net_params */
   if (alternate_clip_bw > 0) {
@@ -2727,7 +3365,7 @@ test_vrs_for_umbw(vote_routerstatus_t *vrs, int voter, time_t now)
     tt_int_op(rs->bandwidth_kb,OP_EQ, max_unmeasured_bw_kb / 2);
     tt_int_op(vrs->measured_bw_kb,OP_EQ, 0);
   } else {
-    tt_assert(0);
+    tt_abort();
   }
 
  done:
@@ -2743,9 +3381,9 @@ test_consensus_for_umbw(networkstatus_t *con, time_t now)
   (void)now;
 
   tt_assert(con);
-  tt_assert(!con->cert);
+  tt_ptr_op(con->cert, OP_EQ, NULL);
   // tt_assert(con->consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB);
-  tt_assert(con->consensus_method >= 16);
+  tt_int_op(con->consensus_method, OP_GE, 16);
   tt_int_op(4,OP_EQ, smartlist_len(con->routerstatus_list));
   /* There should be four listed routers; all voters saw the same in this */
 
@@ -2780,9 +3418,9 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now)
     tt_assert(!rs->is_fast);
     tt_assert(!rs->is_possible_guard);
     tt_assert(!rs->is_stable);
-    /* (If it wasn't running it wouldn't be here) */
+    /* (If it wasn't running and valid it wouldn't be here) */
     tt_assert(rs->is_flagged_running);
-    tt_assert(!rs->is_valid);
+    tt_assert(rs->is_valid);
     tt_assert(!rs->is_named);
     /* This one should have measured bandwidth below the clip cutoff */
     tt_assert(rs->has_bandwidth);
@@ -2842,7 +3480,7 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now)
     tt_assert(rs->bw_is_unmeasured);
   } else {
     /* Weren't expecting this... */
-    tt_assert(0);
+    tt_abort();
   }
 
  done:
@@ -2949,7 +3587,7 @@ mock_get_options(void)
 static void
 reset_routerstatus(routerstatus_t *rs,
                    const char *hex_identity_digest,
-                   int32_t ipv4_addr)
+                   uint32_t ipv4_addr)
 {
   memset(rs, 0, sizeof(routerstatus_t));
   base16_decode(rs->identity_digest, sizeof(rs->identity_digest),
@@ -3010,15 +3648,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
    * Return values are {2, 3, 4} */
 
   /* We want 3 ("*" means match all addresses) */
-  tt_assert(routerset_contains_routerstatus(routerset_all,  rs_a, 0) == 3);
-  tt_assert(routerset_contains_routerstatus(routerset_all,  rs_b, 0) == 3);
+  tt_int_op(routerset_contains_routerstatus(routerset_all, rs_a, 0), OP_EQ, 3);
+  tt_int_op(routerset_contains_routerstatus(routerset_all, rs_b, 0), OP_EQ, 3);
 
   /* We want 4 (match id_digest [or nickname]) */
-  tt_assert(routerset_contains_routerstatus(routerset_a,    rs_a, 0) == 4);
-  tt_assert(routerset_contains_routerstatus(routerset_a,    rs_b, 0) == 0);
+  tt_int_op(routerset_contains_routerstatus(routerset_a, rs_a, 0), OP_EQ, 4);
+  tt_int_op(routerset_contains_routerstatus(routerset_a, rs_b, 0), OP_EQ, 0);
 
-  tt_assert(routerset_contains_routerstatus(routerset_none, rs_a, 0) == 0);
-  tt_assert(routerset_contains_routerstatus(routerset_none, rs_b, 0) == 0);
+  tt_int_op(routerset_contains_routerstatus(routerset_none, rs_a, 0), OP_EQ,
+            0);
+  tt_int_op(routerset_contains_routerstatus(routerset_none, rs_b, 0), OP_EQ,
+            0);
 
   /* Check that "*" sets flags on all routers: Exit
    * Check the flags aren't being confused with each other */
@@ -3030,17 +3670,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   mock_options->TestingDirAuthVoteExitIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_exit == 1);
-  tt_assert(rs_b->is_exit == 1);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 1);
   /* Be paranoid - check no other flags are set */
-  tt_assert(rs_a->is_possible_guard == 0);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_a->is_hs_dir == 0);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 0);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check that "*" sets flags on all routers: Guard & HSDir
    * Cover the remaining flags in one test */
@@ -3054,17 +3694,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_possible_guard == 1);
-  tt_assert(rs_b->is_possible_guard == 1);
-  tt_assert(rs_a->is_hs_dir == 1);
-  tt_assert(rs_b->is_hs_dir == 1);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 1);
   /* Be paranoid - check exit isn't set */
-  tt_assert(rs_a->is_exit == 0);
-  tt_assert(rs_b->is_exit == 0);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
 
   /* Check routerset A sets all flags on router A,
    * but leaves router B unmodified */
@@ -3080,16 +3720,16 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_exit == 1);
-  tt_assert(rs_b->is_exit == 0);
-  tt_assert(rs_a->is_possible_guard == 1);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_a->is_hs_dir == 1);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check routerset A unsets all flags on router B when Strict is set */
   reset_options(mock_options, &mock_get_options_calls);
@@ -3107,11 +3747,11 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
-  tt_assert(rs_b->is_exit == 0);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check routerset A doesn't modify flags on router B without Strict set */
   reset_options(mock_options, &mock_get_options_calls);
@@ -3129,11 +3769,11 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
-  tt_assert(rs_b->is_exit == 1);
-  tt_assert(rs_b->is_possible_guard == 1);
-  tt_assert(rs_b->is_hs_dir == 1);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 1);
 
   /* Check the empty routerset zeroes all flags
    * on routers A & B with Strict set */
@@ -3152,11 +3792,11 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
-  tt_assert(rs_b->is_exit == 0);
-  tt_assert(rs_b->is_possible_guard == 0);
-  tt_assert(rs_b->is_hs_dir == 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 0);
 
   /* Check the empty routerset doesn't modify any flags
    * on A or B without Strict set */
@@ -3176,16 +3816,16 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_assert(mock_get_options_calls == 1);
+  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_assert(mock_get_options_calls == 2);
+  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
-  tt_assert(rs_a->is_exit == 0);
-  tt_assert(rs_a->is_possible_guard == 0);
-  tt_assert(rs_a->is_hs_dir == 0);
-  tt_assert(rs_b->is_exit == 1);
-  tt_assert(rs_b->is_possible_guard == 1);
-  tt_assert(rs_b->is_hs_dir == 1);
+  tt_uint_op(rs_a->is_exit, OP_EQ, 0);
+  tt_uint_op(rs_a->is_possible_guard, OP_EQ, 0);
+  tt_uint_op(rs_a->is_hs_dir, OP_EQ, 0);
+  tt_uint_op(rs_b->is_exit, OP_EQ, 1);
+  tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
+  tt_uint_op(rs_b->is_hs_dir, OP_EQ, 1);
 
  done:
   tor_free(mock_options);
@@ -3241,7 +3881,7 @@ test_dir_http_handling(void *args)
                            "User-Agent: Mozilla/5.0 (Windows;"
                            " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n",
                            &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   /* Bad headers */
   tt_int_op(parse_http_url("GET /a/b/c.txt\r\n"
@@ -3249,40 +3889,113 @@ test_dir_http_handling(void *args)
                            "User-Agent: Mozilla/5.0 (Windows;"
                            " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n",
                            &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1x\r\n", &url),
             OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
   tt_int_op(parse_http_url("GET /tor/a/b/c.txt HTTP/1.\r", &url),OP_EQ, -1);
-  tt_assert(!url);
+  tt_ptr_op(url, OP_EQ, NULL);
 
  done:
   tor_free(url);
 }
 
 static void
-test_dir_purpose_needs_anonymity(void *arg)
+test_dir_purpose_needs_anonymity_returns_true_by_default(void *arg)
+{
+  (void)arg;
+
+  tor_capture_bugs_(1);
+  setup_full_capture_of_logs(LOG_WARN);
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, 0, NULL));
+  tt_int_op(1, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
+  expect_single_log_msg_containing("Called with dir_purpose=0");
+
+  tor_end_capture_bugs_();
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_true_for_bridges(void *arg)
 {
   (void)arg;
-  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE));
-  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_GENERAL));
-  tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC,
-                                            ROUTER_PURPOSE_GENERAL));
+
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE, NULL));
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE,
+                                           "foobar"));
+  tt_int_op(1, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2,
+                                    ROUTER_PURPOSE_BRIDGE, NULL));
  done: ;
 }
 
 static void
+test_dir_purpose_needs_anonymity_returns_false_for_own_bridge_desc(void *arg)
+{
+  (void)arg;
+  tt_int_op(0, OP_EQ, purpose_needs_anonymity(DIR_PURPOSE_FETCH_SERVERDESC,
+                                           ROUTER_PURPOSE_BRIDGE,
+                                           "authority.z"));
+ done: ;
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_true_for_sensitive_purpose(void *arg)
+{
+  (void)arg;
+
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2,
+                    ROUTER_PURPOSE_GENERAL, NULL));
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_UPLOAD_RENDDESC_V2, 0,  NULL));
+  tt_int_op(1, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_FETCH_RENDDESC_V2, 0, NULL));
+ done: ;
+}
+
+static void
+test_dir_purpose_needs_anonymity_ret_false_for_non_sensitive_conn(void *arg)
+{
+  (void)arg;
+
+  tt_int_op(0, OP_EQ, purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_DIR,
+                                           ROUTER_PURPOSE_GENERAL, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_VOTE, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_SIGNATURES, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL));
+  tt_int_op(0, OP_EQ, purpose_needs_anonymity(
+                    DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_CONSENSUS, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_SERVERDESC, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_EXTRAINFO, 0, NULL));
+  tt_int_op(0, OP_EQ,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC, 0, NULL));
+  done: ;
+}
+
+static void
 test_dir_fetch_type(void *arg)
 {
   (void)arg;
@@ -3330,9 +4043,9 @@ test_dir_packages(void *arg)
   (void)arg;
 
 #define BAD(s) \
-  tt_int_op(0, ==, validate_recommended_package_line(s));
+  tt_int_op(0, OP_EQ, validate_recommended_package_line(s));
 #define GOOD(s) \
-  tt_int_op(1, ==, validate_recommended_package_line(s));
+  tt_int_op(1, OP_EQ, validate_recommended_package_line(s));
   GOOD("tor 0.2.6.3-alpha "
        "http://torproject.example.com/dist/tor-0.2.6.3-alpha.tar.gz "
        "sha256=sssdlkfjdsklfjdskfljasdklfj");
@@ -3449,7 +4162,7 @@ test_dir_packages(void *arg)
 
   res = compute_consensus_package_lines(votes);
   tt_assert(res);
-  tt_str_op(res, ==,
+  tt_str_op(res, OP_EQ,
     "package cbc 99.1.11.1.1 http://example.com/cbc/ cubehash=ahooy sha512=m\n"
     "package clownshoes 22alpha3 http://quumble.example.com/ blake2=fooz\n"
     "package clownshoes 22alpha4 http://quumble.example.cam/ blake2=fooa\n"
@@ -3468,490 +4181,156 @@ test_dir_packages(void *arg)
 }
 
 static void
-test_dir_download_status_schedule(void *arg)
-{
-  (void)arg;
-  download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC,
-                                             DL_WANT_AUTHORITY,
-                                             DL_SCHED_INCREMENT_FAILURE,
-                                             DL_SCHED_DETERMINISTIC, 0, 0 };
-  download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_CONSENSUS,
-                                             DL_WANT_ANY_DIRSERVER,
-                                             DL_SCHED_INCREMENT_ATTEMPT,
-                                             DL_SCHED_DETERMINISTIC, 0, 0 };
-  download_status_t dls_bridge  = { 0, 0, 0, DL_SCHED_BRIDGE,
-                                             DL_WANT_AUTHORITY,
-                                             DL_SCHED_INCREMENT_FAILURE,
-                                             DL_SCHED_DETERMINISTIC, 0, 0 };
-  int increment = -1;
-  int expected_increment = -1;
-  time_t current_time = time(NULL);
-  int delay1 = -1;
-  int delay2 = -1;
-  smartlist_t *schedule = smartlist_new();
-
-  /* Make a dummy schedule */
-  smartlist_add(schedule, (void *)&delay1);
-  smartlist_add(schedule, (void *)&delay2);
-
-  /* check a range of values */
-  delay1 = 1000;
-  increment = download_status_schedule_get_delay(&dls_failure,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 TIME_MIN);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_failure.next_attempt_at == TIME_MIN + expected_increment);
-
-  delay1 = INT_MAX;
-  increment =  download_status_schedule_get_delay(&dls_failure,
-                                                  schedule,
-                                                  0, INT_MAX,
-                                                  -1);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_failure.next_attempt_at == TIME_MAX);
-
-  delay1 = 0;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 0);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == 0 + expected_increment);
-
-  delay1 = 1000;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 1);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == 1 + expected_increment);
-
-  delay1 = INT_MAX;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == TIME_MAX);
-
-  delay1 = 1;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 TIME_MAX);
-  expected_increment = delay1;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == TIME_MAX);
-
-  /* see what happens when we reach the end */
-  dls_attempt.n_download_attempts++;
-  dls_bridge.n_download_failures++;
-
-  delay2 = 100;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == current_time + delay2);
-
-  delay2 = 1;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == current_time + delay2);
-
-  /* see what happens when we try to go off the end */
-  dls_attempt.n_download_attempts++;
-  dls_bridge.n_download_failures++;
-
-  delay2 = 5;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == current_time + delay2);
-
-  delay2 = 17;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = delay2;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == current_time + delay2);
-
-  /* see what happens when we reach IMPOSSIBLE_TO_DOWNLOAD */
-  dls_attempt.n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
-  dls_bridge.n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
-
-  delay2 = 35;
-  increment = download_status_schedule_get_delay(&dls_attempt,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = INT_MAX;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_attempt.next_attempt_at == TIME_MAX);
-
-  delay2 = 99;
-  increment = download_status_schedule_get_delay(&dls_bridge,
-                                                 schedule,
-                                                 0, INT_MAX,
-                                                 current_time);
-  expected_increment = INT_MAX;
-  tt_assert(increment == expected_increment);
-  tt_assert(dls_bridge.next_attempt_at == TIME_MAX);
-
- done:
-  /* the pointers in schedule are allocated on the stack */
-  smartlist_free(schedule);
-}
-
-static void
-test_dir_download_status_random_backoff(void *arg)
+download_status_random_backoff_helper(int min_delay)
 {
   download_status_t dls_random =
     { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY,
-               DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 };
+               DL_SCHED_INCREMENT_FAILURE, 0, 0 };
   int increment = -1;
-  int old_increment;
+  int old_increment = -1;
   time_t current_time = time(NULL);
-  const int min_delay = 0;
-  const int max_delay = 1000000;
-
-  (void)arg;
 
   /* Check the random backoff cases */
-  old_increment = 0;
+  int n_attempts = 0;
   do {
     increment = download_status_schedule_get_delay(&dls_random,
-                                                   NULL,
-                                                   min_delay, max_delay,
+                                                   min_delay,
                                                    current_time);
+
+    log_debug(LD_DIR, "Min: %d, Inc: %d, Old Inc: %d",
+              min_delay, increment, old_increment);
+
+    /* Regression test for 20534 and friends
+     * increment must always increase after the first */
+    if (dls_random.last_backoff_position > 0) {
+      /* Always increment the exponential backoff */
+      tt_int_op(increment, OP_GE, 1);
+    }
+
     /* Test */
     tt_int_op(increment, OP_GE, min_delay);
-    tt_int_op(increment, OP_LE, max_delay);
-    tt_int_op(increment, OP_GE, old_increment);
-    /* We at most quadruple, and maybe add one */
-    tt_int_op(increment, OP_LE, 4 * old_increment + 1);
 
     /* Advance */
-    current_time += increment;
-    ++(dls_random.n_download_attempts);
-    ++(dls_random.n_download_failures);
+    if (dls_random.n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD - 1) {
+      ++(dls_random.n_download_attempts);
+      ++(dls_random.n_download_failures);
+    }
 
     /* Try another maybe */
     old_increment = increment;
-  } while (increment < max_delay);
+  } while (++n_attempts < 1000);
 
  done:
   return;
 }
 
 static void
+test_dir_download_status_random_backoff(void *arg)
+{
+  (void)arg;
+
+  /* Do a standard test */
+  download_status_random_backoff_helper(0);
+  /* regression tests for 17750: initial delay */
+  download_status_random_backoff_helper(10);
+  download_status_random_backoff_helper(20);
+
+  /* Pathological cases */
+  download_status_random_backoff_helper(INT_MAX/2);
+}
+
+static void
+test_dir_download_status_random_backoff_ranges(void *arg)
+{
+  (void)arg;
+  int lo, hi;
+  next_random_exponential_delay_range(&lo, &hi, 0, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 11);
+
+  next_random_exponential_delay_range(&lo, &hi, 6, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 6*3);
+
+  next_random_exponential_delay_range(&lo, &hi, 13, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 13 * 3);
+
+  next_random_exponential_delay_range(&lo, &hi, 37, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 111);
+
+  next_random_exponential_delay_range(&lo, &hi, 123, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, 369);
+
+  next_random_exponential_delay_range(&lo, &hi, INT_MAX-5, 10);
+  tt_int_op(lo, OP_EQ, 10);
+  tt_int_op(hi, OP_EQ, INT_MAX);
+ done:
+  ;
+}
+
+static void
 test_dir_download_status_increment(void *arg)
 {
   (void)arg;
-  download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC,
-    DL_WANT_AUTHORITY,
-    DL_SCHED_INCREMENT_FAILURE,
-    DL_SCHED_DETERMINISTIC, 0, 0 };
-  download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_BRIDGE,
+  download_status_t dls_exp = { 0, 0, 0, DL_SCHED_GENERIC,
     DL_WANT_ANY_DIRSERVER,
     DL_SCHED_INCREMENT_ATTEMPT,
-    DL_SCHED_DETERMINISTIC, 0, 0 };
-  int delay0 = -1;
-  int delay1 = -1;
-  int delay2 = -1;
-  smartlist_t *schedule = smartlist_new();
+    0, 0 };
   or_options_t test_options;
-  time_t next_at = TIME_MAX;
   time_t current_time = time(NULL);
 
-  /* Provide some values for the schedule */
-  delay0 = 10;
-  delay1 = 99;
-  delay2 = 20;
-
-  /* Make the schedule */
-  smartlist_add(schedule, (void *)&delay0);
-  smartlist_add(schedule, (void *)&delay1);
-  smartlist_add(schedule, (void *)&delay2);
+  const int delay0 = 10;
+  const int no_delay = 0;
+  const int schedule = 10;
+  const int schedule_no_initial_delay = 0;
 
   /* Put it in the options */
   mock_options = &test_options;
   reset_options(mock_options, &mock_get_options_calls);
-  mock_options->TestingClientDownloadSchedule = schedule;
-  mock_options->TestingBridgeDownloadSchedule = schedule;
+  mock_options->TestingBridgeBootstrapDownloadInitialDelay = schedule;
+  mock_options->TestingClientDownloadInitialDelay = schedule;
 
   MOCK(get_options, mock_get_options);
 
-  /* Check that a failure reset works */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_failure);
-  /* we really want to test that it's equal to time(NULL) + delay0, but that's
-   * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 0);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* avoid timing inconsistencies */
-  dls_failure.next_attempt_at = current_time + delay0;
-
-  /* check that a reset schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay0 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay0,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay0 + 1,
-                                     1) == 1);
-
-  /* Check that a failure increment works */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == current_time + delay1);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 1);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 1);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* check that an incremented schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1 + 1,
-                                     1) == 1);
-
-  /* check that a schedule isn't ready if it's had too many failures */
-  tt_assert(download_status_is_ready(&dls_failure,
-                                     current_time + delay1 + 10,
-                                     0) == 0);
-
-  /* Check that failure increments do happen on 503 for clients, and
-   * attempt increments do too. */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 503, "test", 0,
-                                              current_time);
-  tt_i64_op(next_at, ==, current_time + delay2);
-  tt_int_op(download_status_get_n_failures(&dls_failure), ==, 2);
-  tt_int_op(download_status_get_n_attempts(&dls_failure), ==, 2);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that failure increments do happen on 503 for servers */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 503, "test", 1,
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 3);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 3);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check what happens when we run off the end of the schedule */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_failure, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 4);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 4);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check what happens when we hit the failure limit */
-  mock_get_options_calls = 0;
-  download_status_mark_impossible(&dls_failure);
-  next_at = download_status_increment_failure(&dls_failure, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that a failure reset doesn't reset at the limit */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_failure);
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_failure)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls == 0);
-
-  /* Check that a failure reset resets just before the limit */
+  /* Check that the initial value of the schedule is the first value used,
+   * whether or not it was reset before being used */
+
+  /* regression test for 17750: no initial delay */
+  mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay;
   mock_get_options_calls = 0;
-  dls_failure.n_download_failures = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  dls_failure.n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  download_status_reset(&dls_failure);
   /* we really want to test that it's equal to time(NULL) + delay0, but that's
    * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_failure)
-            != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 0);
-  tt_assert(mock_get_options_calls >= 1);
 
-  /* Check that failure increments do happen on attempt-based schedules,
-   * but that the retry is set at the end of time */
+  /* regression test for 17750: exponential, no initial delay */
+  mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay;
   mock_get_options_calls = 0;
-  next_at = download_status_increment_failure(&dls_attempt, 404, "test", 0,
-                                              current_time);
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 1);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 0);
-  tt_assert(mock_get_options_calls == 0);
-
-  /* Check that an attempt reset works */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_attempt);
   /* we really want to test that it's equal to time(NULL) + delay0, but that's
    * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
-            >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
+            >= current_time + no_delay);
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
             != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 0);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* avoid timing inconsistencies */
-  dls_attempt.next_attempt_at = current_time + delay0;
-
-  /* check that a reset schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay0 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay0,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay0 + 1,
-                                     1) == 1);
-
-  /* Check that an attempt increment works */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == current_time + delay1);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 1);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* check that an incremented schedule becomes ready at the right time */
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1 - 1,
-                                     1) == 0);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1,
-                                     1) == 1);
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1 + 1,
-                                     1) == 1);
-
-  /* check that a schedule isn't ready if it's had too many attempts */
-  tt_assert(download_status_is_ready(&dls_attempt,
-                                     current_time + delay1 + 10,
-                                     0) == 0);
-
-  /* Check what happens when we reach then run off the end of the schedule */
-  mock_get_options_calls = 0;
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 2);
-  tt_assert(mock_get_options_calls >= 1);
+  tt_int_op(download_status_get_n_failures(&dls_exp), OP_EQ, 0);
+  tt_int_op(download_status_get_n_attempts(&dls_exp), OP_EQ, 0);
+  tt_int_op(mock_get_options_calls, OP_GE, 1);
 
+  /* regression test for 17750: exponential, initial delay */
+  mock_options->TestingClientDownloadInitialDelay = schedule;
   mock_get_options_calls = 0;
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == current_time + delay2);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 3);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check what happens when we hit the attempt limit */
-  mock_get_options_calls = 0;
-  download_status_mark_impossible(&dls_attempt);
-  next_at = download_status_increment_attempt(&dls_attempt, "test",
-                                              current_time);
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that an attempt reset doesn't reset at the limit */
-  mock_get_options_calls = 0;
-  download_status_reset(&dls_attempt);
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
-            == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(download_status_get_n_attempts(&dls_attempt)
-            == IMPOSSIBLE_TO_DOWNLOAD);
-  tt_assert(mock_get_options_calls == 0);
-
-  /* Check that an attempt reset resets just before the limit */
-  mock_get_options_calls = 0;
-  dls_attempt.n_download_failures = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  dls_attempt.n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD - 1;
-  download_status_reset(&dls_attempt);
   /* we really want to test that it's equal to time(NULL) + delay0, but that's
    * an unrealiable test, because time(NULL) might change. */
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
             >= current_time + delay0);
-  tt_assert(download_status_get_next_attempt_at(&dls_attempt)
+  tt_assert(download_status_get_next_attempt_at(&dls_exp)
             != TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_attempt) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_attempt) == 0);
-  tt_assert(mock_get_options_calls >= 1);
-
-  /* Check that attempt increments don't happen on failure-based schedules,
-   * and that the attempt is set at the end of time */
-  mock_get_options_calls = 0;
-  setup_full_capture_of_logs(LOG_WARN);
-  next_at = download_status_increment_attempt(&dls_failure, "test",
-                                              current_time);
-  expect_single_log_msg_containing(
-    "Tried to launch an attempt-based connection on a failure-based "
-    "schedule.");
-  teardown_capture_of_logs();
-  tt_assert(next_at == TIME_MAX);
-  tt_assert(download_status_get_n_failures(&dls_failure) == 0);
-  tt_assert(download_status_get_n_attempts(&dls_failure) == 0);
-  tt_assert(mock_get_options_calls == 0);
+  tt_int_op(download_status_get_n_failures(&dls_exp), OP_EQ, 0);
+  tt_int_op(download_status_get_n_attempts(&dls_exp), OP_EQ, 0);
+  tt_int_op(mock_get_options_calls, OP_GE, 1);
 
  done:
-  /* the pointers in schedule are allocated on the stack */
-  smartlist_free(schedule);
   UNMOCK(get_options);
   mock_options = NULL;
   mock_get_options_calls = 0;
@@ -4054,7 +4433,6 @@ test_dir_should_use_directory_guards(void *data)
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
   tt_int_op(CALLED(public_server_mode), OP_EQ, 1);
 
-  options->UseEntryGuardsAsDirGuards = 1;
   options->UseEntryGuards = 1;
   options->DownloadExtraInfo = 0;
   options->FetchDirInfoEarly = 0;
@@ -4068,29 +4446,24 @@ test_dir_should_use_directory_guards(void *data)
   tt_int_op(CALLED(public_server_mode), OP_EQ, 3);
   options->UseEntryGuards = 1;
 
-  options->UseEntryGuardsAsDirGuards = 0;
-  tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
-  options->UseEntryGuardsAsDirGuards = 1;
-
   options->DownloadExtraInfo = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
   options->DownloadExtraInfo = 0;
 
   options->FetchDirInfoEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
   options->FetchDirInfoEarly = 0;
 
   options->FetchDirInfoExtraEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
   options->FetchDirInfoExtraEarly = 0;
 
   options->FetchUselessDescriptors = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 8);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
   options->FetchUselessDescriptors = 0;
 
   done:
@@ -4100,14 +4473,7 @@ test_dir_should_use_directory_guards(void *data)
 }
 
 NS_DECL(void,
-directory_initiate_command_routerstatus, (const routerstatus_t *status,
-                                          uint8_t dir_purpose,
-                                          uint8_t router_purpose,
-                                          dir_indirection_t indirection,
-                                          const char *resource,
-                                          const char *payload,
-                                          size_t payload_len,
-                                          time_t if_modified_since));
+directory_initiate_request, (directory_request_t *req));
 
 static void
 test_dir_should_not_init_request_to_ourselves(void *data)
@@ -4117,7 +4483,7 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   crypto_pk_t *key = pk_generate(2);
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4132,15 +4498,15 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   dir_server_add(ourself);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
 
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
     crypto_pk_free(key);
@@ -4154,7 +4520,7 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
                                 | MICRODESC_DIRINFO;
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4165,14 +4531,14 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
@@ -4183,7 +4549,7 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_t *ds = NULL;
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4194,37 +4560,23 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 1);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 1);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 2);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 2);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
 
 void
-NS(directory_initiate_command_routerstatus)(const routerstatus_t *status,
-                                            uint8_t dir_purpose,
-                                            uint8_t router_purpose,
-                                            dir_indirection_t indirection,
-                                            const char *resource,
-                                            const char *payload,
-                                            size_t payload_len,
-                                            time_t if_modified_since)
+NS(directory_initiate_request)(directory_request_t *req)
 {
-  (void)status;
-  (void)dir_purpose;
-  (void)router_purpose;
-  (void)indirection;
-  (void)resource;
-  (void)payload;
-  (void)payload_len;
-  (void)if_modified_since;
-  CALLED(directory_initiate_command_routerstatus)++;
+  (void)req;
+  CALLED(directory_initiate_request)++;
 }
 
 static void
@@ -4278,22 +4630,24 @@ mock_check_private_dir(const char *dirname, cpd_check_t check,
 
 static char *
 mock_get_datadir_fname(const or_options_t *options,
+                       directory_root_t roottype,
                        const char *sub1, const char *sub2,
                        const char *suffix)
 {
+  (void) roottype;
   char *rv = NULL;
 
   /*
    * Assert we were called like get_datadir_fname2() or get_datadir_fname(),
    * since that's all we implement here.
    */
-  tt_assert(options != NULL);
-  tt_assert(sub1 != NULL);
+  tt_ptr_op(options, OP_NE, NULL);
+  tt_ptr_op(sub1, OP_NE, NULL);
   /*
    * No particular assertions about sub2, since we could be in the
    * get_datadir_fname() or get_datadir_fname2() case.
    */
-  tt_assert(suffix == NULL);
+  tt_ptr_op(suffix, OP_EQ, NULL);
 
   /* Just duplicate the basename and return it for this mock */
   if (sub2) {
@@ -4320,7 +4674,7 @@ mock_unlink_reset(void)
 static int
 mock_unlink(const char *path)
 {
-  tt_assert(path != NULL);
+  tt_ptr_op(path, OP_NE, NULL);
 
   tor_free(last_unlinked_path);
   last_unlinked_path = tor_strdup(path);
@@ -4349,8 +4703,8 @@ mock_write_str_to_file(const char *path, const char *str, int bin)
 
   (void)bin;
 
-  tt_assert(path != NULL);
-  tt_assert(str != NULL);
+  tt_ptr_op(path, OP_NE, NULL);
+  tt_ptr_op(str, OP_NE, NULL);
 
   len = strlen(str);
   crypto_digest256((char *)hash, str, len, DIGEST_SHA256);
@@ -4437,7 +4791,7 @@ test_dir_dump_unparseable_descriptors(void *data)
   mock_options->MaxUnparseableDescSizeToLog = 1536;
   MOCK(get_options, mock_get_options);
   MOCK(check_private_dir, mock_check_private_dir);
-  MOCK(options_get_datadir_fname2_suffix,
+  MOCK(options_get_dir_fname2_suffix,
        mock_get_datadir_fname);
 
   /*
@@ -4476,7 +4830,7 @@ test_dir_dump_unparseable_descriptors(void *data)
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4489,21 +4843,21 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4511,8 +4865,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (2) Fire off dump_desc() twice; this still should trigger no cleanup.
@@ -4524,14 +4878,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4540,21 +4894,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4562,8 +4917,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (3) Three calls to dump_desc cause a FIFO cleanup
@@ -4575,14 +4930,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4591,14 +4946,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4) + strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_4) + strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /* Third time - we should unlink the dump of test_desc_4 here */
@@ -4607,21 +4963,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_1) + strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(write_str_count, ==, 3);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(write_str_count, OP_EQ, 3);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4629,8 +4986,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (4) But repeating one (A B B) doesn't overflow and cleanup
@@ -4642,14 +4999,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4658,14 +5015,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4674,21 +5032,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4696,8 +5055,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (5) Same for the (A B A) repetition
@@ -4709,14 +5068,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4725,14 +5084,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_1) + strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4741,21 +5101,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_1) + strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4763,8 +5124,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (6) (A B B C) triggering overflow on C causes A, not B to be unlinked
@@ -4776,14 +5137,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4792,14 +5153,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4808,14 +5170,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_3) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /* Fourth time - we should unlink the dump of test_desc_3 here */
@@ -4824,21 +5187,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4) + strlen(test_desc_1));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_4) + strlen(test_desc_1));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(write_str_count, ==, 3);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(write_str_count, OP_EQ, 3);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4846,8 +5210,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
   /*
    * (7) (A B A C) triggering overflow on C causes B, not A to be unlinked
@@ -4859,14 +5223,14 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2));
+  tt_u64_op(len_descs_dumped, OP_EQ, strlen(test_desc_2));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 1);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 1);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256);
 
   /* Second time */
@@ -4875,14 +5239,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Third time */
@@ -4891,14 +5256,15 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_3));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 2);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 2);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256);
 
   /* Fourth time - we should unlink the dump of test_desc_3 here */
@@ -4907,21 +5273,22 @@ test_dir_dump_unparseable_descriptors(void *data)
   /*
    * Assert things about the FIFO state
    */
-  tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_4));
+  tt_u64_op(len_descs_dumped, OP_EQ,
+            strlen(test_desc_2) + strlen(test_desc_4));
   tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2);
 
   /*
    * Assert things about the mocks
    */
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(write_str_count, ==, 3);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(write_str_count, OP_EQ, 3);
   tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256);
 
   /*
    * Reset the FIFO and check its state
    */
   dump_desc_fifo_cleanup();
-  tt_u64_op(len_descs_dumped, ==, 0);
+  tt_u64_op(len_descs_dumped, OP_EQ, 0);
   tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0);
 
   /*
@@ -4929,8 +5296,8 @@ test_dir_dump_unparseable_descriptors(void *data)
    */
   mock_unlink_reset();
   mock_write_str_to_file_reset();
-  tt_int_op(unlinked_count, ==, 0);
-  tt_int_op(write_str_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
+  tt_int_op(write_str_count, OP_EQ, 0);
 
  done:
 
@@ -4942,7 +5309,7 @@ test_dir_dump_unparseable_descriptors(void *data)
   mock_unlink_reset();
   UNMOCK(write_str_to_file);
   mock_write_str_to_file_reset();
-  UNMOCK(options_get_datadir_fname2_suffix);
+  UNMOCK(options_get_dir_fname2_suffix);
   UNMOCK(check_private_dir);
   UNMOCK(get_options);
   tor_free(mock_options);
@@ -4977,7 +5344,7 @@ read_file_to_str_mock(const char *filename, int flags,
   char *result = NULL;
 
   /* Insist we got a filename */
-  tt_assert(filename != NULL);
+  tt_ptr_op(filename, OP_NE, NULL);
 
   /* We ignore flags */
   (void)flags;
@@ -5040,53 +5407,53 @@ test_dir_populate_dump_desc_fifo(void *data)
   reset_read_file_to_str_mock();
 
   /* Check state of unlink mock */
-  tt_int_op(unlinked_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
 
   /* Some cases that should fail before trying to read the file */
   ent = dump_desc_populate_one_file(dirname, "bar");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 1);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 1);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(dirname, "unparseable-desc");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 2);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 2);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(dirname, "unparseable-desc.baz");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 3);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 3);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(
       dirname,
       "unparseable-desc.08AE85E90461F59E");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 4);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 4);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   ent = dump_desc_populate_one_file(
       dirname,
       "unparseable-desc.08AE85E90461F59EDF0981323F3A70D02B55AB54B44B04F"
       "287D72F7B72F242E85C8CB0EDA8854A99");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 5);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 5);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   /* This is a correct-length digest but base16_decode() will fail */
   ent = dump_desc_populate_one_file(
       dirname,
       "unparseable-desc.68219B8BGE64B705A6FFC728C069DC596216D60A7D7520C"
       "D5ECE250D912E686B");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 6);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 0);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 6);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 0);
 
   /* This one has a correctly formed filename and should try reading */
 
@@ -5095,10 +5462,10 @@ test_dir_populate_dump_desc_fifo(void *data)
       dirname,
       "unparseable-desc.DF0981323F3A70D02B55AB54B44B04F287D72F7B72F242E"
       "85C8CB0EDA8854A99");
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 7);
-  tt_int_op(read_count, ==, 0);
-  tt_int_op(read_call_count, ==, 1);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 7);
+  tt_int_op(read_count, OP_EQ, 0);
+  tt_int_op(read_call_count, OP_EQ, 1);
 
   /* This read will succeed but the digest won't match the file content */
   fname =
@@ -5111,10 +5478,10 @@ test_dir_populate_dump_desc_fifo(void *data)
   file_stat.st_mtime = 123456;
   ent = dump_desc_populate_one_file(dirname, fname);
   enforce_expected_filename = 0;
-  tt_assert(ent == NULL);
-  tt_int_op(unlinked_count, ==, 8);
-  tt_int_op(read_count, ==, 1);
-  tt_int_op(read_call_count, ==, 2);
+  tt_ptr_op(ent, OP_EQ, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 8);
+  tt_int_op(read_count, OP_EQ, 1);
+  tt_int_op(read_call_count, OP_EQ, 2);
   tor_free(expected_filename);
   tor_free(file_content);
 
@@ -5127,13 +5494,13 @@ test_dir_populate_dump_desc_fifo(void *data)
   file_content_len = strlen(file_content);
   file_stat.st_mtime = 789012;
   ent = dump_desc_populate_one_file(dirname, fname);
-  tt_assert(ent != NULL);
-  tt_int_op(unlinked_count, ==, 8);
-  tt_int_op(read_count, ==, 2);
-  tt_int_op(read_call_count, ==, 3);
+  tt_ptr_op(ent, OP_NE, NULL);
+  tt_int_op(unlinked_count, OP_EQ, 8);
+  tt_int_op(read_count, OP_EQ, 2);
+  tt_int_op(read_call_count, OP_EQ, 3);
   tt_str_op(ent->filename, OP_EQ, expected_filename);
-  tt_int_op(ent->len, ==, file_content_len);
-  tt_int_op(ent->when, ==, file_stat.st_mtime);
+  tt_int_op(ent->len, OP_EQ, file_content_len);
+  tt_int_op(ent->when, OP_EQ, file_stat.st_mtime);
   tor_free(ent->filename);
   tor_free(ent);
   tor_free(expected_filename);
@@ -5142,9 +5509,9 @@ test_dir_populate_dump_desc_fifo(void *data)
    * Reset the mocks and check their state
    */
   mock_unlink_reset();
-  tt_int_op(unlinked_count, ==, 0);
+  tt_int_op(unlinked_count, OP_EQ, 0);
   reset_read_file_to_str_mock();
-  tt_int_op(read_count, ==, 0);
+  tt_int_op(read_count, OP_EQ, 0);
 
  done:
 
@@ -5167,9 +5534,9 @@ listdir_mock(const char *dname)
   (void)dname;
 
   l = smartlist_new();
-  smartlist_add(l, tor_strdup("foo"));
-  smartlist_add(l, tor_strdup("bar"));
-  smartlist_add(l, tor_strdup("baz"));
+  smartlist_add_strdup(l, "foo");
+  smartlist_add_strdup(l, "bar");
+  smartlist_add_strdup(l, "baz");
 
   return l;
 }
@@ -5266,17 +5633,26 @@ mock_networkstatus_consensus_can_use_extra_fallbacks(
   return mock_networkstatus_consensus_can_use_extra_fallbacks_value;
 }
 
-/* data is a 2 character nul-terminated string.
+static int mock_num_bridges_usable_value = 0;
+static int
+mock_num_bridges_usable(int use_maybe_reachable)
+{
+  (void)use_maybe_reachable;
+  return mock_num_bridges_usable_value;
+}
+
+/* data is a 3 character nul-terminated string.
  * If data[0] is 'b', set bootstrapping, anything else means not bootstrapping
  * If data[1] is 'f', set extra fallbacks, anything else means no extra
+ * If data[2] is 'f', set running bridges, anything else means no extra
  * fallbacks.
  */
 static void
-test_dir_find_dl_schedule(void* data)
+test_dir_find_dl_min_delay(void* data)
 {
   const char *str = (const char *)data;
 
-  tt_assert(strlen(data) == 2);
+  tt_assert(strlen(data) == 3);
 
   if (str[0] == 'b') {
     mock_networkstatus_consensus_is_bootstrapping_value = 1;
@@ -5290,49 +5666,60 @@ test_dir_find_dl_schedule(void* data)
     mock_networkstatus_consensus_can_use_extra_fallbacks_value = 0;
   }
 
+  if (str[2] == 'r') {
+    /* Any positive, non-zero value should work */
+    mock_num_bridges_usable_value = 2;
+  } else {
+    mock_num_bridges_usable_value = 0;
+  }
+
   MOCK(networkstatus_consensus_is_bootstrapping,
        mock_networkstatus_consensus_is_bootstrapping);
   MOCK(networkstatus_consensus_can_use_extra_fallbacks,
        mock_networkstatus_consensus_can_use_extra_fallbacks);
+  MOCK(num_bridges_usable,
+       mock_num_bridges_usable);
 
   download_status_t dls;
-  smartlist_t server, client, server_cons, client_cons;
-  smartlist_t client_boot_auth_only_cons, client_boot_auth_cons;
-  smartlist_t client_boot_fallback_cons, bridge;
+
+  const int server=10, client=20, server_cons=30, client_cons=40;
+  const int client_boot_auth_only_cons=50, client_boot_auth_cons=60;
+  const int client_boot_fallback_cons=70, bridge=80, bridge_bootstrap=90;
 
   mock_options = tor_malloc(sizeof(or_options_t));
   reset_options(mock_options, &mock_get_options_calls);
   MOCK(get_options, mock_get_options);
 
-  mock_options->TestingServerDownloadSchedule = &server;
-  mock_options->TestingClientDownloadSchedule = &client;
-  mock_options->TestingServerConsensusDownloadSchedule = &server_cons;
-  mock_options->TestingClientConsensusDownloadSchedule = &client_cons;
-  mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule =
-    &client_boot_auth_only_cons;
-  mock_options->ClientBootstrapConsensusAuthorityDownloadSchedule =
-    &client_boot_auth_cons;
-  mock_options->ClientBootstrapConsensusFallbackDownloadSchedule =
-    &client_boot_fallback_cons;
-  mock_options->TestingBridgeDownloadSchedule = &bridge;
+  mock_options->TestingServerDownloadInitialDelay = server;
+  mock_options->TestingClientDownloadInitialDelay = client;
+  mock_options->TestingServerConsensusDownloadInitialDelay = server_cons;
+  mock_options->TestingClientConsensusDownloadInitialDelay = client_cons;
+  mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay =
+    client_boot_auth_only_cons;
+  mock_options->ClientBootstrapConsensusAuthorityDownloadInitialDelay =
+    client_boot_auth_cons;
+  mock_options->ClientBootstrapConsensusFallbackDownloadInitialDelay =
+    client_boot_fallback_cons;
+  mock_options->TestingBridgeDownloadInitialDelay = bridge;
+  mock_options->TestingBridgeBootstrapDownloadInitialDelay = bridge_bootstrap;
 
   dls.schedule = DL_SCHED_GENERIC;
   /* client */
   mock_options->ClientOnly = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &client);
+  tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, client);
   mock_options->ClientOnly = 0;
 
   /* dir mode */
   mock_options->DirPort_set = 1;
   mock_options->DirCache = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server);
+  tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server);
   mock_options->DirPort_set = 0;
   mock_options->DirCache = 0;
 
   dls.schedule = DL_SCHED_CONSENSUS;
   /* public server mode */
   mock_options->ORPort_set = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server_cons);
+  tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server_cons);
   mock_options->ORPort_set = 0;
 
   /* client and bridge modes */
@@ -5341,30 +5728,30 @@ test_dir_find_dl_schedule(void* data)
       dls.want_authority = 1;
       /* client */
       mock_options->ClientOnly = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_cons);
       mock_options->ClientOnly = 0;
 
       /* bridge relay */
       mock_options->ORPort_set = 1;
       mock_options->BridgeRelay = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_cons);
       mock_options->ORPort_set = 0;
       mock_options->BridgeRelay = 0;
 
       dls.want_authority = 0;
       /* client */
       mock_options->ClientOnly = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_fallback_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_fallback_cons);
       mock_options->ClientOnly = 0;
 
       /* bridge relay */
       mock_options->ORPort_set = 1;
       mock_options->BridgeRelay = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_fallback_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_fallback_cons);
       mock_options->ORPort_set = 0;
       mock_options->BridgeRelay = 0;
 
@@ -5372,30 +5759,30 @@ test_dir_find_dl_schedule(void* data)
       /* dls.want_authority is ignored */
       /* client */
       mock_options->ClientOnly = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_only_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_only_cons);
       mock_options->ClientOnly = 0;
 
       /* bridge relay */
       mock_options->ORPort_set = 1;
       mock_options->BridgeRelay = 1;
-      tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-                &client_boot_auth_only_cons);
+      tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+                client_boot_auth_only_cons);
       mock_options->ORPort_set = 0;
       mock_options->BridgeRelay = 0;
     }
   } else {
     /* client */
     mock_options->ClientOnly = 1;
-    tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-              &client_cons);
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+              client_cons);
     mock_options->ClientOnly = 0;
 
     /* bridge relay */
     mock_options->ORPort_set = 1;
     mock_options->BridgeRelay = 1;
-    tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ,
-              &client_cons);
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ,
+              client_cons);
     mock_options->ORPort_set = 0;
     mock_options->BridgeRelay = 0;
   }
@@ -5403,11 +5790,17 @@ test_dir_find_dl_schedule(void* data)
   dls.schedule = DL_SCHED_BRIDGE;
   /* client */
   mock_options->ClientOnly = 1;
-  tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &bridge);
+  mock_options->UseBridges = 1;
+  if (num_bridges_usable(0) > 0) {
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge);
+  } else {
+    tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap);
+  }
 
  done:
   UNMOCK(networkstatus_consensus_is_bootstrapping);
   UNMOCK(networkstatus_consensus_can_use_extra_fallbacks);
+  UNMOCK(num_bridges_usable);
   UNMOCK(get_options);
   tor_free(mock_options);
   mock_options = NULL;
@@ -5421,9 +5814,8 @@ test_dir_assumed_flags(void *arg)
   memarea_t *area = memarea_new();
   routerstatus_t *rs = NULL;
 
-  /* First, we should always assume that the Running flag is set, even
-   * when it isn't listed, since the consensus method is always
-   * higher than 4. */
+  /* We can assume that consensus method is higher than 24, so Running and
+   * Valid are always implicitly set */
   const char *str1 =
     "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 "
        "192.168.0.1 9001 0\n"
@@ -5432,17 +5824,6 @@ test_dir_assumed_flags(void *arg)
 
   const char *cp = str1;
   rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
-                                            23, FLAV_MICRODESC);
-  tt_assert(rs);
-  tt_assert(rs->is_flagged_running);
-  tt_assert(! rs->is_valid);
-  tt_assert(! rs->is_exit);
-  tt_assert(rs->is_fast);
-  routerstatus_free(rs);
-
-  /* With method 24 or later, we can assume "valid" is set. */
-  cp = str1;
-  rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
                                             24, FLAV_MICRODESC);
   tt_assert(rs);
   tt_assert(rs->is_flagged_running);
@@ -5457,6 +5838,182 @@ test_dir_assumed_flags(void *arg)
 }
 
 static void
+test_dir_post_parsing(void *arg)
+{
+  (void) arg;
+
+  /* Test the version parsing from an HS descriptor publish request. */
+  {
+    const char *end;
+    const char *prefix = "/tor/hs/";
+    int version = parse_hs_version_from_post("/tor/hs//publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/a/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, 3);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("/tor/hs/42/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, 42);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("/tor/hs/18163/publish",prefix, &end);
+    tt_int_op(version, OP_EQ, 18163);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("JUNKJUNKJUNK", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish", "blah", &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    /* Missing the '/' at the end of the prefix. */
+    version = parse_hs_version_from_post("/tor/hs/3/publish", "/tor/hs", &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/random/blah/tor/hs/3/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish/random/junk",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, 3);
+    tt_str_op(end, OP_EQ, "/publish/random/junk");
+    version = parse_hs_version_from_post("/tor/hs/-1/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    /* INT_MAX */
+    version = parse_hs_version_from_post("/tor/hs/2147483647/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, INT_MAX);
+    tt_str_op(end, OP_EQ, "/publish");
+    /* INT_MAX + 1*/
+    version = parse_hs_version_from_post("/tor/hs/2147483648/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_dir_platform_str(void *arg)
+{
+  char platform[256];
+  (void)arg;
+  platform[0] = 0;
+  get_platform_str(platform, sizeof(platform));
+  tt_int_op((int)strlen(platform), OP_GT, 0);
+  tt_assert(!strcmpstart(platform, "Tor "));
+
+  tor_version_t ver;
+  // make sure this is a tor version, a real actual tor version.
+  tt_int_op(tor_version_parse_platform(platform, &ver, 1), OP_EQ, 1);
+
+  TT_BLATHER(("%d.%d.%d.%d", ver.major, ver.minor, ver.micro, ver.patchlevel));
+
+  // Handle an example version.
+  tt_int_op(tor_version_parse_platform(
+        "Tor 0.3.3.3 (foo) (git-xyzzy) on a potato", &ver, 1), OP_EQ, 1);
+ done:
+  ;
+}
+
+static networkstatus_t *mock_networkstatus;
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  (void)f;
+  return mock_networkstatus;
+}
+
+static void
+test_dir_networkstatus_consensus_has_ipv6(void *arg)
+{
+  (void)arg;
+
+  int has_ipv6 = 0;
+
+  /* Init options and networkstatus */
+  or_options_t our_options;
+  mock_options = &our_options;
+  reset_options(mock_options, &mock_get_options_calls);
+  MOCK(get_options, mock_get_options);
+
+  networkstatus_t our_networkstatus;
+  mock_networkstatus = &our_networkstatus;
+  memset(mock_networkstatus, 0, sizeof(*mock_networkstatus));
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+
+  /* A live consensus */
+  mock_networkstatus->valid_after = time(NULL) - 3600;
+  mock_networkstatus->valid_until = time(NULL) + 3600;
+
+  /* Test the bounds for A lines in the NS consensus */
+  mock_options->UseMicrodescriptors = 0;
+
+  mock_networkstatus->consensus_method = MIN_SUPPORTED_CONSENSUS_METHOD;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  /* Test the bounds for A lines in the microdesc consensus */
+  mock_options->UseMicrodescriptors = 1;
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD + 20;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 1;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 20;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS - 1;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(!has_ipv6);
+
+  /* Test the edge cases */
+  mock_options->UseMicrodescriptors = 1;
+  mock_networkstatus->consensus_method =
+      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
+
+  /* Reasonably live */
+  mock_networkstatus->valid_until = approx_time() - 60;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(has_ipv6);
+
+  /* Not reasonably live */
+  mock_networkstatus->valid_after = approx_time() - 24*60*60 - 3600;
+  mock_networkstatus->valid_until = approx_time() - 24*60*60 - 60;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(!has_ipv6);
+
+  /* NULL consensus */
+  mock_networkstatus = NULL;
+  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
+  tt_assert(!has_ipv6);
+
+ done:
+  UNMOCK(get_options);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+}
+
+static void
 test_dir_format_versions_list(void *arg)
 {
   (void)arg;
@@ -5525,12 +6082,15 @@ struct testcase_t dir_tests[] = {
   DIR(parse_router_list, TT_FORK),
   DIR(load_routers, TT_FORK),
   DIR(load_extrainfo, TT_FORK),
+  DIR(getinfo_extra, 0),
   DIR_LEGACY(versions),
   DIR_LEGACY(fp_pairs),
   DIR(split_fps, 0),
-  DIR_LEGACY(dirserv_read_measured_bandwidths),
+  DIR_LEGACY(dirserv_read_measured_bandwidths_empty),
   DIR_LEGACY(measured_bw_kb),
+  DIR_LEGACY(measured_bw_kb_line_is_after_headers),
   DIR_LEGACY(measured_bw_kb_cache),
+  DIR_LEGACY(dirserv_read_measured_bandwidths),
   DIR_LEGACY(param_voting),
   DIR(param_voting_lookup, 0),
   DIR_LEGACY(v3_networkstatus),
@@ -5541,12 +6101,17 @@ struct testcase_t dir_tests[] = {
   DIR(fmt_control_ns, 0),
   DIR(dirserv_set_routerstatus_testing, 0),
   DIR(http_handling, 0),
-  DIR(purpose_needs_anonymity, 0),
+  DIR(purpose_needs_anonymity_returns_true_for_bridges, 0),
+  DIR(purpose_needs_anonymity_returns_false_for_own_bridge_desc, 0),
+  DIR(purpose_needs_anonymity_returns_true_by_default, 0),
+  DIR(purpose_needs_anonymity_returns_true_for_sensitive_purpose, 0),
+  DIR(purpose_needs_anonymity_ret_false_for_non_sensitive_conn, 0),
+  DIR(post_parsing, 0),
   DIR(fetch_type, 0),
   DIR(packages, 0),
-  DIR(download_status_schedule, 0),
   DIR(download_status_random_backoff, 0),
-  DIR(download_status_increment, 0),
+  DIR(download_status_random_backoff_ranges, 0),
+  DIR(download_status_increment, TT_FORK),
   DIR(authdir_type_to_string, 0),
   DIR(conn_purpose_to_string, 0),
   DIR(should_use_directory_guards, 0),
@@ -5557,11 +6122,18 @@ struct testcase_t dir_tests[] = {
   DIR(dump_unparseable_descriptors, 0),
   DIR(populate_dump_desc_fifo, 0),
   DIR(populate_dump_desc_fifo_2, 0),
-  DIR_ARG(find_dl_schedule, TT_FORK, "bf"),
-  DIR_ARG(find_dl_schedule, TT_FORK, "ba"),
-  DIR_ARG(find_dl_schedule, TT_FORK, "cf"),
-  DIR_ARG(find_dl_schedule, TT_FORK, "ca"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bfd"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bad"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "cfd"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "cad"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bfr"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "bar"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"),
+  DIR_ARG(find_dl_min_delay, TT_FORK, "car"),
   DIR(assumed_flags, 0),
+  DIR(networkstatus_compute_bw_weights_v10, 0),
+  DIR(platform_str, 0),
+  DIR(networkstatus_consensus_has_ipv6, TT_FORK),
   DIR(format_versions_list, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index ca43dd4c04..230410f7fa 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -1,18 +1,18 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define DIRVOTE_PRIVATE
-#include "crypto.h"
 #include "test.h"
 #include "container.h"
 #include "or.h"
-#include "dirvote.h"
+#include "dirauth/dirvote.h"
 #include "nodelist.h"
 #include "routerlist.h"
 #include "test_dir_common.h"
+#include "voting_schedule.h"
 
 void dir_common_setup_vote(networkstatus_t **vote, time_t now);
 networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote,
@@ -146,7 +146,7 @@ dir_common_gen_routerstatus_for_v3ns(int idx, time_t now)
       break;
     default:
       /* Shouldn't happen */
-      tt_assert(0);
+      tt_abort();
   }
   if (vrs) {
     vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t));
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index 9682b0db49..65b9cf6436 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index a0f22f1f0c..688d26bdc1 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define RENDCOMMON_PRIVATE
@@ -12,9 +12,10 @@
 #include "or.h"
 #include "config.h"
 #include "connection.h"
+#include "consdiffmgr.h"
 #include "directory.h"
 #include "test.h"
-#include "connection.h"
+#include "compress.h"
 #include "rendcommon.h"
 #include "rendcache.h"
 #include "router.h"
@@ -26,17 +27,19 @@
 #include "entrynodes.h"
 #include "routerparse.h"
 #include "networkstatus.h"
+#include "proto_http.h"
 #include "geoip.h"
 #include "dirserv.h"
-#include "torgzip.h"
-#include "dirvote.h"
+#include "dirauth/dirvote.h"
+#include "log_test_helpers.h"
+#include "voting_schedule.h"
 
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 DISABLE_GCC_WARNING(overlength-strings)
@@ -50,22 +53,10 @@ ENABLE_GCC_WARNING(overlength-strings)
 
 #define NS_MODULE dir_handle_get
 
-static void
-connection_write_to_buf_mock(const char *string, size_t len,
-                             connection_t *conn, int zlib)
-{
-  (void) zlib;
-
-  tor_assert(string);
-  tor_assert(conn);
-
-  write_to_buf(string, len, conn->outbuf);
-}
-
-#define GET(path) "GET " path " HTTP/1.0\r\n\r\n"
 #define NOT_FOUND "HTTP/1.0 404 Not found\r\n\r\n"
 #define BAD_REQUEST "HTTP/1.0 400 Bad request\r\n\r\n"
 #define SERVER_BUSY "HTTP/1.0 503 Directory busy, try again later\r\n\r\n"
+#define TOO_OLD "HTTP/1.0 404 Consensus is too old\r\n\r\n"
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
@@ -74,6 +65,7 @@ new_dir_conn(void)
 {
   dir_connection_t *conn = dir_connection_new(AF_INET);
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
   return conn;
 }
 
@@ -96,7 +88,7 @@ test_dir_handle_get_bad_request(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -125,7 +117,7 @@ test_dir_handle_get_v1_command_not_found(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -172,7 +164,7 @@ test_dir_handle_get_v1_command(void *data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_dirportfrontpage);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 }
@@ -198,7 +190,7 @@ test_dir_handle_get_not_found(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -233,7 +225,7 @@ test_dir_handle_get_robots_txt(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 }
@@ -262,7 +254,7 @@ test_dir_handle_get_rendezvous2_not_found_if_not_encrypted(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -290,7 +282,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_with_invalid_desc_id(
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -323,7 +315,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_not_well_formed(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -352,7 +344,7 @@ test_dir_handle_get_rendezvous2_not_found(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     rend_cache_free_all();
 }
@@ -432,7 +424,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
     UNMOCK(connection_write_to_buf_impl_);
     NS_UNMOCK(router_get_my_routerinfo);
 
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     rend_encoded_v2_service_descriptor_free(desc_holder);
@@ -465,7 +457,7 @@ test_dir_handle_get_micro_d_not_found(void *data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
 
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -476,6 +468,9 @@ init_mock_options(void)
   mock_options = tor_malloc(sizeof(or_options_t));
   memset(mock_options, 0, sizeof(or_options_t));
   mock_options->TestingTorNetwork = 1;
+  mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
+  mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory);
+  check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL);
 }
 
 static const or_options_t *
@@ -512,14 +507,6 @@ test_dir_handle_get_micro_d(void *data)
 
   /* SETUP */
   init_mock_options();
-  const char *fn = get_fname("dir_handle_datadir_test1");
-  mock_options->DataDirectory = tor_strdup(fn);
-
-#ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory));
-#else
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700));
-#endif
 
   /* Add microdesc to cache */
   crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256);
@@ -555,7 +542,7 @@ test_dir_handle_get_micro_d(void *data)
     UNMOCK(connection_write_to_buf_impl_);
 
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     smartlist_free(list);
@@ -579,14 +566,6 @@ test_dir_handle_get_micro_d_server_busy(void *data)
 
   /* SETUP */
   init_mock_options();
-  const char *fn = get_fname("dir_handle_datadir_test2");
-  mock_options->DataDirectory = tor_strdup(fn);
-
-#ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory));
-#else
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700));
-#endif
 
   /* Add microdesc to cache */
   crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256);
@@ -617,7 +596,7 @@ test_dir_handle_get_micro_d_server_busy(void *data)
     UNMOCK(connection_write_to_buf_impl_);
 
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     smartlist_free(list);
     microdesc_free_all();
@@ -654,7 +633,7 @@ test_dir_handle_get_networkstatus_bridges_not_found_without_auth(void *data)
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -694,7 +673,7 @@ test_dir_handle_get_networkstatus_bridges(void *data)
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -731,7 +710,7 @@ test_dir_handle_get_networkstatus_bridges_not_found_wrong_auth(void *data)
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -754,12 +733,12 @@ test_dir_handle_get_server_descriptors_not_found(void* data)
                       NULL, NULL, 1, 0);
 
   tt_str_op(NOT_FOUND, OP_EQ, header);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_SERVER_BY_FP);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
     or_options_free(mock_options); mock_options = NULL;
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -784,6 +763,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   tt_int_op(smartlist_len(our_routerlist->routers), OP_GE, 1);
   mock_routerinfo = smartlist_get(our_routerlist->routers, 0);
   set_server_identity_key(mock_routerinfo->identity_pkey);
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   /* Treat "all" requests as if they were unencrypted */
   mock_routerinfo->cache_info.send_unencrypted = 1;
@@ -798,7 +778,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   //which is smaller than that by annotation_len bytes
   fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
                       &body, &body_used,
-                      mock_routerinfo->cache_info.signed_descriptor_len+1, 0);
+                      1024*1024, 0);
 
   tt_assert(header);
   tt_assert(body);
@@ -814,12 +794,12 @@ test_dir_handle_get_server_descriptors_all(void* data)
 
   tt_str_op(body, OP_EQ, mock_routerinfo->cache_info.signed_descriptor_body +
                          mock_routerinfo->cache_info.annotations_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 
@@ -891,8 +871,9 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   mock_routerinfo->cache_info.signed_descriptor_body =
     tor_strdup(TEST_DESCRIPTOR);
   mock_routerinfo->cache_info.signed_descriptor_len =
-    strlen(TEST_DESCRIPTOR) - annotation_len;;
+    strlen(TEST_DESCRIPTOR) - annotation_len;
   mock_routerinfo->cache_info.annotations_len = annotation_len;
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   conn = new_dir_conn();
 
@@ -915,14 +896,14 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   tt_int_op(body_used, OP_EQ, strlen(body));
 
   tt_str_op(body, OP_EQ, TEST_DESCRIPTOR + annotation_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     crypto_pk_free(identity_pkey);
@@ -957,6 +938,7 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   mock_routerinfo->cache_info.signed_descriptor_len =
     strlen(TEST_DESCRIPTOR) - annotation_len;
   mock_routerinfo->cache_info.annotations_len = annotation_len;
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   conn = new_dir_conn();
 
@@ -986,14 +968,14 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   tt_int_op(body_used, OP_EQ, strlen(body));
 
   tt_str_op(body, OP_EQ, TEST_DESCRIPTOR + annotation_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     crypto_pk_free(identity_pkey);
@@ -1052,12 +1034,12 @@ test_dir_handle_get_server_descriptors_d(void* data)
 
   tt_str_op(body, OP_EQ, router->cache_info.signed_descriptor_body +
                          router->cache_info.annotations_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     crypto_pk_free(identity_pkey);
@@ -1107,13 +1089,13 @@ test_dir_handle_get_server_descriptors_busy(void* data)
   tt_assert(header);
   tt_str_op(SERVER_BUSY, OP_EQ, header);
 
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     crypto_pk_free(identity_pkey);
 
@@ -1144,7 +1126,7 @@ test_dir_handle_get_server_keys_bad_req(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1170,7 +1152,7 @@ test_dir_handle_get_server_keys_all_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1229,7 +1211,7 @@ test_dir_handle_get_server_keys_all(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 
@@ -1259,7 +1241,7 @@ test_dir_handle_get_server_keys_authority_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1307,7 +1289,7 @@ test_dir_handle_get_server_keys_authority(void* data)
   done:
     UNMOCK(get_my_v3_authority_cert);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     authority_cert_free(mock_cert); mock_cert = NULL;
@@ -1335,7 +1317,7 @@ test_dir_handle_get_server_keys_fp_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1389,7 +1371,7 @@ test_dir_handle_get_server_keys_fp(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     clear_dir_servers();
@@ -1418,7 +1400,7 @@ test_dir_handle_get_server_keys_sk_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1463,7 +1445,7 @@ test_dir_handle_get_server_keys_sk(void* data)
   done:
     UNMOCK(get_my_v3_authority_cert);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     authority_cert_free(mock_cert); mock_cert = NULL;
     tor_free(header);
     tor_free(body);
@@ -1491,7 +1473,7 @@ test_dir_handle_get_server_keys_fpsk_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1548,7 +1530,7 @@ test_dir_handle_get_server_keys_fpsk(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
 
@@ -1602,7 +1584,7 @@ test_dir_handle_get_server_keys_busy(void* data)
   done:
     UNMOCK(get_options);
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     or_options_free(mock_options); mock_options = NULL;
 
@@ -1629,7 +1611,13 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d)
   /* init mock */
   mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
   mock_ns_val->flavor = FLAV_NS;
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
   mock_ns_val->voters = smartlist_new();
+  mock_ns_val->valid_after = time(NULL) - 1800;
+  mock_ns_val->valid_until = time(NULL) - 60;
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
 
   /* init mock */
   init_mock_options();
@@ -1662,7 +1650,7 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d)
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_options);
 
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(stats);
     smartlist_free(mock_ns_val->voters);
@@ -1703,12 +1691,86 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_found(void* data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_options);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(stats);
     or_options_free(mock_options); mock_options = NULL;
 }
 
+static void
+test_dir_handle_get_status_vote_current_consensus_too_old(void *data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void)data;
+
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
+  mock_ns_val->flavor = FLAV_MICRODESC;
+  mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800);
+  mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900);
+  mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20);
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
+
+  init_mock_options();
+
+  MOCK(get_options, mock_get_options);
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+  MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor);
+
+  conn = new_dir_conn();
+
+  setup_capture_of_logs(LOG_WARN);
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/consensus-microdesc"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(TOO_OLD, OP_EQ, header);
+
+  expect_log_msg_containing("too old");
+
+  tor_free(header);
+  teardown_capture_of_logs();
+  tor_free(mock_ns_val);
+
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
+  mock_ns_val->flavor = FLAV_NS;
+  mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800);
+  mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900);
+  mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20);
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
+
+  setup_capture_of_logs(LOG_WARN);
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/consensus"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(TOO_OLD, OP_EQ, header);
+
+  expect_no_log_entry();
+
+  done:
+    teardown_capture_of_logs();
+    UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+    UNMOCK(connection_write_to_buf_impl_);
+    UNMOCK(get_options);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+    tor_free(mock_ns_val);
+    or_options_free(mock_options); mock_options = NULL;
+}
+
 NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
 
 int
@@ -1723,12 +1785,26 @@ static void
 status_vote_current_consensus_ns_test(char **header, char **body,
                                       size_t *body_len)
 {
-  common_digests_t digests;
   dir_connection_t *conn = NULL;
 
   #define NETWORK_STATUS "some network status string"
+#if 0
+  common_digests_t digests;
+  uint8_t sha3[DIGEST256_LEN];
+  memset(&digests, 0x60, sizeof(digests));
+  memset(sha3, 0x06, sizeof(sha3));
   dirserv_set_cached_consensus_networkstatus(NETWORK_STATUS, "ns", &digests,
+                                             sha3,
                                              time(NULL));
+#endif /* 0 */
+  networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->type = NS_TYPE_CONSENSUS;
+  ns->flavor = FLAV_NS;
+  ns->valid_after = time(NULL) - 1800;
+  ns->fresh_until = time(NULL) - 900;
+  ns->valid_until = time(NULL) - 60;
+  consdiffmgr_add_consensus(NETWORK_STATUS, ns);
+  networkstatus_vote_free(ns);
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -1741,7 +1817,6 @@ status_vote_current_consensus_ns_test(char **header, char **body,
   tt_str_op("ab", OP_EQ, geoip_get_country_name(1));
 
   conn = new_dir_conn();
-  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
 
   tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
     GET("/tor/status-vote/current/consensus-ns"), NULL, 0));
@@ -1751,7 +1826,7 @@ status_vote_current_consensus_ns_test(char **header, char **body,
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -1783,8 +1858,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
                                                             comp_body_used);
   tt_int_op(ZLIB_METHOD, OP_EQ, compression);
 
-  tor_gzip_uncompress(&body, &body_used, comp_body, comp_body_used,
-                      compression, 0, LOG_PROTOCOL_WARN);
+  tor_uncompress(&body, &body_used, comp_body, comp_body_used,
+                 compression, 0, LOG_PROTOCOL_WARN);
 
   tt_str_op(NETWORK_STATUS, OP_EQ, body);
   tt_int_op(strlen(NETWORK_STATUS), OP_EQ, body_used);
@@ -1874,7 +1949,7 @@ test_dir_handle_get_status_vote_current_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -1897,7 +1972,7 @@ status_vote_current_d_test(char **header, char **body, size_t *body_l)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -1917,7 +1992,7 @@ status_vote_next_d_test(char **header, char **body, size_t *body_l)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -1982,7 +2057,7 @@ test_dir_handle_get_status_vote_d(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  dirvote_recalculate_timing(mock_options, now);
+  voting_schedule_recalculate_timing(mock_options, now);
 
   const char *msg_out = NULL;
   int status_out = 0;
@@ -2042,7 +2117,7 @@ test_dir_handle_get_status_vote_next_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -2061,7 +2136,7 @@ status_vote_next_consensus_test(char **header, char **body, size_t *body_used)
                       body, body_used, 18, 0);
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 }
 
 static void
@@ -2101,7 +2176,7 @@ test_dir_handle_get_status_vote_current_authority_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -2125,7 +2200,7 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
 }
 
@@ -2207,7 +2282,7 @@ status_vote_next_consensus_signatures_test(char **header, char **body,
                       body, body_used, 22, 0);
 
   done:
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     UNMOCK(connection_write_to_buf_impl_);
 }
 
@@ -2328,7 +2403,7 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  dirvote_recalculate_timing(mock_options, now);
+  voting_schedule_recalculate_timing(mock_options, now);
 
   struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
                                                  &status_out);
@@ -2355,7 +2430,7 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_my_v3_authority_cert);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     authority_cert_free(mock_cert); mock_cert = NULL;
@@ -2407,7 +2482,7 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455;
-  dirvote_recalculate_timing(mock_options, now-1);
+  voting_schedule_recalculate_timing(mock_options, now-1);
 
   struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
                                                  &status_out);
@@ -2437,7 +2512,7 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   done:
     UNMOCK(connection_write_to_buf_impl_);
     UNMOCK(get_my_v3_authority_cert);
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
     authority_cert_free(mock_cert); mock_cert = NULL;
@@ -2448,6 +2523,53 @@ test_dir_handle_get_status_vote_current_authority(void* data)
     dirvote_free_all();
 }
 
+static void
+test_dir_handle_get_parse_accept_encoding(void *arg)
+{
+  (void)arg;
+  const unsigned B_NONE = 1u << NO_METHOD;
+  const unsigned B_ZLIB = 1u << ZLIB_METHOD;
+  const unsigned B_GZIP = 1u << GZIP_METHOD;
+  const unsigned B_LZMA = 1u << LZMA_METHOD;
+  const unsigned B_ZSTD = 1u << ZSTD_METHOD;
+
+  unsigned encodings;
+
+  encodings = parse_accept_encoding_header("");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("  ");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and howe ");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and gzip");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and, gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header(" gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("x-zstd, deflate, x-tor-lzma");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header(
+                                        "x-zstd, deflate, x-tor-lzma, gzip");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("x-zstd,deflate,x-tor-lzma,gzip");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA|B_GZIP, OP_EQ, encodings);
+
+ done:
+  ;
+}
+
 #define DIR_HANDLE_CMD(name,flags) \
   { #name, test_dir_handle_get_##name, (flags), NULL, NULL }
 
@@ -2492,10 +2614,11 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns, 0),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_d_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_d_not_found, 0),
   DIR_HANDLE_CMD(status_vote_d, 0),
@@ -2505,6 +2628,7 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures_busy, 0),
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures, 0),
+  DIR_HANDLE_CMD(parse_accept_encoding, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_dns.c b/src/test/test_dns.c
index 6a8e92cb47..1fee01d2c0 100644
--- a/src/test/test_dns.c
+++ b/src/test/test_dns.c
@@ -1,3 +1,6 @@
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #include "or.h"
 #include "test.h"
 
@@ -18,9 +21,9 @@ NS(test_main)(void *arg)
 
   uint32_t ttl_mid = MIN_DNS_TTL_AT_EXIT / 2 + MAX_DNS_TTL_AT_EXIT / 2;
 
-  tt_int_op(dns_clip_ttl(MIN_DNS_TTL_AT_EXIT - 1),==,MIN_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(ttl_mid),==,MAX_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(MAX_DNS_TTL_AT_EXIT + 1),==,MAX_DNS_TTL_AT_EXIT);
+  tt_int_op(dns_clip_ttl(MIN_DNS_TTL_AT_EXIT - 1),OP_EQ,MIN_DNS_TTL_AT_EXIT);
+  tt_int_op(dns_clip_ttl(ttl_mid),OP_EQ,MAX_DNS_TTL_AT_EXIT);
+  tt_int_op(dns_clip_ttl(MAX_DNS_TTL_AT_EXIT + 1),OP_EQ,MAX_DNS_TTL_AT_EXIT);
 
   done:
   return;
@@ -121,7 +124,7 @@ static int n_connection_free = 0;
 static connection_t *last_freed_conn = NULL;
 
 static void
-NS(connection_free)(connection_t *conn)
+NS(connection_free_)(connection_t *conn)
 {
    n_connection_free++;
 
@@ -172,10 +175,10 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,1);
-  tt_str_op(resolved_name,==,last_resolved_hostname);
+  tt_int_op(retval,OP_EQ,1);
+  tt_str_op(resolved_name,OP_EQ,last_resolved_hostname);
   tt_assert(conn_for_resolved_cell == exitconn);
-  tt_int_op(n_send_resolved_hostname_cell_replacement,==,
+  tt_int_op(n_send_resolved_hostname_cell_replacement,OP_EQ,
             prev_n_send_resolved_hostname_cell_replacement + 1);
   tt_assert(exitconn->on_circuit == NULL);
 
@@ -201,12 +204,12 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,1);
+  tt_int_op(retval,OP_EQ,1);
   tt_assert(conn_for_resolved_cell == exitconn);
-  tt_int_op(n_send_resolved_cell_replacement,==,
+  tt_int_op(n_send_resolved_cell_replacement,OP_EQ,
             prev_n_send_resolved_cell_replacement + 1);
   tt_assert(last_resolved == fake_resolved);
-  tt_int_op(last_answer_type,==,0xff);
+  tt_int_op(last_answer_type,OP_EQ,0xff);
   tt_assert(exitconn->on_circuit == NULL);
 
   /* CASE 3: The purpose of exit connection is not EXIT_PURPOSE_RESOLVE
@@ -229,12 +232,12 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,1);
+  tt_int_op(retval,OP_EQ,1);
   tt_assert(on_circuit->n_streams == exitconn);
   tt_assert(exitconn->next_stream == nextconn);
-  tt_int_op(prev_n_send_resolved_cell_replacement,==,
+  tt_int_op(prev_n_send_resolved_cell_replacement,OP_EQ,
             n_send_resolved_cell_replacement);
-  tt_int_op(prev_n_send_resolved_hostname_cell_replacement,==,
+  tt_int_op(prev_n_send_resolved_hostname_cell_replacement,OP_EQ,
             n_send_resolved_hostname_cell_replacement);
 
   /* CASE 4: _impl returns 0.
@@ -253,8 +256,8 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(exitconn->base_.state,==,EXIT_CONN_STATE_RESOLVING);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(exitconn->base_.state,OP_EQ,EXIT_CONN_STATE_RESOLVING);
   tt_assert(on_circuit->resolving_streams == exitconn);
   tt_assert(exitconn->next_stream == nextconn);
 
@@ -264,7 +267,7 @@ NS(test_main)(void *arg)
    */
 
   NS_MOCK(dns_cancel_pending_resolve);
-  NS_MOCK(connection_free);
+  NS_MOCK(connection_free_);
 
   exitconn->on_circuit = &(on_circuit->base_);
   exitconn->base_.purpose = EXIT_PURPOSE_RESOLVE;
@@ -278,12 +281,12 @@ NS(test_main)(void *arg)
 
   retval = dns_resolve(exitconn);
 
-  tt_int_op(retval,==,-1);
-  tt_int_op(n_send_resolved_cell_replacement,==,
+  tt_int_op(retval,OP_EQ,-1);
+  tt_int_op(n_send_resolved_cell_replacement,OP_EQ,
             prev_n_send_resolved_cell_replacement + 1);
-  tt_int_op(last_answer_type,==,RESOLVED_TYPE_ERROR);
-  tt_int_op(n_dns_cancel_pending_resolve_replacement,==,1);
-  tt_int_op(n_connection_free,==,prev_n_connection_free + 1);
+  tt_int_op(last_answer_type,OP_EQ,RESOLVED_TYPE_ERROR);
+  tt_int_op(n_dns_cancel_pending_resolve_replacement,OP_EQ,1);
+  tt_int_op(n_connection_free,OP_EQ,prev_n_connection_free + 1);
   tt_assert(last_freed_conn == TO_CONN(exitconn));
 
   done:
@@ -291,7 +294,7 @@ NS(test_main)(void *arg)
   NS_UNMOCK(send_resolved_cell);
   NS_UNMOCK(send_resolved_hostname_cell);
   NS_UNMOCK(dns_cancel_pending_resolve);
-  NS_UNMOCK(connection_free);
+  NS_UNMOCK(connection_free_);
   tor_free(on_circuit);
   tor_free(exitconn);
   tor_free(nextconn);
@@ -351,9 +354,9 @@ NS(test_main)(void *arg)
 
   resolved_addr = &(exitconn->base_.addr);
 
-  tt_int_op(retval,==,1);
+  tt_int_op(retval,OP_EQ,1);
   tt_assert(tor_addr_eq(resolved_addr, (const tor_addr_t *)&addr_to_compare));
-  tt_int_op(exitconn->address_ttl,==,DEFAULT_DNS_TTL);
+  tt_int_op(exitconn->address_ttl,OP_EQ,DEFAULT_DNS_TTL);
 
   done:
   tor_free(on_circ);
@@ -393,7 +396,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   done:
   tor_free(TO_CONN(exitconn)->address);
@@ -436,7 +439,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   done:
   NS_UNMOCK(router_my_exit_policy_is_reject_star);
@@ -478,7 +481,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   tor_free(TO_CONN(exitconn)->address);
 
@@ -488,7 +491,7 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,-1);
+  tt_int_op(retval,OP_EQ,-1);
 
   done:
   NS_UNMOCK(router_my_exit_policy_is_reject_star);
@@ -546,8 +549,8 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(made_pending,==,1);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(made_pending,OP_EQ,1);
 
   pending_conn = cache_entry->pending_connections;
 
@@ -628,8 +631,8 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             &resolve_out);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(made_pending,==,0);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(made_pending,OP_EQ,0);
   tt_assert(resolve_out == cache_entry);
 
   tt_assert(last_exitconn == exitconn);
@@ -699,8 +702,8 @@ NS(test_main)(void *arg)
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
 
-  tt_int_op(retval,==,0);
-  tt_int_op(made_pending,==,1);
+  tt_int_op(retval,OP_EQ,0);
+  tt_int_op(made_pending,OP_EQ,1);
 
   cache_entry = dns_get_cache_entry(&query);
 
@@ -712,7 +715,7 @@ NS(test_main)(void *arg)
   tt_assert(pending_conn->conn == exitconn);
 
   tt_assert(last_launched_resolve == cache_entry);
-  tt_str_op(cache_entry->address,==,TO_CONN(exitconn)->address);
+  tt_str_op(cache_entry->address,OP_EQ,TO_CONN(exitconn)->address);
 
   done:
   NS_UNMOCK(router_my_exit_policy_is_reject_star);
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index cb9d9e559c..8ae967f3ae 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -8,6 +8,7 @@
 #include "or.h"
 #include "dos.h"
 #include "circuitlist.h"
+#include "crypto_rand.h"
 #include "geoip.h"
 #include "channel.h"
 #include "microdesc.h"
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index 9580a1fd3f..9d8a072c77 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -14,6 +14,10 @@
 #include "confparse.h"
 #include "connection.h"
 #include "connection_edge.h"
+#include "nodelist.h"
+
+#include "hs_cache.h"
+#include "rendcache.h"
 
 static void *
 entryconn_rewrite_setup(const struct testcase_t *tc)
@@ -30,7 +34,7 @@ entryconn_rewrite_teardown(const struct testcase_t *tc, void *arg)
   (void)tc;
   entry_connection_t *ec = arg;
   if (ec)
-    connection_free_(ENTRY_TO_CONN(ec));
+    connection_free_minimal(ENTRY_TO_CONN(ec));
   addressmap_free_all();
   return 1;
 }
@@ -72,7 +76,6 @@ test_entryconn_rewrite_bad_dotexit(void *arg)
   entry_connection_t *ec = arg;
   rewrite_result_t rr;
 
-  get_options_mutable()->AllowDotExit = 0;
   tt_assert(ec->socks_request);
   strlcpy(ec->socks_request->address, "www.TORproject.org.foo.exit",
           sizeof(ec->socks_request->address));
@@ -100,7 +103,7 @@ test_entryconn_rewrite_automap_ipv4(void *arg)
   ec3 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes, tor_strdup("."));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes, ".");
   parse_virtual_addr_network("127.202.0.0/16", AF_INET, 0, &msg);
 
   /* Automap this on resolve. */
@@ -153,8 +156,8 @@ test_entryconn_rewrite_automap_ipv4(void *arg)
             ec->socks_request->address);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
 }
 
 /* Automap on resolve, connect to automapped address, resolve again and get
@@ -173,7 +176,7 @@ test_entryconn_rewrite_automap_ipv6(void *arg)
   ec3 = entry_connection_new(CONN_TYPE_AP, AF_INET6);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes, tor_strdup("."));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes, ".");
   parse_virtual_addr_network("FE80::/32", AF_INET6, 0, &msg);
 
   /* Automap this on resolve. */
@@ -227,9 +230,9 @@ test_entryconn_rewrite_automap_ipv6(void *arg)
             ec->socks_request->address);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec));
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
 }
 
 #if 0
@@ -280,9 +283,9 @@ test_entryconn_rewrite_automap_reverse(void *arg)
   tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_NONE);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
-#endif
+#endif /* 0 */
 
 /* Rewrite because of cached DNS entry. */
 static void
@@ -330,7 +333,7 @@ test_entryconn_rewrite_cached_dns_ipv4(void *arg)
   tt_str_op(ec2->socks_request->address, OP_EQ, "240.240.241.241");
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Rewrite because of cached DNS entry. */
@@ -382,8 +385,8 @@ test_entryconn_rewrite_cached_dns_ipv6(void *arg)
   tt_str_op(ec2->socks_request->address, OP_EQ, "[::f00f]");
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec));
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Fail to connect to unmapped address in virtual range. */
@@ -423,7 +426,7 @@ test_entryconn_rewrite_unmapped_virtual(void *arg)
   tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_NONE);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Rewrite because of mapaddress option */
@@ -476,7 +479,7 @@ test_entryconn_rewrite_reject_internal_reverse(void *arg)
   ;
 }
 
-/* Rewrite into .exit because of virtual address mapping */
+/* Rewrite into .exit because of virtual address mapping.  */
 static void
 test_entryconn_rewrite_automap_exit(void *arg)
 {
@@ -487,46 +490,24 @@ test_entryconn_rewrite_automap_exit(void *arg)
 
   ec2 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
-  get_options_mutable()->AutomapHostsOnResolve = 1;
-  get_options_mutable()->AllowDotExit = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".EXIT"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".EXIT");
   parse_virtual_addr_network("127.1.0.0/16", AF_INET, 0, &msg);
 
-  /* Automap this on resolve. */
+  /* Try to automap this on resolve. */
   strlcpy(ec->socks_request->address, "website.example.exit",
           sizeof(ec->socks_request->address));
   ec->socks_request->command = SOCKS_COMMAND_RESOLVE;
   connection_ap_handshake_rewrite(ec, &rr);
 
-  tt_int_op(rr.automap, OP_EQ, 1);
-  tt_int_op(rr.should_close, OP_EQ, 0);
-  tt_int_op(rr.end_reason, OP_EQ, 0);
-  tt_i64_op(rr.map_expires, OP_EQ, TIME_MAX);
-  tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_NONE);
-  tt_str_op(rr.orig_address, OP_EQ, "website.example.exit");
-  tt_str_op(ec->original_dest_address, OP_EQ, "website.example.exit");
-
-  tt_assert(!strcmpstart(ec->socks_request->address,"127.1."));
-
-  /* Connect to it and make sure we get the original address back. */
-  strlcpy(ec2->socks_request->address, ec->socks_request->address,
-          sizeof(ec2->socks_request->address));
-
-  ec2->socks_request->command = SOCKS_COMMAND_CONNECT;
-  connection_ap_handshake_rewrite(ec2, &rr);
-
+  /* Make sure it isn't allowed -- there is no longer an AllowDotExit
+   * option. */
   tt_int_op(rr.automap, OP_EQ, 0);
-  tt_int_op(rr.should_close, OP_EQ, 0);
-  tt_int_op(rr.end_reason, OP_EQ, 0);
-  tt_i64_op(rr.map_expires, OP_EQ, TIME_MAX);
-  tt_int_op(rr.exit_source, OP_EQ, ADDRMAPSRC_AUTOMAP);
-  tt_str_op(rr.orig_address, OP_EQ, ec->socks_request->address);
-  tt_str_op(ec2->original_dest_address, OP_EQ, ec->socks_request->address);
-  tt_str_op(ec2->socks_request->address, OP_EQ, "website.example.exit");
+  tt_int_op(rr.should_close, OP_EQ, 1);
+  tt_int_op(rr.end_reason, OP_EQ, END_STREAM_REASON_TORPROTOCOL);
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
 }
 
 /* Rewrite into .exit because of mapaddress */
@@ -573,9 +554,8 @@ test_entryconn_rewrite_mapaddress_automap_onion(void *arg)
   ec4 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  get_options_mutable()->AllowDotExit = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
   config_line_append(&get_options_mutable()->AddressMap,
                      "MapAddress", "foo.onion abcdefghijklmnop.onion");
@@ -638,9 +618,9 @@ test_entryconn_rewrite_mapaddress_automap_onion(void *arg)
   */
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
-  connection_free_(ENTRY_TO_CONN(ec4));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec4));
 }
 
 static void
@@ -698,8 +678,8 @@ test_entryconn_rewrite_mapaddress_automap_onion_common(entry_connection_t *ec,
                            "abcdefghijklmnop.onion"));
 
  done:
-  connection_free_(ENTRY_TO_CONN(ec2));
-  connection_free_(ENTRY_TO_CONN(ec3));
+  connection_free_minimal(ENTRY_TO_CONN(ec2));
+  connection_free_minimal(ENTRY_TO_CONN(ec3));
 }
 
 /* This time is the same, but we start with a mapping from a non-onion
@@ -709,8 +689,8 @@ test_entryconn_rewrite_mapaddress_automap_onion2(void *arg)
 {
   char *msg = NULL;
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
   config_line_append(&get_options_mutable()->AddressMap,
                      "MapAddress", "irc.example.com abcdefghijklmnop.onion");
@@ -736,13 +716,95 @@ test_entryconn_rewrite_mapaddress_automap_onion4(void *arg)
 {
   char *msg = NULL;
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
 
   test_entryconn_rewrite_mapaddress_automap_onion_common(arg, 0, 1);
 }
 
+/** Test that rewrite functions can handle v2 addresses */
+static void
+test_entryconn_rewrite_onion_v2(void *arg)
+{
+  int retval;
+  entry_connection_t *conn = arg;
+
+  (void) arg;
+
+  rend_cache_init();
+
+  /* Make a SOCKS request */
+  conn->socks_request->command = SOCKS_COMMAND_CONNECT;
+  strlcpy(conn->socks_request->address,
+          "pqeed46efnwmfuid.onion",
+          sizeof(conn->socks_request->address));
+
+  /* Make an onion connection using the SOCKS request */
+  conn->entry_cfg.onion_traffic = 1;
+  ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_SOCKS_WAIT;
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->rend_data);
+
+  /* Handle SOCKS and rewrite! */
+  retval = connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check connection state after rewrite */
+  tt_int_op(ENTRY_TO_CONN(conn)->state, OP_EQ, AP_CONN_STATE_RENDDESC_WAIT);
+  /* check that the address got rewritten */
+  tt_str_op(conn->socks_request->address, OP_EQ,
+            "pqeed46efnwmfuid");
+  /* check that HS information got attached to the connection */
+  tt_assert(ENTRY_TO_EDGE_CONN(conn)->rend_data);
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->hs_ident);
+
+ done:
+  rend_cache_free_all();
+  /* 'conn' is cleaned by handler */
+}
+
+/** Test that rewrite functions can handle v3 onion addresses */
+static void
+test_entryconn_rewrite_onion_v3(void *arg)
+{
+  int retval;
+  entry_connection_t *conn = arg;
+
+  (void) arg;
+
+  hs_cache_init();
+
+  /* Make a SOCKS request */
+  conn->socks_request->command = SOCKS_COMMAND_CONNECT;
+  strlcpy(conn->socks_request->address,
+          "git.25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion",
+          sizeof(conn->socks_request->address));
+
+  /* Make an onion connection using the SOCKS request */
+  conn->entry_cfg.onion_traffic = 1;
+  ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_SOCKS_WAIT;
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->rend_data);
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->hs_ident);
+
+  /* Handle SOCKS and rewrite! */
+  retval = connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check connection state after rewrite. It should be in waiting for
+   * descriptor state. */
+  tt_int_op(ENTRY_TO_CONN(conn)->state, OP_EQ, AP_CONN_STATE_RENDDESC_WAIT);
+  /* check that the address got rewritten */
+  tt_str_op(conn->socks_request->address, OP_EQ,
+            "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
+  /* check that HS information got attached to the connection */
+  tt_assert(ENTRY_TO_EDGE_CONN(conn)->hs_ident);
+  tt_assert(!ENTRY_TO_EDGE_CONN(conn)->rend_data);
+
+ done:
+  hs_free_all();
+  /* 'conn' is cleaned by handler */
+}
+
 #define REWRITE(name)                           \
   { #name, test_entryconn_##name, TT_FORK, &test_rewrite_setup, NULL }
 
@@ -763,6 +825,8 @@ struct testcase_t entryconn_tests[] = {
   REWRITE(rewrite_mapaddress_automap_onion2),
   REWRITE(rewrite_mapaddress_automap_onion3),
   REWRITE(rewrite_mapaddress_automap_onion4),
+  REWRITE(rewrite_onion_v2),
+  REWRITE(rewrite_onion_v3),
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index b1c3accfab..e0162e3058 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -1,18 +1,28 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
+#define CIRCUITLIST_PRIVATE
+#define CIRCUITBUILD_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
+#define DIRECTORY_PRIVATE
 
 #include "or.h"
 #include "test.h"
 
+#include "bridges.h"
+#include "circuitlist.h"
+#include "circuitbuild.h"
 #include "config.h"
+#include "confparse.h"
+#include "crypto_rand.h"
+#include "directory.h"
 #include "entrynodes.h"
 #include "nodelist.h"
+#include "networkstatus.h"
 #include "policies.h"
 #include "routerlist.h"
 #include "routerparse.h"
@@ -21,6 +31,7 @@
 #include "util.h"
 
 #include "test_helpers.h"
+#include "log_test_helpers.h"
 
 /* TODO:
  * choose_random_entry() test with state set.
@@ -39,37 +50,152 @@ get_or_state_replacement(void)
   return dummy_state;
 }
 
+static networkstatus_t *dummy_consensus = NULL;
+
+static smartlist_t *big_fake_net_nodes = NULL;
+
+static smartlist_t *
+bfn_mock_nodelist_get_list(void)
+{
+  return big_fake_net_nodes;
+}
+
+static networkstatus_t *
+bfn_mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void)now;
+  return dummy_consensus;
+}
+
+static const node_t *
+bfn_mock_node_get_by_id(const char *id)
+{
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
+                    if (fast_memeq(n->identity, id, 20))
+                      return n);
+
+  return NULL;
+}
+
+/* Helper function to free a test node. */
+static void
+test_node_free(node_t *n)
+{
+  tor_free(n->rs);
+  tor_free(n->md->onion_curve25519_pkey);
+  short_policy_free(n->md->exit_policy);
+  tor_free(n->md);
+  tor_free(n);
+}
+
 /* Unittest cleanup function: Cleanup the fake network. */
 static int
-fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
+big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
 {
   (void) testcase;
   (void) ptr;
 
-  routerlist_free_all();
-  nodelist_free_all();
-  entry_guards_free_all();
+  if (big_fake_net_nodes) {
+    SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+      test_node_free(n);
+    });
+    smartlist_free(big_fake_net_nodes);
+  }
+
+  UNMOCK(nodelist_get_list);
+  UNMOCK(node_get_by_id);
+  UNMOCK(get_or_state);
+  UNMOCK(networkstatus_get_live_consensus);
   or_state_free(dummy_state);
+  dummy_state = NULL;
+  tor_free(dummy_consensus);
 
   return 1; /* NOP */
 }
 
 /* Unittest setup function: Setup a fake network. */
 static void *
-fake_network_setup(const struct testcase_t *testcase)
+big_fake_network_setup(const struct testcase_t *testcase)
 {
-  (void) testcase;
+  int i;
+
+  /* These are minimal node_t objects that only contain the aspects of node_t
+   * that we need for entrynodes.c. */
+  const int N_NODES = 271;
+
+  big_fake_net_nodes = smartlist_new();
+  for (i = 0; i < N_NODES; ++i) {
+    curve25519_secret_key_t curve25519_secret_key;
+
+    node_t *n = tor_malloc_zero(sizeof(node_t));
+    n->md = tor_malloc_zero(sizeof(microdesc_t));
+
+    /* Generate curve25519 key for this node */
+    n->md->onion_curve25519_pkey =
+      tor_malloc_zero(sizeof(curve25519_public_key_t));
+    curve25519_secret_key_generate(&curve25519_secret_key, 0);
+    curve25519_public_key_generate(n->md->onion_curve25519_pkey,
+                                   &curve25519_secret_key);
+
+    crypto_rand(n->identity, sizeof(n->identity));
+    n->rs = tor_malloc_zero(sizeof(routerstatus_t));
+
+    memcpy(n->rs->identity_digest, n->identity, DIGEST_LEN);
+
+    n->is_running = n->is_valid = n->is_fast = n->is_stable = 1;
+
+    /* Note: all these guards have the same address, so you'll need to
+     * disable EnforceDistinctSubnets when a restriction is applied. */
+    n->rs->addr = 0x04020202;
+    n->rs->or_port = 1234;
+    n->rs->is_v2_dir = 1;
+    n->rs->has_bandwidth = 1;
+    n->rs->bandwidth_kb = 30;
+
+    /* Make a random nickname for each node */
+    {
+      char nickname_binary[8];
+      crypto_rand(nickname_binary, sizeof(nickname_binary));
+      base32_encode(n->rs->nickname, sizeof(n->rs->nickname),
+                    nickname_binary, sizeof(nickname_binary));
+    }
+
+    /* Call half of the nodes a possible guard. */
+    if (i % 2 == 0) {
+      n->is_possible_guard = 1;
+      n->rs->guardfraction_percentage = 100;
+      n->rs->has_guardfraction = 1;
+      n->rs->is_possible_guard = 1;
+    }
+
+    /* Make some of these nodes a possible exit */
+    if (i % 7 == 0) {
+      n->md->exit_policy = parse_short_policy("accept 443");
+    }
+
+    smartlist_add(big_fake_net_nodes, n);
+  }
 
-  /* Setup fake state */
   dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  dummy_consensus->valid_after = approx_time() - 3600;
+  dummy_consensus->valid_until = approx_time() + 3600;
+
+  MOCK(nodelist_get_list, bfn_mock_nodelist_get_list);
+  MOCK(node_get_by_id, bfn_mock_node_get_by_id);
   MOCK(get_or_state,
        get_or_state_replacement);
-
-  /* Setup fake routerlist. */
-  helper_setup_fake_routerlist();
-
+  MOCK(networkstatus_get_live_consensus,
+       bfn_mock_networkstatus_get_live_consensus);
   /* Return anything but NULL (it's interpreted as test fail) */
-  return dummy_state;
+  return (void*)testcase;
+}
+
+static time_t
+mock_randomize_time_no_randomization(time_t a, time_t b)
+{
+  (void) b;
+  return a;
 }
 
 static or_options_t mocked_options;
@@ -80,796 +206,2871 @@ mock_get_options(void)
   return &mocked_options;
 }
 
-/** Test choose_random_entry() with none of our routers being guard nodes. */
+#define TEST_IPV4_ADDR "123.45.67.89"
+#define TEST_IPV6_ADDR "[1234:5678:90ab:cdef::]"
+
 static void
-test_choose_random_entry_no_guards(void *arg)
+test_node_preferred_orport(void *arg)
 {
-  const node_t *chosen_entry = NULL;
-
-  (void) arg;
+  (void)arg;
+  tor_addr_t ipv4_addr;
+  const uint16_t ipv4_port = 4444;
+  tor_addr_t ipv6_addr;
+  const uint16_t ipv6_port = 6666;
+  routerinfo_t node_ri;
+  node_t node;
+  tor_addr_port_t ap;
 
+  /* Setup options */
+  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* We don't test ClientPreferIPv6ORPort here, because it's used in
+   * nodelist_set_consensus to setup node.ipv6_preferred, which we set
+   * directly. */
   MOCK(get_options, mock_get_options);
 
-  /* Check that we get a guard if it passes preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* Setup IP addresses */
+  tor_addr_parse(&ipv4_addr, TEST_IPV4_ADDR);
+  tor_addr_parse(&ipv6_addr, TEST_IPV6_ADDR);
+
+  /* Setup node_ri */
+  memset(&node_ri, 0, sizeof(node_ri));
+  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+  node_ri.or_port = ipv4_port;
+  tor_addr_copy(&node_ri.ipv6_addr, &ipv6_addr);
+  node_ri.ipv6_orport = ipv6_port;
+
+  /* Setup node */
+  memset(&node, 0, sizeof(node));
+  node.ri = &node_ri;
+
+  /* Check the preferred address is IPv4 if we're only using IPv4, regardless
+   * of whether we prefer it or not */
   mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+  mocked_options.ClientUseIPv6 = 0;
+  node.ipv6_preferred = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* Try to pick an entry even though none of our routers are guards. */
-  chosen_entry = choose_random_entry(NULL);
+  node.ipv6_preferred = 1;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* Unintuitively, we actually pick a random node as our entry,
-     because router_choose_random_node() relaxes its constraints if it
-     can't find a proper entry guard. */
-  tt_assert(chosen_entry);
+  /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
+   * don't prefer the IPv6 address */
+  mocked_options.ClientUseIPv4 = 1;
+  mocked_options.ClientUseIPv6 = 1;
+  node.ipv6_preferred = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* And with the other IP version active */
+  /* Check the preferred address is IPv6 if we prefer it and
+   * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
+  mocked_options.ClientUseIPv4 = 1;
   mocked_options.ClientUseIPv6 = 1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  node.ipv6_preferred = 1;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  /* And with the preference on auto */
-  mocked_options.ClientPreferIPv6ORPort = -1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  mocked_options.ClientUseIPv4 = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  /* Check that we don't get a guard if it doesn't pass mandatory address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* Check the preferred address is IPv6 if we don't prefer it, but
+   * ClientUseIPv4 is 0 */
   mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+  mocked_options.ClientUseIPv6 = 1;
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  chosen_entry = choose_random_entry(NULL);
+ done:
+  UNMOCK(get_options);
+}
 
-  /* If we don't allow IPv4 at all, we don't get a guard*/
-  tt_assert(!chosen_entry);
+static void
+test_entry_guard_describe(void *arg)
+{
+  (void)arg;
+  entry_guard_t g;
+  memset(&g, 0, sizeof(g));
+  strlcpy(g.nickname, "okefenokee", sizeof(g.nickname));
+  memcpy(g.identity, "theforestprimeval---", DIGEST_LEN);
 
-  /* Check that we get a guard if it passes allowed but not preferred address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 1;
+  tt_str_op(entry_guard_describe(&g), OP_EQ,
+            "okefenokee ($746865666F726573747072696D6576616C2D2D2D)");
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+ done:
+  ;
+}
 
-  /* Check that we get a guard if it passes preferred address settings when
-   * they're auto */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = -1;
+static void
+test_entry_guard_randomize_time(void *arg)
+{
+  const time_t now = 1479153573;
+  const int delay = 86400;
+  const int N = 1000;
+  (void)arg;
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  time_t t;
+  int i;
+  for (i = 0; i < N; ++i) {
+    t = randomize_time(now, delay);
+    tt_int_op(t, OP_LE, now);
+    tt_int_op(t, OP_GE, now-delay);
+  }
 
-  /* And with IPv6 active */
-  mocked_options.ClientUseIPv6 = 1;
+  /* now try the corner cases */
+  for (i = 0; i < N; ++i) {
+    t = randomize_time(100, delay);
+    tt_int_op(t, OP_GE, 1);
+    tt_int_op(t, OP_LE, 100);
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+    t = randomize_time(0, delay);
+    tt_int_op(t, OP_EQ, 1);
+  }
 
  done:
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  UNMOCK(get_options);
+  ;
 }
 
-/** Test choose_random_entry() with only one of our routers being a
-    guard node. */
 static void
-test_choose_random_entry_one_possible_guard(void *arg)
+test_entry_guard_encode_for_state_minimal(void *arg)
 {
-  const node_t *chosen_entry = NULL;
-  node_t *the_guard = NULL;
-  smartlist_t *our_nodelist = NULL;
+  (void) arg;
+  entry_guard_t *eg = tor_malloc_zero(sizeof(entry_guard_t));
+
+  eg->selection_name = tor_strdup("wubwub");
+  memcpy(eg->identity, "plurpyflurpyslurpydo", DIGEST_LEN);
+  eg->sampled_on_date = 1479081600;
+  eg->confirmed_idx = -1;
+
+  char *s = NULL;
+  s = entry_guard_encode_for_state(eg);
+
+  tt_str_op(s, OP_EQ,
+            "in=wubwub "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "sampled_on=2016-11-14T00:00:00 "
+            "listed=0");
 
+ done:
+  entry_guard_free(eg);
+  tor_free(s);
+}
+
+static void
+test_entry_guard_encode_for_state_maximal(void *arg)
+{
   (void) arg;
+  entry_guard_t *eg = tor_malloc_zero(sizeof(entry_guard_t));
+
+  strlcpy(eg->nickname, "Fred", sizeof(eg->nickname));
+  eg->selection_name = tor_strdup("default");
+  memcpy(eg->identity, "plurpyflurpyslurpydo", DIGEST_LEN);
+  eg->bridge_addr = tor_malloc_zero(sizeof(tor_addr_port_t));
+  tor_addr_from_ipv4h(&eg->bridge_addr->addr, 0x08080404);
+  eg->bridge_addr->port = 9999;
+  eg->sampled_on_date = 1479081600;
+  eg->sampled_by_version = tor_strdup("1.2.3");
+  eg->unlisted_since_date = 1479081645;
+  eg->currently_listed = 1;
+  eg->confirmed_on_date = 1479081690;
+  eg->confirmed_idx = 333;
+  eg->extra_state_fields = tor_strdup("and the green grass grew all around");
+
+  char *s = NULL;
+  s = entry_guard_encode_for_state(eg);
+
+  tt_str_op(s, OP_EQ,
+            "in=default "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=8.8.4.4:9999 "
+            "nickname=Fred "
+            "sampled_on=2016-11-14T00:00:00 "
+            "sampled_by=1.2.3 "
+            "unlisted_since=2016-11-14T00:00:45 "
+            "listed=1 "
+            "confirmed_on=2016-11-14T00:01:30 "
+            "confirmed_idx=333 "
+            "and the green grass grew all around");
 
-  MOCK(get_options, mock_get_options);
+ done:
+  entry_guard_free(eg);
+  tor_free(s);
+}
 
-  /* Set one of the nodes to be a guard. */
-  our_nodelist = nodelist_get_list();
-  the_guard = smartlist_get(our_nodelist, 4); /* chosen by fair dice roll */
-  the_guard->is_possible_guard = 1;
+static void
+test_entry_guard_parse_from_state_minimal(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+  time_t t = approx_time();
+
+  eg = entry_guard_parse_from_state(
+                 "in=default_plus "
+                 "rsa_id=596f75206d6179206e656564206120686f626279");
+  tt_assert(eg);
+
+  tt_str_op(eg->selection_name, OP_EQ, "default_plus");
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "596f75206d6179206e656564206120686f626279");
+  tt_str_op(eg->nickname, OP_EQ, "$596F75206D6179206E656564206120686F626279");
+  tt_ptr_op(eg->bridge_addr, OP_EQ, NULL);
+  tt_i64_op(eg->sampled_on_date, OP_GE, t);
+  tt_i64_op(eg->sampled_on_date, OP_LE, t+86400);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 0);
+  tt_ptr_op(eg->sampled_by_version, OP_EQ, NULL);
+  tt_int_op(eg->currently_listed, OP_EQ, 0);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(eg->confirmed_idx, OP_EQ, -1);
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  /* Check that we get the guard if it passes preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
 
-  /* Pick an entry. Make sure we pick the node we marked as guard. */
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+static void
+test_entry_guard_parse_from_state_maximal(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+
+  eg = entry_guard_parse_from_state(
+            "in=fred "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=[1::3]:9999 "
+            "nickname=Fred "
+            "sampled_on=2016-11-14T00:00:00 "
+            "sampled_by=1.2.3 "
+            "unlisted_since=2016-11-14T00:00:45 "
+            "listed=1 "
+            "confirmed_on=2016-11-14T00:01:30 "
+            "confirmed_idx=333 "
+            "and the green grass grew all around "
+            "rsa_id=all,around");
+  tt_assert(eg);
+
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "706C75727079666C75727079736C75727079646F");
+  tt_str_op(fmt_addr(&eg->bridge_addr->addr), OP_EQ, "1::3");
+  tt_int_op(eg->bridge_addr->port, OP_EQ, 9999);
+  tt_str_op(eg->nickname, OP_EQ, "Fred");
+  tt_i64_op(eg->sampled_on_date, OP_EQ, 1479081600);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 1479081645);
+  tt_str_op(eg->sampled_by_version, OP_EQ, "1.2.3");
+  tt_int_op(eg->currently_listed, OP_EQ, 1);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 1479081690);
+  tt_int_op(eg->confirmed_idx, OP_EQ, 333);
+  tt_str_op(eg->extra_state_fields, OP_EQ,
+            "and the green grass grew all around rsa_id=all,around");
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  /* And with the other IP version active */
-  mocked_options.ClientUseIPv6 = 1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
 
-  /* And with the preference on auto */
-  mocked_options.ClientPreferIPv6ORPort = -1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+static void
+test_entry_guard_parse_from_state_failure(void *arg)
+{
+  (void)arg;
+  entry_guard_t *eg = NULL;
+
+  /* no selection */
+  eg = entry_guard_parse_from_state(
+                 "rsa_id=596f75206d6179206e656564206120686f626270");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* no RSA ID. */
+  eg = entry_guard_parse_from_state("in=default nickname=Fred");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* Bad RSA ID: bad character. */
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e656564206120686f62627q");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* Bad RSA ID: too long.*/
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e656564206120686f6262703");
+  tt_ptr_op(eg, OP_EQ, NULL);
+
+  /* Bad RSA ID: too short.*/
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e65656420612");
+  tt_ptr_op(eg, OP_EQ, NULL);
 
-  /* Check that we don't get a guard if it doesn't pass mandatory address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+ done:
+  entry_guard_free(eg);
+}
 
-  chosen_entry = choose_random_entry(NULL);
+static void
+test_entry_guard_parse_from_state_partial_failure(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+  time_t t = approx_time();
+
+  eg = entry_guard_parse_from_state(
+            "in=default "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=1.2.3.3.4:5 "
+            "nickname=FredIsANodeWithAStrangeNicknameThatIsTooLong "
+            "sampled_on=2016-11-14T00:00:99 "
+            "sampled_by=1.2.3 stuff in the middle "
+            "unlisted_since=2016-xx-14T00:00:45 "
+            "listed=0 "
+            "confirmed_on=2016-11-14T00:01:30zz "
+            "confirmed_idx=idx "
+            "and the green grass grew all around "
+            "rsa_id=all,around");
+  tt_assert(eg);
+
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "706C75727079666C75727079736C75727079646F");
+  tt_str_op(eg->nickname, OP_EQ, "FredIsANodeWithAStrangeNicknameThatIsTooL");
+  tt_ptr_op(eg->bridge_addr, OP_EQ, NULL);
+  tt_i64_op(eg->sampled_on_date, OP_EQ, t);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 0);
+  tt_str_op(eg->sampled_by_version, OP_EQ, "1.2.3");
+  tt_int_op(eg->currently_listed, OP_EQ, 0);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(eg->confirmed_idx, OP_EQ, -1);
+  tt_str_op(eg->extra_state_fields, OP_EQ,
+            "stuff in the middle and the green grass grew all around "
+            "rsa_id=all,around");
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  /* If we don't allow IPv4 at all, we don't get a guard*/
-  tt_assert(!chosen_entry);
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
 
-  /* Check that we get a node if it passes allowed but not preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 1;
+static int
+mock_entry_guard_is_listed(guard_selection_t *gs, const entry_guard_t *guard)
+{
+  (void)gs;
+  (void)guard;
+  return 1;
+}
 
-  chosen_entry = choose_random_entry(NULL);
+static void
+test_entry_guard_parse_from_state_full(void *arg)
+{
+  (void)arg;
+  /* Here's a state I made while testing.  The identities and locations for
+   * the bridges are redacted. */
+  const char STATE[] =
+  "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
+    "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n"
+  "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
+    "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
+    "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
+    "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1 "
+    "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
+    "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
+    "pb_successful_circuits_closed=13.000000\n"
+  "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
+    "bridge_addr=37.218.246.143:28366 "
+    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n";
+
+  config_line_t *lines = NULL;
+  or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
+  int r = config_get_lines(STATE, &lines, 0);
+  char *msg = NULL;
+  smartlist_t *text = smartlist_new();
+  char *joined = NULL;
 
-  /* We disable the guard check and the preferred address check at the same
-   * time, so we can't be sure we get the guard */
-  tt_assert(chosen_entry);
+  // So nodes aren't expired. This is Tue, 13 Dec 2016 09:37:14 GMT
+  update_approx_time(1481621834);
 
-  /* Check that we get a node if it is allowed but not preferred when settings
-   * are auto */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = -1;
+  MOCK(entry_guard_is_listed, mock_entry_guard_is_listed);
 
-  chosen_entry = choose_random_entry(NULL);
+  dummy_state = state;
+  MOCK(get_or_state,
+       get_or_state_replacement);
 
-  /* We disable the guard check and the preferred address check at the same
-   * time, so we can't be sure we get the guard */
-  tt_assert(chosen_entry);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(lines);
+
+  state->Guard = lines;
+
+  /* Try it first without setting the result. */
+  r = entry_guards_parse_state(state, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  guard_selection_t *gs_br =
+    get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  tt_ptr_op(gs_br, OP_EQ, NULL);
+
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  gs_br = get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  guard_selection_t *gs_df =
+    get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  guard_selection_t *gs_wb =
+    get_guard_selection_by_name("wobblesome", GS_TYPE_NORMAL, 0);
+
+  tt_assert(gs_br);
+  tt_assert(gs_df);
+  tt_assert(gs_wb);
+
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 5);
+  tt_int_op(smartlist_len(gs_br->sampled_entry_guards), OP_EQ, 2);
+  tt_int_op(smartlist_len(gs_wb->sampled_entry_guards), OP_EQ, 1);
+
+  /* Try again; make sure it doesn't double-add the guards. */
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  gs_br = get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  gs_df = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_assert(gs_br);
+  tt_assert(gs_df);
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 5);
+  tt_int_op(smartlist_len(gs_br->sampled_entry_guards), OP_EQ, 2);
+
+  /* Re-encode; it should be the same... almost. */
+  {
+    /* (Make a guard nonpersistent first) */
+    entry_guard_t *g = smartlist_get(gs_df->sampled_entry_guards, 0);
+    g->is_persistent = 0;
+  }
+  config_free_lines(lines);
+  lines = state->Guard = NULL; // to prevent double-free.
+  entry_guards_update_state(state);
+  tt_assert(state->Guard);
+  lines = state->Guard;
+
+  config_line_t *ln;
+  for (ln = lines; ln; ln = ln->next) {
+    smartlist_add_asprintf(text, "%s %s\n",ln->key, ln->value);
+  }
+  joined = smartlist_join_strings(text, "", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+  "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=1 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n"
+  "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
+    "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
+    "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
+    "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1 "
+    "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
+    "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
+    "pb_successful_circuits_closed=13.000000\n"
+  "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
+    "bridge_addr=37.218.246.143:28366 "
+    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n");
 
-  /* and with IPv6 active */
-  mocked_options.ClientUseIPv6 = 1;
+ done:
+  config_free_lines(lines);
+  tor_free(state);
+  tor_free(msg);
+  UNMOCK(get_or_state);
+  UNMOCK(entry_guard_is_listed);
+  SMARTLIST_FOREACH(text, char *, cp, tor_free(cp));
+  smartlist_free(text);
+  tor_free(joined);
+}
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+static void
+test_entry_guard_parse_from_state_broken(void *arg)
+{
+  (void)arg;
+  /* Here's a variation on the previous state. Every line but the first is
+   * busted somehow. */
+  const char STATE[] =
+  /* Okay. */
+  "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
+    "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  /* No selection listed. */
+  "Guard rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  /* Selection is "legacy"!! */
+  "Guard in=legacy rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n";
+
+  config_line_t *lines = NULL;
+  or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
+  int r = config_get_lines(STATE, &lines, 0);
+  char *msg = NULL;
+
+  dummy_state = state;
+  MOCK(get_or_state,
+       get_or_state_replacement);
+
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(lines);
+
+  state->Guard = lines;
+
+  /* First, no-set case. we should get an error. */
+  r = entry_guards_parse_state(state, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  /* And we shouldn't have made anything. */
+  guard_selection_t *gs_df =
+    get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs_df, OP_EQ, NULL);
+  tor_free(msg);
+
+  /* Now see about the set case (which shouldn't happen IRL) */
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  gs_df = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs_df, OP_NE, NULL);
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 1);
 
  done:
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  UNMOCK(get_options);
+  config_free_lines(lines);
+  tor_free(state);
+  tor_free(msg);
+  UNMOCK(get_or_state);
 }
 
-/** Helper to conduct tests for populate_live_entry_guards().
+static void
+test_entry_guard_get_guard_selection_by_name(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs1, *gs2, *gs3;
+
+  gs1 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs1, OP_EQ, NULL);
+  gs1 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 1);
+  tt_ptr_op(gs1, OP_NE, NULL);
+  gs2 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 1);
+  tt_assert(gs2 == gs1);
+  gs2 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 0);
+  tt_assert(gs2 == gs1);
+
+  gs2 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs2, OP_EQ, NULL);
+  gs2 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 1);
+  tt_ptr_op(gs2, OP_NE, NULL);
+  tt_assert(gs2 != gs1);
+  gs3 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 0);
+  tt_assert(gs3 == gs2);
+
+  gs3 = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_ptr_op(gs3, OP_EQ, NULL);
+  gs3 = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 1);
+  tt_ptr_op(gs3, OP_NE, NULL);
+  tt_assert(gs3 != gs2);
+  tt_assert(gs3 != gs1);
+  tt_assert(gs3 == get_guard_selection_info());
 
-   This test adds some entry guards to our list, and then tests
-   populate_live_entry_guards() to mke sure it filters them correctly.
+ done:
+  entry_guards_free_all();
+}
 
-   <b>num_needed</b> is the number of guard nodes we support. It's
-   configurable to make sure we function properly with 1 or 3 guard
-   nodes configured.
-*/
 static void
-populate_live_entry_guards_test_helper(int num_needed)
+test_entry_guard_choose_selection_initial(void *arg)
 {
-  smartlist_t *our_nodelist = NULL;
-  smartlist_t *live_entry_guards = smartlist_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  or_options_t *options = get_options_mutable();
-  int retval;
+  /* Tests for picking our initial guard selection (based on having had
+   * no previous selection */
+  (void)arg;
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(get_options(),
+                                            dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "default");
+  tt_int_op(type, OP_EQ, GS_TYPE_NORMAL);
+
+  /* If we're using bridges, we get the bridge selection. */
+  get_options_mutable()->UseBridges = 1;
+  name = choose_guard_selection(get_options(),
+                                dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "bridges");
+  tt_int_op(type, OP_EQ, GS_TYPE_BRIDGE);
+  get_options_mutable()->UseBridges = 0;
+
+  /* If we discard >99% of our guards, though, we should be in the restricted
+   * set. */
+  tt_assert(get_options_mutable()->EntryNodes == NULL);
+  get_options_mutable()->EntryNodes = routerset_new();
+  routerset_parse(get_options_mutable()->EntryNodes, "1.0.0.0/8", "foo");
+  name = choose_guard_selection(get_options(),
+                                dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "restricted");
+  tt_int_op(type, OP_EQ, GS_TYPE_RESTRICTED);
 
-  /* Set NumEntryGuards to the provided number. */
-  options->NumEntryGuards = num_needed;
-  tt_int_op(num_needed, OP_EQ, decide_num_guards(options, 0));
-
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
-
-  /* Walk the nodelist and add all nodes as entry guards. */
-  our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
-
-  SMARTLIST_FOREACH_BEGIN(our_nodelist, const node_t *, node) {
-    const node_t *node_tmp;
-    node_tmp = add_an_entry_guard(node, 0, 1, 0, 0);
-    tt_assert(node_tmp);
-  } SMARTLIST_FOREACH_END(node);
-
-  /* Make sure the nodes were added as entry guards. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ,
-            HELPER_NUMBER_OF_DESCRIPTORS);
-
-  /* Ensure that all the possible entry guards are enough to satisfy us. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_GE, num_needed);
-
-  /* Walk the entry guard list for some sanity checking */
-  SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) {
-    /* Since we called add_an_entry_guard() with 'for_discovery' being
-       False, all guards should have made_contact enabled. */
-    tt_int_op(entry->made_contact, OP_EQ, 1);
-
-  } SMARTLIST_FOREACH_END(entry);
-
-  /* First, try to get some fast guards. This should fail. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO, /* Don't care about DIRINFO*/
-                                      0, 0,
-                                      1); /* We want fast guard! */
-  tt_int_op(retval, OP_EQ, 0);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, 0);
-
-  /* Now try to get some stable guards. This should fail too. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO,
-                                      0,
-                                      1, /* We want stable guard! */
-                                      0);
-  tt_int_op(retval, OP_EQ, 0);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, 0);
-
-  /* Now try to get any guard we can find. This should succeed. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO,
-                                      0, 0, 0); /* No restrictions! */
-
-  /* Since we had more than enough guards in 'all_entry_guards', we
-     should have added 'num_needed' of them to live_entry_guards.
-     'retval' should be 1 since we now have enough live entry guards
-     to pick one.  */
-  tt_int_op(retval, OP_EQ, 1);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, num_needed);
+ done:
+  ;
+}
+
+static void
+test_entry_guard_add_single_guard(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* 1: Add a single guard to the sample. */
+  node_t *n1 = smartlist_get(big_fake_net_nodes, 0);
+  time_t now = approx_time();
+  tt_assert(n1->is_possible_guard == 1);
+  entry_guard_t *g1 = entry_guard_add_to_sample(gs, n1);
+  tt_assert(g1);
+
+  /* Make sure its fields look right. */
+  tt_mem_op(n1->identity, OP_EQ, g1->identity, DIGEST_LEN);
+  tt_i64_op(g1->sampled_on_date, OP_GE, now - 12*86400);
+  tt_i64_op(g1->sampled_on_date, OP_LE, now);
+  tt_str_op(g1->sampled_by_version, OP_EQ, VERSION);
+  tt_uint_op(g1->currently_listed, OP_EQ, 1);
+  tt_i64_op(g1->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(g1->confirmed_idx, OP_EQ, -1);
+  tt_int_op(g1->last_tried_to_connect, OP_EQ, 0);
+  tt_uint_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_i64_op(g1->failing_since, OP_EQ, 0);
+  tt_uint_op(g1->is_filtered_guard, OP_EQ, 1);
+  tt_uint_op(g1->is_usable_filtered_guard, OP_EQ, 1);
+  tt_uint_op(g1->is_primary, OP_EQ, 0);
+  tt_ptr_op(g1->extra_state_fields, OP_EQ, NULL);
+
+  /* Make sure it got added. */
+  tt_int_op(1, OP_EQ, smartlist_len(gs->sampled_entry_guards));
+  tt_ptr_op(g1, OP_EQ, smartlist_get(gs->sampled_entry_guards, 0));
+  tt_ptr_op(g1, OP_EQ, get_sampled_guard_with_id(gs, (uint8_t*)n1->identity));
+  const uint8_t bad_id[20] = {0};
+  tt_ptr_op(NULL, OP_EQ, get_sampled_guard_with_id(gs, bad_id));
 
  done:
-  smartlist_free(live_entry_guards);
+  guard_selection_free(gs);
 }
 
-/* Test populate_live_entry_guards() for 1 guard node. */
 static void
-test_populate_live_entry_guards_1guard(void *arg)
+test_entry_guard_node_filter(void *arg)
 {
-  (void) arg;
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  bridge_line_t *bl = NULL;
+
+  /* Initialize a bunch of node objects that are all guards. */
+#define NUM 7
+  node_t *n[NUM];
+  entry_guard_t *g[NUM];
+  int i;
+  for (i=0; i < NUM; ++i) {
+    n[i] = smartlist_get(big_fake_net_nodes, i*2); // even ones are guards.
+    g[i] = entry_guard_add_to_sample(gs, n[i]);
+
+    // everything starts out filtered-in
+    tt_uint_op(g[i]->is_filtered_guard, OP_EQ, 1);
+    tt_uint_op(g[i]->is_usable_filtered_guard, OP_EQ, 1);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, NUM);
+
+  /* Make sure refiltering doesn't hurt */
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_uint_op(g[i]->is_filtered_guard, OP_EQ, 1);
+    tt_uint_op(g[i]->is_usable_filtered_guard, OP_EQ, 1);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, NUM);
+
+  /* Now start doing things to make the guards get filtered out, 1 by 1. */
+
+  /* 0: Not listed. */
+  g[0]->currently_listed = 0;
+
+  /* 1: path bias says this guard is maybe eeeevil. */
+  g[1]->pb.path_bias_disabled = 1;
+
+  /* 2: Unreachable address. */
+  n[2]->rs->addr = 0;
+
+  /* 3: ExcludeNodes */
+  n[3]->rs->addr = 0x90902020;
+  routerset_free(get_options_mutable()->ExcludeNodes);
+  get_options_mutable()->ExcludeNodes = routerset_new();
+  routerset_parse(get_options_mutable()->ExcludeNodes, "144.144.0.0/16", "");
+
+  /* 4: Bridge. */
+  get_options_mutable()->UseBridges = 1;
+  sweep_bridge_list();
+  bl = tor_malloc_zero(sizeof(bridge_line_t));
+  tor_addr_from_ipv4h(&bl->addr, n[4]->rs->addr);
+  bl->port = n[4]->rs->or_port;
+  memcpy(bl->digest, n[4]->identity, 20);
+  bridge_add_from_config(bl);
+  bl = NULL; // prevent free.
+  get_options_mutable()->UseBridges = 0;
+
+  /* 5: Unreachable. This stays in the filter, but isn't in usable-filtered */
+  g[5]->last_tried_to_connect = approx_time(); // prevent retry.
+  g[5]->is_reachable = GUARD_REACHABLE_NO;
+
+  /* 6: no change. */
+
+  /* Now refilter and inspect. */
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_assert(g[i]->is_filtered_guard == (i == 5 || i == 6));
+    tt_assert(g[i]->is_usable_filtered_guard == (i == 6));
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 1);
+
+  /* Now make sure we have no live consensus, and no nodes.  Nothing should
+   * pass the filter any more. */
+  tor_free(dummy_consensus);
+  dummy_consensus = NULL;
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, node, {
+    memset(node->identity, 0xff, 20);
+  });
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_uint_op(g[i]->is_filtered_guard, OP_EQ, 0);
+    tt_uint_op(g[i]->is_usable_filtered_guard, OP_EQ, 0);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 0);
 
-  populate_live_entry_guards_test_helper(1);
+ done:
+  guard_selection_free(gs);
+  tor_free(bl);
+#undef NUM
 }
 
-/* Test populate_live_entry_guards() for 3 guard nodes. */
 static void
-test_populate_live_entry_guards_3guards(void *arg)
+test_entry_guard_expand_sample(void *arg)
 {
-  (void) arg;
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  digestmap_t *node_by_id = digestmap_new();
+
+  entry_guard_t *guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // the last guard returned.
+
+  // Every sampled guard here should be filtered and reachable for now.
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  /* Make sure we got the right number. */
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Make sure everything we got was from our fake node list, and everything
+  // was unique.
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    const node_t *n = bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    tt_ptr_op(NULL, OP_EQ, digestmap_get(node_by_id, g->identity));
+    digestmap_set(node_by_id, g->identity, (void*) n);
+    int idx = smartlist_pos(big_fake_net_nodes, n);
+    // The even ones are the guards; make sure we got guards.
+    tt_int_op(idx & 1, OP_EQ, 0);
+  } SMARTLIST_FOREACH_END(g);
+
+  // Nothing became unusable/unfiltered, so a subsequent expand should
+  // make no changes.
+  guard = entry_guards_expand_sample(gs);
+  tt_ptr_op(guard, OP_EQ, NULL); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Make a few guards unreachable.
+  guard = smartlist_get(gs->sampled_entry_guards, 0);
+  guard->is_usable_filtered_guard = 0;
+  guard = smartlist_get(gs->sampled_entry_guards, 1);
+  guard->is_usable_filtered_guard = 0;
+  guard = smartlist_get(gs->sampled_entry_guards, 2);
+  guard->is_usable_filtered_guard = 0;
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE - 3, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // This time, expanding the sample will add some more guards.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ,
+            num_reachable_filtered_guards(gs, NULL)+3);
+
+  // Still idempotent.
+  guard = entry_guards_expand_sample(gs);
+  tt_ptr_op(guard, OP_EQ, NULL); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Now, do a nasty trick: tell the filter to exclude 31/32 of the guards.
+  // This will cause the sample size to get reeeeally huge, while the
+  // filtered sample size grows only slowly.
+  routerset_free(get_options_mutable()->ExcludeNodes);
+  get_options_mutable()->ExcludeNodes = routerset_new();
+  routerset_parse(get_options_mutable()->ExcludeNodes, "144.144.0.0/16", "");
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+    if (n_sl_idx % 64 != 0) {
+      n->rs->addr = 0x90903030;
+    }
+  });
+  entry_guards_update_filtered_sets(gs);
+
+  // Surely (p ~ 1-2**-60), one of our guards has been excluded.
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_LT,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+
+  // Try to regenerate the guards.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // no guard was added.
+
+  /* this time, it's possible that we didn't add enough sampled guards. */
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_LE,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+  /* but we definitely didn't exceed the sample maximum. */
+  const int n_guards = 271 / 2;
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_LE,
+            (int)(n_guards * .3));
 
-  populate_live_entry_guards_test_helper(3);
+ done:
+  guard_selection_free(gs);
+  digestmap_free(node_by_id, NULL);
 }
 
-/** Append some EntryGuard lines to the Tor state at <b>state</b>.
+static void
+test_entry_guard_expand_sample_small_net(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* Fun corner case: not enough guards to make up our whole sample size. */
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+    if (n_sl_idx >= 15) {
+      test_node_free(n);
+      SMARTLIST_DEL_CURRENT(big_fake_net_nodes, n);
+    } else {
+      n->rs->addr = 0; // make the filter reject this.
+    }
+  });
+
+  entry_guard_t *guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // the last guard returned -- some guard was added.
+  // half the nodes are guards, so we have 8 guards left.  The set
+  // is small, so we sampled everything.
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, 8);
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 0);
+ done:
+  guard_selection_free(gs);
+}
 
-   <b>entry_guard_lines</b> is a smartlist containing 2-tuple
-   smartlists that carry the key and values of the statefile.
-   As an example:
-   entry_guard_lines =
-     (("EntryGuard", "name 67E72FF33D7D41BF11C569646A0A7B4B188340DF DirCache"),
-      ("EntryGuardDownSince", "2014-06-07 16:02:46 2014-06-07 16:02:46"))
-*/
 static void
-state_insert_entry_guard_helper(or_state_t *state,
-                                smartlist_t *entry_guard_lines)
+test_entry_guard_update_from_consensus_status(void *arg)
 {
-  config_line_t **next, *line;
+  /* Here we're going to have some nodes become un-guardy, and say we got a
+   * new consensus. This should cause those nodes to get detected as
+   * unreachable. */
+
+  (void)arg;
+  int i;
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  networkstatus_t *ns_tmp = NULL;
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  /* At this point, it should be a no-op to do this: */
+  sampled_guards_update_from_consensus(gs);
+
+  /* Now let's make some of our guards become unlisted.  The easiest way to
+   * do that would be to take away their guard flag. */
+  for (i = 0; i < 5; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+  }
 
-  next = &state->EntryGuards;
-  *next = NULL;
+  update_approx_time(start + 30);
+  {
+    /* try this with no live networkstatus. Nothing should happen! */
+    ns_tmp = dummy_consensus;
+    dummy_consensus = NULL;
+    sampled_guards_update_from_consensus(gs);
+    tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+    tt_i64_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre);
+    /* put the networkstatus back. */
+    dummy_consensus = ns_tmp;
+    ns_tmp = NULL;
+  }
+
+  /* Now those guards should become unlisted, and drop off the filter, but
+   * stay in the sample. */
+  update_approx_time(start + 60);
+  sampled_guards_update_from_consensus(gs);
+
+  tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  tt_i64_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre-5);
+  for (i = 0; i < 5; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(! g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, start+60);
+  }
+  for (i = 5; i < n_sampled_pre; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+  }
 
-  /* Loop over all the state lines in the smartlist */
-  SMARTLIST_FOREACH_BEGIN(entry_guard_lines, const smartlist_t *,state_lines) {
-    /* Get key and value for each line */
-    const char *state_key = smartlist_get(state_lines, 0);
-    const char *state_value = smartlist_get(state_lines, 1);
+  /* Now re-list one, and remove one completely. */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 0);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 1;
+  }
+  {
+    /* try removing the node, to make sure we don't crash on an absent node
+     */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 5);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    smartlist_remove(big_fake_net_nodes, n);
+    test_node_free(n);
+  }
+  update_approx_time(start + 300);
+  sampled_guards_update_from_consensus(gs);
+
+  /* guards 1..5 are now unlisted; 0,6,7.. are listed. */
+  tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  for (i = 1; i < 6; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(! g->currently_listed);
+    if (i == 5)
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+300);
+    else
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+60);
+  }
+  for (i = 0; i < n_sampled_pre; i = (!i) ? 6 : i+1) { /* 0,6,7,8, ... */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+  }
 
-    *next = line = tor_malloc_zero(sizeof(config_line_t));
-    line->key = tor_strdup(state_key);
-    tor_asprintf(&line->value, "%s", state_value);
-    next = &(line->next);
-  } SMARTLIST_FOREACH_END(state_lines);
+ done:
+  tor_free(ns_tmp); /* in case we couldn't put it back */
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
 }
 
-/** Free memory occupied by <b>entry_guard_lines</b>. */
 static void
-state_lines_free(smartlist_t *entry_guard_lines)
+test_entry_guard_update_from_consensus_repair(void *arg)
 {
-  SMARTLIST_FOREACH_BEGIN(entry_guard_lines, smartlist_t *, state_lines) {
-    char *state_key = smartlist_get(state_lines, 0);
-    char *state_value = smartlist_get(state_lines, 1);
+  /* Here we'll make sure that our code to repair the unlisted-since
+   * times is correct. */
 
-    tor_free(state_key);
-    tor_free(state_value);
-    smartlist_free(state_lines);
-  } SMARTLIST_FOREACH_END(state_lines);
+  (void)arg;
+  int i;
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  /* Now corrupt the list a bit.  Call some unlisted-since-never, and some
+   * listed-and-unlisted-since-a-time. */
+  update_approx_time(start + 300);
+  for (i = 0; i < 3; ++i) {
+    /* these will get a date. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+  }
+  for (i = 3; i < 6; ++i) {
+    /* these will become listed. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    g->unlisted_since_date = start+100;
+  }
+  setup_full_capture_of_logs(LOG_WARN);
+  sampled_guards_update_from_consensus(gs);
+  expect_log_msg_containing(
+             "was listed, but with unlisted_since_date set");
+  expect_log_msg_containing(
+             "was unlisted, but with unlisted_since_date unset");
+  teardown_capture_of_logs();
+
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre-3);
+  for (i = 3; i < n_sampled_pre; ++i) {
+    /* these will become listed. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    if (i < 3) {
+      tt_assert(! g->currently_listed);
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+300);
+    } else {
+      tt_assert(g->currently_listed);
+      tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+    }
+  }
 
-  smartlist_free(entry_guard_lines);
+ done:
+  teardown_capture_of_logs();
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
 }
 
-/* Tests entry_guards_parse_state(). It creates a fake Tor state with
-   a saved entry guard and makes sure that Tor can parse it and
-   creates the right entry node out of it.
-*/
 static void
-test_entry_guards_parse_state_simple(void *arg)
+test_entry_guard_update_from_consensus_remove(void *arg)
 {
-  or_state_t *state = or_state_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  smartlist_t *entry_state_lines = smartlist_new();
-  char *msg = NULL;
-  int retval;
+  /* Now let's check the logic responsible for removing guards from the
+   * sample entirely. */
+
+  (void)arg;
+  //int i;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  smartlist_t *keep_ids = smartlist_new();
+  smartlist_t *remove_ids = smartlist_new();
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  const time_t one_day_ago = approx_time() - 1*24*60*60;
+  const time_t one_year_ago = approx_time() - 365*24*60*60;
+  const time_t two_years_ago = approx_time() - 2*365*24*60*60;
+  /* 0: unlisted for a day. (keep this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 0);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+    g->unlisted_since_date = one_day_ago;
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 1: unlisted for a year. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 1);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+    g->unlisted_since_date = one_year_ago;
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
+  /* 2: added a day ago, never confirmed. (keep this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 2);
+    g->sampled_on_date = one_day_ago;
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 3: added a year ago, never confirmed. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 3);
+    g->sampled_on_date = one_year_ago;
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
+  /* 4: added two year ago, confirmed yesterday, primary. (keep this.) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 4);
+    g->sampled_on_date = one_year_ago;
+    g->confirmed_on_date = one_day_ago;
+    g->confirmed_idx = 0;
+    g->is_primary = 1;
+    smartlist_add(gs->confirmed_entry_guards, g);
+    smartlist_add(gs->primary_entry_guards, g);
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 5: added two years ago, confirmed a year ago, primary. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 5);
+    g->sampled_on_date = two_years_ago;
+    g->confirmed_on_date = one_year_ago;
+    g->confirmed_idx = 1;
+    g->is_primary = 1;
+    smartlist_add(gs->confirmed_entry_guards, g);
+    smartlist_add(gs->primary_entry_guards, g);
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
 
-  /* Details of our fake guard node */
-  const char *nickname = "hagbard";
-  const char *fpr = "B29D536DD1752D542E1FBB3C9CE4449D51298212";
-  const char *tor_version = "0.2.5.3-alpha-dev";
-  const char *added_at = get_yesterday_date_str();
-  const char *unlisted_since = "2014-06-08 16:16:50";
+  sampled_guards_update_from_consensus(gs);
 
-  (void) arg;
+  /* Did we remove the right ones? */
+  SMARTLIST_FOREACH(keep_ids, uint8_t *, id, {
+      tt_assert(get_sampled_guard_with_id(gs, id) != NULL);
+  });
+  SMARTLIST_FOREACH(remove_ids, uint8_t *, id, {
+    tt_want(get_sampled_guard_with_id(gs, id) == NULL);
+  });
+
+  /* Did we remove the right number? */
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre - 3);
+
+ done:
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
+  SMARTLIST_FOREACH(keep_ids, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(remove_ids, char *, cp, tor_free(cp));
+  smartlist_free(keep_ids);
+  smartlist_free(remove_ids);
+}
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+static void
+test_entry_guard_confirming_guards(void *arg)
+{
+  (void)arg;
+  /* Now let's check the logic responsible for manipulating the list
+   * of confirmed guards */
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* Create the sample. */
+  entry_guards_expand_sample(gs);
+
+  /* Confirm a few  guards. */
+  time_t start = approx_time();
+  entry_guard_t *g1 = smartlist_get(gs->sampled_entry_guards, 0);
+  entry_guard_t *g2 = smartlist_get(gs->sampled_entry_guards, 1);
+  entry_guard_t *g3 = smartlist_get(gs->sampled_entry_guards, 8);
+  make_guard_confirmed(gs, g2);
+  update_approx_time(start + 10);
+  make_guard_confirmed(gs, g1);
+  make_guard_confirmed(gs, g3);
+
+  /* Were the correct dates and indices fed in? */
+  tt_int_op(g1->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_i64_op(g1->confirmed_on_date, OP_EQ, start+10);
+  tt_i64_op(g2->confirmed_on_date, OP_EQ, start);
+  tt_i64_op(g3->confirmed_on_date, OP_EQ, start+10);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
+
+  /* Now make sure we can regenerate the confirmed_entry_guards list. */
+  smartlist_clear(gs->confirmed_entry_guards);
+  g2->confirmed_idx = 0;
+  g1->confirmed_idx = 10;
+  g3->confirmed_idx = 100;
+  entry_guards_update_confirmed(gs);
+  tt_int_op(g1->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
+
+  /* Now make sure we can regenerate the confirmed_entry_guards list if
+   * the indices are messed up. */
+  g1->confirmed_idx = g2->confirmed_idx = g3->confirmed_idx = 999;
+  smartlist_clear(gs->confirmed_entry_guards);
+  entry_guards_update_confirmed(gs);
+  tt_int_op(g1->confirmed_idx, OP_GE, 0);
+  tt_int_op(g2->confirmed_idx, OP_GE, 0);
+  tt_int_op(g3->confirmed_idx, OP_GE, 0);
+  tt_int_op(g1->confirmed_idx, OP_LE, 2);
+  tt_int_op(g2->confirmed_idx, OP_LE, 2);
+  tt_int_op(g3->confirmed_idx, OP_LE, 2);
+  g1 = smartlist_get(gs->confirmed_entry_guards, 0);
+  g2 = smartlist_get(gs->confirmed_entry_guards, 1);
+  g3 = smartlist_get(gs->confirmed_entry_guards, 2);
+  tt_int_op(g1->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_assert(g1 != g2);
+  tt_assert(g1 != g3);
+  tt_assert(g2 != g3);
 
-  { /* Prepare the state entry */
+ done:
+  UNMOCK(randomize_time);
+  guard_selection_free(gs);
+}
 
-    /* Prepare the smartlist to hold the key/value of each line */
-    smartlist_t *state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuard");
-    smartlist_add_asprintf(state_line, "%s %s %s", nickname, fpr, "DirCache");
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_sample_reachable_filtered(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guards_expand_sample(gs);
+  const int N = 10000;
+  bitarray_t *selected = NULL;
+  int i, j;
+
+  /* We've got a sampled list now; let's make one non-usable-filtered; some
+   * confirmed, some primary, some pending.
+   */
+  int n_guards = smartlist_len(gs->sampled_entry_guards);
+  tt_int_op(n_guards, OP_GT, 10);
+  entry_guard_t *g;
+  g = smartlist_get(gs->sampled_entry_guards, 0);
+  g->is_pending = 1;
+  g = smartlist_get(gs->sampled_entry_guards, 1);
+  make_guard_confirmed(gs, g);
+  g = smartlist_get(gs->sampled_entry_guards, 2);
+  g->is_primary = 1;
+  g = smartlist_get(gs->sampled_entry_guards, 3);
+  g->pb.path_bias_disabled = 1;
+
+  entry_guards_update_filtered_sets(gs);
+  gs->primary_guards_up_to_date = 1;
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_guards - 1);
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+
+  // +1 since the one we made disabled will make  another one get added.
+  ++n_guards;
+
+  /* Try a bunch of selections. */
+  const struct {
+    int flag; int idx;
+  } tests[] = {
+    { 0, -1 },
+    { SAMPLE_EXCLUDE_CONFIRMED, 1 },
+    { SAMPLE_EXCLUDE_PRIMARY|SAMPLE_NO_UPDATE_PRIMARY, 2 },
+    { SAMPLE_EXCLUDE_PENDING, 0 },
+    { -1, -1},
+  };
+
+  for (j = 0; tests[j].flag >= 0; ++j) {
+    selected = bitarray_init_zero(n_guards);
+    const int excluded_flags = tests[j].flag;
+    const int excluded_idx = tests[j].idx;
+    for (i = 0; i < N; ++i) {
+      g = sample_reachable_filtered_entry_guards(gs, NULL, excluded_flags);
+      tor_assert(g);
+      int pos = smartlist_pos(gs->sampled_entry_guards, g);
+      tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+      tt_int_op(pos, OP_GE, 0);
+      tt_int_op(pos, OP_LT, n_guards);
+      bitarray_set(selected, pos);
+    }
+    for (i = 0; i < n_guards; ++i) {
+      const int should_be_set = (i != excluded_idx &&
+                                 i != 3); // filtered out.
+      tt_int_op(!!bitarray_is_set(selected, i), OP_EQ, should_be_set);
+    }
+    bitarray_free(selected);
+    selected = NULL;
+  }
+
+ done:
+  guard_selection_free(gs);
+  bitarray_free(selected);
+}
+
+static void
+test_entry_guard_sample_reachable_filtered_empty(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  /* What if we try to sample from a set of 0? */
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
+                    n->is_possible_guard = 0);
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardAddedBy");
-    smartlist_add_asprintf(state_line, "%s %s %s", fpr, tor_version, added_at);
-    smartlist_add(entry_state_lines, state_line);
+  entry_guard_t *g = sample_reachable_filtered_entry_guards(gs, NULL, 0);
+  tt_ptr_op(g, OP_EQ, NULL);
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardUnlistedSince");
-    smartlist_add_asprintf(state_line, "%s", unlisted_since);
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  guard_selection_free(gs);
+}
+
+static void
+test_entry_guard_retry_unreachable(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  entry_guards_expand_sample(gs);
+  /* Let's say that we have two guards, and they're down.
+   */
+  time_t start = approx_time();
+  entry_guard_t *g1 = smartlist_get(gs->sampled_entry_guards, 0);
+  entry_guard_t *g2 = smartlist_get(gs->sampled_entry_guards, 1);
+  entry_guard_t *g3 = smartlist_get(gs->sampled_entry_guards, 2);
+  g1->is_reachable = GUARD_REACHABLE_NO;
+  g2->is_reachable = GUARD_REACHABLE_NO;
+  g1->is_primary = 1;
+  g1->failing_since = g2->failing_since = start;
+  g1->last_tried_to_connect = g2->last_tried_to_connect = start;
+
+  /* Wait 5 minutes.  Nothing will get retried. */
+  update_approx_time(start + 5 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  entry_guard_consider_retry(g3); // just to make sure this doesn't crash.
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g3->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  /* After 30 min, the primary one gets retried */
+  update_approx_time(start + 35 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+
+  g1->is_reachable = GUARD_REACHABLE_NO;
+  g1->last_tried_to_connect = start + 55*60;
+
+  /* After 1 hour, we'll retry the nonprimary one. */
+  update_approx_time(start + 61 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  g2->is_reachable = GUARD_REACHABLE_NO;
+  g2->last_tried_to_connect = start + 61*60;
+
+  /* And then the primary one again. */
+  update_approx_time(start + 66 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+
+ done:
+  guard_selection_free(gs);
+}
+
+static void
+test_entry_guard_manage_primary(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  smartlist_t *prev_guards = smartlist_new();
+
+  /* If no guards are confirmed, we should pick a few reachable guards and
+   * call them all primary. But not confirmed.*/
+  entry_guards_update_primary(gs);
+  int n_primary = smartlist_len(gs->primary_entry_guards);
+  tt_int_op(n_primary, OP_GE, 1);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_assert(g->is_primary);
+    tt_assert(g->confirmed_idx == -1);
+  });
+
+  /* Calling it a second time should leave the guards unchanged. */
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
+  });
+
+  /* If we have one confirmed guard, that guards becomes the first primary
+   * guard, and the other primary guards get kept. */
+
+  /* find a non-primary guard... */
+  entry_guard_t *confirmed = NULL;
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, g, {
+    if (! g->is_primary) {
+      confirmed = g;
+      break;
+    }
+  });
+  tt_assert(confirmed);
+  /* make it confirmed. */
+  make_guard_confirmed(gs, confirmed);
+  /* update the list... */
+  smartlist_clear(prev_guards);
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+
+  /*  and see what's primary now! */
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  tt_ptr_op(smartlist_get(gs->primary_entry_guards, 0), OP_EQ, confirmed);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_assert(g->is_primary);
+    if (g_sl_idx == 0)
+      continue;
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx - 1));
+  });
+  {
+    entry_guard_t *prev_last_guard = smartlist_get(prev_guards, n_primary-1);
+    tt_assert(! prev_last_guard->is_primary);
   }
 
-  /* Inject our lines in the state */
-  state_insert_entry_guard_helper(state, entry_state_lines);
+  /* Calling it a fourth time should leave the guards unchanged. */
+  smartlist_clear(prev_guards);
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
+  });
+
+  /* Do some dirinfo checks */
+  {
+    /* Check that we have all required dirinfo for the primaries (that's done
+     * in big_fake_network_setup()) */
+    char *dir_info_str =
+      guard_selection_get_err_str_if_dir_info_missing(gs, 0, 0, 0);
+    tt_assert(!dir_info_str);
+
+    /* Now artificially remove the first primary's descriptor and re-check */
+    entry_guard_t *first_primary;
+    first_primary = smartlist_get(gs->primary_entry_guards, 0);
+    /* Change the first primary's identity digest so that the mocked functions
+     * can't find its descriptor */
+    memset(first_primary->identity, 9, sizeof(first_primary->identity));
+    dir_info_str =guard_selection_get_err_str_if_dir_info_missing(gs, 1, 2, 3);
+    tt_str_op(dir_info_str, OP_EQ,
+              "We're missing descriptors for 1/2 of our primary entry guards "
+              "(total microdescriptors: 2/3).");
+    tor_free(dir_info_str);
+  }
 
-  /* Parse state */
-  retval = entry_guards_parse_state(state, 1, &msg);
-  tt_int_op(retval, OP_GE, 0);
+ done:
+  guard_selection_free(gs);
+  smartlist_free(prev_guards);
+}
 
-  /* Test that the guard was registered.
-     We need to re-get the entry guard list since its pointer was
-     overwritten in entry_guards_parse_state(). */
-  all_entry_guards = get_entry_guards();
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+static void
+test_entry_guard_guard_preferred(void *arg)
+{
+  (void) arg;
+  entry_guard_t *g1 = tor_malloc_zero(sizeof(entry_guard_t));
+  entry_guard_t *g2 = tor_malloc_zero(sizeof(entry_guard_t));
+
+  g1->confirmed_idx = g2->confirmed_idx = -1;
+  g1->last_tried_to_connect = approx_time();
+  g2->last_tried_to_connect = approx_time();
 
-  { /* Test the entry guard structure */
-    char hex_digest[1024];
-    char str_time[1024];
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g1));
 
-    const entry_guard_t *e = smartlist_get(all_entry_guards, 0);
-    tt_str_op(e->nickname, OP_EQ, nickname); /* Verify nickname */
+  /* Neither is pending; priorities equal. */
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    base16_encode(hex_digest, sizeof(hex_digest),
-                  e->identity, DIGEST_LEN);
-    tt_str_op(hex_digest, OP_EQ, fpr); /* Verify fingerprint */
+  /* If one is pending, the pending one has higher priority */
+  g1->is_pending = 1;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
 
-    tt_assert(e->is_dir_cache); /* Verify dirness */
+  /* If both are pending, and last_tried_to_connect is equal:
+     priorities equal */
+  g2->is_pending = 1;
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-    tt_str_op(e->chosen_by_version, OP_EQ, tor_version); /* Verify version */
+  /* One had a connection that startied earlier: it has higher priority. */
+  g2->last_tried_to_connect -= 10;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+
+  /* Now, say that g1 is confirmed. It will get higher priority. */
+  g1->confirmed_idx = 5;
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+
+  /* But if g2 was confirmed first, it will get priority */
+  g2->confirmed_idx = 2;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+
+ done:
+  tor_free(g1);
+  tor_free(g2);
+}
 
-    tt_assert(e->made_contact); /* All saved guards have been contacted */
+static void
+test_entry_guard_select_for_circuit_no_confirmed(void *arg)
+{
+  /* Simpler cases: no gaurds are confirmed yet. */
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guard_restriction_t *rst = NULL;
+
+  /* simple starting configuration */
+  entry_guards_update_primary(gs);
+  unsigned state = 9999;
+
+  entry_guard_t *g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                    NULL, &state);
+
+  tt_assert(g);
+  tt_assert(g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+  tt_uint_op(g->is_pending, OP_EQ, 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we do that again, we should get the same guard.
+  entry_guard_t *g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                     NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+
+  // if we mark that guard down, we should get a different primary guard.
+  // auto-retry it.
+  g->is_reachable = GUARD_REACHABLE_NO;
+  g->failing_since = approx_time() - 10;
+  g->last_tried_to_connect = approx_time() - 10;
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_assert(g2);
+  tt_assert(g2->is_primary);
+  tt_int_op(g2->confirmed_idx, OP_EQ, -1);
+  tt_uint_op(g2->is_pending, OP_EQ, 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we say that the first primary guard was last tried a long time ago, we
+  // should get an automatic retry on it.
+  g->failing_since = approx_time() - 72*60*60;
+  g->last_tried_to_connect = approx_time() - 72*60*60;
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+  tt_assert(g2);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  // And if we mark ALL the primary guards down, we should get another guard
+  // at random.
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    guard->is_reachable = GUARD_REACHABLE_NO;
+    guard->last_tried_to_connect = approx_time() - 5;
+    guard->failing_since = approx_time() - 30;
+  });
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g2);
+  tt_assert(!g2->is_primary);
+  tt_int_op(g2->confirmed_idx, OP_EQ, -1);
+  tt_uint_op(g2->is_pending, OP_EQ, 1);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  // As a bonus, maybe we should be retrying the primary guards. Let's say so.
+  mark_primary_guards_maybe_reachable(gs);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    tt_int_op(guard->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+    tt_assert(guard->is_usable_filtered_guard == 1);
+    // no change to these fields.
+    tt_i64_op(guard->last_tried_to_connect, OP_EQ, approx_time() - 5);
+    tt_i64_op(guard->failing_since, OP_EQ, approx_time() - 30);
+  });
+
+  /* Let's try again and we should get the first primary guard again */
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+
+  /* But if we impose a restriction, we don't get the same guard */
+  rst = guard_create_exit_restriction((uint8_t*)g->identity);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, rst, &state);
+  tt_ptr_op(g2, OP_NE, g);
 
-    tt_assert(e->bad_since); /* Verify bad_since timestamp */
-    format_iso_time(str_time, e->bad_since);
-    tt_str_op(str_time, OP_EQ, unlisted_since);
+ done:
+  guard_selection_free(gs);
+  entry_guard_restriction_free(rst);
+}
 
-    /* The rest should be unset */
-    tt_assert(!e->unreachable_since);
-    tt_assert(!e->can_retry);
-    tt_assert(!e->path_bias_noticed);
-    tt_assert(!e->path_bias_warned);
-    tt_assert(!e->path_bias_extreme);
-    tt_assert(!e->path_bias_disabled);
-    tt_assert(!e->path_bias_use_noticed);
-    tt_assert(!e->path_bias_use_extreme);
-    tt_assert(!e->last_attempted);
+static void
+test_entry_guard_select_for_circuit_confirmed(void *arg)
+{
+  /* Case 2: if all the primary guards are down, and there are more confirmed
+     guards, we use a confirmed guard. */
+  (void)arg;
+  int i;
+  entry_guard_restriction_t *rst = NULL;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  const int N_CONFIRMED = 10;
+
+  /* slightly more complicated simple starting configuration */
+  entry_guards_update_primary(gs);
+  for (i = 0; i < N_CONFIRMED; ++i) {
+    entry_guard_t *guard = smartlist_get(gs->sampled_entry_guards, i);
+    make_guard_confirmed(gs, guard);
+  }
+  entry_guards_update_primary(gs); // rebuild the primary list.
+
+  unsigned state = 9999;
+
+  // As above, this gives us a primary guard.
+  entry_guard_t *g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                    NULL, &state);
+  tt_assert(g);
+  tt_assert(g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+  tt_uint_op(g->is_pending, OP_EQ, 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+
+  // But if we mark all the primary guards down...
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    guard->last_tried_to_connect = approx_time();
+    entry_guards_note_guard_failure(gs, guard);
+  });
+
+  // ... we should get a confirmed guard.
+  state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g);
+  tt_assert(! g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, smartlist_len(gs->primary_entry_guards));
+  tt_assert(g->is_pending);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+
+  // And if we try again, we should get a different confirmed guard, since
+  // that one is pending.
+  state = 9999;
+  entry_guard_t *g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                     NULL, &state);
+  tt_assert(g2);
+  tt_assert(! g2->is_primary);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_int_op(g2->confirmed_idx, OP_EQ,
+            smartlist_len(gs->primary_entry_guards)+1);
+  tt_assert(g2->is_pending);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we say that the next confirmed guard in order is excluded, and
+  // we disable EnforceDistinctSubnets, we get the guard AFTER the
+  // one we excluded.
+  get_options_mutable()->EnforceDistinctSubnets = 0;
+  g = smartlist_get(gs->confirmed_entry_guards,
+                     smartlist_len(gs->primary_entry_guards)+2);
+  rst = guard_create_exit_restriction((uint8_t*)g->identity);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, rst, &state);
+  tt_ptr_op(g2, OP_NE, NULL);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_int_op(g2->confirmed_idx, OP_EQ,
+            smartlist_len(gs->primary_entry_guards)+3);
+
+  // If we make every confirmed guard become pending then we start poking
+  // other guards.
+  const int n_remaining_confirmed =
+    N_CONFIRMED - 3 - smartlist_len(gs->primary_entry_guards);
+  for (i = 0; i < n_remaining_confirmed; ++i) {
+    g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+    tt_int_op(g->confirmed_idx, OP_GE, 0);
+    tt_assert(g);
   }
+  state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g);
+  tt_assert(g->is_pending);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+
+  // If we EnforceDistinctSubnets and apply a restriction, we get
+  // nothing, since we put all of the nodes in the same /16.
+  // Regression test for bug 22753/TROVE-2017-006.
+  get_options_mutable()->EnforceDistinctSubnets = 1;
+  g = smartlist_get(gs->confirmed_entry_guards, 0);
+  memcpy(rst->exclude_id, g->identity, DIGEST_LEN);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, rst, &state);
+  tt_ptr_op(g2, OP_EQ, NULL);
 
  done:
-  state_lines_free(entry_state_lines);
-  or_state_free(state);
-  tor_free(msg);
+  guard_selection_free(gs);
+  entry_guard_restriction_free(rst);
 }
 
-/** Similar to test_entry_guards_parse_state_simple() but aims to test
-    the PathBias-related details of the entry guard. */
 static void
-test_entry_guards_parse_state_pathbias(void *arg)
+test_entry_guard_select_for_circuit_highlevel_primary(void *arg)
 {
-  or_state_t *state = or_state_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  char *msg = NULL;
-  int retval;
-  smartlist_t *entry_state_lines = smartlist_new();
+  /* Play around with selecting primary guards for circuits and markign
+   * them up and down */
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  time_t start = approx_time();
+
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard = NULL;
+  entry_guard_t *g;
+  guard_usable_t u;
+  /*
+   * Make sure that the pick-for-circuit API basically works.  We'll get
+   * a primary guard, so it'll be usable on completion.
+   */
+  int r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                       &node, &guard);
+
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+
+  /* Call that circuit successful. */
+  update_approx_time(start+15);
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_USABLE_NOW); /* We can use it now. */
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_YES);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+
+  circuit_guard_state_free(guard);
+  guard = NULL;
+  node = NULL;
+  g = NULL;
+
+  /* Try again. We'll also get a primary guard this time. (The same one,
+     in fact.)  But this time, we'll say the connection has failed. */
+  update_approx_time(start+35);
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(guard->state_set_at, OP_EQ, start+35);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start+35);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0); // same one.
+
+  /* It's failed!  What will happen to our poor guard? */
+  update_approx_time(start+45);
+  entry_guard_failed(&guard);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_DEAD);
+  tt_i64_op(guard->state_set_at, OP_EQ, start+45);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_i64_op(g->failing_since, OP_EQ, start+45);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0); // still confirmed.
+
+  circuit_guard_state_free(guard);
+  guard = NULL;
+  node = NULL;
+  entry_guard_t *g_prev = g;
+  g = NULL;
+
+  /* Now try a third time. Since the other one is down, we'll get a different
+   * (still primary) guard.
+   */
+  update_approx_time(start+60);
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_ptr_op(g, OP_NE, g_prev);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_mem_op(g->identity, OP_NE, g_prev->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start+60);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1); // not confirmed now.
+
+  /* Call this one up; watch it get confirmed. */
+  update_approx_time(start+90);
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_USABLE_NOW);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_YES);
+  tt_int_op(g->confirmed_idx, OP_EQ, 1);
 
-  /* Path bias details of the fake guard */
-  const double circ_attempts = 9;
-  const double circ_successes = 8;
-  const double successful_closed = 4;
-  const double collapsed = 2;
-  const double unusable = 0;
-  const double timeouts = 1;
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
+static void
+test_entry_guard_select_for_circuit_highlevel_confirm_other(void *arg)
+{
   (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+
+  /* At the start, we have no confirmed guards.  We'll mark the primary guards
+   * down, then confirm something else.  As soon as we do, it should become
+   * primary, and we should get it next time. */
+
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard = NULL;
+  int i, r;
+  const node_t *node = NULL;
+  guard_usable_t u;
+
+  /* Declare that we're on the internet. */
+  entry_guards_note_internet_connectivity(gs);
+
+  /* Primary guards are down! */
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_assert(node);
+    tt_assert(guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    guard = NULL;
+    node = NULL;
+  }
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  /* Next guard should be non-primary. */
+  node = NULL;
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(r, OP_EQ, 0);
+  entry_guard_t *g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 1);
+  (void)start;
+
+  u = entry_guard_succeeded(&guard);
+  /* We're on the internet (by fiat), so this guard will get called "confirmed"
+   * and should immediately become primary.
+   */
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  tt_assert(u == GUARD_USABLE_NOW);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_int_op(g->is_pending, OP_EQ, 0);
 
-  { /* Prepare the state entry */
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-    /* Prepare the smartlist to hold the key/value of each line */
-    smartlist_t *state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuard");
-    smartlist_add_asprintf(state_line,
-             "givethanks B29D536DD1752D542E1FBB3C9CE4449D51298212 NoDirCache");
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_select_for_circuit_highlevel_primary_retry(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+
+  /* At the start, we have no confirmed guards.  We'll mark the primary guards
+   * down, then confirm something else.  As soon as we do, it should become
+   * primary, and we should get it next time. */
+
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard = NULL, *guard2 = NULL;
+  int i, r;
+  const node_t *node = NULL;
+  entry_guard_t *g;
+  guard_usable_t u;
+
+  /* Declare that we're on the internet. */
+  entry_guards_note_internet_connectivity(gs);
+
+  /* Make primary guards confirmed (so they won't be superseded by a later
+   * guard), then mark them down. */
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_assert(node);
+    tt_assert(guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    g = entry_guard_handle_get(guard->guard);
+    make_guard_confirmed(gs, g);
+    tt_int_op(g->is_primary, OP_EQ, 1);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+    guard = NULL;
+    node = NULL;
+  }
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardAddedBy");
-    smartlist_add_asprintf(state_line,
-      "B29D536DD1752D542E1FBB3C9CE4449D51298212 0.2.5.3-alpha-dev "
-                           "%s", get_yesterday_date_str());
-    smartlist_add(entry_state_lines, state_line);
+  /* Get another guard that we might try. */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+
+  tt_assert(entry_guards_all_primary_guards_are_down(gs));
+
+  /* And an hour has passed ... */
+  update_approx_time(start + 3600);
+
+  /* Say that guard has succeeded! */
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_MAYBE_USABLE_LATER);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+
+  /* The primary guards should have been marked up! */
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, pg, {
+    tt_int_op(pg->is_primary, OP_EQ, 1);
+    tt_ptr_op(g, OP_NE, pg);
+    tt_int_op(pg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  });
+
+  /* Have a circuit to a primary guard succeed. */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard2);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(guard2->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  u = entry_guard_succeeded(&guard2);
+  tt_assert(u == GUARD_USABLE_NOW);
+  tt_int_op(guard2->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  tt_assert(! entry_guards_all_primary_guards_are_down(gs));
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardUnlistedSince");
-    smartlist_add_asprintf(state_line, "2014-06-08 16:16:50");
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+  circuit_guard_state_free(guard2);
+}
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardPathBias");
-    smartlist_add_asprintf(state_line, "%f %f %f %f %f %f",
-                           circ_attempts, circ_successes, successful_closed,
-                           collapsed, unusable, timeouts);
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_select_and_cancel(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+  int i,r;
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guard_t *g;
+
+  /* Once more, we mark all the primary guards down. */
+  entry_guards_note_internet_connectivity(gs);
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    g = entry_guard_handle_get(guard->guard);
+    tt_int_op(g->is_primary, OP_EQ, 1);
+    tt_int_op(g->is_pending, OP_EQ, 0);
+    make_guard_confirmed(gs, g);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    guard = NULL;
+    node = NULL;
   }
 
-  /* Inject our lines in the state */
-  state_insert_entry_guard_helper(state, entry_state_lines);
+  tt_assert(entry_guards_all_primary_guards_are_down(gs));
+
+  /* Now get another guard we could try... */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 1);
+
+  /* Whoops! We should never have asked for this guard. Cancel the request! */
+  entry_guard_cancel(&guard);
+  tt_ptr_op(guard, OP_EQ, NULL);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 0);
 
-  /* Parse state */
-  retval = entry_guards_parse_state(state, 1, &msg);
-  tt_int_op(retval, OP_GE, 0);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-  /* Test that the guard was registered */
-  all_entry_guards = get_entry_guards();
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+static void
+test_entry_guard_drop_guards(void *arg)
+{
+  (void) arg;
+  int r;
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard;
+  guard_selection_t *gs = get_guard_selection_info();
+
+  // Pick a guard, to get things set up.
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_GE,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+  tt_ptr_op(gs, OP_EQ, get_guard_selection_info());
+
+  // Drop all the guards!  (This is a bad idea....)
+  remove_all_entry_guards_for_guard_selection(gs);
+  gs = get_guard_selection_info();
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->confirmed_entry_guards), OP_EQ, 0);
 
-  { /* Test the path bias of this guard */
-    const entry_guard_t *e = smartlist_get(all_entry_guards, 0);
+ done:
+  circuit_guard_state_free(guard);
+  guard_selection_free(gs);
+}
 
-    tt_assert(!e->is_dir_cache);
-    tt_assert(!e->can_retry);
+/* Unit test setup function: Create a fake network, and set everything up
+ * for testing the upgrade-a-waiting-circuit code. */
+typedef struct {
+  guard_selection_t *gs;
+  time_t start;
+  circuit_guard_state_t *guard1_state;
+  circuit_guard_state_t *guard2_state;
+  entry_guard_t *guard1;
+  entry_guard_t *guard2;
+  origin_circuit_t *circ1;
+  origin_circuit_t *circ2;
+  smartlist_t *all_origin_circuits;
+} upgrade_circuits_data_t;
+static void *
+upgrade_circuits_setup(const struct testcase_t *testcase)
+{
+  upgrade_circuits_data_t *data = tor_malloc_zero(sizeof(*data));
+  guard_selection_t *gs = data->gs =
+    guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard;
+  const node_t *node;
+  entry_guard_t *g;
+  int i;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+  const char *argument = testcase->setup_data;
+  const int make_circ1_succeed = strstr(argument, "c1-done") != NULL;
+  const int make_circ2_succeed = strstr(argument, "c2-done") != NULL;
+
+  big_fake_network_setup(testcase);
+
+  /* We're going to set things up in a state where a circuit will be ready to
+   * be upgraded.  Each test can make a single change (or not) that should
+   * block the upgrade.
+   */
+
+  /* First, make all the primary guards confirmed, and down. */
+  data->start = approx_time();
+  entry_guards_note_internet_connectivity(gs);
+  for (i = 0; i < N_PRIMARY; ++i) {
+    entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &node, &guard);
+    g = entry_guard_handle_get(guard->guard);
+    make_guard_confirmed(gs, g);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+  }
 
-    /* XXX tt_double_op doesn't support equality. Cast to int for now. */
-    tt_int_op((int)e->circ_attempts, OP_EQ, (int)circ_attempts);
-    tt_int_op((int)e->circ_successes, OP_EQ, (int)circ_successes);
-    tt_int_op((int)e->successful_circuits_closed, OP_EQ,
-              (int)successful_closed);
-    tt_int_op((int)e->timeouts, OP_EQ, (int)timeouts);
-    tt_int_op((int)e->collapsed_circuits, OP_EQ, (int)collapsed);
-    tt_int_op((int)e->unusable_circuits, OP_EQ, (int)unusable);
+  /* Grab another couple of guards */
+  data->all_origin_circuits = smartlist_new();
+
+  update_approx_time(data->start + 27);
+  entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                               &node, &data->guard1_state);
+  origin_circuit_t *circ;
+  data->circ1 = circ = origin_circuit_new();
+  circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circ->guard_state = data->guard1_state;
+  smartlist_add(data->all_origin_circuits, circ);
+
+  update_approx_time(data->start + 30);
+  entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                               &node, &data->guard2_state);
+  data->circ2 = circ = origin_circuit_new();
+  circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circ->guard_state = data->guard2_state;
+  smartlist_add(data->all_origin_circuits, circ);
+
+  data->guard1 = entry_guard_handle_get(data->guard1_state->guard);
+  data->guard2 = entry_guard_handle_get(data->guard2_state->guard);
+  tor_assert(data->guard1 != data->guard2);
+  tor_assert(data->guard1_state->state ==
+             GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tor_assert(data->guard2_state->state ==
+             GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+
+  guard_usable_t r;
+  update_approx_time(data->start + 32);
+  if (make_circ1_succeed) {
+    r = entry_guard_succeeded(&data->guard1_state);
+    tor_assert(r == GUARD_MAYBE_USABLE_LATER);
+    tor_assert(data->guard1_state->state ==
+               GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  }
+  update_approx_time(data->start + 33);
+  if (make_circ2_succeed) {
+    r = entry_guard_succeeded(&data->guard2_state);
+    tor_assert(r == GUARD_MAYBE_USABLE_LATER);
+    tor_assert(data->guard2_state->state ==
+               GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
   }
 
- done:
-  or_state_free(state);
-  state_lines_free(entry_state_lines);
-  tor_free(msg);
+  return data;
+}
+static int
+upgrade_circuits_cleanup(const struct testcase_t *testcase, void *ptr)
+{
+  upgrade_circuits_data_t *data = ptr;
+  // circuit_guard_state_free(data->guard1_state); // held in circ1
+  // circuit_guard_state_free(data->guard2_state); // held in circ2
+  guard_selection_free(data->gs);
+  smartlist_free(data->all_origin_circuits);
+  circuit_free_(TO_CIRCUIT(data->circ1));
+  circuit_free_(TO_CIRCUIT(data->circ2));
+  tor_free(data);
+  return big_fake_network_cleanup(testcase, NULL);
 }
 
-/* Simple test of entry_guards_set_from_config() by specifying a
-   particular EntryNode and making sure it gets picked. */
 static void
-test_entry_guards_set_from_config(void *arg)
+test_entry_guard_upgrade_a_circuit(void *arg)
 {
-  or_options_t *options = get_options_mutable();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  const char *entrynodes_str = "test003r";
-  const node_t *chosen_entry = NULL;
-  int retval;
+  upgrade_circuits_data_t *data = arg;
 
-  (void) arg;
+  /* This is the easy case: we have no COMPLETED circuits, all the
+   * primary guards are down, we have two WAITING circuits: one will
+   * get upgraded to COMPLETED!  (The one that started first.)
+   */
 
-  /* Prase EntryNodes as a routerset. */
-  options->EntryNodes = routerset_new();
-  retval = routerset_parse(options->EntryNodes,
-                           entrynodes_str,
-                           "test_entrynodes");
-  tt_int_op(retval, OP_GE, 0);
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
 
-  /* Read nodes from EntryNodes */
-  entry_guards_set_from_config(options);
+  /* circ1 was started first, so we'll get told to ugrade it... */
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  /* Test that only one guard was added. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+  /* And the guard state should be complete */
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
 
-  /* Make sure it was the guard we specified. */
-  chosen_entry = choose_random_entry(NULL);
-  tt_str_op(chosen_entry->ri->nickname, OP_EQ, entrynodes_str);
+ done:
+  smartlist_free(result);
+}
+
+static void
+test_entry_guard_upgrade_blocked_by_live_primary_guards(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* If any primary guards might be up, we can't upgrade any waiting
+   * circuits.
+   */
+  mark_primary_guards_maybe_reachable(data->gs);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("not all primary guards were definitely down.");
 
  done:
-  routerset_free(options->EntryNodes);
+  teardown_capture_of_logs();
+  smartlist_free(result);
 }
 
 static void
-test_entry_is_time_to_retry(void *arg)
+test_entry_guard_upgrade_blocked_by_lack_of_waiting_circuits(void *arg)
 {
-  entry_guard_t *test_guard;
-  time_t now;
-  int retval;
-  (void)arg;
+  upgrade_circuits_data_t *data = arg;
+
+  /* If no circuits are waiting, we can't upgrade anything.  (The test
+   * setup in this case was told not to make any of the circuits "waiting".)
+   */
+  smartlist_t *result = smartlist_new();
+  int r;
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("Considered upgrading guard-stalled circuits, "
+                            "but didn't find any.");
 
-  now = time(NULL);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  test_guard = tor_malloc_zero(sizeof(entry_guard_t));
+static void
+test_entry_guard_upgrade_blocked_by_better_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* We'll run through the logic of upgrade_a_circuit below...
+   * and then try again to make sure that circ2 isn't also upgraded.
+   */
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  /* Now, try again. Make sure that circ2 isn't upgraded. */
+  smartlist_clear(result);
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("At least one complete circuit had higher "
+                            "priority, so not upgrading.");
 
-  test_guard->last_attempted = now - 10;
-  test_guard->unreachable_since = now - 1;
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_upgrade_not_blocked_by_restricted_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* Once more, let circ1 become complete. But this time, we'll claim
+   * that circ2 was restricted to not use the same guard as circ1. */
+  data->guard2_state->restrictions =
+    guard_create_exit_restriction((uint8_t*)data->guard1->identity);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  /* Now, we try again. Since circ2 has a restriction that circ1 doesn't obey,
+   * circ2 _is_ eligible for upgrade. */
+  smartlist_clear(result);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc2 = smartlist_get(result, 0);
+  tt_ptr_op(oc2, OP_EQ, data->circ2);
 
-  test_guard->unreachable_since = now - (6*60*60 - 1);
-  test_guard->last_attempted = now - (60*60 + 1);
+ done:
+  smartlist_free(result);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_upgrade_not_blocked_by_worse_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+  smartlist_t *result = smartlist_new();
+  /* here we manually make circ2 COMPLETE, and make sure that circ1
+   * gets made complete anyway, since guard1 has higher priority
+   */
+  update_approx_time(data->start + 300);
+  data->guard2_state->state = GUARD_CIRC_STATE_COMPLETE;
+  data->guard2_state->state_set_at = approx_time();
+  update_approx_time(data->start + 301);
+
+  /* Now, try again. Make sure that circ1 is approved. */
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  test_guard->last_attempted = now - (60*60 - 1);
+ done:
+  smartlist_free(result);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,0);
+static void
+test_entry_guard_upgrade_blocked_by_better_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* circ2 is done, but circ1 is still pending. Since circ1 is better,
+   * we won't upgrade circ2. */
+
+  /* XXXX Prop271 -- this is a kludge.  I'm making sure circ1 _is_ better,
+   * by messing with the guards' confirmed_idx */
+  make_guard_confirmed(data->gs, data->guard1);
+  {
+    int tmp;
+    tmp = data->guard1->confirmed_idx;
+    data->guard1->confirmed_idx = data->guard2->confirmed_idx;
+    data->guard2->confirmed_idx = tmp;
+  }
 
-  test_guard->unreachable_since = now - (6*60*60 + 1);
-  test_guard->last_attempted = now - (4*60*60 + 1);
+  smartlist_t *result = smartlist_new();
+  setup_capture_of_logs(LOG_DEBUG);
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("but 1 pending circuit(s) had higher guard "
+                            "priority, so not upgrading.");
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  test_guard->unreachable_since = now - (3*24*60*60 - 1);
-  test_guard->last_attempted = now - (4*60*60 + 1);
+static void
+test_entry_guard_upgrade_not_blocked_by_restricted_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+  /* circ2 is done, but circ1 is still pending. But when there is a
+     restriction on circ2 that circ1 can't satisfy, circ1 can't block
+     circ2. */
+
+  /* XXXX Prop271 -- this is a kludge.  I'm making sure circ1 _is_ better,
+   * by messing with the guards' confirmed_idx */
+  make_guard_confirmed(data->gs, data->guard1);
+  {
+    int tmp;
+    tmp = data->guard1->confirmed_idx;
+    data->guard1->confirmed_idx = data->guard2->confirmed_idx;
+    data->guard2->confirmed_idx = tmp;
+  }
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  data->guard2_state->restrictions =
+    guard_create_exit_restriction((uint8_t*)data->guard1->identity);
 
-  test_guard->unreachable_since = now - (3*24*60*60 + 1);
-  test_guard->last_attempted = now - (18*60*60 + 1);
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ2);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  smartlist_free(result);
+}
 
-  test_guard->unreachable_since = now - (7*24*60*60 - 1);
-  test_guard->last_attempted = now - (18*60*60 + 1);
+static void
+test_entry_guard_upgrade_not_blocked_by_worse_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* circ1 is done, but circ2 is still pending. Since circ1 is better,
+   * we will upgrade it. */
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  smartlist_free(result);
+}
 
-  test_guard->last_attempted = now - (18*60*60 - 1);
+static void
+test_enty_guard_should_expire_waiting(void *arg)
+{
+  (void)arg;
+  circuit_guard_state_t *fake_state = tor_malloc_zero(sizeof(*fake_state));
+  /* We'll leave "guard" unset -- it won't matter here. */
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,0);
+  /* No state? Can't expire. */
+  tt_assert(! entry_guard_state_should_expire(NULL));
 
-  test_guard->unreachable_since = now - (7*24*60*60 + 1);
-  test_guard->last_attempted = now - (36*60*60 + 1);
+  /* Let's try one that expires. */
+  fake_state->state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+  fake_state->state_set_at =
+    approx_time() - DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT - 1;
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  tt_assert(entry_guard_state_should_expire(fake_state));
 
-  test_guard->unreachable_since = now - (7*24*60*60 + 1);
-  test_guard->last_attempted = now - (36*60*60 + 1);
+  /* But it wouldn't expire if we changed the state. */
+  fake_state->state = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+  tt_assert(! entry_guard_state_should_expire(fake_state));
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  /* And it wouldn't have expired a few seconds ago. */
+  fake_state->state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+  fake_state->state_set_at =
+    approx_time() - DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT + 5;
+  tt_assert(! entry_guard_state_should_expire(fake_state));
 
  done:
-  tor_free(test_guard);
+  tor_free(fake_state);
 }
 
-/** XXX Do some tests that entry_is_live() */
+/** Test that the number of primary guards can be controlled using torrc */
 static void
-test_entry_is_live(void *arg)
+test_entry_guard_number_of_primaries(void *arg)
 {
-  smartlist_t *our_nodelist = NULL;
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  const node_t *test_node = NULL;
-  const entry_guard_t *test_entry = NULL;
-  const char *msg;
-  int which_node;
-
   (void) arg;
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  /* Get default value */
+  tt_int_op(get_n_primary_guards(), OP_EQ, DFLT_N_PRIMARY_GUARDS);
 
-  /* Walk the nodelist and add all nodes as entry guards. */
-  our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
+  /* Set number of primaries using torrc */
+  get_options_mutable()->NumPrimaryGuards = 42;
+  tt_int_op(get_n_primary_guards(), OP_EQ, 42);
 
-  SMARTLIST_FOREACH_BEGIN(our_nodelist, const node_t *, node) {
-    const node_t *node_tmp;
-    node_tmp = add_an_entry_guard(node, 0, 1, 0, 0);
-    tt_assert(node_tmp);
+ done:
+  ;
+}
 
-    tt_int_op(node->is_stable, OP_EQ, 0);
-    tt_int_op(node->is_fast, OP_EQ, 0);
-  } SMARTLIST_FOREACH_END(node);
+static void
+mock_directory_initiate_request(directory_request_t *req)
+{
+  if (req->guard_state) {
+    circuit_guard_state_free(req->guard_state);
+  }
+}
 
-  /* Make sure the nodes were added as entry guards. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ,
-            HELPER_NUMBER_OF_DESCRIPTORS);
+static networkstatus_t *mock_ns_val = NULL;
+static networkstatus_t *
+mock_ns_get_by_flavor(consensus_flavor_t f)
+{
+  (void)f;
+  return mock_ns_val;
+}
 
-  /* Now get a random test entry that we will use for this unit test. */
-  which_node = 3;  /* (chosen by fair dice roll) */
-  test_entry = smartlist_get(all_entry_guards, which_node);
+/** Test that when we fetch microdescriptors we skip guards that have
+ *  previously failed to serve us needed microdescriptors. */
+static void
+test_entry_guard_outdated_dirserver_exclusion(void *arg)
+{
+  int retval;
+  response_handler_args_t *args = NULL;
+  dir_connection_t *conn = NULL;
+  (void) arg;
 
-  /* Let's do some entry_is_live() tests! */
+  /* Test prep: Make a new guard selection */
+  guard_selection_t *gs = get_guard_selection_by_name("default",
+                                                      GS_TYPE_NORMAL, 1);
+
+  /* ... we want to use entry guards */
+  or_options_t *options = get_options_mutable();
+  options->UseEntryGuards = 1;
+  options->UseBridges = 0;
+
+  /* ... prepare some md digests we want to download in the future */
+  smartlist_t *digests = smartlist_new();
+  const char *prose = "unhurried and wise, we perceive.";
+  for (int i = 0; i < 20; i++) {
+    smartlist_add(digests, (char*)prose);
+  }
 
-  /* Require the node to be stable, but it's not. Should fail.
-     Also enable 'assume_reachable' because why not. */
-  test_node = entry_is_live(test_entry,
-                            ENTRY_NEED_UPTIME | ENTRY_ASSUME_REACHABLE,
-                            &msg);
-  tt_assert(!test_node);
+  tt_int_op(smartlist_len(digests), OP_EQ, 20);
+
+  /* ... now mock some functions */
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor);
+  MOCK(directory_initiate_request, mock_directory_initiate_request);
+
+  /* Test logic:
+   *  0. Create a proper guard set and primary guard list.
+   *  1. Pretend to fail microdescriptor fetches from all the primary guards.
+   *  2. Order another microdescriptor fetch and make sure that primary guards
+   *     get skipped since they failed previous fetches.
+   */
+
+  { /* Setup primary guard list */
+    int i;
+    entry_guards_update_primary(gs);
+    for (i = 0; i < DFLT_N_PRIMARY_GUARDS; ++i) {
+      entry_guard_t *guard = smartlist_get(gs->sampled_entry_guards, i);
+      make_guard_confirmed(gs, guard);
+    }
+    entry_guards_update_primary(gs);
+  }
 
-  /* Require the node to be fast, but it's not. Should fail. */
-  test_node = entry_is_live(test_entry,
-                            ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE,
-                            &msg);
-  tt_assert(!test_node);
+  {
+    /* Fail microdesc fetches with all the primary guards */
+    args = tor_malloc_zero(sizeof(response_handler_args_t));
+    args->status_code = 404;
+    args->reason = NULL;
+    args->body = NULL;
+    args->body_len = 0;
+
+    conn = tor_malloc_zero(sizeof(dir_connection_t));
+    conn->requested_resource = tor_strdup("d/jlinblackorigami");
+    conn->base_.purpose = DIR_PURPOSE_FETCH_MICRODESC;
+
+    /* Pretend to fail fetches with all primary guards */
+    SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards,const entry_guard_t *,g) {
+      memcpy(conn->identity_digest, g->identity, DIGEST_LEN);
+
+      retval = handle_response_fetch_microdesc(conn, args);
+      tt_int_op(retval, OP_EQ, 0);
+    } SMARTLIST_FOREACH_END(g);
+  }
 
-  /* Don't impose any restrictions on the node. Should succeed. */
-  test_node = entry_is_live(test_entry, 0, &msg);
-  tt_assert(test_node);
-  tt_ptr_op(test_node, OP_EQ, node_get_by_id(test_entry->identity));
+  {
+    /* Now order the final md download */
+    setup_full_capture_of_logs(LOG_INFO);
+    initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_MICRODESC,
+                                  digests, 3, 7, 0);
 
-  /* Require descriptor for this node. It has one so it should succeed. */
-  test_node = entry_is_live(test_entry, ENTRY_NEED_DESCRIPTOR, &msg);
-  tt_assert(test_node);
-  tt_ptr_op(test_node, OP_EQ, node_get_by_id(test_entry->identity));
+    /* ... and check that because we failed to fetch microdescs from all our
+     * primaries, we didn't end up selecting a primary for fetching dir info */
+    expect_log_msg_containing("No primary or confirmed guards available.");
+    teardown_capture_of_logs();
+  }
 
  done:
-  ; /* XXX */
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+  UNMOCK(directory_initiate_request);
+  smartlist_free(digests);
+  tor_free(mock_ns_val);
+  tor_free(args);
+  if (conn) {
+    tor_free(conn->requested_resource);
+    tor_free(conn);
+  }
 }
 
-#define TEST_IPV4_ADDR "123.45.67.89"
-#define TEST_IPV6_ADDR "[1234:5678:90ab:cdef::]"
+/** Test helper to extend the <b>oc</b> circuit path <b>n</b> times and then
+ *  ensure that the circuit is now complete. */
+static void
+helper_extend_circuit_path_n_times(origin_circuit_t *oc, int n)
+{
+  int retval;
+  int i;
+
+  /* Extend path n times */
+  for (i = 0 ; i < n ; i++) {
+    retval = onion_extend_cpath(oc);
+    tt_int_op(retval, OP_EQ, 0);
+    tt_int_op(circuit_get_cpath_len(oc), OP_EQ, i+1);
+  }
 
+  /* Now do it one last time and see that circ is complete */
+  retval = onion_extend_cpath(oc);
+  tt_int_op(retval, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+/** Test for basic Tor path selection. Makes sure we build 3-hop circuits. */
 static void
-test_node_preferred_orport(void *arg)
+test_entry_guard_basic_path_selection(void *arg)
 {
-  (void)arg;
-  tor_addr_t ipv4_addr;
-  const uint16_t ipv4_port = 4444;
-  tor_addr_t ipv6_addr;
-  const uint16_t ipv6_port = 6666;
-  routerinfo_t node_ri;
-  node_t node;
-  tor_addr_port_t ap;
+  (void) arg;
 
-  /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  /* We don't test ClientPreferIPv6ORPort here, because it's used in
-   * nodelist_set_consensus to setup node.ipv6_preferred, which we set
-   * directly. */
-  MOCK(get_options, mock_get_options);
+  int retval;
 
-  /* Setup IP addresses */
-  tor_addr_parse(&ipv4_addr, TEST_IPV4_ADDR);
-  tor_addr_parse(&ipv6_addr, TEST_IPV6_ADDR);
+  /* Enable entry guards */
+  or_options_t *options = get_options_mutable();
+  options->UseEntryGuards = 1;
 
-  /* Setup node_ri */
-  memset(&node_ri, 0, sizeof(node_ri));
-  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
-  node_ri.or_port = ipv4_port;
-  tor_addr_copy(&node_ri.ipv6_addr, &ipv6_addr);
-  node_ri.ipv6_orport = ipv6_port;
+  /* disables /16 check since all nodes have the same addr... */
+  options->EnforceDistinctSubnets = 0;
 
-  /* Setup node */
-  memset(&node, 0, sizeof(node));
-  node.ri = &node_ri;
+  /* Create our circuit */
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
 
-  /* Check the preferred address is IPv4 if we're only using IPv4, regardless
-   * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
-  node.ipv6_preferred = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+  /* First pick the exit and pin it on the build_state */
+  retval = onion_pick_cpath_exit(oc, NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
 
-  node.ipv6_preferred = 1;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+  /* Extend path 3 times. First we pick guard, then middle, then exit. */
+  helper_extend_circuit_path_n_times(oc, 3);
 
-  /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
-   * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+ done:
+  circuit_free_(circ);
+}
 
-  /* Check the preferred address is IPv6 if we prefer it and
-   * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = 1;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+/** Test helper to build an L2 and L3 vanguard list. The vanguard lists
+ *  produced should be completely disjoint. */
+static void
+helper_setup_vanguard_list(or_options_t *options)
+{
+  int i = 0;
+
+  /* Add some nodes to the vanguard L2 list */
+  options->HSLayer2Nodes = routerset_new();
+  for (i = 0; i < 10 ; i += 2) {
+    node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i);
+    tt_assert(vanguard_node->is_possible_guard);
+    routerset_parse(options->HSLayer2Nodes, vanguard_node->rs->nickname, "l2");
+  }
+  /* also add some nodes to vanguard L3 list
+   * (L2 list and L3 list should be disjoint for this test to work) */
+  options->HSLayer3Nodes = routerset_new();
+  for (i = 10; i < 20 ; i += 2) {
+    node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i);
+    tt_assert(vanguard_node->is_possible_guard);
+    routerset_parse(options->HSLayer3Nodes, vanguard_node->rs->nickname, "l3");
+  }
 
-  mocked_options.ClientUseIPv4 = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+ done:
+  ;
+}
 
-  /* Check the preferred address is IPv6 if we don't prefer it, but
-   * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+/** Test to ensure that vanguard path selection works properly.  Ensures that
+ *  default vanguard circuits are 4 hops, and that path selection works
+ *  correctly given the vanguard settings. */
+static void
+test_entry_guard_vanguard_path_selection(void *arg)
+{
+  (void) arg;
+
+  int retval;
+
+  /* Enable entry guards */
+  or_options_t *options = get_options_mutable();
+  options->UseEntryGuards = 1;
+
+  /* XXX disables /16 check */
+  options->EnforceDistinctSubnets = 0;
+
+  /* Setup our vanguard list */
+  helper_setup_vanguard_list(options);
+
+  /* Create our circuit */
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->is_internal = 1;
+
+  /* Switch circuit purpose to vanguards */
+  circ->purpose = CIRCUIT_PURPOSE_HS_VANGUARDS;
+
+  /* First pick the exit and pin it on the build_state */
+  tt_int_op(oc->build_state->desired_path_len, OP_EQ, 0);
+  retval = onion_pick_cpath_exit(oc, NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Ensure that vanguards make 4-hop circuits by default */
+  tt_int_op(oc->build_state->desired_path_len, OP_EQ, 4);
+
+  /* Extend path as many times as needed to have complete circ. */
+  helper_extend_circuit_path_n_times(oc, oc->build_state->desired_path_len);
+
+  /* Test that the cpath linked list is set correctly. */
+  crypt_path_t *l1_node = oc->cpath;
+  crypt_path_t *l2_node = l1_node->next;
+  crypt_path_t *l3_node = l2_node->next;
+  crypt_path_t *l4_node = l3_node->next;
+  crypt_path_t *l1_node_again = l4_node->next;
+  tt_ptr_op(l1_node, OP_EQ, l1_node_again);
+
+  /* Test that L2 is indeed HSLayer2Node */
+  retval = routerset_contains_extendinfo(options->HSLayer2Nodes,
+                                         l2_node->extend_info);
+  tt_int_op(retval, OP_EQ, 4);
+  /* test that L3 node is _not_ contained in HSLayer2Node */
+  retval = routerset_contains_extendinfo(options->HSLayer2Nodes,
+                                         l3_node->extend_info);
+  tt_int_op(retval, OP_LT, 4);
+
+  /* Test that L3 is indeed HSLayer3Node */
+  retval = routerset_contains_extendinfo(options->HSLayer3Nodes,
+                                         l3_node->extend_info);
+  tt_int_op(retval, OP_EQ, 4);
+  /* test that L2 node is _not_ contained in HSLayer3Node */
+  retval = routerset_contains_extendinfo(options->HSLayer3Nodes,
+                                         l2_node->extend_info);
+  tt_int_op(retval, OP_LT, 4);
+
+  /* TODO: Test that L1 can be the same as exit. To test this we need start
+     enforcing EnforceDistinctSubnets again, which means that we need to give
+     each test node a different address which currently breaks some tests. */
 
  done:
-  UNMOCK(get_options);
+  circuit_free_(circ);
 }
 
-static const struct testcase_setup_t fake_network = {
-  fake_network_setup, fake_network_cleanup
+static const struct testcase_setup_t big_fake_network = {
+  big_fake_network_setup, big_fake_network_cleanup
+};
+
+static const struct testcase_setup_t upgrade_circuits = {
+  upgrade_circuits_setup, upgrade_circuits_cleanup
 };
 
+#define BFN_TEST(name) \
+  { #name, test_entry_guard_ ## name, TT_FORK, &big_fake_network, NULL }
+
+#define UPGRADE_TEST(name, arg)                                         \
+  { #name, test_entry_guard_ ## name, TT_FORK, &upgrade_circuits,       \
+      (void*)(arg) }
+
 struct testcase_t entrynodes_tests[] = {
-  { "entry_is_time_to_retry", test_entry_is_time_to_retry,
-    TT_FORK, NULL, NULL },
-  { "choose_random_entry_no_guards", test_choose_random_entry_no_guards,
-    TT_FORK, &fake_network, NULL },
-  { "choose_random_entry_one_possibleguard",
-    test_choose_random_entry_one_possible_guard,
-    TT_FORK, &fake_network, NULL },
-  { "populate_live_entry_guards_1guard",
-    test_populate_live_entry_guards_1guard,
-    TT_FORK, &fake_network, NULL },
-  { "populate_live_entry_guards_3guards",
-    test_populate_live_entry_guards_3guards,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_parse_state_simple",
-    test_entry_guards_parse_state_simple,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_parse_state_pathbias",
-    test_entry_guards_parse_state_pathbias,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_set_from_config",
-    test_entry_guards_set_from_config,
-    TT_FORK, &fake_network, NULL },
-  { "entry_is_live",
-    test_entry_is_live,
-    TT_FORK, &fake_network, NULL },
   { "node_preferred_orport",
     test_node_preferred_orport,
     0, NULL, NULL },
+  { "entry_guard_describe", test_entry_guard_describe, 0, NULL, NULL },
+  { "randomize_time", test_entry_guard_randomize_time, 0, NULL, NULL },
+  { "encode_for_state_minimal",
+    test_entry_guard_encode_for_state_minimal, 0, NULL, NULL },
+  { "encode_for_state_maximal",
+    test_entry_guard_encode_for_state_maximal, 0, NULL, NULL },
+  { "parse_from_state_minimal",
+    test_entry_guard_parse_from_state_minimal, 0, NULL, NULL },
+  { "parse_from_state_maximal",
+    test_entry_guard_parse_from_state_maximal, 0, NULL, NULL },
+  { "parse_from_state_failure",
+    test_entry_guard_parse_from_state_failure, 0, NULL, NULL },
+  { "parse_from_state_partial_failure",
+    test_entry_guard_parse_from_state_partial_failure, 0, NULL, NULL },
+  { "parse_from_state_full",
+    test_entry_guard_parse_from_state_full, TT_FORK, NULL, NULL },
+  { "parse_from_state_broken",
+    test_entry_guard_parse_from_state_broken, TT_FORK, NULL, NULL },
+  { "get_guard_selection_by_name",
+    test_entry_guard_get_guard_selection_by_name, TT_FORK, NULL, NULL },
+  { "number_of_primaries",
+    test_entry_guard_number_of_primaries, TT_FORK, NULL, NULL },
+  BFN_TEST(choose_selection_initial),
+  BFN_TEST(add_single_guard),
+  BFN_TEST(node_filter),
+  BFN_TEST(expand_sample),
+  BFN_TEST(expand_sample_small_net),
+  BFN_TEST(update_from_consensus_status),
+  BFN_TEST(update_from_consensus_repair),
+  BFN_TEST(update_from_consensus_remove),
+  BFN_TEST(confirming_guards),
+  BFN_TEST(sample_reachable_filtered),
+  BFN_TEST(sample_reachable_filtered_empty),
+  BFN_TEST(retry_unreachable),
+  BFN_TEST(manage_primary),
+  { "guard_preferred", test_entry_guard_guard_preferred, TT_FORK, NULL, NULL },
+  BFN_TEST(select_for_circuit_no_confirmed),
+  BFN_TEST(select_for_circuit_confirmed),
+  BFN_TEST(select_for_circuit_highlevel_primary),
+  BFN_TEST(select_for_circuit_highlevel_confirm_other),
+  BFN_TEST(select_for_circuit_highlevel_primary_retry),
+  BFN_TEST(select_and_cancel),
+  BFN_TEST(drop_guards),
+  BFN_TEST(outdated_dirserver_exclusion),
+  BFN_TEST(basic_path_selection),
+  BFN_TEST(vanguard_path_selection),
+
+  UPGRADE_TEST(upgrade_a_circuit, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_live_primary_guards, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_lack_of_waiting_circuits, ""),
+  UPGRADE_TEST(upgrade_blocked_by_better_circ_complete, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_complete,
+               "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_complete, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_better_circ_pending, "c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_pending,
+               "c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_pending, "c1-done"),
+  { "should_expire_waiting", test_enty_guard_should_expire_waiting, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 1f92780177..e05342cb8a 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
@@ -10,6 +10,7 @@
 #include "connection_or.h"
 #include "config.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "ext_orport.h"
 #include "main.h"
 #include "test.h"
@@ -58,11 +59,11 @@ test_ext_or_id_map(void *arg)
 
  done:
   if (c1)
-    connection_free_(TO_CONN(c1));
+    connection_free_minimal(TO_CONN(c1));
   if (c2)
-    connection_free_(TO_CONN(c2));
+    connection_free_minimal(TO_CONN(c2));
   if (c3)
-    connection_free_(TO_CONN(c3));
+    connection_free_minimal(TO_CONN(c3));
   tor_free(idp);
   tor_free(idp2);
   connection_or_clear_ext_or_id_map();
@@ -72,13 +73,13 @@ test_ext_or_id_map(void *arg)
  * writes to outbuf. */
 static void
 connection_write_to_buf_impl_replacement(const char *string, size_t len,
-                                         connection_t *conn, int zlib)
+                                         connection_t *conn, int compressed)
 {
-  (void) zlib;
+  (void) compressed;
 
   tor_assert(string);
   tor_assert(conn);
-  write_to_buf(string, len, conn->outbuf);
+  buf_add(conn->outbuf, string, len);
 }
 
 static char *
@@ -89,7 +90,7 @@ buf_get_contents(buf_t *buf, size_t *sz_out)
   if (*sz_out >= ULONG_MAX)
     return NULL; /* C'mon, really? */
   out = tor_malloc(*sz_out + 1);
-  if (fetch_from_buf(out, (unsigned long)*sz_out, buf) != 0) {
+  if (buf_get_bytes(buf, out, (unsigned long)*sz_out) != 0) {
     tor_free(out);
     return NULL;
   }
@@ -145,7 +146,7 @@ test_ext_or_write_command(void *arg)
 
  done:
   if (c1)
-    connection_free_(TO_CONN(c1));
+    connection_free_minimal(TO_CONN(c1));
   tor_free(cp);
   tor_free(buf);
   UNMOCK(connection_write_to_buf_impl_);
@@ -399,14 +400,14 @@ handshake_start(or_connection_t *conn, int receiving)
 
 #define WRITE(s,n)                                                      \
   do {                                                                  \
-    write_to_buf((s), (n), TO_CONN(conn)->inbuf);                       \
+    buf_add(TO_CONN(conn)->inbuf, (s), (n));                           \
   } while (0)
 #define CONTAINS(s,n)                                           \
   do {                                                          \
     tt_int_op((n), OP_LE, sizeof(b));                              \
     tt_int_op(buf_datalen(TO_CONN(conn)->outbuf), OP_EQ, (n));     \
     if ((n)) {                                                  \
-      fetch_from_buf(b, (n), TO_CONN(conn)->outbuf);            \
+      buf_get_bytes(TO_CONN(conn)->outbuf, b, (n));                \
       tt_mem_op(b, OP_EQ, (s), (n));                               \
     }                                                           \
   } while (0)
@@ -497,14 +498,14 @@ test_ext_or_handshake(void *arg)
            "te road There is always another ", 64);
   /* Send the wrong response. */
   WRITE("not with a bang but a whimper...", 32);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("\x00", 1);
   tt_assert(TO_CONN(conn)->marked_for_close);
   /* XXXX Hold-open-until-flushed. */
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
   MOCK(connection_start_reading, note_read_started);
   MOCK(connection_stop_reading, note_read_stopped);
@@ -552,26 +553,26 @@ test_ext_or_handshake(void *arg)
   do_ext_or_handshake(conn);
   /* USERADDR command with an extra NUL byte */
   WRITE("\x00\x01\x00\x0d""1.2.3.4:5678\x00", 17);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("", 0);
   tt_assert(TO_CONN(conn)->marked_for_close);
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
   /* Now fail the TRANSPORT command. */
   conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET);
   do_ext_or_handshake(conn);
   /* TRANSPORT command with an extra NUL byte */
   WRITE("\x00\x02\x00\x08""rfc1149\x00", 12);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("", 0);
   tt_assert(TO_CONN(conn)->marked_for_close);
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
   /* Now fail the TRANSPORT command. */
   conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET);
@@ -579,19 +580,19 @@ test_ext_or_handshake(void *arg)
   /* TRANSPORT command with transport name with symbols (not a
      C-identifier) */
   WRITE("\x00\x02\x00\x07""rf*1149", 11);
-  MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem);
+  MOCK(control_event_bootstrap_prob_or, ignore_bootstrap_problem);
   tt_int_op(-1, OP_EQ, connection_ext_or_process_inbuf(conn));
   CONTAINS("", 0);
   tt_assert(TO_CONN(conn)->marked_for_close);
   close_closeable_connections();
   conn = NULL;
-  UNMOCK(control_event_bootstrap_problem);
+  UNMOCK(control_event_bootstrap_prob_or);
 
  done:
   UNMOCK(connection_write_to_buf_impl_);
   UNMOCK(crypto_rand);
   if (conn)
-    connection_free_(TO_CONN(conn));
+    connection_free_minimal(TO_CONN(conn));
 #undef CONTAINS
 #undef WRITE
 }
diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c
new file mode 100644
index 0000000000..6f849f436b
--- /dev/null
+++ b/src/test/test_geoip.c
@@ -0,0 +1,578 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+/* These macros pull in declarations for some functions and structures that
+ * are typically file-private. */
+#define GEOIP_PRIVATE
+#include "or.h"
+#include "config.h"
+#include "geoip.h"
+#include "test.h"
+
+  /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs
+   * using ipv4.  Since our fake geoip database is the same between
+   * ipv4 and ipv6, we should get the same result no matter which
+   * address family we pick for each IP. */
+#define SET_TEST_ADDRESS(i) do {                \
+    if ((i) & 1) {                              \
+      SET_TEST_IPV6(i);                         \
+      tor_addr_from_in6(&addr, &in6);           \
+    } else {                                    \
+      tor_addr_from_ipv4h(&addr, (uint32_t) i); \
+    }                                           \
+  } while (0)
+
+  /* Make sure that country ID actually works. */
+#define SET_TEST_IPV6(i) \
+  do {                                                          \
+    set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i)));        \
+  } while (0)
+#define CHECK_COUNTRY(country, val) do {                                \
+    /* test ipv4 country lookup */                                      \
+    tt_str_op(country, OP_EQ,                                              \
+               geoip_get_country_name(geoip_get_country_by_ipv4(val))); \
+    /* test ipv6 country lookup */                                      \
+    SET_TEST_IPV6(val);                                                 \
+    tt_str_op(country, OP_EQ,                                              \
+               geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \
+  } while (0)
+
+/** Run unit tests for GeoIP code. */
+static void
+test_geoip(void *arg)
+{
+  int i, j;
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  char *s = NULL, *v = NULL;
+  const char *bridge_stats_1 =
+      "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "bridge-ips zz=24,xy=8\n"
+      "bridge-ip-versions v4=16,v6=16\n"
+      "bridge-ip-transports <OR>=24\n",
+  *dirreq_stats_1 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips ab=8\n"
+      "dirreq-v3-reqs ab=8\n"
+      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
+  *dirreq_stats_2 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips \n"
+      "dirreq-v3-reqs \n"
+      "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
+  *dirreq_stats_3 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips \n"
+      "dirreq-v3-reqs \n"
+      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n",
+  *dirreq_stats_4 =
+      "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "dirreq-v3-ips \n"
+      "dirreq-v3-reqs \n"
+      "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0,"
+          "not-modified=0,busy=0\n"
+      "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n"
+      "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n",
+  *entry_stats_1 =
+      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "entry-ips ab=8\n",
+  *entry_stats_2 =
+      "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+      "entry-ips \n";
+  tor_addr_t addr;
+  struct in6_addr in6;
+
+  /* Populate the DB a bit.  Add these in order, since we can't do the final
+   * 'sort' step.  These aren't very good IP addresses, but they're perfectly
+   * fine uint32_t values. */
+  (void)arg;
+  tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET));
+
+  /* Populate the IPv6 DB equivalently with fake IPs in the same range */
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6));
+  tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6));
+
+  /* We should have 4 countries: ??, ab, xy, zz. */
+  tt_int_op(4,OP_EQ, geoip_get_n_countries());
+  memset(&in6, 0, sizeof(in6));
+
+  CHECK_COUNTRY("??", 3);
+  CHECK_COUNTRY("ab", 32);
+  CHECK_COUNTRY("??", 5);
+  CHECK_COUNTRY("??", 51);
+  CHECK_COUNTRY("xy", 150);
+  CHECK_COUNTRY("xy", 190);
+  CHECK_COUNTRY("??", 2000);
+
+  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3));
+  SET_TEST_IPV6(3);
+  tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6));
+
+  get_options_mutable()->BridgeRelay = 1;
+  get_options_mutable()->BridgeRecordUsageByCountry = 1;
+  /* Put 9 observations in AB... */
+  for (i=32; i < 40; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
+  }
+  SET_TEST_ADDRESS(225);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
+  /* and 3 observations in XY, several times. */
+  for (j=0; j < 10; ++j)
+    for (i=52; i < 55; ++i) {
+      SET_TEST_ADDRESS(i);
+      geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600);
+    }
+  /* and 17 observations in ZZ... */
+  for (i=110; i < 127; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  }
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
+  tt_assert(s);
+  tt_assert(v);
+  tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s);
+  tt_str_op("v4=16,v6=16",OP_EQ, v);
+  tor_free(s);
+  tor_free(v);
+
+  /* Now clear out all the AB observations. */
+  geoip_remove_old_clients(now-6000);
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v);
+  tt_assert(s);
+  tt_assert(v);
+  tt_str_op("zz=24,xy=8",OP_EQ, s);
+  tt_str_op("v4=16,v6=16",OP_EQ, v);
+  tor_free(s);
+  tor_free(v);
+
+  /* Start testing bridge statistics by making sure that we don't output
+   * bridge stats without initializing them. */
+  s = geoip_format_bridge_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats and generate the bridge-stats history string out of
+   * the connecting clients added above. */
+  geoip_bridge_stats_init(now);
+  s = geoip_format_bridge_stats(now + 86400);
+  tt_assert(s);
+  tt_str_op(bridge_stats_1,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting bridge stats and make sure we don't write a history
+   * string anymore. */
+  geoip_bridge_stats_term();
+  s = geoip_format_bridge_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Stop being a bridge and start being a directory mirror that gathers
+   * directory request statistics. */
+  geoip_bridge_stats_term();
+  get_options_mutable()->BridgeRelay = 0;
+  get_options_mutable()->BridgeRecordUsageByCountry = 0;
+  get_options_mutable()->DirReqStatistics = 1;
+
+  /* Start testing dirreq statistics by making sure that we don't collect
+   * dirreq stats without initializing them. */
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note one connecting client, and generate the
+   * dirreq-stats history string. */
+  geoip_dirreq_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_1,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add another connecting client, and ensure we
+   * don't generate a history string. */
+  geoip_dirreq_stats_term();
+  SET_TEST_ADDRESS(101);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add a connecting client, reset stats, and make sure
+   * that we get an all empty history string. */
+  geoip_dirreq_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now);
+  geoip_reset_dirreq_stats(now);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_2,OP_EQ, s);
+  tor_free(s);
+
+  /* Note a successful network status response and make sure that it
+   * appears in the history string. */
+  geoip_note_ns_response(GEOIP_SUCCESS);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_3,OP_EQ, s);
+  tor_free(s);
+
+  /* Start a tunneled directory request. */
+  geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED);
+  s = geoip_format_dirreq_stats(now + 86400);
+  tt_str_op(dirreq_stats_4,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting directory request statistics and start gathering
+   * entry stats. */
+  geoip_dirreq_stats_term();
+  get_options_mutable()->DirReqStatistics = 0;
+  get_options_mutable()->EntryStatistics = 1;
+
+  /* Start testing entry statistics by making sure that we don't collect
+   * anything without initializing entry stats. */
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note one connecting client, and generate the
+   * entry-stats history string. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_str_op(entry_stats_1,OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add another connecting client, and ensure we
+   * don't generate a history string. */
+  geoip_entry_stats_term();
+  SET_TEST_ADDRESS(101);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add a connecting client, reset stats, and make sure
+   * that we get an all empty history string. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now);
+  geoip_reset_entry_stats(now);
+  s = geoip_format_entry_stats(now + 86400);
+  tt_str_op(entry_stats_2,OP_EQ, s);
+  tor_free(s);
+
+  /* Test the OOM handler. Add a client, run the OOM. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
+                         now - (12 * 60 * 60));
+  /* We've seen this 12 hours ago. Run the OOM, it should clean the entry
+   * because it is above the minimum cutoff of 4 hours. */
+  size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000);
+  tt_size_op(bytes_removed, OP_GT, 0);
+
+  /* Do it again but this time with an entry with a lower cutoff. */
+  geoip_entry_stats_init(now);
+  SET_TEST_ADDRESS(100);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
+                         now - (3 * 60 * 60));
+  bytes_removed = geoip_client_cache_handle_oom(now, 1000);
+  tt_size_op(bytes_removed, OP_EQ, 0);
+
+  /* Stop collecting entry statistics. */
+  geoip_entry_stats_term();
+  get_options_mutable()->EntryStatistics = 0;
+
+ done:
+  tor_free(s);
+  tor_free(v);
+}
+
+static void
+test_geoip_with_pt(void *arg)
+{
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  char *s = NULL;
+  int i;
+  tor_addr_t addr;
+  struct in6_addr in6;
+
+  (void)arg;
+  get_options_mutable()->BridgeRelay = 1;
+  get_options_mutable()->BridgeRecordUsageByCountry = 1;
+
+  memset(&in6, 0, sizeof(in6));
+
+  /* No clients seen yet. */
+  s = geoip_get_transport_history();
+  tor_assert(!s);
+
+  /* 4 connections without a pluggable transport */
+  for (i=0; i < 4; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200);
+  }
+
+  /* 9 connections with "alpha" */
+  for (i=4; i < 13; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200);
+  }
+
+  /* one connection with "beta" */
+  SET_TEST_ADDRESS(13);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200);
+
+  /* 14 connections with "charlie" */
+  for (i=14; i < 28; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200);
+  }
+
+  /* 131 connections with "ddr" */
+  for (i=28; i < 159; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200);
+  }
+
+  /* 8 connections with "entropy" */
+  for (i=159; i < 167; ++i) {
+    SET_TEST_ADDRESS(i);
+    geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200);
+  }
+
+  /* 2 connections from the same IP with two different transports. */
+  SET_TEST_ADDRESS(++i);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200);
+  geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200);
+
+  /* Test the transport history string. */
+  s = geoip_get_transport_history();
+  tor_assert(s);
+  tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136,"
+             "entropy=8,fire=8,google=8");
+
+  /* Stop collecting entry statistics. */
+  geoip_entry_stats_term();
+  get_options_mutable()->EntryStatistics = 0;
+
+ done:
+  tor_free(s);
+}
+
+#undef SET_TEST_ADDRESS
+#undef SET_TEST_IPV6
+#undef CHECK_COUNTRY
+
+static const char GEOIP_CONTENT[] =
+  "134445936,134445939,MP\n"
+  "134445940,134447103,GU\n"
+  "134447104,134738943,US\n"
+  "134738944,134739199,CA\n"
+  "134739200,135192575,US\n"
+  "135192576,135200767,MX\n"
+  "135200768,135430143,US\n"
+  "135430144,135430399,CA\n"
+  "135430400,135432191,US\n";
+
+static void
+test_geoip_load_file(void *arg)
+{
+  (void)arg;
+  char *contents = NULL;
+  char *dhex = NULL;
+
+  /* A nonexistant filename should fail. */
+  tt_int_op(-1, OP_EQ,
+            geoip_load_file(AF_INET, "/you/did/not/put/a/file/here/I/hope"));
+
+  /* We start out with only "Ningunpartia" in the database. */
+  tt_int_op(1, OP_EQ, geoip_get_n_countries());
+  tt_str_op("??", OP_EQ, geoip_get_country_name(0));
+  /* Any lookup attempt should say "-1" because we have no info */
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304));
+  /* There should be no 'digest' for a nonexistant file */
+  tt_str_op("0000000000000000000000000000000000000000", OP_EQ,
+            geoip_db_digest(AF_INET));
+
+  const char *fname = get_fname("geoip");
+  tt_int_op(0, OP_EQ, write_str_to_file(fname, GEOIP_CONTENT, 1));
+
+  int rv = geoip_load_file(AF_INET, fname);
+  if (rv != 0) {
+    TT_GRIPE(("Unable to load geoip from %s", escaped(fname)));
+  }
+  tt_int_op(0, OP_EQ, rv);
+
+  /* Check that we loaded some countries; this will fail if there are ever
+   * fewer than 5 countries in our test above. */
+  tt_int_op(geoip_get_n_countries(), OP_GE, 5);
+
+  /* Let's see where 8.8.8.8 is. */
+  int country = geoip_get_country_by_ipv4(0x08080808);
+  tt_int_op(country, OP_GE, 1); /* It shouldn't be 'unknown' or 'nowhere' */
+  const char *cc = geoip_get_country_name(country);
+  tt_int_op(strlen(cc), OP_EQ, 2);
+
+  /* The digest should be set.... */
+  tt_str_op("0000000000000000000000000000000000000000", OP_NE,
+            geoip_db_digest(AF_INET));
+
+  /* And it should be set correctly */
+  contents = read_file_to_str(fname, RFTS_BIN, NULL);
+  uint8_t d[DIGEST_LEN];
+  crypto_digest((char*)d, contents, strlen(contents));
+  dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN));
+  tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET));
+
+  /* Make sure geoip_free_all() works. */
+  geoip_free_all();
+  tt_int_op(1, OP_EQ, geoip_get_n_countries());
+  tt_str_op("??", OP_EQ, geoip_get_country_name(0));
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304));
+  tt_str_op("0000000000000000000000000000000000000000", OP_EQ,
+            geoip_db_digest(AF_INET)); // <--- nick bets this will fail.
+
+ done:
+  tor_free(contents);
+  tor_free(dhex);
+}
+
+static void
+test_geoip6_load_file(void *arg)
+{
+  (void)arg;
+  struct in6_addr iaddr6;
+  char *contents = NULL;
+  char *dhex = NULL;
+
+  /* A nonexistant filename should fail. */
+  tt_int_op(-1, OP_EQ,
+            geoip_load_file(AF_INET6, "/you/did/not/put/a/file/here/I/hope"));
+
+  /* Any lookup attempt should say "-1" because we have no info */
+  tor_inet_pton(AF_INET6, "2001:4860:4860::8888", &iaddr6);
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6));
+
+  /* Load geiop6 file */
+  const char *fname6 = get_fname("geoip6");
+  const char CONTENT[] =
+    "2001:4830:6010::,2001:4830:601f:ffff:ffff:ffff:ffff:ffff,GB\n"
+    "2001:4830:6020::,2001:4830:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4838::,2001:4838:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4840::,2001:4840:ffff:ffff:ffff:ffff:ffff:ffff,XY\n"
+    "2001:4848::,2001:4848:ffff:ffff:ffff:ffff:ffff:ffff,ZD\n"
+    "2001:4850::,2001:4850:ffff:ffff:ffff:ffff:ffff:ffff,RO\n"
+    "2001:4858::,2001:4858:ffff:ffff:ffff:ffff:ffff:ffff,TC\n"
+    "2001:4860::,2001:4860:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4868::,2001:4868:ffff:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4870::,2001:4871:ffff:ffff:ffff:ffff:ffff:ffff,NB\n"
+    "2001:4878::,2001:4878:128:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4878:129::,2001:4878:129:ffff:ffff:ffff:ffff:ffff,CR\n"
+    "2001:4878:12a::,2001:4878:203:ffff:ffff:ffff:ffff:ffff,US\n"
+    "2001:4878:204::,2001:4878:204:ffff:ffff:ffff:ffff:ffff,DE\n"
+    "2001:4878:205::,2001:4878:214:ffff:ffff:ffff:ffff:ffff,US\n";
+  tt_int_op(0, OP_EQ, write_str_to_file(fname6, CONTENT, 1));
+
+  tt_int_op(0, OP_EQ, geoip_load_file(AF_INET6, fname6));
+
+  /* Check that we loaded some countries; this will fail if there are ever
+   * fewer than 5 countries in our test data above. */
+  tt_int_op(geoip_get_n_countries(), OP_GE, 5);
+
+  /* Let's see where 2001:4860:4860::8888 (google dns) is. */
+  const char *caddr6 = "2001:4860:4860::8888";
+  tor_inet_pton(AF_INET6, caddr6, &iaddr6);
+  int country6 = geoip_get_country_by_ipv6(&iaddr6);
+  tt_int_op(country6, OP_GE, 1);
+
+  const char *cc6 = geoip_get_country_name(country6);
+  tt_int_op(strlen(cc6), OP_EQ, 2);
+
+  /* The digest should be set.... */
+  tt_str_op("0000000000000000000000000000000000000000", OP_NE,
+            geoip_db_digest(AF_INET6));
+
+  /* And it should be set correctly */
+  contents = read_file_to_str(fname6, RFTS_BIN, NULL);
+  uint8_t d[DIGEST_LEN];
+  crypto_digest((char*)d, contents, strlen(contents));
+  dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN));
+  tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET6));
+
+  /* Make sure geoip_free_all() works. */
+  geoip_free_all();
+  tt_int_op(1, OP_EQ, geoip_get_n_countries());
+  tt_str_op("??", OP_EQ, geoip_get_country_name(0));
+  tor_inet_pton(AF_INET6, "::1:2:3:4", &iaddr6);
+  tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6));
+  tt_str_op("0000000000000000000000000000000000000000", OP_EQ,
+            geoip_db_digest(AF_INET6));
+
+ done:
+  tor_free(contents);
+  tor_free(dhex);
+}
+
+static void
+test_geoip_load_2nd_file(void *arg)
+{
+  (void)arg;
+
+  char *fname_geoip = tor_strdup(get_fname("geoip_data"));
+  char *fname_empty = tor_strdup(get_fname("geoip_empty"));
+
+  tt_int_op(0, OP_EQ, write_str_to_file(fname_geoip, GEOIP_CONTENT, 1));
+  tt_int_op(0, OP_EQ, write_str_to_file(fname_empty, "\n", 1));
+
+  /* Load 1st geoip file */
+  tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_geoip));
+
+  /* Load 2nd geoip (empty) file */
+  /* It has to be the same IP address family */
+  tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_empty));
+
+  /* Check that there is no geoip information for 8.8.8.8, */
+  /* since loading the empty 2nd file should have delete it. */
+  int country = geoip_get_country_by_ipv4(0x08080808);
+  tt_int_op(country, OP_EQ, 0);
+
+ done:
+  tor_free(fname_geoip);
+  tor_free(fname_empty);
+}
+
+#define ENT(name)                                                       \
+  { #name, test_ ## name , 0, NULL, NULL }
+#define FORK(name)                                                      \
+  { #name, test_ ## name , TT_FORK, NULL, NULL }
+
+struct testcase_t geoip_tests[] = {
+  { "geoip", test_geoip, TT_FORK, NULL, NULL },
+  { "geoip_with_pt", test_geoip_with_pt, TT_FORK, NULL, NULL },
+  { "load_file", test_geoip_load_file, TT_FORK, NULL, NULL },
+  { "load_file6", test_geoip6_load_file, TT_FORK, NULL, NULL },
+  { "load_2nd_file", test_geoip_load_2nd_file, TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c
index 8173e44d47..51ca8f08ec 100644
--- a/src/test/test_guardfraction.c
+++ b/src/test/test_guardfraction.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRSERV_PRIVATE
@@ -38,10 +38,10 @@ gen_vote_routerstatus_for_tests(const char *digest_in_hex, int is_guard)
     rs->is_possible_guard = is_guard;
 
     /* Fill in the fpr */
-    tt_int_op(strlen(digest_in_hex), ==, HEX_DIGEST_LEN);
+    tt_int_op(strlen(digest_in_hex), OP_EQ, HEX_DIGEST_LEN);
     retval = base16_decode(digest_tmp, sizeof(digest_tmp),
                            digest_in_hex, HEX_DIGEST_LEN);
-    tt_int_op(retval, ==, sizeof(digest_tmp));
+    tt_int_op(retval, OP_EQ, sizeof(digest_tmp));
     memcpy(rs->identity_digest, digest_tmp, DIGEST_LEN);
   }
 
@@ -88,7 +88,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, -1);
+  tt_int_op(retval, OP_EQ, -1);
   tor_free(guardfraction_bad);
 
   /* This one does not have a date! Parsing should fail. */
@@ -100,7 +100,7 @@ test_parse_guardfraction_file_bad(void *arg)
                "guard-seen 07B5547026DF3E229806E135CFA8552D56AFBABC 5 420\n");
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, -1);
+  tt_int_op(retval, OP_EQ, -1);
   tor_free(guardfraction_bad);
 
   /* This one has an incomplete n-inputs line, but parsing should
@@ -114,7 +114,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 2);
+  tt_int_op(retval, OP_EQ, 2);
   tor_free(guardfraction_bad);
 
   /* This one does not have a fingerprint in the guard line! */
@@ -126,7 +126,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 0);
+  tt_int_op(retval, OP_EQ, 0);
   tor_free(guardfraction_bad);
 
   /* This one does not even have an integer guardfraction value. */
@@ -139,7 +139,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 1);
+  tt_int_op(retval, OP_EQ, 1);
   tor_free(guardfraction_bad);
 
   /* This one is not a percentage (not in [0, 100]) */
@@ -152,7 +152,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 1);
+  tt_int_op(retval, OP_EQ, 1);
   tor_free(guardfraction_bad);
 
   /* This one is not a percentage either (not in [0, 100]) */
@@ -164,7 +164,7 @@ test_parse_guardfraction_file_bad(void *arg)
                yesterday_date_str);
 
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_bad, NULL);
-  tt_int_op(retval, ==, 0);
+  tt_int_op(retval, OP_EQ, 0);
 
  done:
   tor_free(guardfraction_bad);
@@ -216,14 +216,14 @@ test_parse_guardfraction_file_good(void *arg)
   /* Read the guardfraction file */
   retval = dirserv_read_guardfraction_file_from_str(guardfraction_good,
                                                     routerstatuses);
-  tt_int_op(retval, ==, 1);
+  tt_int_op(retval, OP_EQ, 1);
 
   { /* Test that routerstatus fields got filled properly */
 
     /* The guardfraction fields of the guard should be filled. */
     tt_assert(vrs_guard->status.has_guardfraction);
     tt_int_op(vrs_guard->status.guardfraction_percentage,
-              ==,
+              OP_EQ,
               guardfraction_value);
 
     /* The guard that was not in the guardfraction file should not have
@@ -252,12 +252,12 @@ test_get_guardfraction_bandwidth(void *arg)
   guard_get_guardfraction_bandwidth(&gf_bw,
                                     orig_bw, 25);
 
-  tt_int_op(gf_bw.guard_bw, ==, 250);
-  tt_int_op(gf_bw.non_guard_bw, ==, 750);
+  tt_int_op(gf_bw.guard_bw, OP_EQ, 250);
+  tt_int_op(gf_bw.non_guard_bw, OP_EQ, 750);
 
   /* Also check the 'guard_bw + non_guard_bw == original_bw'
    * invariant. */
-  tt_int_op(gf_bw.non_guard_bw + gf_bw.guard_bw, ==, orig_bw);
+  tt_int_op(gf_bw.non_guard_bw + gf_bw.guard_bw, OP_EQ, orig_bw);
 
  done:
   ;
@@ -295,9 +295,9 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_good,
                                               NULL, NULL,
                                               &rs_good);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
     tt_assert(rs_good.has_guardfraction);
-    tt_int_op(rs_good.guardfraction_percentage, ==, 66);
+    tt_int_op(rs_good.guardfraction_percentage, OP_EQ, 66);
   }
 
   { /* Properly formatted GuardFraction but router is not a
@@ -309,7 +309,7 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_good,
                                               NULL, NULL,
                                               &rs_no_guard);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
     tt_assert(!rs_no_guard.has_guardfraction);
     expect_single_log_msg_containing("Got GuardFraction for non-guard . "
                                      "This is not supposed to happen.");
@@ -323,7 +323,7 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_bad1,
                                               NULL, NULL,
                                               &rs_bad1);
-    tt_int_op(retval, ==, -1);
+    tt_int_op(retval, OP_EQ, -1);
     tt_assert(!rs_bad1.has_guardfraction);
   }
 
@@ -334,7 +334,7 @@ test_parse_guardfraction_consensus(void *arg)
     retval = routerstatus_parse_guardfraction(guardfraction_str_bad2,
                                               NULL, NULL,
                                               &rs_bad2);
-    tt_int_op(retval, ==, -1);
+    tt_int_op(retval, OP_EQ, -1);
     tt_assert(!rs_bad2.has_guardfraction);
   }
 
@@ -375,27 +375,27 @@ test_should_apply_guardfraction(void *arg)
   /* If torrc option is set to yes, we should always use
    * guardfraction.*/
   options->UseGuardFraction = 1;
-  tt_int_op(should_apply_guardfraction(&vote_disabled), ==, 1);
+  tt_int_op(should_apply_guardfraction(&vote_disabled), OP_EQ, 1);
 
   /* If torrc option is set to no, we should never use
    * guardfraction.*/
   options->UseGuardFraction = 0;
-  tt_int_op(should_apply_guardfraction(&vote_enabled), ==, 0);
+  tt_int_op(should_apply_guardfraction(&vote_enabled), OP_EQ, 0);
 
   /* Now let's test torrc option set to auto. */
   options->UseGuardFraction = -1;
 
   /* If torrc option is set to auto, and consensus parameter is set to
    * yes, we should use guardfraction. */
-  tt_int_op(should_apply_guardfraction(&vote_enabled), ==, 1);
+  tt_int_op(should_apply_guardfraction(&vote_enabled), OP_EQ, 1);
 
   /* If torrc option is set to auto, and consensus parameter is set to
    * no, we should use guardfraction. */
-  tt_int_op(should_apply_guardfraction(&vote_disabled), ==, 0);
+  tt_int_op(should_apply_guardfraction(&vote_disabled), OP_EQ, 0);
 
   /* If torrc option is set to auto, and consensus parameter is not
    * set, we should fallback to "no". */
-  tt_int_op(should_apply_guardfraction(&vote_missing), ==, 0);
+  tt_int_op(should_apply_guardfraction(&vote_missing), OP_EQ, 0);
 
  done:
   SMARTLIST_FOREACH(vote_enabled.net_params, char *, cp, tor_free(cp));
diff --git a/src/test/test_handles.c b/src/test/test_handles.c
index 536a478689..eb1e1f1bbe 100644
--- a/src/test/test_handles.c
+++ b/src/test/test_handles.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -13,6 +13,8 @@ typedef struct demo_t {
 } demo_t;
 
 HANDLE_DECL(demo, demo_t, static)
+#define demo_handle_free(h)    \
+  FREE_AND_NULL(demo_handle_t, demo_handle_free_, (h))
 HANDLE_IMPL(demo, demo_t, static)
 
 static demo_t *
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index ae9fc7a243..1db5e9064f 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,14 +7,26 @@
  */
 
 #define ROUTERLIST_PRIVATE
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
+#define MAIN_PRIVATE
+
 #include "orconfig.h"
 #include "or.h"
 
-#include "routerlist.h"
+#include "buffers.h"
+#include "config.h"
+#include "confparse.h"
+#include "connection.h"
+#include "crypto_rand.h"
+#include "main.h"
 #include "nodelist.h"
+#include "relay.h"
+#include "routerlist.h"
 
 #include "test.h"
 #include "test_helpers.h"
+#include "test_connection.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 DISABLE_GCC_WARNING(overlength-strings)
@@ -22,6 +34,7 @@ DISABLE_GCC_WARNING(overlength-strings)
  * at large. */
 #endif
 #include "test_descriptors.inc"
+#include "circuitlist.h"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 ENABLE_GCC_WARNING(overlength-strings)
 #endif
@@ -70,16 +83,16 @@ helper_setup_fake_routerlist(void)
   retval = router_load_routers_from_string(TEST_DESCRIPTORS,
                                            NULL, SAVED_IN_JOURNAL,
                                            NULL, 0, NULL);
-  tt_int_op(retval, ==, HELPER_NUMBER_OF_DESCRIPTORS);
+  tt_int_op(retval, OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
 
   /* Sanity checking of routerlist and nodelist. */
   our_routerlist = router_get_routerlist();
-  tt_int_op(smartlist_len(our_routerlist->routers), ==,
+  tt_int_op(smartlist_len(our_routerlist->routers), OP_EQ,
               HELPER_NUMBER_OF_DESCRIPTORS);
   routerlist_assert_ok(our_routerlist);
 
   our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), ==, HELPER_NUMBER_OF_DESCRIPTORS);
+  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
 
   /* Mark all routers as non-guards but up and running! */
   SMARTLIST_FOREACH_BEGIN(our_nodelist, node_t *, node) {
@@ -92,3 +105,175 @@ helper_setup_fake_routerlist(void)
   UNMOCK(router_descriptor_is_older_than);
 }
 
+void
+connection_write_to_buf_mock(const char *string, size_t len,
+                             connection_t *conn, int compressed)
+{
+  (void) compressed;
+
+  tor_assert(string);
+  tor_assert(conn);
+
+  buf_add(conn->outbuf, string, len);
+}
+
+/* Set up a fake origin circuit with the specified number of cells,
+ * Return a pointer to the newly-created dummy circuit */
+circuit_t *
+dummy_origin_circuit_new(int n_cells)
+{
+  origin_circuit_t *circ = origin_circuit_new();
+  int i;
+  cell_t cell;
+
+  for (i=0; i < n_cells; ++i) {
+    crypto_rand((void*)&cell, sizeof(cell));
+    cell_queue_append_packed_copy(TO_CIRCUIT(circ),
+                                  &TO_CIRCUIT(circ)->n_chan_cells,
+                                  1, &cell, 1, 0);
+  }
+
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  return TO_CIRCUIT(circ);
+}
+
+/** Mock-replacement. As tor_addr_lookup, but always fails on any
+ * address containing a !.  This is necessary for running the unit tests
+ * on networks where DNS hijackers think it's helpful to give answers
+ * for things like 1.2.3.4.5 or "invalidstuff!!"
+ */
+int
+mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
+                                        uint16_t family, tor_addr_t *out)
+{
+  if (name && strchr(name, '!')) {
+    return -1;
+  }
+  return tor_addr_lookup__real(name, family, out);
+}
+
+/*********** Helper funcs for making new connections/streams *****************/
+
+/* Helper for test_conn_get_connection() */
+static int
+fake_close_socket(tor_socket_t sock)
+{
+  (void)sock;
+  return 0;
+}
+
+static int mock_connection_connect_sockaddr_called = 0;
+static int fake_socket_number = TEST_CONN_FD_INIT;
+
+/* Helper for test_conn_get_connection() */
+static int
+mock_connection_connect_sockaddr(connection_t *conn,
+                                 const struct sockaddr *sa,
+                                 socklen_t sa_len,
+                                 const struct sockaddr *bindaddr,
+                                 socklen_t bindaddr_len,
+                                 int *socket_error)
+{
+  (void)sa_len;
+  (void)bindaddr;
+  (void)bindaddr_len;
+
+  tor_assert(conn);
+  tor_assert(sa);
+  tor_assert(socket_error);
+
+  mock_connection_connect_sockaddr_called++;
+
+  conn->s = fake_socket_number++;
+  tt_assert(SOCKET_OK(conn->s));
+  /* We really should call tor_libevent_initialize() here. Because we don't,
+   * we are relying on other parts of the code not checking if the_event_base
+   * (and therefore event->ev_base) is NULL.  */
+  tt_int_op(connection_add_connecting(conn), OP_EQ, 0);
+
+ done:
+  /* Fake "connected" status */
+  return 1;
+}
+
+/** Create and return a new connection/stream */
+connection_t *
+test_conn_get_connection(uint8_t state, uint8_t type, uint8_t purpose)
+{
+  connection_t *conn = NULL;
+  tor_addr_t addr;
+  int socket_err = 0;
+  int in_progress = 0;
+
+  MOCK(connection_connect_sockaddr,
+       mock_connection_connect_sockaddr);
+  MOCK(tor_close_socket, fake_close_socket);
+
+  init_connection_lists();
+
+  conn = connection_new(type, TEST_CONN_FAMILY);
+  tt_assert(conn);
+
+  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS, TEST_CONN_FAMILY, &addr);
+  tt_assert(!tor_addr_is_null(&addr));
+
+  tor_addr_copy_tight(&conn->addr, &addr);
+  conn->port = TEST_CONN_PORT;
+  mock_connection_connect_sockaddr_called = 0;
+  in_progress = connection_connect(conn, TEST_CONN_ADDRESS_PORT, &addr,
+                                   TEST_CONN_PORT, &socket_err);
+  tt_int_op(mock_connection_connect_sockaddr_called, OP_EQ, 1);
+  tt_assert(!socket_err);
+  tt_assert(in_progress == 0 || in_progress == 1);
+
+  /* fake some of the attributes so the connection looks OK */
+  conn->state = state;
+  conn->purpose = purpose;
+  assert_connection_ok(conn, time(NULL));
+
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(tor_close_socket);
+  return conn;
+
+  /* On failure */
+ done:
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(tor_close_socket);
+  return NULL;
+}
+
+/* Helper function to parse a set of torrc options in a text format and return
+ * a newly allocated or_options_t object containing the configuration. On
+ * error, NULL is returned indicating that the conf couldn't be parsed
+ * properly. */
+or_options_t *
+helper_parse_options(const char *conf)
+{
+  int ret = 0;
+  char *msg = NULL;
+  or_options_t *opt = NULL;
+  config_line_t *line = NULL;
+
+  /* Kind of pointless to call this with a NULL value. */
+  tt_assert(conf);
+
+  opt = options_new();
+  tt_assert(opt);
+  ret = config_get_lines(conf, &line, 1);
+  if (ret != 0) {
+    goto done;
+  }
+  ret = config_assign(&options_format, opt, line, 0, &msg);
+  if (ret != 0) {
+    goto done;
+  }
+
+ done:
+  config_free_lines(line);
+  if (ret != 0) {
+    or_options_free(opt);
+    opt = NULL;
+  }
+  return opt;
+}
+
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 684375e1b1..9bc8553257 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -1,17 +1,32 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_HELPERS_H
 #define TOR_TEST_HELPERS_H
 
+#include "or.h"
+
 const char *get_yesterday_date_str(void);
 
+circuit_t * dummy_origin_circuit_new(int num_cells);
+
 /* Number of descriptors contained in test_descriptors.txt. */
 #define HELPER_NUMBER_OF_DESCRIPTORS 8
 
 void helper_setup_fake_routerlist(void);
 
+#define GET(path) "GET " path " HTTP/1.0\r\n\r\n"
+void connection_write_to_buf_mock(const char *string, size_t len,
+                                  connection_t *conn, int compressed);
+
+int mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
+                                            uint16_t family, tor_addr_t *out);
+
+connection_t *test_conn_get_connection(uint8_t state,
+                                       uint8_t type, uint8_t purpose);
+or_options_t *helper_parse_options(const char *conf);
+
 extern const char TEST_DESCRIPTORS[];
 
-#endif
+#endif /* !defined(TOR_TEST_HELPERS_H) */
 
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index d572dd8d2e..07daebc164 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,12 +8,15 @@
 
 #define CONTROL_PRIVATE
 #define CIRCUITBUILD_PRIVATE
+#define RENDCOMMON_PRIVATE
 #define RENDSERVICE_PRIVATE
+#define HS_SERVICE_PRIVATE
 
 #include "or.h"
 #include "test.h"
 #include "control.h"
 #include "config.h"
+#include "hs_common.h"
 #include "rendcommon.h"
 #include "rendservice.h"
 #include "routerset.h"
@@ -31,8 +34,9 @@
 #define STR_HSDIR_NONE_EXIST_LONGNAME \
                        "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
 
-/* DuckDuckGo descriptor as an example. */
-static const char *hs_desc_content = "\
+/* DuckDuckGo descriptor as an example. This one has extra "\r" at the end so
+ * the control port is happy. */
+static const char *hs_desc_content_control = "\
 rendezvous-service-descriptor g5ojobzupf275beh5ra72uyhb3dkpxwg\r\n\
 version 2\r\n\
 permanent-key\r\n\
@@ -93,6 +97,68 @@ PcftsZf2ztN0sbNCtPgDL3d0PqvxY3iHTQAI8EbaGq/IAJUZ8U4y963dD5+Bn6JQ\r\n\
 myE3ctmh0vy5+QxSiRjmQBkuEpCyks7LvWvHYrhnmcg=\r\n\
 -----END SIGNATURE-----";
 
+/* DuckDuckGo descriptor as an example. */
+static const char *hs_desc_content = "\
+rendezvous-service-descriptor g5ojobzupf275beh5ra72uyhb3dkpxwg\n\
+version 2\n\
+permanent-key\n\
+-----BEGIN RSA PUBLIC KEY-----\n\
+MIGJAoGBAJ/SzzgrXPxTlFrKVhXh3buCWv2QfcNgncUpDpKouLn3AtPH5Ocys0jE\n\
+aZSKdvaiQ62md2gOwj4x61cFNdi05tdQjS+2thHKEm/KsB9BGLSLBNJYY356bupg\n\
+I5gQozM65ENelfxYlysBjJ52xSDBd8C4f/p9umdzaaaCmzXG/nhzAgMBAAE=\n\
+-----END RSA PUBLIC KEY-----\n\
+secret-id-part anmjoxxwiupreyajjt5yasimfmwcnxlf\n\
+publication-time 2015-03-11 19:00:00\n\
+protocol-versions 2,3\n\
+introduction-points\n\
+-----BEGIN MESSAGE-----\n\
+aW50cm9kdWN0aW9uLXBvaW50IDd1bnd4cmg2dG5kNGh6eWt1Z3EzaGZzdHduc2ll\n\
+cmhyCmlwLWFkZHJlc3MgMTg4LjEzOC4xMjEuMTE4Cm9uaW9uLXBvcnQgOTAwMQpv\n\
+bmlvbi1rZXkKLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pBb0dC\n\
+QUxGRVVyeVpDbk9ROEhURmV5cDVjMTRObWVqL1BhekFLTTBxRENTNElKUWh0Y3g1\n\
+NXpRSFdOVWIKQ2hHZ0JqR1RjV3ZGRnA0N3FkdGF6WUZhVXE2c0lQKzVqeWZ5b0Q4\n\
+UmJ1bzBwQmFWclJjMmNhYUptWWM0RDh6Vgpuby9sZnhzOVVaQnZ1cWY4eHIrMDB2\n\
+S0JJNmFSMlA2OE1WeDhrMExqcUpUU2RKOE9idm9yQWdNQkFBRT0KLS0tLS1FTkQg\n\
+UlNBIFBVQkxJQyBLRVktLS0tLQpzZXJ2aWNlLWtleQotLS0tLUJFR0lOIFJTQSBQ\n\
+VUJMSUMgS0VZLS0tLS0KTUlHSkFvR0JBTnJHb0ozeTlHNXQzN2F2ekI1cTlwN1hG\n\
+VUplRUVYMUNOaExnWmJXWGJhVk5OcXpoZFhyL0xTUQppM1Z6dW5OaUs3cndUVnE2\n\
+K2QyZ1lRckhMMmIvMXBBY3ZKWjJiNSs0bTRRc0NibFpjRENXTktRbHJnRWN5WXRJ\n\
+CkdscXJTbFFEaXA0ZnNrUFMvNDVkWTI0QmJsQ3NGU1k3RzVLVkxJck4zZFpGbmJr\n\
+NEZIS1hBZ01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCmludHJv\n\
+ZHVjdGlvbi1wb2ludCBiNGM3enlxNXNheGZzN2prNXFibG1wN3I1b3pwdHRvagpp\n\
+cC1hZGRyZXNzIDEwOS4xNjkuNDUuMjI2Cm9uaW9uLXBvcnQgOTAwMQpvbmlvbi1r\n\
+ZXkKLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tCk1JR0pBb0dCQU8xSXpw\n\
+WFFUTUY3RXZUb1NEUXpzVnZiRVFRQUQrcGZ6NzczMVRXZzVaUEJZY1EyUkRaeVp4\n\
+OEQKNUVQSU1FeUE1RE83cGd0ak5LaXJvYXJGMC8yempjMkRXTUlSaXZyU29YUWVZ\n\
+ZXlMM1pzKzFIajJhMDlCdkYxZAp6MEswblRFdVhoNVR5V3lyMHdsbGI1SFBnTlI0\n\
+MS9oYkprZzkwZitPVCtIeGhKL1duUml2QWdNQkFBRT0KLS0tLS1FTkQgUlNBIFBV\n\
+QkxJQyBLRVktLS0tLQpzZXJ2aWNlLWtleQotLS0tLUJFR0lOIFJTQSBQVUJMSUMg\n\
+S0VZLS0tLS0KTUlHSkFvR0JBSzNWZEJ2ajFtQllLL3JrcHNwcm9Ub0llNUtHVmth\n\
+QkxvMW1tK1I2YUVJek1VZFE1SjkwNGtyRwpCd3k5NC8rV0lGNFpGYXh5Z2phejl1\n\
+N2pKY1k3ZGJhd1pFeG1hYXFCRlRwL2h2ZG9rcHQ4a1ByRVk4OTJPRHJ1CmJORUox\n\
+N1FPSmVMTVZZZk5Kcjl4TWZCQ3JQai8zOGh2RUdrbWVRNmRVWElvbVFNaUJGOVRB\n\
+Z01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tCmludHJvZHVjdGlv\n\
+bi1wb2ludCBhdjVtcWl0Y2Q3cjJkandsYmN0c2Jlc2R3eGt0ZWtvegppcC1hZGRy\n\
+ZXNzIDE0NC43Ni44LjczCm9uaW9uLXBvcnQgNDQzCm9uaW9uLWtleQotLS0tLUJF\n\
+R0lOIFJTQSBQVUJMSUMgS0VZLS0tLS0KTUlHSkFvR0JBTzVweVZzQmpZQmNmMXBE\n\
+dklHUlpmWXUzQ05nNldka0ZLMGlvdTBXTGZtejZRVDN0NWhzd3cyVwpjejlHMXhx\n\
+MmN0Nkd6VWkrNnVkTDlITTRVOUdHTi9BbW8wRG9GV1hKWHpBQkFXd2YyMVdsd1lW\n\
+eFJQMHRydi9WCkN6UDkzcHc5OG5vSmdGUGRUZ05iMjdKYmVUZENLVFBrTEtscXFt\n\
+b3NveUN2RitRa25vUS9BZ01CQUFFPQotLS0tLUVORCBSU0EgUFVCTElDIEtFWS0t\n\
+LS0tCnNlcnZpY2Uta2V5Ci0tLS0tQkVHSU4gUlNBIFBVQkxJQyBLRVktLS0tLQpN\n\
+SUdKQW9HQkFMVjNKSmtWN3lTNU9jc1lHMHNFYzFQOTVRclFRR3ZzbGJ6Wi9zRGxl\n\
+RlpKYXFSOUYvYjRUVERNClNGcFMxcU1GbldkZDgxVmRGMEdYRmN2WVpLamRJdHU2\n\
+SndBaTRJeEhxeXZtdTRKdUxrcXNaTEFLaXRLVkx4eGsKeERlMjlDNzRWMmJrOTRJ\n\
+MEgybTNKS2tzTHVwc3VxWWRVUmhOVXN0SElKZmgyZmNIalF0bEFnTUJBQUU9Ci0t\n\
+LS0tRU5EIFJTQSBQVUJMSUMgS0VZLS0tLS0KCg==\n\
+-----END MESSAGE-----\n\
+signature\n\
+-----BEGIN SIGNATURE-----\n\
+d4OuCE5OLAOnRB6cQN6WyMEmg/BHem144Vec+eYgeWoKwx3MxXFplUjFxgnMlmwN\n\
+PcftsZf2ztN0sbNCtPgDL3d0PqvxY3iHTQAI8EbaGq/IAJUZ8U4y963dD5+Bn6JQ\n\
+myE3ctmh0vy5+QxSiRjmQBkuEpCyks7LvWvHYrhnmcg=\n\
+-----END SIGNATURE-----";
+
 /* Helper global variable for hidden service descriptor event test.
  * It's used as a pointer to dynamically created message buffer in
  * send_control_event_string_replacement function, which mocks
@@ -124,6 +190,30 @@ node_describe_longname_by_id_replacement(const char *id_digest)
   }
 }
 
+/** Test that we can parse a hardcoded v2 HS desc. */
+static void
+test_hs_parse_static_v2_desc(void *arg)
+{
+  int ret;
+  rend_encoded_v2_service_descriptor_t desc;
+
+  (void) arg;
+
+  /* Test an obviously not parseable string */
+  desc.desc_str = tor_strdup("ceci n'est pas un HS descriptor");
+  ret = rend_desc_v2_is_parsable(&desc);
+  tor_free(desc.desc_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Test an actual descriptor */
+  desc.desc_str = tor_strdup(hs_desc_content);
+  ret = rend_desc_v2_is_parsable(&desc);
+  tor_free(desc.desc_str);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done: ;
+}
+
 /** Make sure each hidden service descriptor async event generation
  *
  * function generates the message in expected format.
@@ -136,7 +226,7 @@ test_hs_desc_event(void *arg)
   #define STR_DESC_ID_BASE32 "hba3gmcgpfivzfhx5rtfqkfdhv65yrj3"
 
   int ret;
-  rend_data_t rend_query;
+  rend_data_v2_t rend_query;
   const char *expected_msg;
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
 
@@ -148,18 +238,19 @@ test_hs_desc_event(void *arg)
 
   /* setup rend_query struct */
   memset(&rend_query, 0, sizeof(rend_query));
+  rend_query.base_.version = 2;
   strncpy(rend_query.onion_address, STR_HS_ADDR,
           REND_SERVICE_ID_LEN_BASE32+1);
   rend_query.auth_type = REND_NO_AUTH;
-  rend_query.hsdirs_fp = smartlist_new();
-  smartlist_add(rend_query.hsdirs_fp, tor_memdup(HSDIR_EXIST_ID,
-                                                 DIGEST_LEN));
+  rend_query.base_.hsdirs_fp = smartlist_new();
+  smartlist_add(rend_query.base_.hsdirs_fp, tor_memdup(HSDIR_EXIST_ID,
+                                                       DIGEST_LEN));
 
   /* Compute descriptor ID for replica 0, should be STR_DESC_ID_BASE32. */
   ret = rend_compute_v2_desc_id(rend_query.descriptor_id[0],
                                 rend_query.onion_address,
                                 NULL, 0, 0);
-  tt_int_op(ret, ==, 0);
+  tt_int_op(ret, OP_EQ, 0);
   base32_encode(desc_id_base32, sizeof(desc_id_base32),
                 rend_query.descriptor_id[0], DIGEST_LEN);
   /* Make sure rend_compute_v2_desc_id works properly. */
@@ -167,8 +258,9 @@ test_hs_desc_event(void *arg)
             sizeof(desc_id_base32));
 
   /* test request event */
-  control_event_hs_descriptor_requested(&rend_query, HSDIR_EXIST_ID,
-                                        STR_DESC_ID_BASE32);
+  control_event_hs_descriptor_requested(rend_query.onion_address,
+                                        rend_query.auth_type, HSDIR_EXIST_ID,
+                                        STR_DESC_ID_BASE32, NULL);
   expected_msg = "650 HS_DESC REQUESTED "STR_HS_ADDR" NO_AUTH "\
                   STR_HSDIR_EXIST_LONGNAME " " STR_DESC_ID_BASE32 "\r\n";
   tt_assert(received_msg);
@@ -177,8 +269,8 @@ test_hs_desc_event(void *arg)
 
   /* test received event */
   rend_query.auth_type = REND_BASIC_AUTH;
-  control_event_hs_descriptor_received(rend_query.onion_address,
-                                       &rend_query, HSDIR_EXIST_ID);
+  control_event_hsv2_descriptor_received(rend_query.onion_address,
+                                         &rend_query.base_, HSDIR_EXIST_ID);
   expected_msg = "650 HS_DESC RECEIVED "STR_HS_ADDR" BASIC_AUTH "\
                   STR_HSDIR_EXIST_LONGNAME " " STR_DESC_ID_BASE32"\r\n";
   tt_assert(received_msg);
@@ -187,7 +279,7 @@ test_hs_desc_event(void *arg)
 
   /* test failed event */
   rend_query.auth_type = REND_STEALTH_AUTH;
-  control_event_hs_descriptor_failed(&rend_query,
+  control_event_hsv2_descriptor_failed(&rend_query.base_,
                                      HSDIR_NONE_EXIST_ID,
                                      "QUERY_REJECTED");
   expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" STEALTH_AUTH "\
@@ -198,7 +290,7 @@ test_hs_desc_event(void *arg)
 
   /* test invalid auth type */
   rend_query.auth_type = 999;
-  control_event_hs_descriptor_failed(&rend_query,
+  control_event_hsv2_descriptor_failed(&rend_query.base_,
                                      HSDIR_EXIST_ID,
                                      "QUERY_REJECTED");
   expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" UNKNOWN "\
@@ -208,21 +300,42 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
-  /* test valid content. */
+  /* test no HSDir fingerprint type */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hsv2_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_NO_HSDIR");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_NO_HSDIR\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
+  /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
+                                      STR_HS_CONTENT_DESC_ID, NULL, NULL);
+  tor_asprintf(&exp_msg, "650+HS_DESC_CONTENT " STR_HS_ADDR " "\
+               STR_HS_CONTENT_DESC_ID " UNKNOWN" \
+               "\r\n\r\n.\r\n650 OK\r\n");
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, exp_msg);
+  tor_free(received_msg);
+  tor_free(exp_msg);
+
+  /* test valid content. */
+  control_event_hs_descriptor_content(rend_query.onion_address,
                                       STR_HS_CONTENT_DESC_ID, HSDIR_EXIST_ID,
-                                      hs_desc_content);
+                                      hs_desc_content_control);
   tor_asprintf(&exp_msg, "650+HS_DESC_CONTENT " STR_HS_ADDR " "\
                STR_HS_CONTENT_DESC_ID " " STR_HSDIR_EXIST_LONGNAME\
-               "\r\n%s\r\n.\r\n650 OK\r\n", hs_desc_content);
+               "\r\n%s\r\n.\r\n650 OK\r\n", hs_desc_content_control);
 
   tt_assert(received_msg);
   tt_str_op(received_msg, OP_EQ, exp_msg);
   tor_free(received_msg);
   tor_free(exp_msg);
-  SMARTLIST_FOREACH(rend_query.hsdirs_fp, char *, d, tor_free(d));
-  smartlist_free(rend_query.hsdirs_fp);
+  SMARTLIST_FOREACH(rend_query.base_.hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(rend_query.base_.hsdirs_fp);
 
  done:
   UNMOCK(queue_control_event_string);
@@ -248,17 +361,18 @@ test_pick_tor2web_rendezvous_node(void *arg)
 
   /* Parse Tor2webRendezvousPoints as a routerset. */
   options->Tor2webRendezvousPoints = routerset_new();
+  options->UseMicrodescriptors = 0;
   retval = routerset_parse(options->Tor2webRendezvousPoints,
                            tor2web_rendezvous_str,
                            "test_tor2web_rp");
-  tt_int_op(retval, >=, 0);
+  tt_int_op(retval, OP_GE, 0);
 
   /* Pick rendezvous point. Make sure the correct one is
      picked. Repeat many times to make sure it works properly. */
   for (i = 0; i < 50 ; i++) {
     chosen_rp = pick_tor2web_rendezvous_node(flags, options);
     tt_assert(chosen_rp);
-    tt_str_op(chosen_rp->ri->nickname, ==, tor2web_rendezvous_str);
+    tt_str_op(chosen_rp->ri->nickname, OP_EQ, tor2web_rendezvous_str);
   }
 
  done:
@@ -286,13 +400,13 @@ test_pick_bad_tor2web_rendezvous_node(void *arg)
   retval = routerset_parse(options->Tor2webRendezvousPoints,
                            tor2web_rendezvous_str,
                            "test_tor2web_rp");
-  tt_int_op(retval, >=, 0);
+  tt_int_op(retval, OP_GE, 0);
 
   /* Pick rendezvous point. Since Tor2webRendezvousPoints was set to a
      dummy value, we shouldn't find any eligible RPs. */
   for (i = 0; i < 50 ; i++) {
     chosen_rp = pick_tor2web_rendezvous_node(flags, options);
-    tt_assert(!chosen_rp);
+    tt_ptr_op(chosen_rp, OP_EQ, NULL);
   }
 
  done:
@@ -322,43 +436,47 @@ test_hs_rend_data(void *arg)
   client = rend_data_client_create(STR_HS_ADDR, desc_id, client_cookie,
                                    REND_NO_AUTH);
   tt_assert(client);
-  tt_int_op(client->auth_type, ==, REND_NO_AUTH);
-  tt_str_op(client->onion_address, OP_EQ, STR_HS_ADDR);
-  tt_mem_op(client->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_mem_op(client->descriptor_cookie, OP_EQ, client_cookie,
+  rend_data_v2_t *client_v2 = TO_REND_DATA_V2(client);
+  tt_int_op(client_v2->auth_type, OP_EQ, REND_NO_AUTH);
+  tt_str_op(client_v2->onion_address, OP_EQ, STR_HS_ADDR);
+  tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
+  tt_mem_op(client_v2->descriptor_cookie, OP_EQ, client_cookie,
             sizeof(client_cookie));
   tt_assert(client->hsdirs_fp);
-  tt_int_op(smartlist_len(client->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    int ret = rend_compute_v2_desc_id(desc_id, client->onion_address,
-                                      client->descriptor_cookie, now, rep);
+    int ret = rend_compute_v2_desc_id(desc_id, client_v2->onion_address,
+                                      client_v2->descriptor_cookie, now, rep);
     /* That shouldn't never fail. */
-    tt_int_op(ret, ==, 0);
-    tt_mem_op(client->descriptor_id[rep], OP_EQ, desc_id, sizeof(desc_id));
+    tt_int_op(ret, OP_EQ, 0);
+    tt_mem_op(client_v2->descriptor_id[rep], OP_EQ, desc_id,
+              sizeof(desc_id));
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client->rend_pk_digest), ==, 1);
-  tt_int_op(tor_digest_is_zero(client->rend_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_v2->rend_pk_digest), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(client->rend_cookie), OP_EQ, 1);
 
   /* Test dup(). */
   client_dup = rend_data_dup(client);
   tt_assert(client_dup);
-  tt_int_op(client_dup->auth_type, ==, client->auth_type);
-  tt_str_op(client_dup->onion_address, OP_EQ, client->onion_address);
-  tt_mem_op(client_dup->desc_id_fetch, OP_EQ, client->desc_id_fetch,
-            sizeof(client_dup->desc_id_fetch));
-  tt_mem_op(client_dup->descriptor_cookie, OP_EQ, client->descriptor_cookie,
-            sizeof(client_dup->descriptor_cookie));
+  rend_data_v2_t *client_dup_v2 = TO_REND_DATA_V2(client_dup);
+  tt_int_op(client_dup_v2->auth_type, OP_EQ, client_v2->auth_type);
+  tt_str_op(client_dup_v2->onion_address, OP_EQ, client_v2->onion_address);
+  tt_mem_op(client_dup_v2->desc_id_fetch, OP_EQ, client_v2->desc_id_fetch,
+            sizeof(client_dup_v2->desc_id_fetch));
+  tt_mem_op(client_dup_v2->descriptor_cookie, OP_EQ,
+            client_v2->descriptor_cookie,
+            sizeof(client_dup_v2->descriptor_cookie));
 
   tt_assert(client_dup->hsdirs_fp);
-  tt_int_op(smartlist_len(client_dup->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(client_dup->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_mem_op(client_dup->descriptor_id[rep], OP_EQ,
-              client->descriptor_id[rep], DIGEST_LEN);
+    tt_mem_op(client_dup_v2->descriptor_id[rep], OP_EQ,
+              client_v2->descriptor_id[rep], DIGEST_LEN);
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client_dup->rend_pk_digest), ==, 1);
-  tt_int_op(tor_digest_is_zero(client_dup->rend_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_dup_v2->rend_pk_digest), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(client_dup->rend_cookie), OP_EQ, 1);
   rend_data_free(client);
   client = NULL;
   rend_data_free(client_dup);
@@ -373,19 +491,20 @@ test_hs_rend_data(void *arg)
    * zeroed out. */
   client = rend_data_client_create(NULL, desc_id, NULL, REND_BASIC_AUTH);
   tt_assert(client);
-  tt_int_op(client->auth_type, ==, REND_BASIC_AUTH);
-  tt_int_op(strlen(client->onion_address), ==, 0);
-  tt_mem_op(client->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client->descriptor_cookie,
-                            sizeof(client->descriptor_cookie)), ==, 1);
+  client_v2 = TO_REND_DATA_V2(client);
+  tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH);
+  tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0);
+  tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
+  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+                            sizeof(client_v2->descriptor_cookie)), OP_EQ, 1);
   tt_assert(client->hsdirs_fp);
-  tt_int_op(smartlist_len(client->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(client->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(client_v2->descriptor_id[rep]), OP_EQ, 1);
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client->rend_pk_digest), ==, 1);
-  tt_int_op(tor_digest_is_zero(client->rend_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_v2->rend_pk_digest), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(client->rend_cookie), OP_EQ, 1);
   rend_data_free(client);
   client = NULL;
 
@@ -398,37 +517,39 @@ test_hs_rend_data(void *arg)
   service = rend_data_service_create(STR_HS_ADDR, rend_pk_digest,
                                      rend_cookie, REND_NO_AUTH);
   tt_assert(service);
-  tt_int_op(service->auth_type, ==, REND_NO_AUTH);
-  tt_str_op(service->onion_address, OP_EQ, STR_HS_ADDR);
-  tt_mem_op(service->rend_pk_digest, OP_EQ, rend_pk_digest,
+  rend_data_v2_t *service_v2 = TO_REND_DATA_V2(service);
+  tt_int_op(service_v2->auth_type, OP_EQ, REND_NO_AUTH);
+  tt_str_op(service_v2->onion_address, OP_EQ, STR_HS_ADDR);
+  tt_mem_op(service_v2->rend_pk_digest, OP_EQ, rend_pk_digest,
             sizeof(rend_pk_digest));
   tt_mem_op(service->rend_cookie, OP_EQ, rend_cookie, sizeof(rend_cookie));
   tt_assert(service->hsdirs_fp);
-  tt_int_op(smartlist_len(service->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(service->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(service->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(service_v2->descriptor_id[rep]), OP_EQ, 1);
   }
   /* The rest should be zeroed because this is a service request. */
-  tt_int_op(tor_digest_is_zero(service->descriptor_cookie), ==, 1);
-  tt_int_op(tor_digest_is_zero(service->desc_id_fetch), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_v2->descriptor_cookie), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(service_v2->desc_id_fetch), OP_EQ, 1);
 
   /* Test dup(). */
   service_dup = rend_data_dup(service);
+  rend_data_v2_t *service_dup_v2 = TO_REND_DATA_V2(service_dup);
   tt_assert(service_dup);
-  tt_int_op(service_dup->auth_type, ==, service->auth_type);
-  tt_str_op(service_dup->onion_address, OP_EQ, service->onion_address);
-  tt_mem_op(service_dup->rend_pk_digest, OP_EQ, service->rend_pk_digest,
-            sizeof(service_dup->rend_pk_digest));
+  tt_int_op(service_dup_v2->auth_type, OP_EQ, service_v2->auth_type);
+  tt_str_op(service_dup_v2->onion_address, OP_EQ, service_v2->onion_address);
+  tt_mem_op(service_dup_v2->rend_pk_digest, OP_EQ, service_v2->rend_pk_digest,
+            sizeof(service_dup_v2->rend_pk_digest));
   tt_mem_op(service_dup->rend_cookie, OP_EQ, service->rend_cookie,
             sizeof(service_dup->rend_cookie));
   tt_assert(service_dup->hsdirs_fp);
-  tt_int_op(smartlist_len(service_dup->hsdirs_fp), ==, 0);
+  tt_int_op(smartlist_len(service_dup->hsdirs_fp), OP_EQ, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(service_dup->descriptor_id[rep]), ==, 1);
+    tt_assert(tor_digest_is_zero(service_dup_v2->descriptor_id[rep]));
   }
   /* The rest should be zeroed because this is a service request. */
-  tt_int_op(tor_digest_is_zero(service_dup->descriptor_cookie), ==, 1);
-  tt_int_op(tor_digest_is_zero(service_dup->desc_id_fetch), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_dup_v2->descriptor_cookie), OP_EQ, 1);
+  tt_int_op(tor_digest_is_zero(service_dup_v2->desc_id_fetch), OP_EQ, 1);
 
  done:
   rend_data_free(service);
@@ -467,7 +588,7 @@ test_hs_auth_cookies(void *arg)
   re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED, raw_cookie, &auth_type,
                                &err_msg);
   tt_assert(!re);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN);
   tt_int_op(auth_type, OP_EQ, REND_BASIC_AUTH);
   memset(raw_cookie, 0, sizeof(raw_cookie));
@@ -475,7 +596,7 @@ test_hs_auth_cookies(void *arg)
   re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED_STEALTH, raw_cookie,
                                &auth_type, &err_msg);
   tt_assert(!re);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN);
   tt_int_op(auth_type, OP_EQ, REND_STEALTH_AUTH);
   memset(raw_cookie, 0, sizeof(raw_cookie));
@@ -484,7 +605,7 @@ test_hs_auth_cookies(void *arg)
   re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED "==", raw_cookie, NULL,
                                &err_msg);
   tt_assert(!re);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN);
 
   /* Decoding with an unknown type should fail */
@@ -545,32 +666,24 @@ test_single_onion_poisoning(void *arg)
   char *dir2 = tor_strdup(get_fname_rnd("test_hs_dir2"));
   smartlist_t *services = smartlist_new();
   char *poison_path = NULL;
+  char *err_msg = NULL;
 
-  /* No services, no service to verify, no problem! */
-  mock_options->HiddenServiceSingleHopMode = 0;
-  mock_options->HiddenServiceNonAnonymousMode = 0;
-  ret = rend_config_services(mock_options, 1);
-  tt_assert(ret == 0);
-
-  /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
-  ret = rend_config_services(mock_options, 1);
-  tt_assert(ret == 0);
 
   /* Create the data directory, and, if the correct bit in arg is set,
    * create a directory for that service.
    * The data directory is required for the lockfile, which is used when
    * loading keys. */
   ret = check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   if (create_dir_mask & CREATE_HS_DIR1) {
     ret = check_private_dir(dir1, CPD_CREATE, NULL);
-    tt_assert(ret == 0);
+    tt_int_op(ret, OP_EQ, 0);
   }
   if (create_dir_mask & CREATE_HS_DIR2) {
     ret = check_private_dir(dir2, CPD_CREATE, NULL);
-    tt_assert(ret == 0);
+    tt_int_op(ret, OP_EQ, 0);
   }
 
   service_1->directory = dir1;
@@ -580,194 +693,197 @@ test_single_onion_poisoning(void *arg)
   /* Add port to service 1 */
   service_1->ports = smartlist_new();
   service_2->ports = smartlist_new();
-  char *err_msg = NULL;
   rend_service_port_config_t *port1 = rend_service_parse_port_config("80", " ",
                                                                      &err_msg);
   tt_assert(port1);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   smartlist_add(service_1->ports, port1);
 
   rend_service_port_config_t *port2 = rend_service_parse_port_config("90", " ",
                                                                      &err_msg);
   /* Add port to service 2 */
   tt_assert(port2);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   smartlist_add(service_2->ports, port2);
 
   /* No services, a service to verify, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Add the first service */
-  ret = rend_service_check_dir_and_add(services, mock_options, service_1, 0);
-  tt_assert(ret == 0);
+  ret = hs_check_service_private_dir(mock_options->User, service_1->directory,
+                                     service_1->dir_group_readable, 1);
+  tt_int_op(ret, OP_EQ, 0);
+  smartlist_add(services, service_1);
   /* But don't add the second service yet. */
 
   /* Service directories, but no previous keys, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poison! Poison! Poison!
    * This can only be done in HiddenServiceSingleHopMode. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* Poisoning twice is a no-op. */
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poisoned service directories, but no previous keys, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Either way, no problem. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Now add some keys, and we'll have a problem. */
   ret = rend_service_load_all_keys(services);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poisoned service directories with previous keys are not allowed. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* But they are allowed if we're in non-anonymous mode. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Re-poisoning directories with existing keys is a no-op, because
    * directories with existing keys are ignored. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* And it keeps the poison. */
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Now add the second service: it has no key and no poison file */
-  ret = rend_service_check_dir_and_add(services, mock_options, service_2, 0);
-  tt_assert(ret == 0);
+  ret = hs_check_service_private_dir(mock_options->User, service_2->directory,
+                                     service_2->dir_group_readable, 1);
+  tt_int_op(ret, OP_EQ, 0);
+  smartlist_add(services, service_2);
 
   /* A new service, and an existing poisoned service. Not ok. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* But ok to add in non-anonymous mode. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Now remove the poisoning from the first service, and we have the opposite
    * problem. */
   poison_path = rend_service_sos_poison_path(service_1);
   tt_assert(poison_path);
   ret = unlink(poison_path);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Unpoisoned service directories with previous keys are ok, as are empty
    * directories. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* But the existing unpoisoned key is not ok in non-anonymous mode, even if
    * there is an empty service. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Poisoning directories with existing keys is a no-op, because directories
    * with existing keys are ignored. But the new directory should poison. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_poison_new_single_onion_dir(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* And the old directory remains unpoisoned. */
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* And the new directory should be ignored, because it has no key. */
   mock_options->HiddenServiceSingleHopMode = 0;
   mock_options->HiddenServiceNonAnonymousMode = 0;
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Re-poisoning directories without existing keys is a no-op. */
   mock_options->HiddenServiceSingleHopMode = 1;
   mock_options->HiddenServiceNonAnonymousMode = 1;
   ret = rend_service_poison_new_single_onion_dir(service_1, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = rend_service_poison_new_single_onion_dir(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   /* And the old directory remains unpoisoned. */
   ret = rend_service_verify_single_onion_poison(service_1, mock_options);
-  tt_assert(ret < 0);
+  tt_int_op(ret, OP_LT, 0);
   ret = rend_service_verify_single_onion_poison(service_2, mock_options);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
  done:
   /* The test harness deletes the directories at exit */
@@ -779,11 +895,144 @@ test_single_onion_poisoning(void *arg)
   rend_service_free(service_2);
   UNMOCK(get_options);
   tor_free(mock_options->DataDirectory);
+  tor_free(err_msg);
+}
+
+static rend_service_t *
+helper_create_rend_service(const char *path)
+{
+  rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t));
+  s->ports = smartlist_new();
+  s->intro_nodes = smartlist_new();
+  s->expiring_nodes = smartlist_new();
+  if (path) {
+    s->directory = tor_strdup(path);
+  }
+  return s;
+}
+
+static void
+test_prune_services_on_reload(void *arg)
+{
+  smartlist_t *new = smartlist_new(), *old = smartlist_new();
+  /* Non ephemeral service. */
+  rend_service_t *s1 = helper_create_rend_service("SomePath");
+  /* Create a non ephemeral service with the _same_ path as so we can test the
+   * transfer of introduction point between the same services on reload. */
+  rend_service_t *s2 = helper_create_rend_service(s1->directory);
+  /* Ephemeral service (directory is NULL). */
+  rend_service_t *e1 = helper_create_rend_service(NULL);
+  rend_service_t *e2 = helper_create_rend_service(NULL);
+
+  (void) arg;
+
+  {
+    /* Add both services to the old list. */
+    smartlist_add(old, s1);
+    smartlist_add(old, e1);
+    /* Only put the non ephemeral in the new list. */
+    smartlist_add(new, s1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    /* We expect that the ephemeral one is in the new list but removed from
+     * the old one. */
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    tt_assert(smartlist_get(old, 0) == s1);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+    tt_assert(smartlist_get(new, 0) == s1);
+    tt_assert(smartlist_get(new, 1) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* This test will make sure that only the ephemeral service is kept if the
+     * new list is empty. The old list should contain only the non ephemeral
+     * one. */
+    smartlist_add(old, s1);
+    smartlist_add(old, e1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    tt_assert(smartlist_get(old, 0) == s1);
+    tt_int_op(smartlist_len(new), OP_EQ, 1);
+    tt_assert(smartlist_get(new, 0) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* This test makes sure that the new list stays the same even from the old
+     * list being completely different. */
+    smartlist_add(new, s1);
+    smartlist_add(new, e1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+    tt_assert(smartlist_get(new, 0) == s1);
+    tt_assert(smartlist_get(new, 1) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+  }
+
+  {
+    rend_intro_point_t ip1;
+    /* This IP should be found in the s2 service after pruning. */
+    smartlist_add(s1->intro_nodes, &ip1);
+    /* Setup our list. */
+    smartlist_add(old, s1);
+    smartlist_add(new, s2);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    /* Intro nodes have been moved to the s2 in theory so it must be empty. */
+    tt_int_op(smartlist_len(s1->intro_nodes), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 1);
+    rend_service_t *elem = smartlist_get(new, 0);
+    tt_assert(elem);
+    tt_assert(elem == s2);
+    tt_int_op(smartlist_len(elem->intro_nodes), OP_EQ, 1);
+    tt_assert(smartlist_get(elem->intro_nodes, 0) == &ip1);
+    smartlist_clear(s1->intro_nodes);
+    smartlist_clear(s2->intro_nodes);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* Test two ephemeral services. */
+    smartlist_add(old, e1);
+    smartlist_add(old, e2);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    /* Check if they've all been transferred. */
+    tt_int_op(smartlist_len(old), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+  }
+
+ done:
+  rend_service_free(s1);
+  rend_service_free(s2);
+  rend_service_free(e1);
+  rend_service_free(e2);
+  smartlist_free(new);
+  smartlist_free(old);
 }
 
 struct testcase_t hs_tests[] = {
   { "hs_rend_data", test_hs_rend_data, TT_FORK,
     NULL, NULL },
+  { "hs_parse_static_v2_desc", test_hs_parse_static_v2_desc, TT_FORK,
+    NULL, NULL },
   { "hs_desc_event", test_hs_desc_event, TT_FORK,
     NULL, NULL },
   { "pick_tor2web_rendezvous_node", test_pick_tor2web_rendezvous_node, TT_FORK,
@@ -801,6 +1050,9 @@ struct testcase_t hs_tests[] = {
     TT_FORK, &passthrough_setup, (void*)(CREATE_HS_DIR2) },
   { "single_onion_poisoning_create_dir_both", test_single_onion_poisoning,
     TT_FORK, &passthrough_setup, (void*)(CREATE_HS_DIR1 | CREATE_HS_DIR2) },
+  { "prune_services_on_reload", test_prune_services_on_reload, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
new file mode 100644
index 0000000000..458ce1a92e
--- /dev/null
+++ b/src/test/test_hs_cache.c
@@ -0,0 +1,561 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cache.c
+ * \brief Test hidden service caches.
+ */
+
+#define CONNECTION_PRIVATE
+#define DIRECTORY_PRIVATE
+#define HS_CACHE_PRIVATE
+
+#include "ed25519_cert.h"
+#include "hs_cache.h"
+#include "rendcache.h"
+#include "directory.h"
+#include "networkstatus.h"
+#include "connection.h"
+#include "proto_http.h"
+
+#include "hs_test_helpers.h"
+#include "test_helpers.h"
+#include "test.h"
+
+/* Static variable used to encoded the HSDir query. */
+static char query_b64[256];
+
+/* Build an HSDir query using a ed25519 public key. */
+static const char *
+helper_get_hsdir_query(const hs_descriptor_t *desc)
+{
+  ed25519_public_to_base64(query_b64, &desc->plaintext_data.blinded_pubkey);
+  return query_b64;
+}
+
+static void
+init_test(void)
+{
+  /* Always needed. Initialize the subsystem. */
+  hs_cache_init();
+  /* We need the v2 cache since our OOM and cache cleanup does poke at it. */
+  rend_cache_init();
+}
+
+static void
+test_directory(void *arg)
+{
+  int ret;
+  size_t oom_size;
+  char *desc1_str = NULL;
+  const char *desc_out;
+  ed25519_keypair_t signing_kp1;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  init_test();
+  /* Generate a valid descriptor with normal values. */
+  ret = ed25519_keypair_generate(&signing_kp1, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc1 = hs_helper_build_hs_desc_with_ip(&signing_kp1);
+  tt_assert(desc1);
+  ret = hs_desc_encode_descriptor(desc1, &signing_kp1, &desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Very first basic test, should be able to be stored, survive a
+   * clean, found with a lookup and then cleaned by our OOM. */
+  {
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Re-add, it should fail since we already have it. */
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, -1);
+    /* Try to clean now which should be fine, there is at worst few seconds
+     * between the store and this call. */
+    hs_cache_clean_as_dir(time(NULL));
+    /* We should find it in our cache. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, desc1_str);
+    /* Tell our OOM to run and to at least remove a byte which will result in
+     * removing the descriptor from our cache. */
+    oom_size = hs_cache_handle_oom(time(NULL), 1);
+    tt_int_op(oom_size, OP_GE, 1);
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Store two descriptors and remove the expiring one only. */
+  {
+    ed25519_keypair_t signing_kp_zero;
+    ret = ed25519_keypair_generate(&signing_kp_zero, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    hs_descriptor_t *desc_zero_lifetime;
+    desc_zero_lifetime = hs_helper_build_hs_desc_with_ip(&signing_kp_zero);
+    tt_assert(desc_zero_lifetime);
+    desc_zero_lifetime->plaintext_data.revision_counter = 1;
+    desc_zero_lifetime->plaintext_data.lifetime_sec = 0;
+    char *desc_zero_lifetime_str;
+    ret = hs_desc_encode_descriptor(desc_zero_lifetime, &signing_kp_zero,
+                                    &desc_zero_lifetime_str);
+    tt_int_op(ret, OP_EQ, 0);
+
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_store_as_dir(desc_zero_lifetime_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* This one should clear out our zero lifetime desc. */
+    hs_cache_clean_as_dir(time(NULL));
+    /* We should find desc1 in our cache. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, desc1_str);
+    /* We should NOT find our zero lifetime desc in our cache. */
+    ret = hs_cache_lookup_as_dir(3,
+                                 helper_get_hsdir_query(desc_zero_lifetime),
+                                 NULL);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Cleanup our entire cache. */
+    oom_size = hs_cache_handle_oom(time(NULL), 1);
+    tt_int_op(oom_size, OP_GE, 1);
+    hs_descriptor_free(desc_zero_lifetime);
+    tor_free(desc_zero_lifetime_str);
+  }
+
+  /* Throw junk at it. */
+  {
+    ret = hs_cache_store_as_dir("blah");
+    tt_int_op(ret, OP_EQ, -1);
+    /* Poor attempt at tricking the decoding. */
+    ret = hs_cache_store_as_dir("hs-descriptor 3\nJUNK");
+    tt_int_op(ret, OP_EQ, -1);
+    /* Undecodable base64 query. */
+    ret = hs_cache_lookup_as_dir(3, "blah", NULL);
+    tt_int_op(ret, OP_EQ, -1);
+    /* Decodable base64 query but wrong ed25519 size. */
+    ret = hs_cache_lookup_as_dir(3, "dW5pY29ybg==", NULL);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Test descriptor replacement with revision counter. */
+  {
+    char *new_desc_str;
+
+    /* Add a descriptor. */
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    /* Bump revision counter. */
+    desc1->plaintext_data.revision_counter++;
+    ret = hs_desc_encode_descriptor(desc1, &signing_kp1, &new_desc_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_store_as_dir(new_desc_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Look it up, it should have been replaced. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, new_desc_str);
+    tor_free(new_desc_str);
+  }
+
+ done:
+  hs_descriptor_free(desc1);
+  tor_free(desc1_str);
+}
+
+static void
+test_clean_as_dir(void *arg)
+{
+  size_t ret;
+  char *desc1_str = NULL;
+  time_t now = time(NULL);
+  hs_descriptor_t *desc1 = NULL;
+  ed25519_keypair_t signing_kp1;
+
+  (void) arg;
+
+  init_test();
+
+  /* Generate a valid descriptor with values. */
+  ret = ed25519_keypair_generate(&signing_kp1, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc1 = hs_helper_build_hs_desc_with_ip(&signing_kp1);
+  tt_assert(desc1);
+  ret = hs_desc_encode_descriptor(desc1, &signing_kp1, &desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = hs_cache_store_as_dir(desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* With the lifetime being 3 hours, a cleanup shouldn't remove it. */
+  ret = cache_clean_v3_as_dir(now, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 1);
+  /* Set a cutoff 100 seconds in the past. It should not remove the entry
+   * since the entry is still recent enough. */
+  ret = cache_clean_v3_as_dir(now, now - 100);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 1);
+  /* Set a cutoff of 100 seconds in the future. It should remove the entry
+   * that we've just added since it's not too old for the cutoff. */
+  ret = cache_clean_v3_as_dir(now, now + 100);
+  tt_int_op(ret, OP_GT, 0);
+  /* Shouldn't be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  hs_descriptor_free(desc1);
+  tor_free(desc1_str);
+}
+
+/* Test helper: Fetch an HS descriptor from an HSDir (for the hidden service
+   with <b>blinded_key</b>. Return the received descriptor string. */
+static char *
+helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
+{
+  int retval;
+
+  char *received_desc = NULL;
+  char *hsdir_query_str = NULL;
+
+  /* The dir conn we are going to simulate */
+  dir_connection_t *conn = NULL;
+
+  /* First extract the blinded public key that we are going to use in our
+     query, and then build the actual query string. */
+  {
+    char hsdir_cache_key[ED25519_BASE64_LEN+1];
+
+    retval = ed25519_public_to_base64(hsdir_cache_key,
+                                      blinded_key);
+    tt_int_op(retval, OP_EQ, 0);
+    tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
+  }
+
+  /* Simulate an HTTP GET request to the HSDir */
+  conn = dir_connection_new(AF_INET);
+  tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+  retval = directory_handle_command_get(conn, hsdir_query_str,
+                                        NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Read the descriptor that the HSDir just served us */
+  {
+    char *headers = NULL;
+    size_t body_used = 0;
+
+    fetch_from_buf_http(TO_CONN(conn)->outbuf, &headers, MAX_HEADERS_SIZE,
+                        &received_desc, &body_used, HS_DESC_MAX_LEN, 0);
+    tor_free(headers);
+  }
+
+ done:
+  tor_free(hsdir_query_str);
+  if (conn)
+    connection_free_minimal(TO_CONN(conn));
+
+  return received_desc;
+}
+
+/* Publish a descriptor to the HSDir, then fetch it. Check that the received
+   descriptor matches the published one. */
+static void
+test_upload_and_download_hs_desc(void *arg)
+{
+  int retval;
+  hs_descriptor_t *published_desc = NULL;
+
+  char *published_desc_str = NULL;
+  char *received_desc_str = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  /* Test a descriptor not found in the directory cache. */
+  {
+    ed25519_public_key_t blinded_key;
+    memset(&blinded_key.pubkey, 'A', sizeof(blinded_key.pubkey));
+    received_desc_str = helper_fetch_desc_from_hsdir(&blinded_key);
+    tt_int_op(strlen(received_desc_str), OP_EQ, 0);
+    tor_free(received_desc_str);
+  }
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    ed25519_keypair_t signing_kp;
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* Publish descriptor to the HSDir */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 200);
+  }
+
+  /* Simulate a fetch of the previously published descriptor */
+  {
+    const ed25519_public_key_t *blinded_key;
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+  }
+
+  /* Verify we received the exact same descriptor we published earlier */
+  tt_str_op(received_desc_str, OP_EQ, published_desc_str);
+  tor_free(received_desc_str);
+
+  /* With a valid descriptor in the directory cache, try again an invalid. */
+  {
+    ed25519_public_key_t blinded_key;
+    memset(&blinded_key.pubkey, 'A', sizeof(blinded_key.pubkey));
+    received_desc_str = helper_fetch_desc_from_hsdir(&blinded_key);
+    tt_int_op(strlen(received_desc_str), OP_EQ, 0);
+  }
+
+ done:
+  tor_free(received_desc_str);
+  tor_free(published_desc_str);
+  hs_descriptor_free(published_desc);
+}
+
+/* Test that HSDirs reject outdated descriptors based on their revision
+ * counter. Also test that HSDirs correctly replace old descriptors with newer
+ * descriptors. */
+static void
+test_hsdir_revision_counter_check(void *arg)
+{
+  int retval;
+
+  ed25519_keypair_t signing_kp;
+
+  hs_descriptor_t *published_desc = NULL;
+  char *published_desc_str = NULL;
+
+  uint8_t subcredential[DIGEST256_LEN];
+  char *received_desc_str = NULL;
+  hs_descriptor_t *received_desc = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* Publish descriptor to the HSDir */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 200);
+  }
+
+  /* Try publishing again with the same revision counter: Should fail. */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 400);
+  }
+
+  /* Fetch the published descriptor and validate the revision counter. */
+  {
+    const ed25519_public_key_t *blinded_key;
+
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    hs_get_subcredential(&signing_kp.pubkey, blinded_key, subcredential);
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+
+    retval = hs_desc_decode_descriptor(received_desc_str,
+                                       subcredential, &received_desc);
+    tt_int_op(retval, OP_EQ, 0);
+    tt_assert(received_desc);
+
+    /* Check that the revision counter is correct */
+    tt_u64_op(received_desc->plaintext_data.revision_counter, OP_EQ, 42);
+
+    hs_descriptor_free(received_desc);
+    received_desc = NULL;
+    tor_free(received_desc_str);
+  }
+
+  /* Increment the revision counter and try again. Should work. */
+  {
+    published_desc->plaintext_data.revision_counter = 1313;
+    tor_free(published_desc_str);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, OP_EQ, 200);
+  }
+
+  /* Again, fetch the published descriptor and perform the revision counter
+     validation. The revision counter must have changed. */
+  {
+    const ed25519_public_key_t *blinded_key;
+
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+
+    retval = hs_desc_decode_descriptor(received_desc_str,
+                                       subcredential, &received_desc);
+    tt_int_op(retval, OP_EQ, 0);
+    tt_assert(received_desc);
+
+    /* Check that the revision counter is the latest */
+    tt_u64_op(received_desc->plaintext_data.revision_counter, OP_EQ, 1313);
+  }
+
+ done:
+  hs_descriptor_free(published_desc);
+  hs_descriptor_free(received_desc);
+  tor_free(received_desc_str);
+  tor_free(published_desc_str);
+}
+
+static networkstatus_t mock_ns;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+/** Test that we can store HS descriptors in the client HS cache. */
+static void
+test_client_cache(void *arg)
+{
+  int retval;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *published_desc = NULL;
+  char *published_desc_str = NULL;
+  uint8_t wanted_subcredential[DIGEST256_LEN];
+  response_handler_args_t *args = NULL;
+  dir_connection_t *conn = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                           &mock_ns.valid_until);
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+    memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN);
+    tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
+  }
+
+  /* Test handle_response_fetch_hsdesc_v3() */
+  {
+    args = tor_malloc_zero(sizeof(response_handler_args_t));
+    args->status_code = 200;
+    args->reason = NULL;
+    args->body = published_desc_str;
+    args->body_len = strlen(published_desc_str);
+
+    conn = tor_malloc_zero(sizeof(dir_connection_t));
+    conn->hs_ident = tor_malloc_zero(sizeof(hs_ident_dir_conn_t));
+    ed25519_pubkey_copy(&conn->hs_ident->identity_pk, &signing_kp.pubkey);
+  }
+
+  /* store the descriptor! */
+  retval = handle_response_fetch_hsdesc_v3(conn, args);
+  tt_int_op(retval, == , 0);
+
+  /* Progress time a bit and attempt to clean cache: our desc should not be
+   * cleaned since we still in the same TP. */
+  {
+    parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC",
+                       &mock_ns.valid_after);
+    parse_rfc1123_time("Sat, 27 Oct 1985 03:00:00 UTC",
+                       &mock_ns.fresh_until);
+    parse_rfc1123_time("Sat, 27 Oct 1985 05:00:00 UTC",
+                       &mock_ns.valid_until);
+
+    /* fetch the descriptor and make sure it's there */
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
+    tt_assert(cached_desc);
+    tt_mem_op(cached_desc->subcredential, OP_EQ, wanted_subcredential,
+              DIGEST256_LEN);
+  }
+
+  /* Progress time to next TP and check that desc was cleaned */
+  {
+    parse_rfc1123_time("Sat, 27 Oct 1985 12:00:00 UTC",
+                       &mock_ns.valid_after);
+    parse_rfc1123_time("Sat, 27 Oct 1985 13:00:00 UTC",
+                       &mock_ns.fresh_until);
+    parse_rfc1123_time("Sat, 27 Oct 1985 15:00:00 UTC",
+                       &mock_ns.valid_until);
+
+    const hs_descriptor_t *cached_desc = NULL;
+    cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
+    tt_assert(!cached_desc);
+  }
+
+ done:
+  tor_free(args);
+  hs_descriptor_free(published_desc);
+  tor_free(published_desc_str);
+  if (conn) {
+    tor_free(conn->hs_ident);
+    tor_free(conn);
+  }
+}
+
+struct testcase_t hs_cache[] = {
+  /* Encoding tests. */
+  { "directory", test_directory, TT_FORK,
+    NULL, NULL },
+  { "clean_as_dir", test_clean_as_dir, TT_FORK,
+    NULL, NULL },
+  { "hsdir_revision_counter_check", test_hsdir_revision_counter_check, TT_FORK,
+    NULL, NULL },
+  { "upload_and_download_hs_desc", test_upload_and_download_hs_desc, TT_FORK,
+    NULL, NULL },
+  { "client_cache", test_client_cache, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c
new file mode 100644
index 0000000000..8e15184c2a
--- /dev/null
+++ b/src/test/test_hs_cell.c
@@ -0,0 +1,131 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cell.c
+ * \brief Test hidden service cell functionality.
+ */
+
+#define HS_INTROPOINT_PRIVATE
+#define HS_SERVICE_PRIVATE
+
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+
+#include "crypto_ed25519.h"
+#include "crypto_rand.h"
+#include "hs_cell.h"
+#include "hs_intropoint.h"
+#include "hs_service.h"
+
+/* Trunnel. */
+#include "hs/cell_establish_intro.h"
+
+/** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
+ *  parse it from the receiver side. */
+static void
+test_gen_establish_intro_cell(void *arg)
+{
+  (void) arg;
+  ssize_t ret;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t buf[RELAY_PAYLOAD_SIZE];
+  trn_cell_establish_intro_t *cell_in = NULL;
+
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  {
+    /* We only need the auth key pair here. */
+    hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+    /* Auth key pair is generated in the constructor so we are all set for
+     * using this IP object. */
+    ret = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+    service_intro_point_free(ip);
+    tt_u64_op(ret, OP_GT, 0);
+  }
+
+  /* Check the contents of the cell */
+  {
+    /* First byte is the auth key type: make sure its correct */
+    tt_int_op(buf[0], OP_EQ, HS_INTRO_AUTH_KEY_TYPE_ED25519);
+    /* Next two bytes is auth key len */
+    tt_int_op(ntohs(get_uint16(buf+1)), OP_EQ, ED25519_PUBKEY_LEN);
+    /* Skip to the number of extensions: no extensions */
+    tt_int_op(buf[35], OP_EQ, 0);
+    /* Skip to the sig len. Make sure it's the size of an ed25519 sig */
+    tt_int_op(ntohs(get_uint16(buf+35+1+32)), OP_EQ, ED25519_SIG_LEN);
+  }
+
+  /* Parse it as the receiver */
+  {
+    ret = trn_cell_establish_intro_parse(&cell_in, buf, sizeof(buf));
+    tt_u64_op(ret, OP_GT, 0);
+
+    ret = verify_establish_intro_cell(cell_in,
+                                      (const uint8_t *) circ_nonce,
+                                      sizeof(circ_nonce));
+    tt_u64_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  trn_cell_establish_intro_free(cell_in);
+}
+
+/* Mocked ed25519_sign_prefixed() function that always fails :) */
+static int
+mock_ed25519_sign_prefixed(ed25519_signature_t *signature_out,
+                           const uint8_t *msg, size_t msg_len,
+                           const char *prefix_str,
+                           const ed25519_keypair_t *keypair) {
+  (void) signature_out;
+  (void) msg;
+  (void) msg_len;
+  (void) prefix_str;
+  (void) keypair;
+  return -1;
+}
+
+/** We simulate a failure to create an ESTABLISH_INTRO cell */
+static void
+test_gen_establish_intro_cell_bad(void *arg)
+{
+  (void) arg;
+  ssize_t cell_len = 0;
+  trn_cell_establish_intro_t *cell = NULL;
+  char circ_nonce[DIGEST_LEN] = {0};
+  hs_service_intro_point_t *ip = NULL;
+
+  MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
+
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+
+  setup_full_capture_of_logs(LOG_WARN);
+  /* Easiest way to make that function fail is to mock the
+     ed25519_sign_prefixed() function and make it fail. */
+  cell = trn_cell_establish_intro_new();
+  tt_assert(cell);
+  ip = service_intro_point_new(NULL, 0, 0);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL);
+  service_intro_point_free(ip);
+  expect_log_msg_containing("Unable to make signature for "
+                            "ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_i64_op(cell_len, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  UNMOCK(ed25519_sign_prefixed);
+}
+
+struct testcase_t hs_cell_tests[] = {
+  { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
+    NULL, NULL },
+  { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
new file mode 100644
index 0000000000..50dca588ed
--- /dev/null
+++ b/src/test/test_hs_client.c
@@ -0,0 +1,603 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_client.c
+ * \brief Test prop224 HS client functionality.
+ */
+
+#define CRYPTO_PRIVATE
+#define MAIN_PRIVATE
+#define HS_CLIENT_PRIVATE
+#define TOR_CHANNEL_INTERNAL_
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CONNECTION_PRIVATE
+
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+#include "rend_test_helpers.h"
+#include "hs_test_helpers.h"
+
+#include "config.h"
+#include "crypto.h"
+#include "channeltls.h"
+#include "main.h"
+#include "nodelist.h"
+#include "routerset.h"
+
+#include "hs_circuit.h"
+#include "hs_client.h"
+#include "hs_ident.h"
+#include "hs_cache.h"
+#include "circuitlist.h"
+#include "circuitbuild.h"
+#include "connection.h"
+#include "connection_edge.h"
+#include "networkstatus.h"
+
+static int
+mock_connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
+{
+  (void) ap_conn;
+  return 0;
+}
+
+static networkstatus_t mock_ns;
+
+/* Always return NULL. */
+static networkstatus_t *
+mock_networkstatus_get_live_consensus_false(time_t now)
+{
+  (void) now;
+  return NULL;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+/* Test helper function: Setup a circuit and a stream with the same hidden
+ * service destination, and put them in <b>circ_out</b> and
+ * <b>conn_out</b>. Make the stream wait for circuits to be established to the
+ * hidden service. */
+static int
+helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,
+                                    connection_t **conn_out,
+                                    int is_legacy)
+{
+  int retval;
+  channel_tls_t *n_chan=NULL;
+  rend_data_t *conn_rend_data = NULL;
+  origin_circuit_t *or_circ = NULL;
+  connection_t *conn = NULL;
+  ed25519_public_key_t service_pk;
+
+  /* Make a dummy connection stream and make it wait for our circuit */
+  conn = test_conn_get_connection(AP_CONN_STATE_CIRCUIT_WAIT,
+                                  CONN_TYPE_AP /* ??? */,
+                                  0);
+  if (is_legacy) {
+    /* Legacy: Setup rend_data of stream */
+    char service_id[REND_SERVICE_ID_LEN_BASE32+1] = {0};
+    TO_EDGE_CONN(conn)->rend_data = mock_rend_data(service_id);
+    conn_rend_data = TO_EDGE_CONN(conn)->rend_data;
+  } else {
+    /* prop224: Setup hs conn identifier on the stream */
+    ed25519_secret_key_t sk;
+    tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
+    tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
+
+    /* Setup hs_conn_identifier of stream */
+    TO_EDGE_CONN(conn)->hs_ident = hs_ident_edge_conn_new(&service_pk);
+  }
+
+  /* Make it wait for circuit */
+  connection_ap_mark_as_pending_circuit(TO_ENTRY_CONN(conn));
+
+  /* This is needed to silence a BUG warning from
+     connection_edge_update_circuit_isolation() */
+  TO_ENTRY_CONN(conn)->original_dest_address =
+    tor_strdup(TO_ENTRY_CONN(conn)->socks_request->address);
+
+  /****************************************************/
+
+  /* Now make dummy circuit */
+  or_circ = origin_circuit_new();
+
+  or_circ->base_.purpose = CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED;
+
+  or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  or_circ->build_state->is_internal = 1;
+
+  if (is_legacy) {
+    /* Legacy: Setup rend data and final cpath */
+    or_circ->build_state->pending_final_cpath =
+      tor_malloc_zero(sizeof(crypt_path_t));
+    or_circ->build_state->pending_final_cpath->magic = CRYPT_PATH_MAGIC;
+    or_circ->build_state->pending_final_cpath->rend_dh_handshake_state =
+      crypto_dh_new(DH_TYPE_REND);
+    tt_assert(
+         or_circ->build_state->pending_final_cpath->rend_dh_handshake_state);
+    retval = crypto_dh_generate_public(
+           or_circ->build_state->pending_final_cpath->rend_dh_handshake_state);
+    tt_int_op(retval, OP_EQ, 0);
+    or_circ->rend_data = rend_data_dup(conn_rend_data);
+  } else {
+    /* prop224: Setup hs ident on the circuit */
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
+                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+  }
+
+  TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
+
+  /* fake n_chan */
+  n_chan = tor_malloc_zero(sizeof(channel_tls_t));
+  n_chan->base_.global_identifier = 1;
+  or_circ->base_.n_chan = &(n_chan->base_);
+
+  *circ_out = or_circ;
+  *conn_out = conn;
+
+  return 0;
+
+ done:
+  /* something failed */
+  return -1;
+}
+
+/* Test: Ensure that setting up legacy e2e rendezvous circuits works
+ * correctly. */
+static void
+test_e2e_rend_circuit_setup_legacy(void *arg)
+{
+  ssize_t retval;
+  origin_circuit_t *or_circ = NULL;
+  connection_t *conn = NULL;
+
+  (void) arg;
+
+  /** In this test we create a v2 legacy HS stream and a circuit with the same
+   *  hidden service destination. We make the stream wait for circuits to be
+   *  established to the hidden service, and then we complete the circuit using
+   *  the hs_circuit_setup_e2e_rend_circ_legacy_client() function. We then
+   *  check that the end-to-end cpath was setup correctly and that the stream
+   *  was attached to the circuit as expected. */
+
+  MOCK(connection_ap_handshake_send_begin,
+       mock_connection_ap_handshake_send_begin);
+
+  /* Setup */
+  retval = helper_get_circ_and_stream_for_test( &or_circ, &conn, 1);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(or_circ);
+  tt_assert(conn);
+
+  /* Check number of hops */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check that our stream is not attached on any circuits */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, NULL);
+
+  /********************************************** */
+
+  /* Make a good RENDEZVOUS1 cell body because it needs to pass key exchange
+   * digest verification... */
+  uint8_t rend_cell_body[DH_KEY_LEN+DIGEST_LEN] = {2};
+  {
+    char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN];
+    crypto_dh_t *dh_state =
+      or_circ->build_state->pending_final_cpath->rend_dh_handshake_state;
+    /* compute and overwrite digest of cell body with the right value */
+    retval = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh_state,
+                                      (char*)rend_cell_body, DH_KEY_LEN,
+                                      keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN);
+    tt_int_op(retval, OP_GT, 0);
+    memcpy(rend_cell_body+DH_KEY_LEN, keys, DIGEST_LEN);
+  }
+
+  /* Setup the circuit */
+  retval = hs_circuit_setup_e2e_rend_circ_legacy_client(or_circ,
+                                                        rend_cell_body);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /**********************************************/
+
+  /* See that a hop was added to the circuit's cpath */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Check the digest algo */
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+            OP_EQ, DIGEST_SHA1);
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+            OP_EQ, DIGEST_SHA1);
+  tt_assert(or_circ->cpath->crypto.f_crypto);
+  tt_assert(or_circ->cpath->crypto.b_crypto);
+
+  /* Ensure that circ purpose was changed */
+  tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
+
+  /* Test that stream got attached */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, TO_CIRCUIT(or_circ));
+
+ done:
+  connection_free_minimal(conn);
+  if (or_circ)
+    tor_free(TO_CIRCUIT(or_circ)->n_chan);
+  circuit_free_(TO_CIRCUIT(or_circ));
+}
+
+/* Test: Ensure that setting up v3 rendezvous circuits works correctly. */
+static void
+test_e2e_rend_circuit_setup(void *arg)
+{
+  uint8_t ntor_key_seed[DIGEST256_LEN] = {0};
+  origin_circuit_t *or_circ = NULL;
+  int retval;
+  connection_t *conn = NULL;
+
+  (void) arg;
+
+  /** In this test we create a prop224 v3 HS stream and a circuit with the same
+   *  hidden service destination. We make the stream wait for circuits to be
+   *  established to the hidden service, and then we complete the circuit using
+   *  the hs_circuit_setup_e2e_rend_circ() function. We then check that the
+   *  end-to-end cpath was setup correctly and that the stream was attached to
+   *  the circuit as expected. */
+
+  MOCK(connection_ap_handshake_send_begin,
+       mock_connection_ap_handshake_send_begin);
+
+  /* Setup */
+  retval = helper_get_circ_and_stream_for_test( &or_circ, &conn, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(or_circ);
+  tt_assert(conn);
+
+  /* Check number of hops: There should be no hops yet to this circ */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_ptr_op(or_circ->cpath, OP_EQ, NULL);
+
+  /* Check that our stream is not attached on any circuits */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, NULL);
+
+  /**********************************************/
+
+  /* Setup the circuit */
+  retval = hs_circuit_setup_e2e_rend_circ(or_circ,
+                                          ntor_key_seed, sizeof(ntor_key_seed),
+                                          0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /**********************************************/
+
+  /* See that a hop was added to the circuit's cpath */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Check that the crypt path has prop224 algorithm parameters */
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_assert(or_circ->cpath->crypto.f_crypto);
+  tt_assert(or_circ->cpath->crypto.b_crypto);
+
+  /* Ensure that circ purpose was changed */
+  tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
+
+  /* Test that stream got attached */
+  tt_ptr_op(TO_EDGE_CONN(conn)->on_circuit, OP_EQ, TO_CIRCUIT(or_circ));
+
+ done:
+  connection_free_minimal(conn);
+  if (or_circ)
+    tor_free(TO_CIRCUIT(or_circ)->n_chan);
+  circuit_free_(TO_CIRCUIT(or_circ));
+}
+
+/** Test client logic for picking intro points from a descriptor. Also test how
+ *  ExcludeNodes and intro point failures affect picking intro points. */
+static void
+test_client_pick_intro(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  hs_descriptor_t *desc = NULL;
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  (void) arg;
+
+  hs_init();
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Set time */
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  update_approx_time(mock_ns.fresh_until-10);
+  time_t now = approx_time();
+
+  /* Test logic:
+   *
+   * 1) Add our desc with intro points to the HS cache.
+   *
+   * 2) Mark all descriptor intro points except _the chosen one_ as
+   *    failed. Then query the desc to get a random intro: check that we got
+   *    _the chosen one_. Then fail the chosen one as well, and see that no
+   *    intros are returned.
+   *
+   * 3) Then clean the intro state cache and get an intro point.
+   *
+   * 4) Try fetching an intro with the wrong service key: shouldn't work
+   *
+   * 5) Set StrictNodes and put all our intro points in ExcludeNodes: see that
+   *    nothing is returned.
+   */
+
+  /* 1) Add desc to HS cache */
+  {
+    char *encoded = NULL;
+    desc = hs_helper_build_hs_desc_with_ip(&service_kp);
+    ret = hs_desc_encode_descriptor(desc, &service_kp, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* store it */
+    hs_cache_store_as_client(encoded, &service_kp.pubkey);
+
+    /* fetch it to make sure it works */
+    const hs_descriptor_t *fetched_desc =
+      hs_cache_lookup_as_client(&service_kp.pubkey);
+    tt_assert(fetched_desc);
+    tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential,
+              DIGEST256_LEN);
+    tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential,
+                               DIGEST256_LEN));
+    tor_free(encoded);
+  }
+
+  /* 2) Mark all intro points except _the chosen one_ as failed. Then query the
+   *   desc and get a random intro: check that we got _the chosen one_. */
+  {
+    /* Pick the chosen intro point and get its ei */
+    hs_desc_intro_point_t *chosen_intro_point =
+      smartlist_get(desc->encrypted_data.intro_points, 0);
+    extend_info_t *chosen_intro_ei =
+      desc_intro_point_to_extend_info(chosen_intro_point);
+    tt_assert(chosen_intro_point);
+    tt_assert(chosen_intro_ei);
+
+    /* Now mark all other intro points as failed */
+    SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                            hs_desc_intro_point_t *, ip) {
+      /* Skip the chosen intro point */
+      if (ip == chosen_intro_point) {
+        continue;
+      }
+      ed25519_public_key_t *intro_auth_key = &ip->auth_key_cert->signed_key;
+      hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                       intro_auth_key,
+                                       INTRO_POINT_FAILURE_GENERIC);
+    } SMARTLIST_FOREACH_END(ip);
+
+    /* Try to get a random intro: Should return the chosen one! */
+    /* (We try several times, to make sure this behavior is consistent, and to
+     * cover the different cases of client_get_random_intro().) */
+    for (int i = 0; i < 64; ++i) {
+      extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+      tor_assert(ip);
+      tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
+      tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest,
+                DIGEST_LEN);
+      extend_info_free(ip);
+    }
+
+    extend_info_free(chosen_intro_ei);
+
+    /* Now also mark the chosen one as failed: See that we can't get any intro
+       points anymore. */
+    hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                &chosen_intro_point->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_TIMEOUT);
+    extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+    tor_assert(!ip);
+  }
+
+  /* 3) Clean the intro state cache and get an intro point */
+  {
+    /* Pretend we are 5 mins in the future and order a cleanup of the intro
+     * state. This should clean up the intro point failures and allow us to get
+     * an intro. */
+    hs_cache_client_intro_state_clean(now + 5*60);
+
+    /* Get an intro. It should work! */
+    extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+    tor_assert(ip);
+    extend_info_free(ip);
+  }
+
+  /* 4) Try fetching an intro with the wrong service key: shouldn't work */
+  {
+    ed25519_keypair_t dummy_kp;
+    tt_int_op(0, OP_EQ, ed25519_keypair_generate(&dummy_kp, 0));
+    extend_info_t *ip = client_get_random_intro(&dummy_kp.pubkey);
+    tor_assert(!ip);
+  }
+
+  /* 5) Set StrictNodes and put all our intro points in ExcludeNodes: see that
+   *    nothing is returned. */
+  {
+    get_options_mutable()->ExcludeNodes = routerset_new();
+    get_options_mutable()->StrictNodes = 1;
+    SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                            hs_desc_intro_point_t *, ip) {
+      extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip);
+      if (intro_ei) {
+        const char *ptr;
+        char ip_addr[TOR_ADDR_BUF_LEN];
+        /* We need to decorate in case it is an IPv6 else routerset_parse()
+         * doesn't like it. */
+        ptr = tor_addr_to_str(ip_addr, &intro_ei->addr, sizeof(ip_addr), 1);
+        tt_assert(ptr == ip_addr);
+        ret = routerset_parse(get_options_mutable()->ExcludeNodes,
+                              ip_addr, "");
+        tt_int_op(ret, OP_EQ, 0);
+        extend_info_free(intro_ei);
+      }
+    } SMARTLIST_FOREACH_END(ip);
+
+    extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
+    tt_assert(!ip);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+}
+
+static int
+mock_router_have_minimum_dir_info_false(void)
+{
+  return 0;
+}
+static int
+mock_router_have_minimum_dir_info_true(void)
+{
+  return 1;
+}
+
+static hs_client_fetch_status_t
+mock_fetch_v3_desc_error(const ed25519_public_key_t *key)
+{
+  (void) key;
+  return HS_CLIENT_FETCH_ERROR;
+}
+
+static void
+mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
+                                    int line, const char *file)
+{
+  (void) line;
+  (void) file;
+  conn->edge_.end_reason = endreason;
+}
+
+static void
+test_descriptor_fetch(void *arg)
+{
+  int ret;
+  entry_connection_t *ec = NULL;
+  ed25519_public_key_t service_pk;
+  ed25519_secret_key_t service_sk;
+
+  (void) arg;
+
+  hs_init();
+  memset(&service_sk, 'A', sizeof(service_sk));
+  ret = ed25519_public_key_generate(&service_pk, &service_sk);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Initialize this so get_voting_interval() doesn't freak out. */
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ec = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  tt_assert(ec);
+  ENTRY_TO_EDGE_CONN(ec)->hs_ident = hs_ident_edge_conn_new(&service_pk);
+  tt_assert(ENTRY_TO_EDGE_CONN(ec)->hs_ident);
+  TO_CONN(ENTRY_TO_EDGE_CONN(ec))->state = AP_CONN_STATE_RENDDESC_WAIT;
+  smartlist_add(get_connection_array(), &ec->edge_.base_);
+
+  /* 1. FetchHidServDescriptors is false so we shouldn't be able to fetch. */
+  get_options_mutable()->FetchHidServDescriptors = 0;
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_NOT_ALLOWED);
+  get_options_mutable()->FetchHidServDescriptors = 1;
+
+  /* 2. We don't have a live consensus. */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus_false);
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  UNMOCK(networkstatus_get_live_consensus);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_MISSING_INFO);
+
+  /* From now on, return a live consensus. */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* 3. Not enough dir information. */
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_false);
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  UNMOCK(router_have_minimum_dir_info);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_MISSING_INFO);
+
+  /* From now on, we do have enough directory information. */
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_true);
+
+  /* 4. We do have a pending directory request. */
+  {
+    dir_connection_t *dir_conn = dir_connection_new(AF_INET);
+    dir_conn->hs_ident = tor_malloc_zero(sizeof(hs_ident_dir_conn_t));
+    TO_CONN(dir_conn)->purpose = DIR_PURPOSE_FETCH_HSDESC;
+    ed25519_pubkey_copy(&dir_conn->hs_ident->identity_pk, &service_pk);
+    smartlist_add(get_connection_array(), TO_CONN(dir_conn));
+    ret = hs_client_refetch_hsdesc(&service_pk);
+    smartlist_remove(get_connection_array(), TO_CONN(dir_conn));
+    connection_free_minimal(TO_CONN(dir_conn));
+    tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_PENDING);
+  }
+
+  /* 5. We'll trigger an error on the fetch_desc_v3 and force to close all
+   *    pending SOCKS request. */
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info_true);
+  MOCK(fetch_v3_desc, mock_fetch_v3_desc_error);
+  MOCK(connection_mark_unattached_ap_,
+       mock_connection_mark_unattached_ap_);
+  ret = hs_client_refetch_hsdesc(&service_pk);
+  UNMOCK(fetch_v3_desc);
+  UNMOCK(connection_mark_unattached_ap_);
+  tt_int_op(ret, OP_EQ, HS_CLIENT_FETCH_ERROR);
+  /* The close waiting for descriptor function has been called. */
+  tt_int_op(ec->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
+
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(router_have_minimum_dir_info);
+  hs_free_all();
+}
+
+struct testcase_t hs_client_tests[] = {
+  { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy,
+    TT_FORK, NULL, NULL },
+  { "e2e_rend_circuit_setup", test_e2e_rend_circuit_setup,
+    TT_FORK, NULL, NULL },
+  { "client_pick_intro", test_client_pick_intro,
+    TT_FORK, NULL, NULL },
+  { "descriptor_fetch", test_descriptor_fetch,
+    TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
new file mode 100644
index 0000000000..7348eb746c
--- /dev/null
+++ b/src/test/test_hs_common.c
@@ -0,0 +1,1829 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_common.c
+ * \brief Test hidden service common functionalities.
+ */
+
+#define HS_COMMON_PRIVATE
+#define HS_CLIENT_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define NODELIST_PRIVATE
+
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+#include "hs_test_helpers.h"
+
+#include "connection_edge.h"
+#include "crypto_rand.h"
+#include "hs_common.h"
+#include "hs_client.h"
+#include "hs_service.h"
+#include "config.h"
+#include "networkstatus.h"
+#include "directory.h"
+#include "dirauth/dirvote.h"
+#include "nodelist.h"
+#include "routerlist.h"
+#include "statefile.h"
+#include "circuitlist.h"
+#include "dirauth/shared_random.h"
+#include "util.h"
+#include "voting_schedule.h"
+
+/** Test the validation of HS v3 addresses */
+static void
+test_validate_address(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Address too short and too long. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid("blah");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("has an invalid length");
+  teardown_capture_of_logs();
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "p3xnclpu4mu22dwaurjtsybyqk4xfjmcfz6z62yl24uwmhjatiwnlnadb");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("has an invalid length");
+  teardown_capture_of_logs();
+
+  /* Invalid checksum (taken from prop224) */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "l5satjgud6gucryazcyvyvhuxhr74u6ygigiuyixe3a6ysis67ororad");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("invalid checksum");
+  teardown_capture_of_logs();
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "btojiu7nu5y5iwut64eufevogqdw4wmqzugnoluw232r4t3ecsfv37ad");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("invalid checksum");
+  teardown_capture_of_logs();
+
+  /* Non base32 decodable string. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_address_is_valid(
+           "????????????????????????????????????????????????????????");
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg_containing("can't be decoded");
+  teardown_capture_of_logs();
+
+  /* Valid address. */
+  ret = hs_address_is_valid(
+           "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static int
+mock_write_str_to_file(const char *path, const char *str, int bin)
+{
+  (void)bin;
+  tt_str_op(path, OP_EQ, "/double/five"PATH_SEPARATOR"squared");
+  tt_str_op(str, OP_EQ,
+           "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion\n");
+
+ done:
+  return 0;
+}
+
+/** Test building HS v3 onion addresses. Uses test vectors from the
+ *  ./hs_build_address.py script. */
+static void
+test_build_address(void *arg)
+{
+  int ret;
+  char onion_addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  ed25519_public_key_t pubkey;
+  /* hex-encoded ed25519 pubkey used in hs_build_address.py */
+  char pubkey_hex[] =
+    "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a";
+  hs_service_t *service = NULL;
+
+  (void) arg;
+
+  MOCK(write_str_to_file, mock_write_str_to_file);
+
+  /* The following has been created with hs_build_address.py script that
+   * follows proposal 224 specification to build an onion address. */
+  static const char *test_addr =
+    "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid";
+
+  /* Let's try to build the same onion address as the script */
+  base16_decode((char*)pubkey.pubkey, sizeof(pubkey.pubkey),
+                pubkey_hex, strlen(pubkey_hex));
+  hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
+  tt_str_op(test_addr, OP_EQ, onion_addr);
+  /* Validate that address. */
+  ret = hs_address_is_valid(onion_addr);
+  tt_int_op(ret, OP_EQ, 1);
+
+  service = tor_malloc_zero(sizeof(hs_service_t));
+  memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
+  tor_asprintf(&service->config.directory_path, "/double/five");
+  ret = write_address_to_file(service, "squared");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  hs_service_free(service);
+}
+
+/** Test that our HS time period calculation functions work properly */
+static void
+test_time_period(void *arg)
+{
+  (void) arg;
+  uint64_t tn;
+  int retval;
+  time_t fake_time, correct_time, start_time;
+
+  /* Let's do the example in prop224 section [TIME-PERIODS] */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check that the time period number is right */
+  tn = hs_get_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16903);
+
+  /* Increase current time to 11:59:59 UTC and check that the time period
+     number is still the same */
+  fake_time += 3599;
+  tn = hs_get_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16903);
+
+  { /* Check start time of next time period */
+    retval = parse_rfc1123_time("Wed, 13 Apr 2016 12:00:00 UTC",
+                                &correct_time);
+    tt_int_op(retval, OP_EQ, 0);
+
+    start_time = hs_get_start_time_of_next_time_period(fake_time);
+    tt_int_op(start_time, OP_EQ, correct_time);
+  }
+
+  /* Now take time to 12:00:00 UTC and check that the time period rotated */
+  fake_time += 1;
+  tn = hs_get_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16904);
+
+  /* Now also check our hs_get_next_time_period_num() function */
+  tn = hs_get_next_time_period_num(fake_time);
+  tt_u64_op(tn, OP_EQ, 16905);
+
+  { /* Check start time of next time period again */
+    retval = parse_rfc1123_time("Wed, 14 Apr 2016 12:00:00 UTC",
+                                &correct_time);
+    tt_int_op(retval, OP_EQ, 0);
+
+    start_time = hs_get_start_time_of_next_time_period(fake_time);
+    tt_int_op(start_time, OP_EQ, correct_time);
+  }
+
+  /* Now do another sanity check: The time period number at the start of the
+   * next time period, must be the same time period number as the one returned
+   * from hs_get_next_time_period_num() */
+  {
+    time_t next_tp_start = hs_get_start_time_of_next_time_period(fake_time);
+    tt_u64_op(hs_get_time_period_num(next_tp_start), OP_EQ,
+              hs_get_next_time_period_num(fake_time));
+  }
+
+ done:
+  ;
+}
+
+/** Test that we can correctly find the start time of the next time period */
+static void
+test_start_time_of_next_time_period(void *arg)
+{
+  (void) arg;
+  int retval;
+  time_t fake_time;
+  char tbuf[ISO_TIME_LEN + 1];
+  time_t next_tp_start_time;
+
+  /* Do some basic tests */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, OP_EQ, 0);
+  next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+  /* Compare it with the correct result */
+  format_iso_time(tbuf, next_tp_start_time);
+  tt_str_op("2016-04-13 12:00:00", OP_EQ, tbuf);
+
+  /* Another test with an edge-case time (start of TP) */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 12:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, OP_EQ, 0);
+  next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+  format_iso_time(tbuf, next_tp_start_time);
+  tt_str_op("2016-04-14 12:00:00", OP_EQ, tbuf);
+
+  {
+    /* Now pretend we are on a testing network and alter the voting schedule to
+       be every 10 seconds. This means that a time period has length 10*24
+       seconds (4 minutes). It also means that we apply a rotational offset of
+       120 seconds to the time period, so that it starts at 00:02:00 instead of
+       00:00:00. */
+    or_options_t *options = get_options_mutable();
+    options->TestingTorNetwork = 1;
+    options->V3AuthVotingInterval = 10;
+    options->TestingV3AuthInitialVotingInterval = 10;
+
+    retval = parse_rfc1123_time("Wed, 13 Apr 2016 00:00:00 UTC",
+                                &fake_time);
+    tt_int_op(retval, OP_EQ, 0);
+    next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, next_tp_start_time);
+    tt_str_op("2016-04-13 00:02:00", OP_EQ, tbuf);
+
+    retval = parse_rfc1123_time("Wed, 13 Apr 2016 00:02:00 UTC",
+                                &fake_time);
+    tt_int_op(retval, OP_EQ, 0);
+    next_tp_start_time = hs_get_start_time_of_next_time_period(fake_time);
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, next_tp_start_time);
+    tt_str_op("2016-04-13 00:06:00", OP_EQ, tbuf);
+  }
+
+ done:
+  ;
+}
+
+/* Cleanup the global nodelist. It also frees the "md" in the node_t because
+ * we allocate the memory in helper_add_hsdir_to_networkstatus(). */
+static void
+cleanup_nodelist(void)
+{
+  smartlist_t *nodelist = nodelist_get_list();
+  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
+    tor_free(node->md);
+    node->md = NULL;
+  } SMARTLIST_FOREACH_END(node);
+  nodelist_free_all();
+}
+
+static void
+helper_add_hsdir_to_networkstatus(networkstatus_t *ns,
+                                  int identity_idx,
+                                  const char *nickname,
+                                  int is_hsdir)
+{
+  routerstatus_t *rs = tor_malloc_zero(sizeof(routerstatus_t));
+  routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t));
+  uint8_t identity[DIGEST_LEN];
+  tor_addr_t ipv4_addr;
+  node_t *node = NULL;
+
+  memset(identity, identity_idx, sizeof(identity));
+
+  memcpy(rs->identity_digest, identity, DIGEST_LEN);
+  rs->is_hs_dir = is_hsdir;
+  rs->pv.supports_v3_hsdir = 1;
+  strlcpy(rs->nickname, nickname, sizeof(rs->nickname));
+  tor_addr_parse(&ipv4_addr, "1.2.3.4");
+  ri->addr = tor_addr_to_ipv4h(&ipv4_addr);
+  rs->addr = tor_addr_to_ipv4h(&ipv4_addr);
+  ri->nickname = tor_strdup(nickname);
+  ri->protocol_list = tor_strdup("HSDir=1-2 LinkAuth=3");
+  memcpy(ri->cache_info.identity_digest, identity, DIGEST_LEN);
+  ri->cache_info.signing_key_cert = tor_malloc_zero(sizeof(tor_cert_t));
+  /* Needed for the HSDir index computation. */
+  memset(&ri->cache_info.signing_key_cert->signing_key,
+         identity_idx, ED25519_PUBKEY_LEN);
+  tt_assert(nodelist_set_routerinfo(ri, NULL));
+
+  node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+  tt_assert(node);
+  node->rs = rs;
+  /* We need this to exist for node_has_preferred_descriptor() to return
+   * true. */
+  node->md = tor_malloc_zero(sizeof(microdesc_t));
+  /* Do this now the nodelist_set_routerinfo() function needs a "rs" to set
+   * the indexes which it doesn't have when it is called. */
+  node_set_hsdir_index(node, ns);
+  node->ri = NULL;
+  smartlist_add(ns->routerstatus_list, rs);
+
+ done:
+  if (node == NULL)
+    routerstatus_free(rs);
+
+  routerinfo_free(ri);
+}
+
+static networkstatus_t *mock_ns = NULL;
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  time_t now = approx_time();
+
+  /* If initialized, return it */
+  if (mock_ns) {
+    return mock_ns;
+  }
+
+  /* Initialize fake consensus */
+  mock_ns = tor_malloc_zero(sizeof(networkstatus_t));
+
+  /* This consensus is live */
+  mock_ns->valid_after = now-1;
+  mock_ns->fresh_until = now+1;
+  mock_ns->valid_until = now+2;
+  /* Create routerstatus list */
+  mock_ns->routerstatus_list = smartlist_new();
+  mock_ns->type = NS_TYPE_CONSENSUS;
+
+  return mock_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+
+  tt_assert(mock_ns);
+
+ done:
+  return mock_ns;
+}
+
+/** Test the responsible HSDirs calculation function */
+static void
+test_responsible_hsdirs(void *arg)
+{
+  smartlist_t *responsible_dirs = smartlist_new();
+  networkstatus_t *ns = NULL;
+  (void) arg;
+
+  hs_init();
+
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+
+  ns = networkstatus_get_latest_consensus();
+
+  { /* First router: HSdir */
+    helper_add_hsdir_to_networkstatus(ns, 1, "igor", 1);
+  }
+
+  { /* Second HSDir */
+    helper_add_hsdir_to_networkstatus(ns, 2, "victor", 1);
+  }
+
+  { /* Third relay but not HSDir */
+    helper_add_hsdir_to_networkstatus(ns, 3, "spyro", 0);
+  }
+
+  /* Use a fixed time period and pub key so we always take the same path */
+  ed25519_public_key_t pubkey;
+  uint64_t time_period_num = 17653; // 2 May, 2018, 14:00.
+  memset(&pubkey, 42, sizeof(pubkey));
+
+  hs_get_responsible_hsdirs(&pubkey, time_period_num,
+                            0, 0, responsible_dirs);
+
+  /* Make sure that we only found 2 responsible HSDirs.
+   * The third relay was not an hsdir! */
+  tt_int_op(smartlist_len(responsible_dirs), OP_EQ, 2);
+
+  /** TODO: Build a bigger network and do more tests here */
+
+ done:
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_free(responsible_dirs);
+  smartlist_clear(ns->routerstatus_list);
+  networkstatus_vote_free(mock_ns);
+  cleanup_nodelist();
+}
+
+static void
+mock_directory_initiate_request(directory_request_t *req)
+{
+  (void)req;
+  return;
+}
+
+static int
+mock_hs_desc_encode_descriptor(const hs_descriptor_t *desc,
+                           const ed25519_keypair_t *signing_kp,
+                           char **encoded_out)
+{
+  (void)desc;
+  (void)signing_kp;
+
+  tor_asprintf(encoded_out, "lulu");
+  return 0;
+}
+
+static or_state_t dummy_state;
+
+/* Mock function to get fake or state (used for rev counters) */
+static or_state_t *
+get_or_state_replacement(void)
+{
+  return &dummy_state;
+}
+
+static int
+mock_router_have_minimum_dir_info(void)
+{
+  return 1;
+}
+
+/** Test that we correctly detect when the HSDir hash ring changes so that we
+ *  reupload our descriptor. */
+static void
+test_desc_reupload_logic(void *arg)
+{
+  networkstatus_t *ns = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(router_have_minimum_dir_info,
+       mock_router_have_minimum_dir_info);
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(directory_initiate_request,
+       mock_directory_initiate_request);
+  MOCK(hs_desc_encode_descriptor,
+       mock_hs_desc_encode_descriptor);
+
+  ns = networkstatus_get_latest_consensus();
+
+  /** Test logic:
+   *  1) Upload descriptor to HSDirs
+   *     CHECK that previous_hsdirs list was populated.
+   *  2) Then call router_dir_info_changed() without an HSDir set change.
+   *     CHECK that no reuplod occurs.
+   *  3) Now change the HSDir set, and call dir_info_changed() again.
+   *     CHECK that reupload occurs.
+   *  4) Finally call service_desc_schedule_upload().
+   *     CHECK that previous_hsdirs list was cleared.
+   **/
+
+  /* Let's start by building our descriptor and service */
+  hs_service_descriptor_t *desc = service_descriptor_new();
+  hs_service_t *service = NULL;
+  /* hex-encoded ed25519 pubkey used in hs_build_address.py */
+  char pubkey_hex[] =
+    "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a";
+  char onion_addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  ed25519_public_key_t pubkey;
+  base16_decode((char*)pubkey.pubkey, sizeof(pubkey.pubkey),
+                pubkey_hex, strlen(pubkey_hex));
+  hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
+  service = tor_malloc_zero(sizeof(hs_service_t));
+  memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  service->desc_current = desc;
+  /* Also add service to service map */
+  hs_service_ht *service_map = get_hs_service_map();
+  tt_assert(service_map);
+  tt_int_op(hs_service_get_num_services(), OP_EQ, 0);
+  register_service(service_map, service);
+  tt_int_op(hs_service_get_num_services(), OP_EQ, 1);
+
+  /* Now let's create our hash ring: */
+  {
+    helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1);
+    helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1);
+    helper_add_hsdir_to_networkstatus(ns, 3, "aaron", 1);
+    helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1);
+    helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1);
+    helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1);
+  }
+
+  /* Now let's upload our desc to all hsdirs */
+  upload_descriptor_to_all(service, desc);
+  /* Check that previous hsdirs were populated */
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+  /* Poison next upload time so that we can see if it was changed by
+   * router_dir_info_changed(). No changes in hash ring so far, so the upload
+   * time should stay as is. */
+  desc->next_upload_time = 42;
+  router_dir_info_changed();
+  tt_int_op(desc->next_upload_time, OP_EQ, 42);
+
+  /* Now change the HSDir hash ring by swapping nora for aaron.
+   * Start by clearing the hash ring */
+  {
+    SMARTLIST_FOREACH(ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(ns->routerstatus_list);
+    cleanup_nodelist();
+    routerlist_free_all();
+  }
+
+  { /* Now add back all the nodes */
+    helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1);
+    helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1);
+    helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1);
+    helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1);
+    helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1);
+    helper_add_hsdir_to_networkstatus(ns, 7, "nora", 1);
+  }
+
+  /* Now call service_desc_hsdirs_changed() and see that it detected the hash
+     ring change */
+  time_t now = approx_time();
+  tt_assert(now);
+  tt_int_op(service_desc_hsdirs_changed(service, desc), OP_EQ, 1);
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+  /* Now order another upload and see that we keep having 6 prev hsdirs */
+  upload_descriptor_to_all(service, desc);
+  /* Check that previous hsdirs were populated */
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+  /* Now restore the HSDir hash ring to its original state by swapping back
+     aaron for nora */
+  /* First clear up the hash ring */
+  {
+    SMARTLIST_FOREACH(ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(ns->routerstatus_list);
+    cleanup_nodelist();
+    routerlist_free_all();
+  }
+
+  { /* Now populate the hash ring again */
+    helper_add_hsdir_to_networkstatus(ns, 1, "dingus", 1);
+    helper_add_hsdir_to_networkstatus(ns, 2, "clive", 1);
+    helper_add_hsdir_to_networkstatus(ns, 3, "aaron", 1);
+    helper_add_hsdir_to_networkstatus(ns, 4, "lizzie", 1);
+    helper_add_hsdir_to_networkstatus(ns, 5, "daewon", 1);
+    helper_add_hsdir_to_networkstatus(ns, 6, "clarke", 1);
+  }
+
+  /* Check that our algorithm catches this change of hsdirs */
+  tt_int_op(service_desc_hsdirs_changed(service, desc), OP_EQ, 1);
+
+  /* Now pretend that the descriptor changed, and order a reupload to all
+     HSDirs. Make sure that the set of previous HSDirs was cleared. */
+  service_desc_schedule_upload(desc, now, 1);
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 0);
+
+  /* Now reupload again: see that the prev hsdir set got populated again. */
+  upload_descriptor_to_all(service, desc);
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 6);
+
+ done:
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_clear(ns->routerstatus_list);
+  networkstatus_vote_free(ns);
+  cleanup_nodelist();
+  hs_free_all();
+}
+
+/** Test disaster SRV computation and caching */
+static void
+test_disaster_srv(void *arg)
+{
+  uint8_t *cached_disaster_srv_one = NULL;
+  uint8_t *cached_disaster_srv_two = NULL;
+  uint8_t srv_one[DIGEST256_LEN] = {0};
+  uint8_t srv_two[DIGEST256_LEN] = {0};
+  uint8_t srv_three[DIGEST256_LEN] = {0};
+  uint8_t srv_four[DIGEST256_LEN] = {0};
+  uint8_t srv_five[DIGEST256_LEN] = {0};
+
+  (void) arg;
+
+  /* Get the cached SRVs: we gonna use them later for verification */
+  cached_disaster_srv_one = get_first_cached_disaster_srv();
+  cached_disaster_srv_two = get_second_cached_disaster_srv();
+
+  /* Compute some srvs */
+  get_disaster_srv(1, srv_one);
+  get_disaster_srv(2, srv_two);
+
+  /* Check that the cached ones where updated */
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
+
+  /* Ask for an SRV that has already been computed */
+  get_disaster_srv(2, srv_two);
+  /* and check that the cache entries have not changed */
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
+
+  /* Ask for a new SRV */
+  get_disaster_srv(3, srv_three);
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_three, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
+
+  /* Ask for another SRV: none of the original SRVs should now be cached */
+  get_disaster_srv(4, srv_four);
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_three, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_four, DIGEST256_LEN);
+
+  /* Ask for yet another SRV */
+  get_disaster_srv(5, srv_five);
+  tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_five, DIGEST256_LEN);
+  tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_four, DIGEST256_LEN);
+
+ done:
+  ;
+}
+
+/** Test our HS descriptor request tracker by making various requests and
+ *  checking whether they get tracked properly. */
+static void
+test_hid_serv_request_tracker(void *arg)
+{
+  (void) arg;
+  time_t retval;
+  routerstatus_t *hsdir = NULL, *hsdir2 = NULL, *hsdir3 = NULL;
+  time_t now = approx_time();
+
+  const char *req_key_str_first =
+ "vd4zb6zesaubtrjvdqcr2w7x7lhw2up4Xnw4526ThUNbL5o1go+EdUuEqlKxHkNbnK41pRzizzs";
+  const char *req_key_str_second =
+ "g53o7iavcd62oihswhr24u6czmqws5kpXnw4526ThUNbL5o1go+EdUuEqlKxHkNbnK41pRzizzs";
+  const char *req_key_str_small = "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ";
+
+  /*************************** basic test *******************************/
+
+  /* Get request tracker and make sure it's empty */
+  strmap_t *request_tracker = get_last_hid_serv_requests();
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 0);
+
+  /* Let's register a hid serv request */
+  hsdir = tor_malloc_zero(sizeof(routerstatus_t));
+  memset(hsdir->identity_digest, 'Z', DIGEST_LEN);
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_first,
+                                           now, 1);
+  tt_int_op(retval, OP_EQ, now);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+
+  /* Let's lookup a non-existent hidserv request */
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_second,
+                                           now+1, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+
+  /* Let's lookup a real hidserv request */
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_first,
+                                           now+2, 0);
+  tt_int_op(retval, OP_EQ, now); /* we got it */
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+
+  /**********************************************************************/
+
+  /* Let's add another request for the same HS but on a different HSDir. */
+  hsdir2 = tor_malloc_zero(sizeof(routerstatus_t));
+  memset(hsdir2->identity_digest, 2, DIGEST_LEN);
+  retval = hs_lookup_last_hid_serv_request(hsdir2, req_key_str_first,
+                                           now+3, 1);
+  tt_int_op(retval, OP_EQ, now+3);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 2);
+
+  /* Check that we can clean the first request based on time */
+  hs_clean_last_hid_serv_requests(now+3+REND_HID_SERV_DIR_REQUERY_PERIOD);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 1);
+  /* Check that it doesn't exist anymore */
+  retval = hs_lookup_last_hid_serv_request(hsdir, req_key_str_first,
+                                           now+2, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now let's add a smaller req key str */
+  hsdir3 = tor_malloc_zero(sizeof(routerstatus_t));
+  memset(hsdir3->identity_digest, 3, DIGEST_LEN);
+  retval = hs_lookup_last_hid_serv_request(hsdir3, req_key_str_small,
+                                           now+4, 1);
+  tt_int_op(retval, OP_EQ, now+4);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 2);
+
+  /*************************** deleting entries **************************/
+
+  /* Add another request with very short key */
+  retval = hs_lookup_last_hid_serv_request(hsdir, "l",  now, 1);
+  tt_int_op(retval, OP_EQ, now);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+
+  /* Try deleting entries with a dummy key. Check that our previous requests
+   * are still there */
+  tor_capture_bugs_(1);
+  hs_purge_hid_serv_from_last_hid_serv_requests("a");
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+  tor_end_capture_bugs_();
+
+  /* Try another dummy key. Check that requests are still there */
+  {
+    char dummy[2000];
+    memset(dummy, 'Z', 2000);
+    dummy[1999] = '\x00';
+    hs_purge_hid_serv_from_last_hid_serv_requests(dummy);
+    tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+  }
+
+  /* Another dummy key! */
+  hs_purge_hid_serv_from_last_hid_serv_requests(req_key_str_second);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 3);
+
+  /* Now actually delete a request! */
+  hs_purge_hid_serv_from_last_hid_serv_requests(req_key_str_first);
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 2);
+
+  /* Purge it all! */
+  hs_purge_last_hid_serv_requests();
+  request_tracker = get_last_hid_serv_requests();
+  tt_int_op(strmap_size(request_tracker),OP_EQ, 0);
+
+ done:
+  tor_free(hsdir);
+  tor_free(hsdir2);
+  tor_free(hsdir3);
+}
+
+static void
+test_parse_extended_hostname(void *arg)
+{
+  (void) arg;
+
+  char address1[] = "fooaddress.onion";
+  char address2[] = "aaaaaaaaaaaaaaaa.onion";
+  char address3[] = "fooaddress.exit";
+  char address4[] = "www.torproject.org";
+  char address5[] = "foo.abcdefghijklmnop.onion";
+  char address6[] = "foo.bar.abcdefghijklmnop.onion";
+  char address7[] = ".abcdefghijklmnop.onion";
+  char address8[] =
+    "www.25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion";
+
+  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address1));
+  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address2));
+  tt_str_op(address2,OP_EQ, "aaaaaaaaaaaaaaaa");
+  tt_assert(EXIT_HOSTNAME == parse_extended_hostname(address3));
+  tt_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4));
+  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address5));
+  tt_str_op(address5,OP_EQ, "abcdefghijklmnop");
+  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address6));
+  tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
+  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
+  tt_assert(ONION_V3_HOSTNAME == parse_extended_hostname(address8));
+  tt_str_op(address8, OP_EQ,
+            "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
+
+ done: ;
+}
+
+static void
+test_time_between_tp_and_srv(void *arg)
+{
+  int ret;
+  networkstatus_t ns;
+  (void) arg;
+
+  /* This function should be returning true where "^" are:
+   *
+   *    +------------------------------------------------------------------+
+   *    |                                                                  |
+   *    | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *    | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *    |                                                                  |
+   *    |  $==========|-----------$===========|-----------$===========|    |
+   *    |             ^^^^^^^^^^^^            ^^^^^^^^^^^^                 |
+   *    |                                                                  |
+   *    +------------------------------------------------------------------+
+   */
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 00:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 01:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 11:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 23:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 1);
+
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC", &ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  ret = hs_in_period_between_tp_and_srv(&ns, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+/************ Reachability Test (it is huge) ****************/
+
+/* Simulate different consensus for client and service. Used by the
+ * reachability test. The SRV and responsible HSDir list are used by all
+ * reachability tests so make them common to simplify setup and teardown. */
+static networkstatus_t *mock_service_ns = NULL;
+static networkstatus_t *mock_client_ns = NULL;
+static sr_srv_t current_srv, previous_srv;
+static smartlist_t *service_responsible_hsdirs = NULL;
+static smartlist_t *client_responsible_hsdirs = NULL;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus_service(time_t now)
+{
+  (void) now;
+
+  if (mock_service_ns) {
+    return mock_service_ns;
+  }
+
+  mock_service_ns = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_service_ns->routerstatus_list = smartlist_new();
+  mock_service_ns->type = NS_TYPE_CONSENSUS;
+
+  return mock_service_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_service(void)
+{
+  return mock_networkstatus_get_live_consensus_service(0);
+}
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus_client(time_t now)
+{
+  (void) now;
+
+  if (mock_client_ns) {
+    return mock_client_ns;
+  }
+
+  mock_client_ns = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_client_ns->routerstatus_list = smartlist_new();
+  mock_client_ns->type = NS_TYPE_CONSENSUS;
+
+  return mock_client_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_client(void)
+{
+  return mock_networkstatus_get_live_consensus_client(0);
+}
+
+/* Mock function because we are not trying to test the close circuit that does
+ * an awful lot of checks on the circuit object. */
+static void
+mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
+                            const char *file)
+{
+  (void) circ;
+  (void) reason;
+  (void) line;
+  (void) file;
+  return;
+}
+
+/* Initialize a big HSDir V3 hash ring. */
+static void
+helper_initialize_big_hash_ring(networkstatus_t *ns)
+{
+  int ret;
+
+  /* Generate 250 hsdirs! :) */
+  for (int counter = 1 ; counter < 251 ; counter++) {
+    /* Let's generate random nickname for each hsdir... */
+    char nickname_binary[8];
+    char nickname_str[13] = {0};
+    crypto_rand(nickname_binary, sizeof(nickname_binary));
+    ret = base64_encode(nickname_str, sizeof(nickname_str),
+                        nickname_binary, sizeof(nickname_binary), 0);
+    tt_int_op(ret, OP_EQ, 12);
+    helper_add_hsdir_to_networkstatus(ns, counter, nickname_str, 1);
+  }
+
+  /* Make sure we have 200 hsdirs in our list */
+  tt_int_op(smartlist_len(ns->routerstatus_list), OP_EQ, 250);
+
+ done:
+  ;
+}
+
+/** Initialize service and publish its descriptor as needed. Return the newly
+ *  allocated service object to the caller. */
+static hs_service_t *
+helper_init_service(time_t now)
+{
+  int retval;
+  hs_service_t *service = hs_service_new(get_options());
+  tt_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  /* Register service to global map. */
+  retval = register_service(get_hs_service_map(), service);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Initialize service descriptor */
+  build_all_descriptors(now);
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_next);
+
+ done:
+  return service;
+}
+
+/* Helper function to set the RFC 1123 time string into t. */
+static void
+set_consensus_times(const char *timestr, time_t *t)
+{
+  tt_assert(timestr);
+  tt_assert(t);
+
+  int ret = parse_rfc1123_time(timestr, t);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  return;
+}
+
+/* Helper function to cleanup the mock consensus (client and service) */
+static void
+cleanup_mock_ns(void)
+{
+  if (mock_service_ns) {
+    SMARTLIST_FOREACH(mock_service_ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(mock_service_ns->routerstatus_list);
+    mock_service_ns->sr_info.current_srv = NULL;
+    mock_service_ns->sr_info.previous_srv = NULL;
+    networkstatus_vote_free(mock_service_ns);
+    mock_service_ns = NULL;
+  }
+
+  if (mock_client_ns) {
+    SMARTLIST_FOREACH(mock_client_ns->routerstatus_list,
+                      routerstatus_t *, rs, routerstatus_free(rs));
+    smartlist_clear(mock_client_ns->routerstatus_list);
+    mock_client_ns->sr_info.current_srv = NULL;
+    mock_client_ns->sr_info.previous_srv = NULL;
+    networkstatus_vote_free(mock_client_ns);
+    mock_client_ns = NULL;
+  }
+}
+
+/* Helper function to setup a reachability test. Once called, the
+ * cleanup_reachability_test MUST be called at the end. */
+static void
+setup_reachability_test(void)
+{
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(get_or_state, get_or_state_replacement);
+
+  hs_init();
+
+  /* Baseline to start with. */
+  memset(&current_srv, 0, sizeof(current_srv));
+  memset(&previous_srv, 1, sizeof(previous_srv));
+
+  /* Initialize the consensuses. */
+  mock_networkstatus_get_latest_consensus_service();
+  mock_networkstatus_get_latest_consensus_client();
+
+  service_responsible_hsdirs = smartlist_new();
+  client_responsible_hsdirs = smartlist_new();
+}
+
+/* Helper function to cleanup a reachability test initial setup. */
+static void
+cleanup_reachability_test(void)
+{
+  smartlist_free(service_responsible_hsdirs);
+  service_responsible_hsdirs = NULL;
+  smartlist_free(client_responsible_hsdirs);
+  client_responsible_hsdirs = NULL;
+  hs_free_all();
+  cleanup_mock_ns();
+  UNMOCK(get_or_state);
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/* A reachability test always check if the resulting service and client
+ * responsible HSDir for the given parameters are equal.
+ *
+ * Return true iff the same exact nodes are in both list. */
+static int
+are_responsible_hsdirs_equal(void)
+{
+  int count = 0;
+  tt_int_op(smartlist_len(client_responsible_hsdirs), OP_EQ, 6);
+  tt_int_op(smartlist_len(service_responsible_hsdirs), OP_EQ, 8);
+
+  SMARTLIST_FOREACH_BEGIN(client_responsible_hsdirs,
+                          const routerstatus_t *, c_rs) {
+    SMARTLIST_FOREACH_BEGIN(service_responsible_hsdirs,
+                            const routerstatus_t *, s_rs) {
+      if (tor_memeq(c_rs->identity_digest, s_rs->identity_digest,
+                    DIGEST_LEN)) {
+        count++;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(s_rs);
+  } SMARTLIST_FOREACH_END(c_rs);
+
+ done:
+  return (count == 6);
+}
+
+/* Tor doesn't use such a function to get the previous HSDir, it is only used
+ * in node_set_hsdir_index(). We need it here so we can test the reachability
+ * scenario 6 that requires the previous time period to compute the list of
+ * responsible HSDir because of the client state timing. */
+static uint64_t
+get_previous_time_period(time_t now)
+{
+  return hs_get_time_period_num(now) - 1;
+}
+
+/* Configuration of a reachability test scenario. */
+typedef struct reachability_cfg_t {
+  /* Consensus timings to be set. They have to be compliant with
+   * RFC 1123 time format. */
+  const char *service_valid_after;
+  const char *service_valid_until;
+  const char *client_valid_after;
+  const char *client_valid_until;
+
+  /* SRVs that the service and client should use. */
+  sr_srv_t *service_current_srv;
+  sr_srv_t *service_previous_srv;
+  sr_srv_t *client_current_srv;
+  sr_srv_t *client_previous_srv;
+
+  /* A time period function for the service to use for this scenario. For a
+   * successful reachability test, the client always use the current time
+   * period thus why no client function. */
+  uint64_t (*service_time_period_fn)(time_t);
+
+  /* Is the client and service expected to be in a new time period. After
+   * setting the consensus time, the reachability test checks
+   * hs_in_period_between_tp_and_srv() and test the returned value against
+   * this. */
+  unsigned int service_in_new_tp;
+  unsigned int client_in_new_tp;
+
+  /* Some scenario requires a hint that the client, because of its consensus
+   * time, will request the "next" service descriptor so this indicates if it
+   * is the case or not. */
+  unsigned int client_fetch_next_desc;
+} reachability_cfg_t;
+
+/* Some defines to help with semantic while reading a configuration below. */
+#define NOT_IN_NEW_TP 0
+#define IN_NEW_TP 1
+#define DONT_NEED_NEXT_DESC 0
+#define NEED_NEXT_DESC 1
+
+static reachability_cfg_t reachability_scenarios[] = {
+  /* Scenario 1
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |              ^ ^                                                 |
+   *  |              S C                                                 |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 13:00 and client to 15:00,
+   *  both are after TP#1 thus have access to SRV#1. Service and client should
+   *  be using TP#1.
+   */
+
+  { "Sat, 26 Oct 1985 13:00:00 UTC", /* Service valid_after */
+    "Sat, 26 Oct 1985 14:00:00 UTC", /* Service valid_until */
+    "Sat, 26 Oct 1985 15:00:00 UTC", /* Client valid_after */
+    "Sat, 26 Oct 1985 16:00:00 UTC", /* Client valid_until. */
+    &current_srv, NULL, /* Service current and previous SRV */
+    &current_srv, NULL, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    IN_NEW_TP, /* Is service in new TP? */
+    IN_NEW_TP, /* Is client in new TP? */
+    NEED_NEXT_DESC },
+
+  /* Scenario 2
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        S C                                       |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 23:00 and client to 01:00,
+   *  which makes the client after the SRV#2 and the service just before. The
+   *  service should only be using TP#1. The client should be using TP#1.
+   */
+
+  { "Sat, 26 Oct 1985 23:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 00:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 01:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 02:00:00 UTC", /* Client valid_until. */
+    &previous_srv, NULL, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    IN_NEW_TP, /* Is service in new TP? */
+    NOT_IN_NEW_TP, /* Is client in new TP? */
+    NEED_NEXT_DESC },
+
+  /* Scenario 3
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|----------$===========|     |
+   *  |                            ^ ^                                   |
+   *  |                            S C                                   |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 03:00 and client to 05:00,
+   *  which makes both after SRV#2. The service should be using TP#1 as its
+   *  current time period. The client should be using TP#1.
+   */
+
+  { "Sat, 27 Oct 1985 03:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 04:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 05:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 06:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    NOT_IN_NEW_TP, /* Is service in new TP? */
+    NOT_IN_NEW_TP, /* Is client in new TP? */
+    DONT_NEED_NEXT_DESC },
+
+  /* Scenario 4
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    S C                           |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 11:00 and client to 13:00,
+   *  which makes the service before TP#2 and the client just after. The
+   *  service should be using TP#1 as its current time period and TP#2 as the
+   *  next. The client should be using TP#2 time period.
+   */
+
+  { "Sat, 27 Oct 1985 11:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 12:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 13:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 14:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    hs_get_next_time_period_num, /* Service time period function. */
+    NOT_IN_NEW_TP, /* Is service in new TP? */
+    IN_NEW_TP, /* Is client in new TP? */
+    NEED_NEXT_DESC },
+
+  /* Scenario 5
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        C S                                       |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 01:00 and client to 23:00,
+   *  which makes the service after SRV#2 and the client just before. The
+   *  service should be using TP#1 as its current time period and TP#2 as the
+   *  next. The client should be using TP#1 time period.
+   */
+
+  { "Sat, 27 Oct 1985 01:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 02:00:00 UTC", /* Service valid_until */
+    "Sat, 26 Oct 1985 23:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 00:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &previous_srv, NULL, /* Client current and previous SRV */
+    hs_get_time_period_num, /* Service time period function. */
+    NOT_IN_NEW_TP, /* Is service in new TP? */
+    IN_NEW_TP, /* Is client in new TP? */
+    DONT_NEED_NEXT_DESC },
+
+  /* Scenario 6
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    C S                           |
+   *  +------------------------------------------------------------------+
+   *
+   *  S: Service, C: Client
+   *
+   *  Service consensus valid_after time is set to 13:00 and client to 11:00,
+   *  which makes the service outside after TP#2 and the client just before.
+   *  The service should be using TP#1 as its current time period and TP#2 as
+   *  its next. The client should be using TP#1 time period.
+   */
+
+  { "Sat, 27 Oct 1985 13:00:00 UTC", /* Service valid_after */
+    "Sat, 27 Oct 1985 14:00:00 UTC", /* Service valid_until */
+    "Sat, 27 Oct 1985 11:00:00 UTC", /* Client valid_after */
+    "Sat, 27 Oct 1985 12:00:00 UTC", /* Client valid_until. */
+    &current_srv, &previous_srv, /* Service current and previous SRV */
+    &current_srv, &previous_srv, /* Client current and previous SRV */
+    get_previous_time_period, /* Service time period function. */
+    IN_NEW_TP, /* Is service in new TP? */
+    NOT_IN_NEW_TP, /* Is client in new TP? */
+    DONT_NEED_NEXT_DESC },
+
+  /* End marker. */
+  { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0}
+};
+
+/* Run a single reachability scenario. num_scenario is the corresponding
+ * scenario number from the documentation. It is used to log it in case of
+ * failure so we know which scenario fails. */
+static int
+run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario)
+{
+  int ret = -1;
+  hs_service_t *service;
+  uint64_t service_tp, client_tp;
+  ed25519_public_key_t service_blinded_pk, client_blinded_pk;
+
+  setup_reachability_test();
+
+  tt_assert(cfg);
+
+  /* Set service consensus time. */
+  set_consensus_times(cfg->service_valid_after,
+                      &mock_service_ns->valid_after);
+  set_consensus_times(cfg->service_valid_until,
+                      &mock_service_ns->valid_until);
+  set_consensus_times(cfg->service_valid_until,
+                      &mock_service_ns->fresh_until);
+  voting_schedule_recalculate_timing(get_options(),
+                                     mock_service_ns->valid_after);
+  /* Set client consensus time. */
+  set_consensus_times(cfg->client_valid_after,
+                      &mock_client_ns->valid_after);
+  set_consensus_times(cfg->client_valid_until,
+                      &mock_client_ns->valid_until);
+  set_consensus_times(cfg->client_valid_until,
+                      &mock_client_ns->fresh_until);
+  voting_schedule_recalculate_timing(get_options(),
+                                     mock_client_ns->valid_after);
+
+  /* New time period checks for this scenario. */
+  tt_int_op(hs_in_period_between_tp_and_srv(mock_service_ns, 0), OP_EQ,
+            cfg->service_in_new_tp);
+  tt_int_op(hs_in_period_between_tp_and_srv(mock_client_ns, 0), OP_EQ,
+            cfg->client_in_new_tp);
+
+  /* Set the SRVs for this scenario. */
+  mock_client_ns->sr_info.current_srv = cfg->client_current_srv;
+  mock_client_ns->sr_info.previous_srv = cfg->client_previous_srv;
+  mock_service_ns->sr_info.current_srv = cfg->service_current_srv;
+  mock_service_ns->sr_info.previous_srv = cfg->service_previous_srv;
+
+  /* Initialize a service to get keys. */
+  service = helper_init_service(time(NULL));
+
+  /*
+   * === Client setup ===
+   */
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus_client);
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus_client);
+
+  /* Make networkstatus_is_live() happy. */
+  update_approx_time(mock_client_ns->valid_after);
+  /* Initialize a big hashring for this consensus with the hsdir index set. */
+  helper_initialize_big_hash_ring(mock_client_ns);
+
+  /* Client ONLY use the current time period. This is the whole point of these
+   * reachability test that is to make sure the client can always reach the
+   * service using only its current time period. */
+  client_tp = hs_get_time_period_num(0);
+
+  hs_build_blinded_pubkey(&service->keys.identity_pk, NULL, 0,
+                          client_tp, &client_blinded_pk);
+  hs_get_responsible_hsdirs(&client_blinded_pk, client_tp, 0, 1,
+                            client_responsible_hsdirs);
+  /* Cleanup the nodelist so we can let the service computes its own set of
+   * node with its own hashring. */
+  cleanup_nodelist();
+  tt_int_op(smartlist_len(client_responsible_hsdirs), OP_EQ, 6);
+
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_live_consensus);
+
+  /*
+   * === Service setup ===
+   */
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus_service);
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus_service);
+
+  /* Make networkstatus_is_live() happy. */
+  update_approx_time(mock_service_ns->valid_after);
+  /* Initialize a big hashring for this consensus with the hsdir index set. */
+  helper_initialize_big_hash_ring(mock_service_ns);
+
+  service_tp = cfg->service_time_period_fn(0);
+
+  hs_build_blinded_pubkey(&service->keys.identity_pk, NULL, 0,
+                          service_tp, &service_blinded_pk);
+
+  /* A service builds two lists of responsible HSDir, for the current and the
+   * next descriptor. Depending on the scenario, the client timing indicate if
+   * it is fetching the current or the next descriptor so we use the
+   * "client_fetch_next_desc" to know which one the client is trying to get to
+   * confirm that the service computes the same hashring for the same blinded
+   * key and service time period function. */
+  hs_get_responsible_hsdirs(&service_blinded_pk, service_tp,
+                            cfg->client_fetch_next_desc, 0,
+                            service_responsible_hsdirs);
+  cleanup_nodelist();
+  tt_int_op(smartlist_len(service_responsible_hsdirs), OP_EQ, 8);
+
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_live_consensus);
+
+  /* Some testing of the values we just got from the client and service. */
+  tt_mem_op(&client_blinded_pk, OP_EQ, &service_blinded_pk,
+            ED25519_PUBKEY_LEN);
+  tt_int_op(are_responsible_hsdirs_equal(), OP_EQ, 1);
+
+  /* Everything went well. */
+  ret = 0;
+
+ done:
+  cleanup_reachability_test();
+  if (ret == -1) {
+    /* Do this so we can know which scenario failed. */
+    char msg[32];
+    tor_snprintf(msg, sizeof(msg), "Scenario %d failed", num_scenario);
+    tt_fail_msg(msg);
+  }
+  return ret;
+}
+
+static void
+test_reachability(void *arg)
+{
+  (void) arg;
+
+  /* NOTE: An important axiom to understand here is that SRV#N must only be
+   * used with TP#N value. For example, SRV#2 with TP#1 should NEVER be used
+   * together. The HSDir index computation is based on this axiom.*/
+
+  for (int i = 0; reachability_scenarios[i].service_valid_after; ++i) {
+    int ret = run_reachability_scenario(&reachability_scenarios[i], i + 1);
+    if (ret < 0) {
+      return;
+    }
+  }
+}
+
+/** Pick an HSDir for service with <b>onion_identity_pk</b> as a client. Put
+ *  its identity digest in <b>hsdir_digest_out</b>. */
+static void
+helper_client_pick_hsdir(const ed25519_public_key_t *onion_identity_pk,
+                        char *hsdir_digest_out)
+{
+  tt_assert(onion_identity_pk);
+
+  routerstatus_t *client_hsdir = pick_hsdir_v3(onion_identity_pk);
+  tt_assert(client_hsdir);
+  digest_to_base64(hsdir_digest_out, client_hsdir->identity_digest);
+
+ done:
+  ;
+}
+
+static void
+test_hs_indexes(void *arg)
+{
+  int ret;
+  uint64_t period_num = 42;
+  ed25519_public_key_t pubkey;
+
+  (void) arg;
+
+  /* Build the hs_index */
+  {
+    uint8_t hs_index[DIGEST256_LEN];
+    const char *b32_test_vector =
+      "37e5cbbd56a22823714f18f1623ece5983a0d64c78495a8cfab854245e5f9a8a";
+    char test_vector[DIGEST256_LEN];
+    ret = base16_decode(test_vector, sizeof(test_vector), b32_test_vector,
+                        strlen(b32_test_vector));
+    tt_int_op(ret, OP_EQ, sizeof(test_vector));
+    /* Our test vector uses a public key set to 32 bytes of \x42. */
+    memset(&pubkey, '\x42', sizeof(pubkey));
+    hs_build_hs_index(1, &pubkey, period_num, hs_index);
+    tt_mem_op(hs_index, OP_EQ, test_vector, sizeof(hs_index));
+  }
+
+  /* Build the hsdir_index */
+  {
+    uint8_t srv[DIGEST256_LEN];
+    uint8_t hsdir_index[DIGEST256_LEN];
+    const char *b32_test_vector =
+      "db475361014a09965e7e5e4d4a25b8f8d4b8f16cb1d8a7e95eed50249cc1a2d5";
+    char test_vector[DIGEST256_LEN];
+    ret = base16_decode(test_vector, sizeof(test_vector), b32_test_vector,
+                        strlen(b32_test_vector));
+    tt_int_op(ret, OP_EQ, sizeof(test_vector));
+    /* Our test vector uses a public key set to 32 bytes of \x42. */
+    memset(&pubkey, '\x42', sizeof(pubkey));
+    memset(srv, '\x43', sizeof(srv));
+    hs_build_hsdir_index(&pubkey, srv, period_num, hsdir_index);
+    tt_mem_op(hsdir_index, OP_EQ, test_vector, sizeof(hsdir_index));
+  }
+
+ done:
+  ;
+}
+
+#define EARLY_IN_SRV_TO_TP 0
+#define LATE_IN_SRV_TO_TP 1
+#define EARLY_IN_TP_TO_SRV 2
+#define LATE_IN_TP_TO_SRV 3
+
+/** Set the consensus and system time based on <b>position</b>. See the
+ *    following diagram for details:
+ *
+ *  +------------------------------------------------------------------+
+ *  |                                                                  |
+ *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+ *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+ *  |                                                                  |
+ *  |  $==========|-----------$===========|----------$===========|     |
+ *  |                                                                  |
+ *  |                                                                  |
+ *  +------------------------------------------------------------------+
+ */
+static time_t
+helper_set_consensus_and_system_time(networkstatus_t *ns, int position)
+{
+  time_t real_time = 0;
+
+  /* The period between SRV#N and TP#N is from 00:00 to 12:00 UTC. Consensus
+   * valid_after is what matters here, the rest is just to specify the voting
+   * period correctly. */
+  if (position == LATE_IN_SRV_TO_TP) {
+    parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 13 Apr 2016 12:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 13 Apr 2016 14:00:00 UTC", &ns->valid_until);
+  } else if (position == EARLY_IN_TP_TO_SRV) {
+    parse_rfc1123_time("Wed, 13 Apr 2016 13:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 13 Apr 2016 14:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 13 Apr 2016 16:00:00 UTC", &ns->valid_until);
+  } else if (position == LATE_IN_TP_TO_SRV) {
+    parse_rfc1123_time("Wed, 13 Apr 2016 23:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 14 Apr 2016 00:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 14 Apr 2016 02:00:00 UTC", &ns->valid_until);
+  } else if (position == EARLY_IN_SRV_TO_TP) {
+    parse_rfc1123_time("Wed, 14 Apr 2016 01:00:00 UTC", &ns->valid_after);
+    parse_rfc1123_time("Wed, 14 Apr 2016 02:00:00 UTC", &ns->fresh_until);
+    parse_rfc1123_time("Wed, 14 Apr 2016 04:00:00 UTC", &ns->valid_until);
+  } else {
+    tt_assert(0);
+  }
+  voting_schedule_recalculate_timing(get_options(), ns->valid_after);
+
+  /* Set system time: pretend to be just 2 minutes before consensus expiry */
+  real_time = ns->valid_until - 120;
+  update_approx_time(real_time);
+
+ done:
+  return real_time;
+}
+
+/** Helper function that carries out the actual test for
+ *  test_client_service_sync() */
+static void
+helper_test_hsdir_sync(networkstatus_t *ns,
+                       int service_position, int client_position,
+                       int client_fetches_next_desc)
+{
+  hs_service_descriptor_t *desc;
+  int retval;
+
+  /** Test logic:
+   *   1) Initialize service time: consensus and system time.
+   *   1.1) Initialize service hash ring
+   *   2) Initialize service and publish descriptors.
+   *   3) Initialize client time: consensus and system time.
+   *   3.1) Initialize client hash ring
+   *   4) Try to fetch descriptor as client, and CHECK that the HSDir picked by
+   *      the client was also picked by service.
+   */
+
+  /* 1) Initialize service time: consensus and real time */
+  time_t now = helper_set_consensus_and_system_time(ns, service_position);
+  helper_initialize_big_hash_ring(ns);
+
+  /* 2) Initialize service */
+  hs_service_t *service = helper_init_service(now);
+  desc = client_fetches_next_desc ? service->desc_next : service->desc_current;
+
+  /* Now let's upload our desc to all hsdirs */
+  upload_descriptor_to_all(service, desc);
+  /* Cleanup right now so we don't memleak on error. */
+  cleanup_nodelist();
+  /* Check that previous hsdirs were populated */
+  tt_int_op(smartlist_len(desc->previous_hsdirs), OP_EQ, 8);
+
+  /* 3) Initialize client time */
+  helper_set_consensus_and_system_time(ns, client_position);
+
+  cleanup_nodelist();
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_clear(ns->routerstatus_list);
+  helper_initialize_big_hash_ring(ns);
+
+  /* 4) Pick 6 HSDirs as a client and check that they were also chosen by the
+        service. */
+  for (int y = 0 ; y < 6 ; y++) {
+    char client_hsdir_b64_digest[BASE64_DIGEST_LEN+1] = {0};
+    helper_client_pick_hsdir(&service->keys.identity_pk,
+                             client_hsdir_b64_digest);
+
+    /* CHECK: Go through the hsdirs chosen by the service and make sure that it
+     * contains the one picked by the client! */
+    retval = smartlist_contains_string(desc->previous_hsdirs,
+                                       client_hsdir_b64_digest);
+    tt_int_op(retval, OP_EQ, 1);
+  }
+
+  /* Finally, try to pick a 7th hsdir and see that NULL is returned since we
+   * exhausted all of them: */
+  tt_assert(!pick_hsdir_v3(&service->keys.identity_pk));
+
+ done:
+  /* At the end: free all services and initialize the subsystem again, we will
+   * need it for next scenario. */
+  cleanup_nodelist();
+  hs_service_free_all();
+  hs_service_init();
+  SMARTLIST_FOREACH(ns->routerstatus_list,
+                    routerstatus_t *, rs, routerstatus_free(rs));
+  smartlist_clear(ns->routerstatus_list);
+}
+
+/** This test ensures that client and service will pick the same HSDirs, under
+ *  various timing scenarios:
+ *  a) Scenario where both client and service are in the time segment between
+ *     SRV#N and TP#N:
+ *  b) Scenario where both client and service are in the time segment between
+ *     TP#N and SRV#N+1.
+ *  c) Scenario where service is between SRV#N and TP#N, but client is between
+ *     TP#N and SRV#N+1.
+ *  d) Scenario where service is between TP#N and SRV#N+1, but client is
+ *     between SRV#N and TP#N.
+ *
+ * This test is important because it tests that upload_descriptor_to_all() is
+ * in synch with pick_hsdir_v3(). That's not the case for the
+ * test_reachability() test which only compares the responsible hsdir sets.
+ */
+static void
+test_client_service_hsdir_set_sync(void *arg)
+{
+  networkstatus_t *ns = NULL;
+
+  (void) arg;
+
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(hs_desc_encode_descriptor,
+       mock_hs_desc_encode_descriptor);
+  MOCK(directory_initiate_request,
+       mock_directory_initiate_request);
+
+  hs_init();
+
+  /* Initialize a big hash ring: we want it to be big so that client and
+   * service cannot accidentally select the same HSDirs */
+  ns = networkstatus_get_latest_consensus();
+  tt_assert(ns);
+
+  /** Now test the various synch scenarios. See the helper function for more
+      details: */
+
+  /*  a) Scenario where both client and service are in the time segment between
+   *     SRV#N and TP#N. At this time the client fetches the first HS desc:
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|----------$===========|     |
+   *  |                                  ^ ^                             |
+   *  |                                  S C                             |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_SRV_TO_TP, LATE_IN_SRV_TO_TP, 0);
+
+  /*  b) Scenario where both client and service are in the time segment between
+   *     TP#N and SRV#N+1. At this time the client fetches the second HS
+   *     desc:
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                      ^ ^                                         |
+   *  |                      S C                                         |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_TP_TO_SRV, LATE_IN_TP_TO_SRV, 1);
+
+  /*  c) Scenario where service is between SRV#N and TP#N, but client is
+   *     between TP#N and SRV#N+1. Client is forward in time so it fetches the
+   *     second HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    S C                           |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_SRV_TO_TP, EARLY_IN_TP_TO_SRV, 1);
+
+  /*  d) Scenario where service is between TP#N and SRV#N+1, but client is
+   *     between SRV#N and TP#N. Client is backwards in time so it fetches the
+   *     first HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                                    ^ ^                           |
+   *  |                                    C S                           |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, EARLY_IN_TP_TO_SRV, LATE_IN_SRV_TO_TP, 0);
+
+  /*  e) Scenario where service is between SRV#N and TP#N, but client is
+   *     between TP#N-1 and SRV#3. Client is backwards in time so it fetches
+   *     the first HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        C S                                       |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, EARLY_IN_SRV_TO_TP, LATE_IN_TP_TO_SRV, 0);
+
+  /*  f) Scenario where service is between TP#N and SRV#N+1, but client is
+   *     between SRV#N+1 and TP#N+1. Client is forward in time so it fetches
+   *     the second HS desc.
+   *
+   *  +------------------------------------------------------------------+
+   *  |                                                                  |
+   *  | 00:00      12:00       00:00       12:00       00:00       12:00 |
+   *  | SRV#1      TP#1        SRV#2       TP#2        SRV#3       TP#3  |
+   *  |                                                                  |
+   *  |  $==========|-----------$===========|-----------$===========|    |
+   *  |                        ^ ^                                       |
+   *  |                        S C                                       |
+   *  +------------------------------------------------------------------+
+   */
+  helper_test_hsdir_sync(ns, LATE_IN_TP_TO_SRV, EARLY_IN_SRV_TO_TP, 1);
+
+ done:
+  networkstatus_vote_free(ns);
+  nodelist_free_all();
+  hs_free_all();
+}
+
+struct testcase_t hs_common_tests[] = {
+  { "build_address", test_build_address, TT_FORK,
+    NULL, NULL },
+  { "validate_address", test_validate_address, TT_FORK,
+    NULL, NULL },
+  { "time_period", test_time_period, TT_FORK,
+    NULL, NULL },
+  { "start_time_of_next_time_period", test_start_time_of_next_time_period,
+    TT_FORK, NULL, NULL },
+  { "responsible_hsdirs", test_responsible_hsdirs, TT_FORK,
+    NULL, NULL },
+  { "desc_reupload_logic", test_desc_reupload_logic, TT_FORK,
+    NULL, NULL },
+  { "disaster_srv", test_disaster_srv, TT_FORK,
+    NULL, NULL },
+  { "hid_serv_request_tracker", test_hid_serv_request_tracker, TT_FORK,
+    NULL, NULL },
+  { "parse_extended_hostname", test_parse_extended_hostname, TT_FORK,
+    NULL, NULL },
+  { "time_between_tp_and_srv", test_time_between_tp_and_srv, TT_FORK,
+    NULL, NULL },
+  { "reachability", test_reachability, TT_FORK,
+    NULL, NULL },
+  { "client_service_hsdir_set_sync", test_client_service_hsdir_set_sync,
+    TT_FORK, NULL, NULL },
+  { "hs_indexes", test_hs_indexes, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c
new file mode 100644
index 0000000000..a76be301d3
--- /dev/null
+++ b/src/test/test_hs_config.c
@@ -0,0 +1,487 @@
+/* Copyright (c) 2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_config.c
+ * \brief Test hidden service configuration functionality.
+ */
+
+#define CONFIG_PRIVATE
+#define HS_SERVICE_PRIVATE
+
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+
+#include "config.h"
+#include "hs_common.h"
+#include "hs_config.h"
+#include "hs_service.h"
+#include "rendservice.h"
+
+static int
+helper_config_service(const char *conf, int validate_only)
+{
+  int ret = 0;
+  or_options_t *options = NULL;
+  tt_assert(conf);
+  options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_service_all(options, validate_only);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+static void
+test_invalid_service(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Try with a missing port configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 1\n"; /* Wrong not supported version. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceVersion must be between 2 and 3");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad value of HiddenServiceAllowUnknownPorts. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceAllowUnknownPorts 2\n"; /* Should be 0 or 1. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceAllowUnknownPorts must be "
+                              "between 0 and 1, not 2");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad value of HiddenServiceDirGroupReadable */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceDirGroupReadable 2\n"; /* Should be 0 or 1. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceDirGroupReadable must be "
+                              "between 0 and 1, not 2");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad value of HiddenServiceMaxStreamsCloseCircuit */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceMaxStreamsCloseCircuit 2\n"; /* Should be 0 or 1. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceMaxStreamsCloseCircuit must "
+                              "be between 0 and 1, not 2");
+    teardown_capture_of_logs();
+  }
+
+  /* Too much max streams. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceMaxStreams 65536\n"; /* One too many. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceMaxStreams must be between "
+                              "0 and 65535, not 65536");
+    teardown_capture_of_logs();
+  }
+
+  /* Duplicate directory directive. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 81\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Another hidden service is already "
+                              "configured for directory");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad port. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 65536\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Missing or invalid port");
+    teardown_capture_of_logs();
+  }
+
+  /* Out of order directives. */
+  {
+    const char *conf =
+      "HiddenServiceVersion 2\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServicePort 80\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceVersion with no preceding "
+                              "HiddenServiceDir directive");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  ;
+}
+
+static void
+test_valid_service(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Mix of v2 and v3. Still valid. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 81\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 82\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_invalid_service_v2(void *arg)
+{
+  int validate_only = 1, ret;
+
+  (void) arg;
+
+  /* Try with a missing port configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("with no ports configured.");
+    teardown_capture_of_logs();
+  }
+
+  /* Too many introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints 11\n"; /* One too many. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
+                              "be between 0 and 10, not 11");
+    teardown_capture_of_logs();
+  }
+
+  /* Too little introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints -1\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
+                              "be between 0 and 10, not -1");
+    teardown_capture_of_logs();
+  }
+
+  /* Bad authorized client type. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceAuthorizeClient blah alice,bob\n"; /* blah is no good. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceAuthorizeClient contains "
+                              "unrecognized auth-type");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  ;
+}
+
+static void
+test_valid_service_v2(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Valid complex configuration. Basic client authorization. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServicePort 22 localhost:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 42 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAuthorizeClient basic alice,bob,eve\n"
+      "HiddenServiceAllowUnknownPorts 1\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 7\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid complex configuration. Stealth client authorization. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 65535\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 9000 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAuthorizeClient stealth charlie,romeo\n"
+      "HiddenServiceAllowUnknownPorts 0\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 8\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_invalid_service_v3(void *arg)
+{
+  int validate_only = 1, ret;
+
+  (void) arg;
+
+  /* Try with a missing port configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("with no ports configured.");
+    teardown_capture_of_logs();
+  }
+
+  /* Too many introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints 21\n"; /* One too many. */
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints must "
+                              "be between 3 and 20, not 21.");
+    teardown_capture_of_logs();
+  }
+
+  /* Too little introduction points. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceNumIntroductionPoints 1\n";
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, validate_only);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints must "
+                              "be between 3 and 20, not 1.");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  ;
+}
+
+static void
+test_valid_service_v3(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Valid complex configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 80\n"
+      "HiddenServicePort 22 localhost:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 42 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAllowUnknownPorts 1\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 7\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid complex configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 65535\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+#ifdef HAVE_SYS_UN_H
+      "HiddenServicePort 9000 unix:/path/to/socket\n"
+#endif
+      "HiddenServiceAllowUnknownPorts 0\n"
+      "HiddenServiceMaxStreams 42\n"
+      "HiddenServiceMaxStreamsCloseCircuit 0\n"
+      "HiddenServiceDirGroupReadable 1\n"
+      "HiddenServiceNumIntroductionPoints 20\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Mix of v2 and v3. Still valid. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs1\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 80\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 81\n"
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 2\n"
+      "HiddenServicePort 82\n";
+    ret = helper_config_service(conf, 1);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_staging_service_v3(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* We don't validate a service object, this is the service test that are in
+   * charge of doing so. We just check for the stable state after
+   * registration. */
+
+  hs_init();
+
+  /* Time for a valid v3 service that should get staged. */
+  const char *conf =
+    "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs2\n"
+    "HiddenServiceVersion 3\n"
+    "HiddenServicePort 65535\n"
+    "HiddenServicePort 22 1.1.1.1:22\n"
+#ifdef HAVE_SYS_UN_H
+    "HiddenServicePort 9000 unix:/path/to/socket\n"
+#endif
+    "HiddenServiceAllowUnknownPorts 0\n"
+    "HiddenServiceMaxStreams 42\n"
+    "HiddenServiceMaxStreamsCloseCircuit 0\n"
+    "HiddenServiceDirGroupReadable 1\n"
+    "HiddenServiceNumIntroductionPoints 20\n";
+  ret = helper_config_service(conf, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Ok, we have a service in our map! Registration went well. */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+  /* Make sure we don't have a magic v2 service out of this. */
+  tt_int_op(rend_num_services(), OP_EQ, 0);
+
+ done:
+  hs_free_all();
+}
+
+struct testcase_t hs_config_tests[] = {
+  /* Invalid service not specific to any version. */
+  { "invalid_service", test_invalid_service, TT_FORK,
+    NULL, NULL },
+  { "valid_service", test_valid_service, TT_FORK,
+    NULL, NULL },
+
+  /* Test case only for version 2. */
+  { "invalid_service_v2", test_invalid_service_v2, TT_FORK,
+    NULL, NULL },
+  { "valid_service_v2", test_valid_service_v2, TT_FORK,
+    NULL, NULL },
+
+  /* Test case only for version 3. */
+  { "invalid_service_v3", test_invalid_service_v3, TT_FORK,
+    NULL, NULL },
+  { "valid_service_v3", test_valid_service_v3, TT_FORK,
+    NULL, NULL },
+
+  /* Test service staging. */
+  { "staging_service_v3", test_staging_service_v3, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c
new file mode 100644
index 0000000000..308843e9b8
--- /dev/null
+++ b/src/test/test_hs_control.c
@@ -0,0 +1,198 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_control.c
+ * \brief Unit tests for hidden service control port event and command.
+ **/
+
+#define CONTROL_PRIVATE
+#define CIRCUITBUILD_PRIVATE
+#define RENDCOMMON_PRIVATE
+#define RENDSERVICE_PRIVATE
+#define HS_SERVICE_PRIVATE
+
+#include "or.h"
+#include "test.h"
+#include "control.h"
+#include "config.h"
+#include "hs_common.h"
+#include "hs_control.h"
+#include "nodelist.h"
+//#include "rendcommon.h"
+//#include "rendservice.h"
+//#include "routerset.h"
+//#include "circuitbuild.h"
+#include "test_helpers.h"
+
+/* mock ID digest and longname for node that's in nodelist */
+#define HSDIR_EXIST_ID \
+  "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" \
+  "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+#define STR_HSDIR_EXIST_LONGNAME \
+  "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=TestDir"
+#define STR_HSDIR_NONE_EXIST_LONGNAME \
+  "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
+
+/* Helper global variable for hidden service descriptor event test.
+ * It's used as a pointer to dynamically created message buffer in
+ * send_control_event_string_replacement function, which mocks
+ * send_control_event_string function.
+ *
+ * Always free it after use! */
+static char *received_msg = NULL;
+
+/** Mock function for send_control_event_string
+ */
+static void
+queue_control_event_string_replacement(uint16_t event, char *msg)
+{
+  (void) event;
+  tor_free(received_msg);
+  received_msg = msg;
+}
+
+/** Mock function for node_describe_longname_by_id, it returns either
+ * STR_HSDIR_EXIST_LONGNAME or STR_HSDIR_NONE_EXIST_LONGNAME
+ */
+static const char *
+node_describe_longname_by_id_replacement(const char *id_digest)
+{
+  if (!strcmp(id_digest, HSDIR_EXIST_ID)) {
+    return STR_HSDIR_EXIST_LONGNAME;
+  } else {
+    return STR_HSDIR_NONE_EXIST_LONGNAME;
+  }
+}
+
+/* HSDir fetch index is a series of 'D' */
+#define HSDIR_INDEX_FETCH_HEX \
+  "4343434343434343434343434343434343434343434343434343434343434343"
+#define HSDIR_INDEX_STORE_HEX \
+  "4444444444444444444444444444444444444444444444444444444444444444"
+
+static const node_t *
+mock_node_get_by_id(const char *digest)
+{
+  static node_t node;
+  memcpy(node.identity, digest, DIGEST_LEN);
+  memset(node.hsdir_index.fetch, 'C', DIGEST256_LEN);
+  memset(node.hsdir_index.store_first, 'D', DIGEST256_LEN);
+  return &node;
+}
+
+static void
+test_hs_desc_event(void *arg)
+{
+  int ret;
+  char *expected_msg = NULL;
+  char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+  ed25519_keypair_t identity_kp;
+  ed25519_public_key_t blinded_pk;
+  char base64_blinded_pk[ED25519_BASE64_LEN + 1];
+  routerstatus_t hsdir_rs;
+  hs_ident_dir_conn_t ident;
+
+  (void) arg;
+  MOCK(queue_control_event_string,
+       queue_control_event_string_replacement);
+  MOCK(node_describe_longname_by_id,
+       node_describe_longname_by_id_replacement);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  /* Setup what we need for this test. */
+  ed25519_keypair_generate(&identity_kp, 0);
+  hs_build_address(&identity_kp.pubkey, HS_VERSION_THREE, onion_address);
+  ret = hs_address_is_valid(onion_address);
+  tt_int_op(ret, OP_EQ, 1);
+  memset(&blinded_pk, 'B', sizeof(blinded_pk));
+  memset(&hsdir_rs, 0, sizeof(hsdir_rs));
+  memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN);
+  ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
+  tt_int_op(ret, OP_EQ, 0);
+  memcpy(&ident.identity_pk, &identity_kp.pubkey,
+         sizeof(ed25519_public_key_t));
+  memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk));
+
+  /* HS_DESC REQUESTED ... */
+  hs_control_desc_event_requested(&identity_kp.pubkey, base64_blinded_pk,
+                                  &hsdir_rs);
+  tor_asprintf(&expected_msg, "650 HS_DESC REQUESTED %s NO_AUTH "
+               STR_HSDIR_EXIST_LONGNAME " %s HSDIR_INDEX="
+               HSDIR_INDEX_FETCH_HEX "\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC CREATED... */
+  hs_control_desc_event_created(onion_address, &blinded_pk);
+  tor_asprintf(&expected_msg, "650 HS_DESC CREATED %s UNKNOWN "
+                              "UNKNOWN %s\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC UPLOAD... */
+  uint8_t hsdir_index_store[DIGEST256_LEN];
+  memset(hsdir_index_store, 'D', sizeof(hsdir_index_store));
+  hs_control_desc_event_upload(onion_address, HSDIR_EXIST_ID,
+                               &blinded_pk, hsdir_index_store);
+  tor_asprintf(&expected_msg, "650 HS_DESC UPLOAD %s UNKNOWN "
+                              STR_HSDIR_EXIST_LONGNAME " %s "
+                              "HSDIR_INDEX=" HSDIR_INDEX_STORE_HEX "\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC FAILED... */
+  hs_control_desc_event_failed(&ident, HSDIR_EXIST_ID, "BAD_DESC");
+  tor_asprintf(&expected_msg, "650 HS_DESC FAILED %s NO_AUTH "
+                              STR_HSDIR_EXIST_LONGNAME " %s "
+                              "REASON=BAD_DESC\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC RECEIVED... */
+  hs_control_desc_event_received(&ident, HSDIR_EXIST_ID);
+  tor_asprintf(&expected_msg, "650 HS_DESC RECEIVED %s NO_AUTH "
+                              STR_HSDIR_EXIST_LONGNAME " %s\r\n",
+               onion_address, base64_blinded_pk);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+  /* HS_DESC UPLOADED... */
+  hs_control_desc_event_uploaded(&ident, HSDIR_EXIST_ID);
+  tor_asprintf(&expected_msg, "650 HS_DESC UPLOADED %s UNKNOWN "
+                              STR_HSDIR_EXIST_LONGNAME "\r\n",
+               onion_address);
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, expected_msg);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+
+ done:
+  UNMOCK(queue_control_event_string);
+  UNMOCK(node_describe_longname_by_id);
+  UNMOCK(node_get_by_id);
+  tor_free(received_msg);
+  tor_free(expected_msg);
+}
+
+struct testcase_t hs_control_tests[] = {
+  { "hs_desc_event", test_hs_desc_event, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
new file mode 100644
index 0000000000..14f1a664e7
--- /dev/null
+++ b/src/test/test_hs_descriptor.c
@@ -0,0 +1,899 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_descriptor.c
+ * \brief Test hidden service descriptor encoding and decoding.
+ */
+
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "crypto_ed25519.h"
+#include "crypto_digest.h"
+#include "crypto_rand.h"
+#include "ed25519_cert.h"
+#include "or.h"
+#include "hs_descriptor.h"
+#include "test.h"
+#include "torcert.h"
+
+#include "hs_test_helpers.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+
+#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
+DISABLE_GCC_WARNING(overlength-strings)
+/* We allow huge string constants in the unit tests, but not in the code
+ * at large. */
+#endif
+#include "test_hs_descriptor.inc"
+ENABLE_GCC_WARNING(overlength-strings)
+
+/* Test certificate encoding put in a descriptor. */
+static void
+test_cert_encoding(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  time_t now = time(NULL);
+  ed25519_keypair_t kp;
+  ed25519_public_key_t signed_key;
+  ed25519_secret_key_t secret_key;
+  tor_cert_t *cert = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&kp, 0);
+  tt_int_op(ret, == , 0);
+  ret = ed25519_secret_key_generate(&secret_key, 0);
+  tt_int_op(ret, == , 0);
+  ret = ed25519_public_key_generate(&signed_key, &secret_key);
+  tt_int_op(ret, == , 0);
+
+  cert = tor_cert_create(&kp, CERT_TYPE_SIGNING_AUTH, &signed_key,
+                         now, 3600 * 2, CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(cert);
+
+  /* Test the certificate encoding function. */
+  ret = tor_cert_encode_ed22519(cert, &encoded);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Validated the certificate string. */
+  {
+    char *end, *pos = encoded;
+    char *b64_cert, buf[256];
+    size_t b64_cert_len;
+    tor_cert_t *parsed_cert;
+
+    tt_int_op(strcmpstart(pos, "-----BEGIN ED25519 CERT-----\n"), OP_EQ, 0);
+    pos += strlen("-----BEGIN ED25519 CERT-----\n");
+
+    /* Isolate the base64 encoded certificate and try to decode it. */
+    end = strstr(pos, "-----END ED25519 CERT-----");
+    tt_assert(end);
+    b64_cert = pos;
+    b64_cert_len = end - pos;
+    ret = base64_decode(buf, sizeof(buf), b64_cert, b64_cert_len);
+    tt_int_op(ret, OP_GT, 0);
+    /* Parseable? */
+    parsed_cert = tor_cert_parse((uint8_t *) buf, ret);
+    tt_assert(parsed_cert);
+    /* Signature is valid? */
+    ret = tor_cert_checksig(parsed_cert, &kp.pubkey, now + 10);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = tor_cert_eq(cert, parsed_cert);
+    tt_int_op(ret, OP_EQ, 1);
+    /* The cert did have the signing key? */
+    ret= ed25519_pubkey_eq(&parsed_cert->signing_key, &kp.pubkey);
+    tt_int_op(ret, OP_EQ, 1);
+    tor_cert_free(parsed_cert);
+
+    /* Get to the end part of the certificate. */
+    pos += b64_cert_len;
+    tt_int_op(strcmpstart(pos, "-----END ED25519 CERT-----"), OP_EQ, 0);
+    pos += strlen("-----END ED25519 CERT-----");
+    tt_str_op(pos, OP_EQ, "");
+  }
+
+ done:
+  tor_cert_free(cert);
+  tor_free(encoded);
+}
+
+/* Test the descriptor padding. */
+static void
+test_descriptor_padding(void *arg)
+{
+  char *plaintext;
+  size_t plaintext_len, padded_len;
+  uint8_t *padded_plaintext = NULL;
+
+/* Example: if l = 129, the ceiled division gives 2 and then multiplied by 128
+ * to give 256. With l = 127, ceiled division gives 1 then times 128. */
+#define PADDING_EXPECTED_LEN(l) \
+  CEIL_DIV(l, HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE) * \
+  HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE
+
+  (void) arg;
+
+  { /* test #1: no padding */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+    plaintext = tor_malloc(plaintext_len);
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+  { /* test #2: one byte padding? */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE - 1;
+    plaintext = tor_malloc(plaintext_len);
+    padded_plaintext = NULL;
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+  { /* test #3: Lots more bytes of padding? */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE + 1;
+    plaintext = tor_malloc(plaintext_len);
+    padded_plaintext = NULL;
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+ done:
+  return;
+}
+
+static void
+test_link_specifier(void *arg)
+{
+  ssize_t ret;
+  hs_desc_link_specifier_t spec;
+  smartlist_t *link_specifiers = smartlist_new();
+  char buf[256];
+  char *b64 = NULL;
+  link_specifier_t *ls = NULL;
+
+  (void) arg;
+
+  /* Always this port. */
+  spec.u.ap.port = 42;
+  smartlist_add(link_specifiers, &spec);
+
+  /* Test IPv4 for starter. */
+  {
+    uint32_t ipv4;
+
+    spec.type = LS_IPV4;
+    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
+    tt_int_op(ret, OP_EQ, AF_INET);
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, OP_GT, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), OP_EQ, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    tt_int_op(ret, OP_EQ, 8);
+    /* Should be 2 bytes for port and 4 bytes for IPv4. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6);
+    ipv4 = link_specifier_get_un_ipv4_addr(ls);
+    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4);
+    tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+  /* Test IPv6. */
+  {
+    uint8_t ipv6[16];
+
+    spec.type = LS_IPV6;
+    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
+    tt_int_op(ret, OP_EQ, AF_INET6);
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, OP_GT, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), OP_EQ, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    tt_int_op(ret, OP_EQ, 20);
+    /* Should be 2 bytes for port and 16 bytes for IPv6. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18);
+    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
+      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
+    }
+    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6,
+              sizeof(ipv6));
+    tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+  /* Test legacy. */
+  {
+    uint8_t *id;
+
+    spec.type = LS_LEGACY_ID;
+    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, OP_GT, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), OP_EQ, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    /* 20 bytes digest + 1 byte type + 1 byte len. */
+    tt_int_op(ret, OP_EQ, 22);
+    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
+    /* Digest length is 20 bytes. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
+    id = link_specifier_getarray_un_legacy_id(ls);
+    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+ done:
+  link_specifier_free(ls);
+  tor_free(b64);
+  smartlist_free(link_specifiers);
+}
+
+static void
+test_encode_descriptor(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+  ret = hs_desc_encode_descriptor(desc, &signing_kp, &encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(encoded);
+
+ done:
+  hs_descriptor_free(desc);
+  tor_free(encoded);
+}
+
+static void
+test_decode_descriptor(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+  hs_descriptor_t *decoded = NULL;
+  hs_descriptor_t *desc_no_ip = NULL;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+
+  hs_helper_get_subcred_from_identity_keypair(&signing_kp,
+                                              subcredential);
+
+  /* Give some bad stuff to the decoding function. */
+  ret = hs_desc_decode_descriptor("hladfjlkjadf", subcredential, &decoded);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = hs_desc_encode_descriptor(desc, &signing_kp, &encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(encoded);
+
+  ret = hs_desc_decode_descriptor(encoded, subcredential, &decoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(decoded);
+
+  hs_helper_desc_equal(desc, decoded);
+
+  /* Decode a descriptor with _no_ introduction points. */
+  {
+    ed25519_keypair_t signing_kp_no_ip;
+    ret = ed25519_keypair_generate(&signing_kp_no_ip, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    hs_helper_get_subcred_from_identity_keypair(&signing_kp_no_ip,
+                                                subcredential);
+    desc_no_ip = hs_helper_build_hs_desc_no_ip(&signing_kp_no_ip);
+    tt_assert(desc_no_ip);
+    tor_free(encoded);
+    ret = hs_desc_encode_descriptor(desc_no_ip, &signing_kp_no_ip, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+    hs_descriptor_free(decoded);
+    ret = hs_desc_decode_descriptor(encoded, subcredential, &decoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(decoded);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+  hs_descriptor_free(desc_no_ip);
+  hs_descriptor_free(decoded);
+  tor_free(encoded);
+}
+
+static void
+test_supported_version(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Unsupported. */
+  ret = hs_desc_is_supported_version(42);
+  tt_int_op(ret, OP_EQ, 0);
+  /* To early. */
+  ret = hs_desc_is_supported_version(HS_DESC_SUPPORTED_FORMAT_VERSION_MIN - 1);
+  tt_int_op(ret, OP_EQ, 0);
+  /* One too new. */
+  ret = hs_desc_is_supported_version(HS_DESC_SUPPORTED_FORMAT_VERSION_MAX + 1);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Valid version. */
+  ret = hs_desc_is_supported_version(3);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_encrypted_data_len(void *arg)
+{
+  int ret;
+  size_t value;
+
+  (void) arg;
+
+  /* No length, error. */
+  ret = encrypted_data_length_is_valid(0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Valid value. */
+  value = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN + 1;
+  ret = encrypted_data_length_is_valid(value);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_decode_invalid_intro_point(void *arg)
+{
+  int ret;
+  char *encoded_ip = NULL;
+  size_t len_out;
+  hs_desc_intro_point_t *ip = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+
+  (void) arg;
+
+  /* Separate pieces of a valid encoded introduction point. */
+  const char *intro_point =
+    "introduction-point AQIUMDI5OUYyNjhGQ0E5RDU1Q0QxNTc=";
+  const char *auth_key =
+    "auth-key\n"
+    "-----BEGIN ED25519 CERT-----\n"
+    "AQkACOhAAQW8ltYZMIWpyrfyE/b4Iyi8CNybCwYs6ADk7XfBaxsFAQAgBAD3/BE4\n"
+    "XojGE/N2bW/wgnS9r2qlrkydGyuCKIGayYx3haZ39LD4ZTmSMRxwmplMAqzG/XNP\n"
+    "0Kkpg4p2/VnLFJRdU1SMFo1lgQ4P0bqw7Tgx200fulZ4KUM5z5V7m+a/mgY=\n"
+    "-----END ED25519 CERT-----";
+  const char *enc_key =
+    "enc-key ntor bpZKLsuhxP6woDQ3yVyjm5gUKSk7RjfAijT2qrzbQk0=";
+  const char *enc_key_cert =
+    "enc-key-cert\n"
+    "-----BEGIN ED25519 CERT-----\n"
+    "AQsACOhZAUpNvCZ1aJaaR49lS6MCdsVkhVGVrRqoj0Y2T4SzroAtAQAgBABFOcGg\n"
+    "lbTt1DF5nKTE/gU3Fr8ZtlCIOhu1A+F5LM7fqCUupfesg0KTHwyIZOYQbJuM5/he\n"
+    "/jDNyLy9woPJdjkxywaY2RPUxGjLYtMQV0E8PUxWyICV+7y52fTCYaKpYQw=\n"
+    "-----END ED25519 CERT-----";
+
+  /* Try to decode a junk string. */
+  {
+    hs_descriptor_free(desc);
+    desc = NULL;
+    ret = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    const char *junk = "this is not a descriptor";
+    ip = decode_introduction_point(desc, junk);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    hs_desc_intro_point_free(ip);
+    ip = NULL;
+  }
+
+  /* Invalid link specifiers. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "introduction-point blah";
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) enc_key);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+    hs_desc_intro_point_free(ip);
+    ip = NULL;
+  }
+
+  /* Invalid auth key type. */
+  {
+    smartlist_t *lines = smartlist_new();
+    /* Try to put a valid object that our tokenize function will be able to
+     * parse but that has nothing to do with the auth_key. */
+    const char *bad_line =
+      "auth-key\n"
+      "-----BEGIN UNICORN CERT-----\n"
+      "MIGJAoGBAO4bATcW8kW4h6RQQAKEgg+aXCpF4JwbcO6vGZtzXTDB+HdPVQzwqkbh\n"
+      "XzFM6VGArhYw4m31wcP1Z7IwULir7UMnAFd7Zi62aYfU6l+Y1yAoZ1wzu1XBaAMK\n"
+      "ejpwQinW9nzJn7c2f69fVke3pkhxpNdUZ+vplSA/l9iY+y+v+415AgMBAAE=\n"
+      "-----END UNICORN CERT-----";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line =
+      "enc-key unicorn bpZKLsuhxP6woDQ3yVyjm5gUKSk7RjfAijT2qrzbQk0=";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key object. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "enc-key ntor";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key base64 curv25519 key. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "enc-key ntor blah===";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key invalid legacy. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "legacy-key blah===";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_ptr_op(ip, OP_EQ, NULL);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+  hs_desc_intro_point_free(ip);
+}
+
+/** Make sure we fail gracefully when decoding the bad desc from #23233. */
+static void
+test_decode_bad_signature(void *arg)
+{
+  hs_desc_plaintext_data_t desc_plaintext;
+  int ret;
+
+  (void) arg;
+
+  /* Update approx time to dodge cert expiration */
+  update_approx_time(1502661599);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_desc_decode_plaintext(HS_DESC_BAD_SIG, &desc_plaintext);
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Malformed signature line. Rejecting.");
+  teardown_capture_of_logs();
+
+ done:
+  desc_plaintext_data_free_contents(&desc_plaintext);
+}
+
+static void
+test_decode_plaintext(void *arg)
+{
+  int ret;
+  hs_desc_plaintext_data_t desc_plaintext;
+  const char *bad_value = "unicorn";
+
+  (void) arg;
+
+#define template \
+    "hs-descriptor %s\n" \
+    "descriptor-lifetime %s\n" \
+    "descriptor-signing-key-cert\n" \
+    "-----BEGIN ED25519 CERT-----\n" \
+    "AQgABjvPAQaG3g+dc6oV/oJV4ODAtkvx56uBnPtBT9mYVuHVOhn7AQAgBABUg3mQ\n" \
+    "myBr4bu5LCr53wUEbW2EXui01CbUgU7pfo9LvJG3AcXRojj6HlfsUs9BkzYzYdjF\n" \
+    "A69Apikgu0ewHYkFFASt7Il+gB3w6J8YstQJZT7dtbtl+doM7ug8B68Qdg8=\n" \
+    "-----END ED25519 CERT-----\n" \
+    "revision-counter %s\n" \
+    "encrypted\n" \
+    "-----BEGIN %s-----\n" \
+    "UNICORN\n" \
+    "-----END MESSAGE-----\n" \
+    "signature m20WJH5agqvwhq7QeuEZ1mYyPWQDO+eJOZUjLhAiKu8DbL17DsDfJE6kXbWy" \
+    "HimbNj2we0enV3cCOOAsmPOaAw\n"
+
+  /* Invalid version. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, bad_value, "180", "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Missing fields. */
+  {
+    const char *plaintext = "hs-descriptor 3\n";
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Max length. */
+  {
+    size_t big = 64000;
+    /* Must always be bigger than HS_DESC_MAX_LEN. */
+    tt_int_op(HS_DESC_MAX_LEN, OP_LT, big);
+    char *plaintext = tor_malloc_zero(big);
+    memset(plaintext, 'a', big);
+    plaintext[big - 1] = '\0';
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Bad lifetime value. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", bad_value, "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Huge lifetime value. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "7181615", "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Invalid encrypted section. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "180", "42", bad_value);
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Invalid revision counter. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "180", bad_value, "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_validate_cert(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+  ed25519_keypair_t kp;
+  tor_cert_t *cert = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Cert of type CERT_TYPE_AUTH_HS_IP_KEY. */
+  cert = tor_cert_create(&kp, CERT_TYPE_AUTH_HS_IP_KEY,
+                                     &kp.pubkey, now, 3600,
+                                     CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(cert);
+  /* Test with empty certificate. */
+  ret = cert_is_valid(NULL, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  /* Test with a bad type. */
+  ret = cert_is_valid(cert, CERT_TYPE_SIGNING_HS_DESC, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  /* Normal validation. */
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 1);
+  /* Break signing key so signature verification will fails. */
+  memset(&cert->signing_key, 0, sizeof(cert->signing_key));
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  tor_cert_free(cert);
+
+  /* Try a cert without including the signing key. */
+  cert = tor_cert_create(&kp, CERT_TYPE_AUTH_HS_IP_KEY, &kp.pubkey, now,
+                         3600, 0);
+  tt_assert(cert);
+  /* Test with a bad type. */
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_cert_free(cert);
+}
+
+static void
+test_desc_signature(void *arg)
+{
+  int ret;
+  char *data = NULL, *desc = NULL;
+  char sig_b64[ED25519_SIG_BASE64_LEN + 1];
+  ed25519_keypair_t kp;
+  ed25519_signature_t sig;
+
+  (void) arg;
+
+  ed25519_keypair_generate(&kp, 0);
+  /* Setup a phoony descriptor but with a valid signature token that is the
+   * signature is verifiable. */
+  tor_asprintf(&data, "This is a signed descriptor\n");
+  ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
+                              "Tor onion service descriptor sig v3", &kp);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = ed25519_signature_to_base64(sig_b64, &sig);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Build the descriptor that should be valid. */
+  tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
+  ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
+  tt_int_op(ret, OP_EQ, 1);
+  /* Junk signature. */
+  ret = desc_sig_is_valid("JUNK", &kp.pubkey, desc, strlen(desc));
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_free(desc);
+  tor_free(data);
+}
+
+/* bad desc auth type */
+static const char bad_superencrypted_text1[] = "desc-auth-type scoobysnack\n"
+  "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC"
+  "BiYWQgYXQgYWxs\n"
+  "-----END MESSAGE-----\n";
+
+/* bad ephemeral key */
+static const char bad_superencrypted_text2[] = "desc-auth-type x25519\n"
+  "desc-auth-ephemeral-key differentalphabet\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC"
+  "BiYWQgYXQgYWxs\n"
+  "-----END MESSAGE-----\n";
+
+/* bad encrypted msg */
+static const char bad_superencrypted_text3[] = "desc-auth-type x25519\n"
+  "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "SO SMALL NOT GOOD\n"
+  "-----END MESSAGE-----\n";
+
+static const char correct_superencrypted_text[] = "desc-auth-type x25519\n"
+  "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "auth-client Od09Qu636Qo /PKLzqewAdS/+0+vZC+MvQ dpw4NFo13zDnuPz45rxrOg\n"
+  "auth-client JRr840iGYN0 8s8cxYqF7Lx23+NducC4Qg zAafl4wPLURkuEjJreZq1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC"
+  "BiYWQgYXQgYWxs\n"
+  "-----END MESSAGE-----\n";
+
+static const char correct_encrypted_plaintext[] = "being on mountains, "
+  "thinking about computers, is not bad at all";
+
+static void
+test_parse_hs_desc_superencrypted(void *arg)
+{
+  (void) arg;
+  size_t retval;
+  uint8_t *encrypted_out = NULL;
+
+  {
+    setup_full_capture_of_logs(LOG_WARN);
+    retval = decode_superencrypted(bad_superencrypted_text1,
+                                   strlen(bad_superencrypted_text1),
+                                   &encrypted_out);
+    tt_u64_op(retval, OP_EQ, 0);
+    tt_ptr_op(encrypted_out, OP_EQ, NULL);
+    expect_log_msg_containing("Unrecognized desc auth type");
+    teardown_capture_of_logs();
+  }
+
+  {
+    setup_full_capture_of_logs(LOG_WARN);
+    retval = decode_superencrypted(bad_superencrypted_text2,
+                                   strlen(bad_superencrypted_text2),
+                                   &encrypted_out);
+    tt_u64_op(retval, OP_EQ, 0);
+    tt_ptr_op(encrypted_out, OP_EQ, NULL);
+    expect_log_msg_containing("Bogus desc auth key in HS desc");
+    teardown_capture_of_logs();
+  }
+
+  {
+    setup_full_capture_of_logs(LOG_WARN);
+    retval = decode_superencrypted(bad_superencrypted_text3,
+                                   strlen(bad_superencrypted_text3),
+                                   &encrypted_out);
+    tt_u64_op(retval, OP_EQ, 0);
+    tt_ptr_op(encrypted_out, OP_EQ, NULL);
+    expect_log_msg_containing("Length of descriptor\'s encrypted data "
+                              "is too small.");
+    teardown_capture_of_logs();
+  }
+
+  /* Now finally the good one */
+  retval = decode_superencrypted(correct_superencrypted_text,
+                                 strlen(correct_superencrypted_text),
+                                 &encrypted_out);
+
+  tt_u64_op(retval, OP_EQ, strlen(correct_encrypted_plaintext));
+  tt_mem_op(encrypted_out, OP_EQ, correct_encrypted_plaintext,
+            strlen(correct_encrypted_plaintext));
+
+ done:
+  tor_free(encrypted_out);
+}
+
+struct testcase_t hs_descriptor[] = {
+  /* Encoding tests. */
+  { "cert_encoding", test_cert_encoding, TT_FORK,
+    NULL, NULL },
+  { "link_specifier", test_link_specifier, TT_FORK,
+    NULL, NULL },
+  { "encode_descriptor", test_encode_descriptor, TT_FORK,
+    NULL, NULL },
+  { "descriptor_padding", test_descriptor_padding, TT_FORK,
+    NULL, NULL },
+
+  /* Decoding tests. */
+  { "decode_descriptor", test_decode_descriptor, TT_FORK,
+    NULL, NULL },
+  { "encrypted_data_len", test_encrypted_data_len, TT_FORK,
+    NULL, NULL },
+  { "decode_invalid_intro_point", test_decode_invalid_intro_point, TT_FORK,
+    NULL, NULL },
+  { "decode_plaintext", test_decode_plaintext, TT_FORK,
+    NULL, NULL },
+  { "decode_bad_signature", test_decode_bad_signature, TT_FORK,
+    NULL, NULL },
+
+  /* Misc. */
+  { "version", test_supported_version, TT_FORK,
+    NULL, NULL },
+  { "validate_cert", test_validate_cert, TT_FORK,
+    NULL, NULL },
+  { "desc_signature", test_desc_signature, TT_FORK,
+    NULL, NULL },
+
+  { "parse_hs_desc_superencrypted", test_parse_hs_desc_superencrypted,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_descriptor.inc b/src/test/test_hs_descriptor.inc
new file mode 100644
index 0000000000..70a2c7c2f7
--- /dev/null
+++ b/src/test/test_hs_descriptor.inc
@@ -0,0 +1,224 @@
+static const char* HS_DESC_BAD_SIG =
+"hs-descriptor 3\n"
+"descriptor-lifetime 180\n"
+"descriptor-signing-key-cert\n"
+"-----BEGIN ED25519 CERT-----\n"
+"AQgABl5+AQoPXRnCGEOxIup3AcjQXb8npNiUFm2Qv7A6JKk/K+EuAQAgBAD18iUD\n"
+"nbkUblnUvTHzipq4bcr6aPyFVB42Ptobg4xr8s3VjHiJtjs9MDEdr6nXS7UlyhEl\n"
+"78vsuFEvLp7cvAgGxYY1xGXdn5RdHMCdi8W9yZLKMQX9OuJckmp1C6q+cA4=\n"
+"-----END ED25519 CERT-----\n"
+"revision-counter 42\n"
+"superencrypted\n"
+"-----BEGIN MESSAGE-----\n"
+"BxzghAOjM4De6Z6eGTvBrTP2SJDdQOYV/u9qtvlFsa2FRQWk20Adv3zJ/AI10CQO\n"
+"mUP4DNXM8FWQYGTvmD7wGz2/cXGjKwBXg1qO7zF5eP/D/My1sXsIfCcb41mkheNt\n"
+"xn1I5eKXcnghtd4lw7OkPVjSb/Z+VARUMmf+0qSNgmHLgEVnAoGJsn8W8B4qtIay\n"
+"4h4PuV0jPPlqJx6jMFOOEW72uqnfmqeNvClENXXW60xhnaxsf0up62fuW8ktu6Wf\n"
+"lnX/lvTstBFZZQ8/XI1+G+BPf8TZf7mxu0WYVg1s/KWYasYMSw46as59nkqdq2Ii\n"
+"qJnqHX/R20mWBhgpLse6wO0aNpky/rozEnikaPqyO1DShf6a6jXY8ADBg7spnK2/\n"
+"h7sf1+F1xfi2dy2WGxc1EUMP1kTVUmbft7kOo2nA7+3YZwQuSJHaN/66HrzU2x5z\n"
+"ayRUJ8+qDtfpEf17xthc/Uh253blFK96IoJJiqBfI6xt3IqOdHJq0OOC9zBbF6Rj\n"
+"vKMsaxmc/nc6uOB2WePYSgkZ0qs/dRKBJs6+Ahn1KdGkadyd8mDKL86Oe8lncHdB\n"
+"m/6sQjhKqFgngkCDOIlEJyWizqfN84AGqD5Zyxq0rbsN+9KLsHFfEbCRjgqjO5nS\n"
+"FYSFtuKgCZl2gaYEslL1pIEYE6BD2Whjn/HWTRyWiULJr6SuavgcbxeNEQDuVCC+\n"
+"fm0X7Z+qERaMAMR0vTMJK/NzT4GifrDpgmgbxc+34CtNBF5TriM8aXTNZZlsW00k\n"
+"d0XRxFbbbtiT5VOaEHbny7R3MdTVutEc9E/BhLBvjSSrGX7vrryh6Oj++nthIIzm\n"
+"F4M7I11S0TyA+UE06qF1C8rKmhcqU9MWy1SiccJ9KOWhJ5xwlsXBIID7wVygUhVl\n"
+"ovzfKkDDPfRoBch6NdVkxNJx3gb63CUmC2TzfwOMh973nntMVzqqw9A7jYkro9ln\n"
+"217kHUwMk3e83UgFL4nn7NCf3Kj0zhJ4jSfAsuQpV6e7dhzrlNya0lqrUsY2zFXP\n"
+"xv8wUtg6Vo1KewgVQas4oElkgFjDN8RJ7uBAwfuE/b9NnYJoQd76G8DHei/1PHbu\n"
+"tbtwN9I5RHaTvEOfetsJFnIAkCG6O4CQpzwHu1DdvEP4s6/el10b/4awBJ4VwOVZ\n"
+"YHSe4X0DStTV4Cu6aLh5OvrOmGbieRj6HdGQ6syYCaEBTuxbBUUpjIAfVlReAIph\n"
+"6aOrY6HNcCmeVmL5qm4dKr2XXOREsnUFuMqmfQuQd9pN3zlmS+RqCgSJuFrguFpd\n"
+"mjo6UxZvbjE7yJjtCih38HRe5BaigP5RDRkXmiXjqJ4koLJpyjQh19k3BYGcdxUC\n"
+"RCcYXydbGF7qHlnoaX9HnX7y6ZRsyKQpt91PMTGOUsB4fS8NhsqPpl2gdp4poLNs\n"
+"+hqjWZJ3uuLotXBcgM39Dtq9tqqu9vM12T80UAfWnVEHrBphmukh49EhEr2sx/la\n"
+"kAzRoTbLyTdlGVei8hI7/RtZIaIcOkzlhcFI5zmBlydyrv6/79vzt6WI/w9GVGpM\n"
+"OuSM0NS2CDJ7Iw412nz3CV1pEXB551ZBmbme6NHUe4EtEsDbgkP1Z201H4j51yVz\n"
+"wNoIksE5Bh5XRKuu4We5f9KZb+AEG9kxKJ5DbJk2YGJEQFTyfv0H68pl9urstPXD\n"
+"aMQF806COe2uhGm5gV/skvPVTeEvStE3K8DxZgcWNcTMVk8ZjrUHNfguVVToP8hT\n"
+"Fl4Iqo3r+JZEAGXnAbTpxUVC2Xxspf3jsT5xhUfB/NOexZxrXWnQZ+pscsbow0ba\n"
+"GATtakD3TF2WBqq5WscmOex+lrJcBCWVIzVWdwi5ngAtm1S7efkJlFUvmi4OuYnN\n"
+"RyZfxVIpoer8f2/xPXvxkOWFminDy5sFEvlh2/pnymfKOUV+CKih9ZApt+izlRJn\n"
+"+sMIOW6Jhf/WYyjeN6KQpwi6CDpclQJXA1SVoOVVL5A3lotLjs0x7ThIcBoxCZBq\n"
+"rFBhBu1gJgJ8guMySAHssIvhHHwXJsYEwzWCVAg/zIUXy4PLwIkgHApl+vGcldGv\n"
+"Br5HNCuqQ2pD9z2RvzNneB/LrYB214i+BP2piO5HbmeJBhby93blGXVfQewQT6aF\n"
+"dBlK8/jQM0rvb+LkmvQm2ypOttRpX2kyQXooJHYTTusaUr4jVmgngCvGtgqAQVqD\n"
+"HULXfHWvugZbAh6dXF7gKnnsyDOWwAgy4OJRi8i0jCaZ8aWSFRUjeGKT26dg/ayB\n"
+"U4QfMb8vL8tMdXVBfQLGcBgvrzQYrY69//pV6bX3SbLfUfWXV9eqUVWVPqVyPEwa\n"
+"Tz/aGVnGv/dY8h2cVnrgSXJGlOO+mCwSl+k9nk7VcEaKYuNlaOP3ZlKJvVj1LefM\n"
+"FODh4qTDBo5NkyfKu5fcZcOqDMBeGWXZzltE7CmvY7fOpDNMsuAoXYWI7q9gK82F\n"
+"w+nS0tVFCIWYa9DgGMv9GKTOk4Ia9elkbWypdRE/4oz4QxmHsArEsK4gDI+wmcp7\n"
+"/NsAZeuy96r2YDIUam4uASKOiAqrEfCv6B6cYctdYwZbAEXdo4fkGrCIjNRZmZGv\n"
+"kcZzHzIymnAmKRTkPt/LQ7Rx27Qd/Vt++B3zt2ORFuopqowOP0ocGZtkm0daK3Fc\n"
+"YDXMwIpf6Z8PwvvsG1bQHcSR+cUZi7vK7+hj/LGhMPafHM7HmFUbAxpJYr5CvR6y\n"
+"V1pZQYltT8xWayCeMHlLAAg10RgDkqCnY4dHnrY4GdwI2O7Wpxomni7qVHMjn+cN\n"
+"UTrd7EeVw+dxAIYosuqG7ua7ee3VGoOs+XMLrscAqHahfGbyYC+j+6Tow4qwWBdU\n"
+"/W3NJXnRWaHTXFHllpClnxggPRQx4yPtgTOmBBVl/O0T6i4Bv0ygsJeZAqC3VmAJ\n"
+"QodQTzGf2jwqsZf4uHKQa0EKGQvTGjFVgAFNpHmAuzyqh0b1pq5JeXiFERGsKC3j\n"
+"xcJilq1XeIx4SL38YNuCxi4pnyJyLnGGHpNjdjeFO5lvgCaKPegsPo4hpNpTvBJ1\n"
+"D7+o3E5CqxzjRt9kQmtwBbuH/SQX2T0x8aQ6vhwjj8ftDfw+FbjpMR9zfU0Lf8V7\n"
+"UjVGIl2yiVBGScBZu1nSD83PxjFy3XdFtBYoU5OrlXwBEYQs91jwK7UCiGtjI2Ao\n"
+"ZGkJaBd4AqP6voyJiGnC3LWFcmeMyzfExgiclQwfhFqqf762TX5JwG6xGqtdcNKS\n"
+"k54LlcI/RfvJw3ncSs9YsodZr6Jz5irpRTHX5WwCrX9mLukP96SXo29bIXEZAqEr\n"
+"ZxEcF0zlYE+km5bRfRCRcVVrScugCshSNLOdQp6fOAtHCl7rdQ/8Rz7oHuqieLVi\n"
+"UldRsAmpk9fIfRLphXj4j24jRP0VtL/LoJwakWTa0xO8K7eBAMVITI+HgFfN4wSO\n"
+"Yh1B+bGD5WKxFsWSgBMmW+YLF5ZtxVmmbg7wK2dIpJs4pjg2YO/MTO7SifJ9kjcb\n"
+"bCc74Tjs5mLLGGjGCIoXfda6WXbt2it40XhFk2zUAcPPsgjbctftkaWph7JSZpmZ\n"
+"fVcPqKdhmA1U0LA2XEOMTxGyCAeseH6pJXZm9LdBozc1CwyWP8XEDHHJf35vfPKY\n"
+"JDe2PanFepIOHaoRTgE7ZkGWKzOIKlS0Ucr1ezVfcxiFgQUNM+MYXXbUz51BVVq1\n"
+"Dulg4VvX104nt/ULijcfa/TsE+uklEnkyk1mhavH337NQg38XF4cAngNlUF4nSW/\n"
+"j0jizbAtaSx1f7q6xqPm3zPRlHrGQizHXLyl+SLzDUVPOXbPwcoeev97YeeyB6h5\n"
+"NBbIK9hmekNDmYIwI0bmlrg6IXhC5pyvRe8sQlV+9wBY2liF0M1mq5onW3a55afp\n"
+"+ynxXfQucb1HxZLXvRIGMBgWSQ7HfIPASqSE90Vu6qQCfkOW5PDqONr4BM65V4+g\n"
+"AYsVEgaosgHw9CF7yKgkvmZpToOtGpCHVcdUeeY2/rrQnAQeSy19gj/baJ+OKl6Q\n"
+"i1EGU8Yqo2r0d4XDFp/eKgC4sv57qp1PwkYQ/HKqoelJ09IAZL2sQWc05BGwt1A0\n"
+"11qDIEdkZBjzK3qUnY3QlOuoZtALZrnPg56SlF1RGDOPqbcF+3opqsvzBoiikh4V\n"
+"WV5OUYjRDMUDLQqf/OkuktdYf5N3RcbYP0XsAvY0ZWG3Gp068b3p8peCpkDzrF9p\n"
+"bQ2ZvS304tN7+p0hif3+JyZy5/sxl17RxTeg5I3mo2+J0ptQDYwF/WadONO8r7uU\n"
+"YlRltFtQfyMzyVzHON4NHGjZh7dDGtWp0MGeHRBHQsC8bEChhvWme19VXhgZoWpl\n"
+"dUIZkSuvRwiURXjhKbZrEdJbVmr9FX6zoyOahv3VnmcEARoR+umxzvo3hGQPbHyH\n"
+"jTsQtSBjs75/9fCxcYmBWkh3JHVDVsCbV+z+5KZpk3m50J4Y1hC8hvepC9CaBqOM\n"
+"DjfyXh58x1yKiueEbcjSWsRuF7CjcrYnFUBHOs9U1j9WytCI3fhOWPMgR4UZpGuU\n"
+"WlcR1BXg1wYxX273xOS/jYn9MLAVlbRpPTUMIH9VRP+sc8+XaxKpJSCl4C+vcwNY\n"
+"1YdKD2QiuoBJ3fXGtqMVRtn9eZvatSJuY9CnRKRbf0hWmFD4D5RkiwE0WkdtwoHR\n"
+"uEXJ47RlF0/JDU1fY1mXBkq3usvB4Absy78qL06vh45xkk9bHbdf+7Ao1RQKmqiB\n"
+"NL5XnjBu+YX535WG7t7Su3mTCJXYHvn72ATxry8yhSLgWqt81STkRwc14HmrOGG8\n"
+"Gw7bz7y5vikj/rnPyr7ry+QRgNNDDayAqenAu2vEAzWir0RQC/iZ9rc/r7YQWGgL\n"
+"Xrd4TQ6rTZePARhwB3VomnLDDvLvi2oq/jPzLKSYM2a7qj/vBSbJ/NnNaDW5Ccew\n"
+"RjMI1lIHeedqYTVAW/CKoSEPcFSAzi/Ija0gcWLgX5xsFDGIYBepAX0KS9426kMu\n"
+"0r/V66zmPMusMilqRTx7KW+jZMVxXVc2zClcdmohMmtjsbqLkczprfSbdGswMv9Q\n"
+"I1ktHJHIRD0vPeZXnvKZsRKZw3sKb0ltZi33ZxCJFQPeGGtM5aAFthj6awcXy6Tt\n"
+"DPUQdCU/vh1zmGRAX17/Xb0irfvN+GhQLEl42pzhigJXc/rCG3a4Na8wT+xAIZVf\n"
+"WUI7hMslx5wA+iB4lrAjCq0YIrjINI/lHYpotXUZGmz5wz0jOciTmXMSx9du4cpk\n"
+"fIQJfR+fr5tG3fjHMgSP+p+RewHkd/7RUAmHC2k3cuk5pCJvUVJrhUIqsi1fa0LG\n"
+"GA0UU6Nr9tpYdNr1WkbKQjxTg0D//AXe61jmUS5XUU4AQf6zQVfN0TMtmuYeacbK\n"
+"4r6Z1CSIRbsgcnL1BN8GSd4KddkCqSk941aJUCoX+77ou4t0btVSB9FnLKipigtE\n"
+"E/Rpmv+81lA4fLiIag62/pcJ3uppsZ9aaHdR10SMmuCjAVLYHqhJfrHHn32dyqLK\n"
+"UI8kEZJ6GQzHLUXcGbbdnk1Qm6JwO8TeF/oQvh9y9py+oAyFy0qzP2UeUMUI2yRQ\n"
+"mlWSy+wX1DbVDQ3UHwJjWp65CgyYXuW8eCB0AbyF0kF4KGf7/7Ae7tEGbmYSm5MA\n"
+"71z+Azxtv5gRyRb787V2dyo0wcmbRlL7iUBVXNM/czQo31tAZIwLc+lKNp0SPH6g\n"
+"gJ2yX/GeDSFNAeEVUZ/f4KZIa7QQsnGWrUr+agSnQFkySmIjWYjwC/abJwah0v0d\n"
+"ulwr3tECaaXtoWVdYXa3utEclBz9umBwMJ9MQCm4Kx7dTYUWFT3bMM/ESTkGPcfm\n"
+"m+C4FsqFBs80WY0ududu50vTDSdJt1RqZ7Sg6DNH6acBvWyXOpT5mPJKUjnSFwyG\n"
+"oVLgv0aDDx7lLZdCkhyz/Ff5LNmBgQsjGllPszJ2gTZxZ5LD68S4kUirQG/qtzlS\n"
+"PGfDOC79SMZGgsoAnr4wV3RUTxsTVFlxVHsBMB+EXOFHAr3wHTVxUGBbGzxBlQ9w\n"
+"I/jlu8LIIexXAU75HS5KCGGfg0Z7BLqEzqpMKqcBQC7BD7GnCXrDSQ2DCXnl7bLN\n"
+"lIrQ/z2Y8AgSdED46R40MqyyN6CPPNiOCjONHZ30fLEXuEgCp4R/+x0WWsWpjGk2\n"
+"Ydkc03cx/X6moUYxB5HTqTodBmAQuWMX0rxFDrnR0SWghWjdWth9gjd+dvZ82tt1\n"
+"UMUywDPhcYchtUi2lnqnYJm5p00GN9Mk14MC5ZC5qP57IJVqxu0ktOMpks+CLPnz\n"
+"qp9OBpI4sIzd0y0aUJC2Gd+E9aAhlREIiicyBDmxLdk1i37QeeCralI3eubLNmE+\n"
+"CjDjD8t8FUGPpKglSD3lfLTqbp2TUvyWfvJC6ulFPNsAbeLHTnPnpyPQmWxhMNGt\n"
+"h67B9tbYww2TvNwqIgmB4+YIR4/pSs15TpAqvuUvjpmRwGklqgiSmrQrlIxCxux/\n"
+"mfsaL3KE97wm8BsaMpMkjUL7ByTIFhFZ/gHPTxaFpbqTZ4G+lABLgp3bIsB9Dl/P\n"
+"ovoqX+qL2Mq9T0GrVJGfRBuA5hISw63hx5zdsj2Cj3A3khHPqR+GRN/rVYUuOpLm\n"
+"z3v5pU/74vZRmNMAIhyhmweSEPNtyVkgSdgbFErqvhxN0om2Cd/7cWh2g5BXHyUL\n"
+"PBr7ZkgfsE9TnuDH7Z0JoBqXJki+MO6nqz73oH2Mm86yxcXp6O/ieKTollrUJ3yQ\n"
+"P6hLcEbYPzUV99del7Va5Wi0nn0wbRXCGVQdwY+iWc7pT+VVlncyg0TvLXi0OtOt\n"
+"O8xbT2DAzVXxMwOsKV9ZgS/0dtwzwICpnTzBI/47V8GYhHbOUNTBPZ52GaXMeWlX\n"
+"cuRGb0+7OkKWuriyOQ5z5xaASCVfqgnOwSZYiAk0gcDoK+JHdr64/sMoJhH87R4i\n"
+"2TO90whkScgiGR7A06Ba42bT1nJtI6pxvzdB2b4BDAs2Lr2OdcB3BY1dtzKjFkw/\n"
+"qfIw3F55UQwcs84ZEFQDAB/tmfNHajblDFpXR4N5QvU/PdWVWJUub7oNyhIX6ruu\n"
+"ln4H7lpTUHJZ7jkr1qpnvkztZtHGlpJ0QdUHgyMYER1xU58Hg77yzIW3EdAa2PyK\n"
+"1t4udKbQKChShlShIMzwzj57ss/69QobrpYAHYi6IRMaMUGBfipGBACK3yeXsXz0\n"
+"c3Q2J5vI6QbxNsiJ5t7Ry1IqotbJcU7HND/yVUAUbEg5CpEDOSeSOW/ulyLuFxEV\n"
+"lRTwIO/68BoIoR7umlP23/1N5OYzaBHhH2nThILBovHeJRXnGXSgeFfwSj7LIYEV\n"
+"c1MdDSg/HzoADPXyEPLzqFzHRHeNiqEolmOPnFh0hRzbMZ0W5TQPDGWJdF21g816\n"
+"vA0WW4UQjLM+vnX9kKKLA1ut+9JWk1dGKsmWtdWUDfJjUP/L6dS4OYEl6O6+SjM9\n"
+"GcyGvHTiC5OpJllYpvELP/NjtTf9or8Bmruuga/axeOuS5ocYLK/sGRlmO6Z96da\n"
+"QSlyGWEQAnM2D1cDmdd4CetPslOVIcQ41+coWCi2xg3UjO/bFK1CA4R1rb4ekXfs\n"
+"s5U2XChyHhUPgl57y1r0ILXRXWJTJ0/F9hhu4aYQVFeIV/IuzJbmTKKkAcCOH6ys\n"
+"qnu2BXz8Pm2tU10JFfRcuZ8rHuUyUErA40ESsLijON98GMwL4Rat9ZSCNS5hlK7y\n"
+"yRJdr0ITp8oTbduAoulgWOvtcw1L87QBVojWz3cbhXra+WITirYuGNbzfmZn1WQM\n"
+"kukEZUEHSypGOrHr1XiuY4Rw/DBaJSLyZ+VybEOfXqXkDBh5s1ayypBvzrzFZCIn\n"
+"PJxIVsvrkhrpEbTJ9d7zLWjhOa9ZWw8lAubllbGm+7qCfdHmGsfBtvJdzx6zhB1Y\n"
+"otL/PCis2XVTBEDJeB8pGqKFOZjNz8PC5qP+ymtAfy2ktl/u4HsFlxV7CsEKGYPm\n"
+"p3LqnhPUy5M5gin4E4uPPyzzD2kcM3way49FKWUKlblQU0SyWtHRmMB3vcVmyT85\n"
+"BRULXF7jgog7XR/EMltwQyJI6GcUCrnWZu+G0BEwXG+CsgCzE7assDavc1NSGLZM\n"
+"rmzXiFFyfk7CE6lW2Lm+oWaFwKdvpmNZJFGGX8ZHRE9ZvkFMnfw9MYf2W7xa0jf7\n"
+"k3c6X5wMuk9mznVtq5itNFVXh1mT1ujeWOiiqyH5UhQQjj6O+ZXt4gqt/jT6dd1i\n"
+"jRuhhxaUGOlhpVBW/ySXhZ+HgOy9aCJ/bgjRGaqGixogk4f4rcgigHruwTpOQuDn\n"
+"xDZ3Xns70S40WtHSYN+Gbl9nIh4yl78aNnA4FVtTAuLlVKEKlMJi9OBFuP5TEczG\n"
+"+0HTwL/VPSCI+8FUZBhlz3YwecYq6dY5mS46+luPW+5Wl+5jtzb8V9oxVnRx2hQq\n"
+"B5HJsM5FOOhHDHMXoCsevj7N/ufK7cU7Wbr0DkgYRwvb0ZJB5WYgcaQ0W7aduhGb\n"
+"MQsandhP8Ajb2cmLobi3mHHPbcEkvjT8JP9Sim5xtfF+oCMMB5ByA5bI2aIFybZm\n"
+"jX9e/V8wNgtpDKDVKPjB3+9dj5gU1N5JsrjQwQDB0kVRMWdpJCtD4hZ2+T/QE3SI\n"
+"f8Rdk8pj8qBzRPbnhW6qsoWZdjMRC8qixZqHw4jol09UF7Ab9hjEF5ZDTfNGXwy8\n"
+"/hz8su+mr8hhrlCrOF2vBYUayAA96zhbDWfg3Pdxo9bTn3/DmyAngL4J5Gu679xK\n"
+"rWN4j7uQG4bzTa8WJb09/lW49UzWvmrz0c6/yexk3T//xDD067FafdnP5pYs4Cvp\n"
+"rCoHpXbKjxx99DJmb5iXW0JRLSpFSCbf1HPHbmzST3minSXap5FCWDJcSgExKIJp\n"
+"DXZ9rk0LMnQA74MWC5gjjM+5t0AHKuNRhJbQSwYWTKqeApXho53T/COlfDlSs2tb\n"
+"Vz1Ia5z7IOfu1QheE93huNAHT3Ob+mSmUq782SqFPr6uwud/l5uP3HpcuwugdlFm\n"
+"Jw8uBBOQ53W4lLbYfQYTVgieClVhmYMu7Ye0xYZ5B2jf714sjZRMa0LCbsyj58xH\n"
+"uzs8ddNN1fLMzb0JRBE8JWj5PbxhA/sTwMkD7SnEMBUTtP0obmuQ982aTfyvQCH/\n"
+"ve8OUPtYf5XWNv18mpR+h+riMt1Y8Eb6BJzTMFNWagMJAe3JV6A6upHroNFo2FxY\n"
+"1XPRM1Rt0zKo7GD+oXnixfpl1aG8yqZhYo1ZC9buaHwH6zvM+xoiGD0iujeDtpVy\n"
+"Vp6cAqqaGmrNwcPVBLc7hNKrJnbFKyhjL5/xp9j6jQov1aWQ8HsaNvh0p2ljmlwb\n"
+"daTYZcwLgSgPna7HhiqnOSAmXZ7St/qe/b9TqBtIVzwzmtevgMyG98QV0syFP5X6\n"
+"2Jc1g9733sTZp7njq4Cu07JhpICpinhLWR3nkODJbjk/mpLcQZgtV6W749AUo8oT\n"
+"jRVEJ8MpCo1h0bVDxsRnA3DrMneD88L8/b10aHs+bPm1HKbCmT+kJAFaUQNa8JvJ\n"
+"pReN37qTWvZCte7vaPAIP5cboATMu/J4t3izpm+YJoJlWcIegGx3kQ+17P4MbgDl\n"
+"S93U4sOLvTk9+MoyPo9yGWU/zHgzcQ6wCFdzWMDRswuh+/4TJ2+yg6maq3iBtj39\n"
+"gNLMR+sRgGGvYisqE9bfvNQy5IWrABBKcSBTXeTM1DmW6jv3TI8DoCzCbpjqcIwT\n"
+"u2J+7k8wJEHPcAwnBjlyWphVvwNwM0cXqOnlJZ/4z7OGgjiNEem7TMuvxk+YkiXK\n"
+"OzftdTjeIpzBwsGRP8/teMBpjS95M7GloKtxO+muBVxXbmsq8GBRC9vtNJ2Ma/xP\n"
+"bXvd+7caytD3ob6ZfOzCpi4ZS8uByEfIMxlgZ5Sn3jhgEkcIU+YW9b3teMZOuWdA\n"
+"QpDCoMpXaHVyRqwVV59JjmftiBnNBEo1/QzRj2UxRi7fHMfmNxL5LRM4CHSLUSCq\n"
+"Y3A3pkxvBHUzemhynSFvtCPa8GHiUpe9so0V/2hlgaENAVELPjMlWytaYufRllgy\n"
+"tUnCd32C5PrrmYzMKnxKRPXLcxLgziruJGSks9vIspoPk0pWgkZm+M9fRpJKlWHF\n"
+"yT9OOGBW2yynw/yvXssxJmdUDxVcWL4uS2bZc4s0Zc6RSL9uQPjZVX0JLj+cXfx4\n"
+"93Gn5bDhMgm+CGM6j3RiAAD7tT5V0sytNFjXd1A4U1u8yj3wzhKqOtZpDmuGUlMn\n"
+"EODu7I5KtWxOTPThy7TecI5r+F/6KL+2MOtRhj2PmlT/Xed6PaAmDkQeiXGps08x\n"
+"u0JIpuB61axvT4PAsKZNUd4ExbzNxRDAARUMgY8krpmyKZyHVFIQ19uHM2lGl9/i\n"
+"h3PKlLHYI8RsHutHElzq+F5tWd5AA99LVRZX4axAVIQNiqRg8IMSoCwUaCCbjUMz\n"
+"sJCo2t36GYk5S2BRnfrCqYoZRHw+ENYN0tDEMhXq1OqjvNHW3TzL3DsUhM6EZU5n\n"
+"cRR4ynUvPqqWFphLefRW10vCtaW9roJQZyFYf9kd8xgW/BhcDNbTTaQ1U6xCHgX+\n"
+"78DKee/NvY1WIEBR8X0iVk5XlSJb14eRtxNawXFyebVdmC/DiMNgnTBncMbePnZi\n"
+"KCl1r5xqo7tSIoJ6Z0l6qINd89T9fcg9mujTVwsfQ+5/kdEy0Iw7CQcTOGvMaoPX\n"
+"IAJlWSVeZ8eu8kmsD1Z8ewoPufMKiY4cPRAK5bCDgsrK6bAExOlCwPnNNM8Ym1Hz\n"
+"aYFeGs5sW468Qww+Nbl5xcNFKtwUKZ6EebRHjwttiyTgCdAhv9wL1u2WFydWWgkG\n"
+"rwUbNpSLKls+pijCeJAscvxzbZz96iOaYrY8IyzGBFwfgFAESfnzBc8SQjZzMzoO\n"
+"vmYIRon2m/5w5AZA2IjQ4VxXJDK6XExD/ZLsxNXzMnROD++hE+s8DvPlRPmN4egF\n"
+"gAzJs/9t7IyE/dDf7gSSBqzEBbwduD8ozzYHwELUc4ERdRzjEdBM0azT61g7Yilr\n"
+"iT5Hy+2iw/pNwiqVOYiAbj2lwcoMlFZmdxviD4IMXdsNVWsCAVJL0PqIh1UDDb3z\n"
+"Urv3idBJeSBuuFr6AFS6kAgvrwV/pEGoBoHuyii/rZxVugGKeuMynKEvSHuFNuQU\n"
+"qIHcNgqQR34v2Ut5pQ1R8s7K3Rae/AhE5GncJa6FJmB9TF8MYMu9PlSZV/eGv8UL\n"
+"IDWQ7sY3NdhZini//xtwPqIw29yOeZ0X6Aqsek9tfh21UwKSpHb7T+PwXYmoB+23\n"
+"p3FXkP/rv4AGRq1xJqFYzKJvwsXqTFuNFWP74yhTg6rC90w2p5TeH1rJMAnv4u0L\n"
+"hGtG/NL+D1Tzdf00TYAjno5Ia5dQJDd/eO+Ygqnhl6hAqGtS6r9JhIEXw1nQD7SC\n"
+"lj96ZuKdUWO8rpIiAtvHAsn++xvMVPm/S1SwA8oE049iVwS8/eNNiMKoSlTlYc7o\n"
+"pusBZQrVF4We4HHYFjysBbcXlvoXDd8LkZ8Nh63VQPnoIGNKH2U6aXCnQcJ8dZqO\n"
+"DNxL4uyM4A578FUUR6vxqt2asnLHQ0Z7pPE4uqtz/WgbiHI/i2oHS8oe1clsifCw\n"
+"3ZY33kflqLftkTNka1oiftDb0OqFLjkS7/AUorqHazw53gM3gqJY5EXA3Px9+nhu\n"
+"NzxSK/t41JoCfgQJHMkIWb3yUcO4OFZeGCeAxIJY95hv/brt6/WNielXjNaohYvc\n"
+"lsSUHEJRHwVxQmWK0LS+g13HAgOI7cNt3MA8sSkzTneHGFgEvmrSyb0wCEmushC9\n"
+"mjQThvaxfQk9douA/cR2bHr7axXqv9vjztmxUr0a30a7lvLMBQbJmFtJJylW+tJe\n"
+"v/vKNOB+9mK793cttr2JFnMhwUKFKWiFDQJtxw/eLQWY4BJ19Rs2x4BJgmV+u1jB\n"
+"zR8uvxuArG/cqVEJsoC6uuSzhAWSwdvumijO6yuyWF6nHY6aAcy8dyFQlDFHAd+/\n"
+"J05Lrbzj4N9lcI7hPalh0uMdERGvtUdT8QRm5ebP1zogYEkZk/1GOU29dMawkAt/\n"
+"SWhp2yWdjLt8f5HQKu72vUF/yyTfzfdqQqJwfthP7+vp+sHDO85AMF45uU9g3pxW\n"
+"IbXSbZ4fFGC1/41db/2GOHFgaheMXj0SIWHqQE1jtihr3BBBO4b3Ccz5QCnrn48J\n"
+"8L+QRdh4a/cAx4ty/oHEiXwpSBBSFRl5+y2NijC8GITA5dRjCRWP+Y0zuTrJ7j1a\n"
+"h+3kGs1kxqskhaEuhXnXyknGLjXrU+ewRGhHzP23o5betVhX+c1XjVqmJNZ5OPn/\n"
+"wrqx/XwoIl/3F5lMmGDG9mPtyg0E227nKl9Sy0Vbwx2tu1unjOlzSCa7lpoD4TIX\n"
+"PBJ5+Zb0CE6HEt3V0ec1m4uUe/xObAnzyr4UbzdqLaMy8vTcF/qsncXyPBjwqdjR\n"
+"ReDAtt99bAPY4roPKGt8dgKUPE0t/XoY+SlmUp75TkZDXrOIJXpEW0GpLPf53T+W\n"
+"Ex3KtfLAnZzrw8+dIageY7IgoQ85h3sYE7uEI8QlcO/o4udqUzTp4Sn4sWvdTLrx\n"
+"W7ImvK2rsU5ubVdsEaFKM7+7nxGn2JyMpIWFz0SbP34CkXHhrXxyRD+GhMIDHFxV\n"
+"uBnZnjJsw+ooIm1rL4I7/VMWEwmVegreT6w9Gsmb5igw+zu9v2YBgTOhysA9XZd4\n"
+"7O3VjqKkhTXcBqdpRWuz8gPQ+4rfwij28Gg2alG04Eh3G3868NOCFJhhaHVmwYR2\n"
+"ygRm6N9eDW1bHhYSN75HSEb6aIebk+1AT4S1QtJaPSH0EduIXO++JYAs+jIFKy2c\n"
+"jCVFlO/LbXl7iCdXurJHpSbMNmZFNUri6zEolENODLwke836jBOKiVrWzLnEMxHI\n"
+"WDDTpLTYhR3C7sEprpEQm9SX2Eik3WxVb4ZTb7SZFU1y1d4tWnjGu3U1D+vO9wVq\n"
+"Sss9lDipbkhQ9k4j1/Pqozaxvi8lYLbh3WEjK3Iwpr66Bk6Ai2oRg4b+7vzV4o+6\n"
+"L47JPJhajdHac0CIlmupyA4eejECS6OpoLDf5Wr/616k3dxM//3kAWGUnXVw9GSo\n"
+"UF5W8AaKlaGZ6EZk09NyGSFRjEs18z+g5ckviGF0EhZI7ZPWQQmlqWUsL9O0S4GO\n"
+"ZZ9f0UhNmHEspcugbs7e1yfjwGVyxIBkrmxpkmfHE4Gb47UGlJevg2OvZOPT3wMH\n"
+"vOds2BtqdT3tuss9k+7hsISGse7isEOb7TN5MHb6yyzqnCUZhp5m3Iag7TUkiyfU\n"
+"jKH5R13tHqKUoJ2rofWoLO2H5xSfp/lqF9sLd4rJ+Pbjhiuvfwz5copYsuTNL4kB\n"
+"SPUikHlTxSOgTBYNV77qxpsqOI3+iziCrSqHsxNdlaA1T3fiq6SeZBNdD822AYm9\n"
+"L5hbcgpDPEEwT/n5kWNbRNueerJkJwboaOnT1ZX1601Pwj5QDi+YM1NYy5PsdWxb\n"
+"bPGpQyZ+uf917q9gV7Ykr5cic10YD11khAghr0n6fYfb8Ijc22uP6m47KItDqQc1\n"
+"eFym149F56B0yg5FR85Arg==\n"
+"-----END MESSAGE-----\n"
+" signature Of+jvQKzH9ot2NV5twlDO2CFbzLSB4absWTwG58TCHb+TWgQi3z6SZIoTnGGY/uicJgEkCN++bZZR49GiyHyCQ\n";
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
new file mode 100644
index 0000000000..faa14d9015
--- /dev/null
+++ b/src/test/test_hs_intropoint.c
@@ -0,0 +1,928 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_service.c
+ * \brief Test hidden service functionality.
+ */
+
+#define HS_SERVICE_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+#define RENDSERVICE_PRIVATE
+#define CIRCUITLIST_PRIVATE
+
+#include "test.h"
+#include "log_test_helpers.h"
+#include "crypto_rand.h"
+
+#include "or.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "ht.h"
+#include "relay.h"
+#include "rendservice.h"
+
+#include "hs_cell.h"
+#include "hs_circuitmap.h"
+#include "hs_common.h"
+#include "hs_intropoint.h"
+#include "hs_service.h"
+
+/* Trunnel. */
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+#include "hs/cell_common.h"
+
+static size_t
+new_establish_intro_cell(const char *circ_nonce,
+                         trn_cell_establish_intro_t **cell_out)
+{
+  ssize_t cell_len = 0;
+  uint8_t buf[RELAY_PAYLOAD_SIZE] = {0};
+  trn_cell_establish_intro_t *cell = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  /* Ensure that *cell_out is NULL such that we can use to check if we need to
+   * free `cell` in case of an error. */
+  *cell_out = NULL;
+
+  /* Auth key pair is generated in the constructor so we are all set for
+   * using this IP object. */
+  ip = service_intro_point_new(NULL, 0, 0);
+  tt_assert(ip);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf));
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+  *cell_out = cell;
+
+ done:
+  if (*cell_out == NULL)
+    trn_cell_establish_intro_free(cell);
+
+  service_intro_point_free(ip);
+  return cell_len;
+}
+
+static ssize_t
+new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out)
+{
+  ssize_t cell_len = 0;
+  hs_service_intro_point_t *ip = NULL;
+
+  /* Auth key pair is generated in the constructor so we are all set for
+   * using this IP object. */
+  ip = service_intro_point_new(NULL, 0, 0);
+  tt_assert(ip);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+ done:
+  service_intro_point_free(ip);
+  return cell_len;
+}
+
+/* Mock function to avoid networking in unittests */
+static int
+mock_send_intro_established_cell(or_circuit_t *circ)
+{
+  (void) circ;
+  return 0;
+}
+
+static int
+mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
+                                  uint8_t relay_command, const char *payload,
+                                  size_t payload_len,
+                                  crypt_path_t *cpath_layer,
+                                  const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+  return 0;
+}
+
+static or_circuit_t *
+helper_create_intro_circuit(void)
+{
+  or_circuit_t *circ = or_circuit_new(0, NULL);
+  tt_assert(circ);
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+ done:
+  return circ;
+}
+
+static trn_cell_introduce1_t *
+helper_create_introduce1_cell(void)
+{
+  trn_cell_introduce1_t *cell = NULL;
+  ed25519_keypair_t auth_key_kp;
+
+  /* Generate the auth_key of the cell. */
+  if (ed25519_keypair_generate(&auth_key_kp, 0) < 0) {
+    goto err;
+  }
+
+  cell = trn_cell_introduce1_new();
+  tt_assert(cell);
+
+  /* Set the auth key. */
+  {
+    size_t auth_key_len = sizeof(auth_key_kp.pubkey);
+    trn_cell_introduce1_set_auth_key_type(cell,
+                                         HS_INTRO_AUTH_KEY_TYPE_ED25519);
+    trn_cell_introduce1_set_auth_key_len(cell, auth_key_len);
+    trn_cell_introduce1_setlen_auth_key(cell, auth_key_len);
+    uint8_t *auth_key_ptr = trn_cell_introduce1_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, auth_key_kp.pubkey.pubkey, auth_key_len);
+  }
+
+  /* Set the cell extensions to none. */
+  {
+    trn_cell_extension_t *ext = trn_cell_extension_new();
+    trn_cell_extension_set_num(ext, 0);
+    trn_cell_introduce1_set_extensions(cell, ext);
+  }
+
+  /* Set the encrypted section to some data. */
+  {
+    size_t enc_len = 128;
+    trn_cell_introduce1_setlen_encrypted(cell, enc_len);
+    uint8_t *enc_ptr = trn_cell_introduce1_getarray_encrypted(cell);
+    memset(enc_ptr, 'a', enc_len);
+  }
+
+  return cell;
+ err:
+ done:
+  trn_cell_introduce1_free(cell);
+  return NULL;
+}
+
+/* Try sending an ESTABLISH_INTRO cell on a circuit that is already an intro
+ * point. Should fail. */
+static void
+test_establish_intro_wrong_purpose(void *arg)
+{
+  int retval;
+  ssize_t cell_len = 0;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  memcpy(intro_circ->rend_circ_nonce, circ_nonce, DIGEST_LEN);
+
+  /* Set a bad circuit purpose!! :) */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  cell_len = new_establish_intro_encoded_cell(circ_nonce, cell_body);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Rejecting ESTABLISH_INTRO on non-OR circuit.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Prepare a circuit for accepting an ESTABLISH_INTRO cell */
+static void
+helper_prepare_circ_for_intro(or_circuit_t *circ, const char *circ_nonce)
+{
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  memcpy(circ->rend_circ_nonce, circ_nonce, DIGEST_LEN);
+}
+
+/* Send an empty ESTABLISH_INTRO cell. Should fail. */
+static void
+test_establish_intro_wrong_keytype(void *arg)
+{
+  int retval;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+  char circ_nonce[DIGEST_LEN] = {0};
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, (uint8_t *) "", 0);
+  expect_log_msg_containing("Empty ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send an ESTABLISH_INTRO cell with an unknown auth key type. Should fail. */
+static void
+test_establish_intro_wrong_keytype2(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_encoded_cell(circ_nonce, cell_body);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  /* Mutate the auth key type! :) */
+  cell_body[0] = 42;
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Unrecognized AUTH_KEY_TYPE 42.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong MAC. Should fail. */
+static void
+test_establish_intro_wrong_mac(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  ssize_t cell_len = 0;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  trn_cell_establish_intro_t *cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+
+  /* Mangle one byte of the MAC. */
+  uint8_t *handshake_ptr =
+    trn_cell_establish_intro_getarray_handshake_mac(cell);
+  handshake_ptr[TRUNNEL_SHA3_256_LEN - 1]++;
+  /* We need to resign the payload with that change. */
+  {
+    ed25519_signature_t sig;
+    ed25519_keypair_t key_struct;
+    /* New keypair for the signature since we don't have access to the private
+     * key material generated earlier when creating the cell. */
+    retval = ed25519_keypair_generate(&key_struct, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    uint8_t *auth_key_ptr =
+      trn_cell_establish_intro_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, key_struct.pubkey.pubkey, ED25519_PUBKEY_LEN);
+    /* Encode payload so we can sign it. */
+    cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                               cell);
+    tt_i64_op(cell_len, OP_GT, 0);
+
+    retval = ed25519_sign_prefixed(&sig, cell_body,
+                                   cell_len -
+                                   (ED25519_SIG_LEN + sizeof(cell->sig_len)),
+                                   ESTABLISH_INTRO_SIG_PREFIX, &key_struct);
+    tt_int_op(retval, OP_EQ, 0);
+    /* And write the signature to the cell */
+    uint8_t *sig_ptr =
+      trn_cell_establish_intro_getarray_sig(cell);
+    memcpy(sig_ptr, sig.sig, cell->sig_len);
+    /* Re-encode with the new signature. */
+    cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                               cell);
+    tt_i64_op(cell_len, OP_GT, 0);
+  }
+
+  /* Receive the cell. Should fail because our MAC is wrong. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO handshake_auth not as expected");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong auth key length. Should
+ * fail. */
+static void
+test_establish_intro_wrong_auth_key_len(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  size_t bad_auth_key_len = ED25519_PUBKEY_LEN - 1;
+  trn_cell_establish_intro_t *cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+
+  /* Mangle the auth key length. */
+  trn_cell_establish_intro_set_auth_key_len(cell, bad_auth_key_len);
+  trn_cell_establish_intro_setlen_auth_key(cell, bad_auth_key_len);
+  /* Encode cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                             cell);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO auth key length is invalid");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong sig length. Should
+ * fail. */
+static void
+test_establish_intro_wrong_sig_len(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  size_t bad_sig_len = ED25519_SIG_LEN - 1;
+  trn_cell_establish_intro_t *cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+
+  /* Mangle the signature length. */
+  trn_cell_establish_intro_set_sig_len(cell, bad_sig_len);
+  trn_cell_establish_intro_setlen_sig(cell, bad_sig_len);
+  /* Encode cell. */
+  cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                             cell);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO sig len is invalid");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but slightly change the signature. Should
+ * fail. */
+static void
+test_establish_intro_wrong_sig(void *arg)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  cell_len = new_establish_intro_encoded_cell(circ_nonce, cell_body);
+  tt_i64_op(cell_len, OP_GT, 0);
+
+  /* Mutate the last byte (signature)! :) */
+  cell_body[cell_len - 1]++;
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body,
+                                             (size_t)cell_len);
+  expect_log_msg_containing("Failed to verify ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+}
+
+/* Helper function: Send a well-formed v3 ESTABLISH_INTRO cell to
+ * <b>intro_circ</b>. Return the cell. */
+static trn_cell_establish_intro_t *
+helper_establish_intro_v3(or_circuit_t *intro_circ)
+{
+  int retval;
+  char circ_nonce[DIGEST_LEN] = {0};
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  trn_cell_establish_intro_t *cell = NULL;
+
+  tt_assert(intro_circ);
+
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+   * attempt to parse it. */
+  cell_len = new_establish_intro_cell(circ_nonce, &cell);
+  tt_i64_op(cell_len, OP_GT, 0);
+  tt_assert(cell);
+  cell_len = trn_cell_establish_intro_encode(cell_body, sizeof(cell_body),
+                                             cell);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive the cell */
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body,
+                                             (size_t) cell_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  return cell;
+}
+
+/* Helper function: Send a well-formed v2 ESTABLISH_INTRO cell to
+ * <b>intro_circ</b>. Return the public key advertised in the cell. */
+static crypto_pk_t *
+helper_establish_intro_v2(or_circuit_t *intro_circ)
+{
+  crypto_pk_t *key1 = NULL;
+  int retval;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  char circ_nonce[DIGEST_LEN] = {0};
+
+  tt_assert(intro_circ);
+
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Send legacy establish_intro */
+  key1 = pk_generate(0);
+
+  /* Use old circ_nonce why not */
+  cell_len = rend_service_encode_establish_intro_cell(
+                                           (char*)cell_body,
+                                           sizeof(cell_body), key1,
+                                           circ_nonce);
+  tt_int_op(cell_len, OP_GT, 0);
+
+  /* Receive legacy establish_intro */
+  retval = hs_intro_received_establish_intro(intro_circ,
+                                             cell_body, (size_t) cell_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  return key1;
+}
+
+/* Helper function: test circuitmap free_all function outside of
+ * test_intro_point_registration to prevent Coverity from seeing a
+ * double free if the assertion hypothetically fails.
+ */
+static void
+test_circuitmap_free_all(void)
+{
+  hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+  the_hs_circuitmap = get_hs_circuitmap();
+  tt_assert(the_hs_circuitmap);
+  hs_circuitmap_free_all();
+  the_hs_circuitmap = get_hs_circuitmap();
+  tt_ptr_op(the_hs_circuitmap, OP_EQ, NULL);
+ done:
+  ;
+}
+
+/** Successfully register a v2 intro point and a v3 intro point. Ensure that HS
+ *  circuitmap is maintained properly. */
+static void
+test_intro_point_registration(void *arg)
+{
+  int retval;
+  hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+  or_circuit_t *intro_circ = NULL;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  ed25519_public_key_t auth_key;
+
+  crypto_pk_t *legacy_auth_key = NULL;
+  or_circuit_t *legacy_intro_circ = NULL;
+
+  or_circuit_t *returned_intro_circ = NULL;
+
+  (void) arg;
+
+  MOCK(hs_intro_send_intro_established_cell, mock_send_intro_established_cell);
+
+  hs_circuitmap_init();
+
+  /* Check that the circuitmap is currently empty */
+  {
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(0, OP_EQ, HT_SIZE(the_hs_circuitmap));
+    /* Do a circuitmap query in any case */
+    returned_intro_circ =hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    tt_ptr_op(returned_intro_circ, OP_EQ, NULL);
+  }
+
+  /* Create a v3 intro point */
+  {
+    intro_circ = or_circuit_new(0, NULL);
+    tt_assert(intro_circ);
+    establish_intro_cell = helper_establish_intro_v3(intro_circ);
+
+    /* Check that the intro point was registered on the HS circuitmap */
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(1, OP_EQ, HT_SIZE(the_hs_circuitmap));
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                           establish_intro_cell);
+    returned_intro_circ =
+      hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    tt_ptr_op(intro_circ, OP_EQ, returned_intro_circ);
+  }
+
+  /* Create a v2 intro point */
+  {
+    char key_digest[DIGEST_LEN];
+
+    legacy_intro_circ = or_circuit_new(1, NULL);
+    tt_assert(legacy_intro_circ);
+    legacy_auth_key = helper_establish_intro_v2(legacy_intro_circ);
+    tt_assert(legacy_auth_key);
+
+    /* Check that the circuitmap now has two elements */
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(2, OP_EQ, HT_SIZE(the_hs_circuitmap));
+
+    /* Check that the new element is our legacy intro circuit. */
+    retval = crypto_pk_get_digest(legacy_auth_key, key_digest);
+    tt_int_op(retval, OP_EQ, 0);
+    returned_intro_circ =
+      hs_circuitmap_get_intro_circ_v2_relay_side((uint8_t*)key_digest);
+    tt_ptr_op(legacy_intro_circ, OP_EQ, returned_intro_circ);
+  }
+
+  /* XXX Continue test and try to register a second v3 intro point with the
+   * same auth key. Make sure that old intro circuit gets closed. */
+
+ done:
+  crypto_pk_free(legacy_auth_key);
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  circuit_free_(TO_CIRCUIT(legacy_intro_circ));
+  trn_cell_establish_intro_free(establish_intro_cell);
+  test_circuitmap_free_all();
+
+  UNMOCK(hs_intro_send_intro_established_cell);
+}
+
+static void
+test_introduce1_suitable_circuit(void *arg)
+{
+  int ret;
+  or_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Valid suitable circuit. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 1);
+  }
+
+  /* Test if the circuit purpose safeguard works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Test the non-edge circuit safeguard works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    /* Bogus pointer, the check is against NULL on n_chan. */
+    circ->base_.n_chan = (channel_t *) circ;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Mangle the circuit a bit more so see if our only one INTRODUCE1 cell
+   * limit works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    circ->already_received_introduce1 = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_introduce1_is_legacy(void *arg)
+{
+  int ret;
+  uint8_t request[256];
+
+  (void) arg;
+
+  /* For a cell to be considered legacy, according to the specification, the
+   * first 20 bytes MUST BE non-zero else it's a v3 cell. */
+  memset(request, 'a', DIGEST_LEN);
+  memset(request + DIGEST_LEN, 0, sizeof(request) - DIGEST_LEN);
+  ret = introduce1_cell_is_legacy(request);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* This is a NON legacy cell. */
+  memset(request, 0, DIGEST_LEN);
+  memset(request + DIGEST_LEN, 'a', sizeof(request) - DIGEST_LEN);
+  ret = introduce1_cell_is_legacy(request);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+static void
+test_introduce1_validation(void *arg)
+{
+  int ret;
+  trn_cell_introduce1_t *cell = NULL;
+
+  (void) arg;
+
+  /* Create our decoy cell that we'll modify as we go to test the validation
+   * function of that parsed cell. */
+  cell = helper_create_introduce1_cell();
+  tt_assert(cell);
+
+  /* It should NOT be a legacy cell which will trigger a BUG(). */
+  memset(cell->legacy_key_id, 'a', sizeof(cell->legacy_key_id));
+  tor_capture_bugs_(1);
+  ret = validate_introduce1_parsed_cell(cell);
+  tor_end_capture_bugs_();
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset legacy ID and make sure it's correct. */
+  memset(cell->legacy_key_id, 0, sizeof(cell->legacy_key_id));
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Non existing auth key type. */
+  cell->auth_key_type = 42;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset is to correct value and make sure it's correct. */
+  cell->auth_key_type = HS_INTRO_AUTH_KEY_TYPE_ED25519;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Really bad key length. */
+  cell->auth_key_len = 0;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  cell->auth_key_len = UINT16_MAX;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Correct size, let's try that. */
+  cell->auth_key_len = sizeof(ed25519_public_key_t);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Set an invalid size of the auth key buffer. */
+  trn_cell_introduce1_setlen_auth_key(cell, 3);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset auth key buffer and make sure it works. */
+  trn_cell_introduce1_setlen_auth_key(cell, sizeof(ed25519_public_key_t));
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Empty encrypted section. */
+  trn_cell_introduce1_setlen_encrypted(cell, 0);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset it to some non zero bytes and validate. */
+  trn_cell_introduce1_setlen_encrypted(cell, 1);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  trn_cell_introduce1_free(cell);
+}
+
+static void
+test_received_introduce1_handling(void *arg)
+{
+  int ret;
+  uint8_t *request = NULL, buf[128];
+  trn_cell_introduce1_t *cell = NULL;
+  or_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Too small request length. An INTRODUCE1 expect at the very least a
+   * DIGEST_LEN size. */
+  {
+    memset(buf, 0, sizeof(buf));
+    circ = helper_create_intro_circuit();
+    ret = hs_intro_received_introduce1(circ, buf, DIGEST_LEN - 1);
+    tt_int_op(ret, OP_EQ, -1);
+    circuit_free_(TO_CIRCUIT(circ));
+  }
+
+  /* We have a unit test only for the suitability of a circuit to receive an
+   * INTRODUCE1 cell so from now on we'll only test the handling of a cell. */
+
+  /* Bad request. */
+  {
+    circ = helper_create_intro_circuit();
+    uint8_t test[2]; /* Too small request. */
+    memset(test, 0, sizeof(test));
+    ret = handle_introduce1(circ, test, sizeof(test));
+    tor_free(circ->p_chan);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Valid case. */
+  {
+    cell = helper_create_introduce1_cell();
+    ssize_t request_len = trn_cell_introduce1_encoded_len(cell);
+    tt_int_op((int)request_len, OP_GT, 0);
+    request = tor_malloc_zero(request_len);
+    ssize_t encoded_len =
+      trn_cell_introduce1_encode(request, request_len, cell);
+    tt_int_op((int)encoded_len, OP_GT, 0);
+
+    circ = helper_create_intro_circuit();
+    or_circuit_t *service_circ = helper_create_intro_circuit();
+    circuit_change_purpose(TO_CIRCUIT(service_circ),
+                           CIRCUIT_PURPOSE_INTRO_POINT);
+    /* Register the circuit in the map for the auth key of the cell. */
+    ed25519_public_key_t auth_key;
+    const uint8_t *cell_auth_key =
+      trn_cell_introduce1_getconstarray_auth_key(cell);
+    memcpy(auth_key.pubkey, cell_auth_key, ED25519_PUBKEY_LEN);
+    hs_circuitmap_register_intro_circ_v3_relay_side(service_circ, &auth_key);
+    ret = hs_intro_received_introduce1(circ, request, request_len);
+    circuit_free_(TO_CIRCUIT(circ));
+    circuit_free_(TO_CIRCUIT(service_circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid legacy cell. */
+  {
+    tor_free(request);
+    trn_cell_introduce1_free(cell);
+    cell = helper_create_introduce1_cell();
+    uint8_t *legacy_key_id = trn_cell_introduce1_getarray_legacy_key_id(cell);
+    memset(legacy_key_id, 'a', DIGEST_LEN);
+    /* Add an arbitrary amount of data for the payload of a v2 cell. */
+    size_t request_len = trn_cell_introduce1_encoded_len(cell) + 256;
+    tt_size_op(request_len, OP_GT, 0);
+    request = tor_malloc_zero(request_len + 256);
+    ssize_t encoded_len =
+      trn_cell_introduce1_encode(request, request_len, cell);
+    tt_int_op((int)encoded_len, OP_GT, 0);
+
+    circ = helper_create_intro_circuit();
+    or_circuit_t *service_circ = helper_create_intro_circuit();
+    circuit_change_purpose(TO_CIRCUIT(service_circ),
+                           CIRCUIT_PURPOSE_INTRO_POINT);
+    /* Register the circuit in the map for the auth key of the cell. */
+    uint8_t token[REND_TOKEN_LEN];
+    memcpy(token, legacy_key_id, sizeof(token));
+    hs_circuitmap_register_intro_circ_v2_relay_side(service_circ, token);
+    ret = hs_intro_received_introduce1(circ, request, request_len);
+    circuit_free_(TO_CIRCUIT(circ));
+    circuit_free_(TO_CIRCUIT(service_circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  trn_cell_introduce1_free(cell);
+  tor_free(request);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+struct testcase_t hs_intropoint_tests[] = {
+  { "intro_point_registration",
+    test_intro_point_registration, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_keytype",
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_keytype2",
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_purpose",
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_sig",
+    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_sig_len",
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_auth_key_len",
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_mac",
+    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+
+  { "introduce1_suitable_circuit",
+    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+
+  { "introduce1_is_legacy",
+    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+
+  { "introduce1_validation",
+    test_introduce1_validation, TT_FORK, NULL, NULL },
+
+  { "received_introduce1_handling",
+    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_ntor.c b/src/test/test_hs_ntor.c
new file mode 100644
index 0000000000..8eee54d4b4
--- /dev/null
+++ b/src/test/test_hs_ntor.c
@@ -0,0 +1,114 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_ntor.c
+ * \brief Test hidden service ntor functionality.
+ */
+
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+
+#include "hs_ntor.h"
+
+/* Test the HS ntor handshake. Simulate the sending of an encrypted INTRODUCE1
+ * cell, and verify the proper derivation of decryption keys on the other end.
+ * Then simulate the sending of an authenticated RENDEZVOUS1 cell and verify
+ * the proper verification on the other end. */
+static void
+test_hs_ntor(void *arg)
+{
+  int retval;
+
+  uint8_t subcredential[DIGEST256_LEN];
+
+  ed25519_keypair_t service_intro_auth_keypair;
+  curve25519_keypair_t service_intro_enc_keypair;
+  curve25519_keypair_t service_ephemeral_rend_keypair;
+
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+
+  hs_ntor_intro_cell_keys_t client_hs_ntor_intro_cell_keys;
+  hs_ntor_intro_cell_keys_t service_hs_ntor_intro_cell_keys;
+
+  hs_ntor_rend_cell_keys_t service_hs_ntor_rend_cell_keys;
+  hs_ntor_rend_cell_keys_t client_hs_ntor_rend_cell_keys;
+
+  (void) arg;
+
+  /* Generate fake data for this unittest */
+  {
+    /* Generate fake subcredential */
+    memset(subcredential, 'Z', DIGEST256_LEN);
+
+    /* service */
+    curve25519_keypair_generate(&service_intro_enc_keypair, 0);
+    ed25519_keypair_generate(&service_intro_auth_keypair, 0);
+    curve25519_keypair_generate(&service_ephemeral_rend_keypair, 0);
+    /* client */
+    curve25519_keypair_generate(&client_ephemeral_enc_keypair, 0);
+  }
+
+  /* Client: Simulate the sending of an encrypted INTRODUCE1 cell */
+  retval =
+    hs_ntor_client_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
+                                       &service_intro_enc_keypair.pubkey,
+                                       &client_ephemeral_enc_keypair,
+                                       subcredential,
+                                       &client_hs_ntor_intro_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Service: Simulate the decryption of the received INTRODUCE1 */
+  retval =
+    hs_ntor_service_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
+                                        &service_intro_enc_keypair,
+                                        &client_ephemeral_enc_keypair.pubkey,
+                                        subcredential,
+                                        &service_hs_ntor_intro_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Test that the INTRODUCE1 encryption/mac keys match! */
+  tt_mem_op(client_hs_ntor_intro_cell_keys.enc_key, OP_EQ,
+            service_hs_ntor_intro_cell_keys.enc_key,
+            CIPHER256_KEY_LEN);
+  tt_mem_op(client_hs_ntor_intro_cell_keys.mac_key, OP_EQ,
+            service_hs_ntor_intro_cell_keys.mac_key,
+            DIGEST256_LEN);
+
+  /* Service: Simulate creation of RENDEZVOUS1 key material. */
+  retval =
+    hs_ntor_service_get_rendezvous1_keys(&service_intro_auth_keypair.pubkey,
+                                         &service_intro_enc_keypair,
+                                         &service_ephemeral_rend_keypair,
+                                         &client_ephemeral_enc_keypair.pubkey,
+                                         &service_hs_ntor_rend_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Client: Simulate the verification of a received RENDEZVOUS1 cell */
+  retval =
+    hs_ntor_client_get_rendezvous1_keys(&service_intro_auth_keypair.pubkey,
+                                        &client_ephemeral_enc_keypair,
+                                        &service_intro_enc_keypair.pubkey,
+                                        &service_ephemeral_rend_keypair.pubkey,
+                                        &client_hs_ntor_rend_cell_keys);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Test that the RENDEZVOUS1 key material match! */
+  tt_mem_op(client_hs_ntor_rend_cell_keys.rend_cell_auth_mac, OP_EQ,
+            service_hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+            DIGEST256_LEN);
+  tt_mem_op(client_hs_ntor_rend_cell_keys.ntor_key_seed, OP_EQ,
+            service_hs_ntor_rend_cell_keys.ntor_key_seed,
+            DIGEST256_LEN);
+ done:
+  ;
+}
+
+struct testcase_t hs_ntor_tests[] = {
+  { "hs_ntor", test_hs_ntor, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_ntor.sh b/src/test/test_hs_ntor.sh
new file mode 100755
index 0000000000..8a0003d44a
--- /dev/null
+++ b/src/test/test_hs_ntor.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+# Validate Tor's ntor implementation.
+
+exitcode=0
+
+# Run the python integration test sand return the exitcode of the python
+# script.  The python script might ask the testsuite to skip it if not all
+# python dependencies are covered.
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/hs_ntor_ref.py" || exitcode=$?
+
+exit ${exitcode}
diff --git a/src/test/test_hs_ntor_cl.c b/src/test/test_hs_ntor_cl.c
new file mode 100644
index 0000000000..ed1eda58ea
--- /dev/null
+++ b/src/test/test_hs_ntor_cl.c
@@ -0,0 +1,255 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** This is a wrapper over the little-t-tor HS ntor functions. The wrapper is
+ *  used by src/test/hs_ntor_ref.py to conduct the HS ntor integration
+ *  tests.
+ *
+ *  The logic of this wrapper is basically copied from src/test/test_ntor_cl.c
+ */
+
+#include "orconfig.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ONION_NTOR_PRIVATE
+#include "or.h"
+#include "util.h"
+#include "compat.h"
+#include "crypto.h"
+#include "crypto_curve25519.h"
+#include "hs_ntor.h"
+#include "onion_ntor.h"
+
+#define N_ARGS(n) STMT_BEGIN {                                  \
+    if (argc < (n)) {                                           \
+      fprintf(stderr, "%s needs %d arguments.\n",argv[1],n);    \
+      return 1;                                                 \
+    }                                                           \
+  } STMT_END
+#define BASE16(idx, var, n) STMT_BEGIN {                                \
+    const char *s = argv[(idx)];                                        \
+    if (base16_decode((char*)var, n, s, strlen(s)) < (int)n ) {              \
+      fprintf(stderr, "couldn't decode argument %d (%s)\n",idx,s);      \
+      return 1;                                                         \
+    }                                                                   \
+  } STMT_END
+#define INT(idx, var) STMT_BEGIN {                                      \
+    var = atoi(argv[(idx)]);                                            \
+    if (var <= 0) {                                                     \
+      fprintf(stderr, "bad integer argument %d (%s)\n",idx,argv[(idx)]); \
+    }                                                                   \
+  } STMT_END
+
+/** The first part of the HS ntor protocol. The client-side computes all
+    necessary key material and sends the appropriate message to the service. */
+static int
+client1(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_public_key_t intro_enc_pubkey;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
+
+  char buf[256];
+
+  N_ARGS(6);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(4, client_ephemeral_enc_keypair.seckey.secret_key,
+         CURVE25519_SECKEY_LEN);
+  BASE16(5, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
+                                 &client_ephemeral_enc_keypair.seckey);
+
+  retval = hs_ntor_client_get_introduce1_keys(&intro_auth_pubkey,
+                                              &intro_enc_pubkey,
+                                              &client_ephemeral_enc_keypair,
+                                              subcredential,
+                                              &hs_ntor_intro_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send ENC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.enc_key,
+                sizeof(hs_ntor_intro_cell_keys.enc_key));
+  printf("%s\n", buf);
+  /* Send MAC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.mac_key,
+                sizeof(hs_ntor_intro_cell_keys.mac_key));
+  printf("%s\n", buf);
+
+ done:
+  return retval;
+}
+
+/** The second part of the HS ntor protocol. The service-side computes all
+    necessary key material and sends the appropriate message to the client */
+static int
+server1(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_keypair_t intro_enc_keypair;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_public_key_t client_ephemeral_enc_pubkey;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
+  hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+  curve25519_keypair_t service_ephemeral_rend_keypair;
+
+  char buf[256];
+
+  N_ARGS(6);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, intro_enc_keypair.seckey.secret_key, CURVE25519_SECKEY_LEN);
+  BASE16(4, client_ephemeral_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(5, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&intro_enc_keypair.pubkey,
+                                 &intro_enc_keypair.seckey);
+  curve25519_keypair_generate(&service_ephemeral_rend_keypair, 0);
+
+  /* Get INTRODUCE1 keys */
+  retval = hs_ntor_service_get_introduce1_keys(&intro_auth_pubkey,
+                                               &intro_enc_keypair,
+                                               &client_ephemeral_enc_pubkey,
+                                               subcredential,
+                                               &hs_ntor_intro_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Get RENDEZVOUS1 keys */
+  retval = hs_ntor_service_get_rendezvous1_keys(&intro_auth_pubkey,
+                                               &intro_enc_keypair,
+                                               &service_ephemeral_rend_keypair,
+                                               &client_ephemeral_enc_pubkey,
+                                               &hs_ntor_rend_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send ENC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.enc_key,
+                sizeof(hs_ntor_intro_cell_keys.enc_key));
+  printf("%s\n", buf);
+  /* Send MAC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.mac_key,
+                sizeof(hs_ntor_intro_cell_keys.mac_key));
+  printf("%s\n", buf);
+  /* Send AUTH_MAC */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+                sizeof(hs_ntor_rend_cell_keys.rend_cell_auth_mac));
+  printf("%s\n", buf);
+  /* Send NTOR_KEY_SEED */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.ntor_key_seed,
+                sizeof(hs_ntor_rend_cell_keys.ntor_key_seed));
+  printf("%s\n", buf);
+  /* Send service ephemeral pubkey (Y) */
+  base16_encode(buf, sizeof(buf),
+                (const char*)service_ephemeral_rend_keypair.pubkey.public_key,
+                sizeof(service_ephemeral_rend_keypair.pubkey.public_key));
+  printf("%s\n", buf);
+
+ done:
+  return retval;
+}
+
+/** The final step of the ntor protocol, the client computes and returns the
+ *  rendezvous key material. */
+static int
+client2(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_public_key_t intro_enc_pubkey;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+  curve25519_public_key_t service_ephemeral_rend_pubkey;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+
+  char buf[256];
+
+  N_ARGS(7);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, client_ephemeral_enc_keypair.seckey.secret_key,
+         CURVE25519_SECKEY_LEN);
+  BASE16(4, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(5, service_ephemeral_rend_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(6, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
+                                 &client_ephemeral_enc_keypair.seckey);
+
+  /* Get RENDEZVOUS1 keys */
+  retval = hs_ntor_client_get_rendezvous1_keys(&intro_auth_pubkey,
+                                               &client_ephemeral_enc_keypair,
+                                               &intro_enc_pubkey,
+                                               &service_ephemeral_rend_pubkey,
+                                               &hs_ntor_rend_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send AUTH_MAC */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+                sizeof(hs_ntor_rend_cell_keys.rend_cell_auth_mac));
+  printf("%s\n", buf);
+  /* Send NTOR_KEY_SEED */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.ntor_key_seed,
+                sizeof(hs_ntor_rend_cell_keys.ntor_key_seed));
+  printf("%s\n", buf);
+
+ done:
+  return 1;
+}
+
+/** Perform a different part of the protocol depdning on the argv used. */
+int
+main(int argc, char **argv)
+{
+  if (argc < 2) {
+    fprintf(stderr, "I need arguments. Read source for more info.\n");
+    return 1;
+  }
+
+  curve25519_init();
+  if (!strcmp(argv[1], "client1")) {
+    return client1(argc, argv);
+  } else if (!strcmp(argv[1], "server1")) {
+    return server1(argc, argv);
+  } else if (!strcmp(argv[1], "client2")) {
+    return client2(argc, argv);
+  } else {
+    fprintf(stderr, "What's a %s?\n", argv[1]);
+    return 1;
+  }
+}
+
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
new file mode 100644
index 0000000000..7972434d69
--- /dev/null
+++ b/src/test/test_hs_service.c
@@ -0,0 +1,1630 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_service.c
+ * \brief Test hidden service functionality.
+ */
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
+#define CRYPTO_PRIVATE
+#define HS_COMMON_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+#define HS_CIRCUIT_PRIVATE
+#define MAIN_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define STATEFILE_PRIVATE
+#define TOR_CHANNEL_INTERNAL_
+#define HS_CLIENT_PRIVATE
+#define ROUTERPARSE_PRIVATE
+
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+#include "rend_test_helpers.h"
+#include "hs_test_helpers.h"
+
+#include "or.h"
+#include "config.h"
+#include "circuitbuild.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "crypto_rand.h"
+#include "dirauth/dirvote.h"
+#include "networkstatus.h"
+#include "nodelist.h"
+#include "relay.h"
+#include "routerparse.h"
+#include "hs_common.h"
+#include "hs_config.h"
+#include "hs_ident.h"
+#include "hs_intropoint.h"
+#include "hs_ntor.h"
+#include "hs_circuit.h"
+#include "hs_service.h"
+#include "hs_client.h"
+#include "main.h"
+#include "rendservice.h"
+#include "statefile.h"
+#include "dirauth/shared_random_state.h"
+#include "voting_schedule.h"
+
+/* Trunnel */
+#include "hs/cell_establish_intro.h"
+
+static networkstatus_t mock_ns;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+static or_state_t *dummy_state = NULL;
+
+/* Mock function to get fake or state (used for rev counters) */
+static or_state_t *
+get_or_state_replacement(void)
+{
+  return dummy_state;
+}
+
+/* Mock function because we are not trying to test the close circuit that does
+ * an awful lot of checks on the circuit object. */
+static void
+mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
+                            const char *file)
+{
+  (void) circ;
+  (void) reason;
+  (void) line;
+  (void) file;
+  return;
+}
+
+static int
+mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
+                                  uint8_t relay_command, const char *payload,
+                                  size_t payload_len,
+                                  crypt_path_t *cpath_layer,
+                                  const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+  return 0;
+}
+
+/* Helper: from a set of options in conf, configure a service which will add
+ * it to the staging list of the HS subsytem. */
+static int
+helper_config_service(const char *conf)
+{
+  int ret = 0;
+  or_options_t *options = NULL;
+  tt_assert(conf);
+  options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_service_all(options, 0);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+/* Test: Ensure that setting up rendezvous circuits works correctly. */
+static void
+test_e2e_rend_circuit_setup(void *arg)
+{
+  ed25519_public_key_t service_pk;
+  origin_circuit_t *or_circ;
+  int retval;
+
+  /** In this test we create a v3 prop224 service-side rendezvous circuit.
+   *  We simulate an HS ntor key exchange with a client, and check that
+   *  the circuit was setup correctly and is ready to accept rendezvous data */
+
+  (void) arg;
+
+  /* Now make dummy circuit */
+  {
+    or_circ = origin_circuit_new();
+
+    or_circ->base_.purpose = CIRCUIT_PURPOSE_S_CONNECT_REND;
+
+    or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+    or_circ->build_state->is_internal = 1;
+
+    /* prop224: Setup hs conn identifier on the stream */
+    ed25519_secret_key_t sk;
+    tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
+    tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
+
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
+                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+
+    TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
+  }
+
+  /* Check number of hops */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Setup the circuit: do the ntor key exchange */
+  {
+    uint8_t ntor_key_seed[DIGEST256_LEN] = {2};
+    retval = hs_circuit_setup_e2e_rend_circ(or_circ,
+                                       ntor_key_seed, sizeof(ntor_key_seed),
+                                       1);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* See that a hop was added to the circuit's cpath */
+  retval = cpath_get_n_hops(&or_circ->cpath);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Check the digest algo */
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+            OP_EQ, DIGEST_SHA3_256);
+  tt_assert(or_circ->cpath->crypto.f_crypto);
+  tt_assert(or_circ->cpath->crypto.b_crypto);
+
+  /* Ensure that circ purpose was changed */
+  tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
+
+ done:
+  circuit_free_(TO_CIRCUIT(or_circ));
+}
+
+/* Helper: Return a newly allocated and initialized origin circuit with
+ * purpose and flags. A default HS identifier is set to an ed25519
+ * authentication key for introduction point. */
+static origin_circuit_t *
+helper_create_origin_circuit(int purpose, int flags)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = origin_circuit_init(purpose, flags);
+  tor_assert(circ);
+  circ->cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  circ->cpath->magic = CRYPT_PATH_MAGIC;
+  circ->cpath->state = CPATH_STATE_OPEN;
+  circ->cpath->package_window = circuit_initial_package_window();
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+  circ->cpath->prev = circ->cpath;
+  /* Random nonce. */
+  crypto_rand(circ->cpath->prev->rend_circ_nonce, DIGEST_LEN);
+  /* Create a default HS identifier. */
+  circ->hs_ident = tor_malloc_zero(sizeof(hs_ident_circuit_t));
+
+  return circ;
+}
+
+/* Helper: Return a newly allocated service object with the identity keypair
+ * sets and the current descriptor. Then register it to the global map.
+ * Caller should us hs_free_all() to free this service or remove it from the
+ * global map before freeing. */
+static hs_service_t *
+helper_create_service(void)
+{
+  /* Set a service for this circuit. */
+  hs_service_t *service = hs_service_new(get_options());
+  tor_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  service->desc_current = service_descriptor_new();
+  tt_assert(service->desc_current);
+  /* Register service to global map. */
+  int ret = register_service(get_hs_service_map(), service);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  return service;
+}
+
+/* Helper: Return a newly allocated service intro point with two link
+ * specifiers, one IPv4 and one legacy ID set to As. */
+static hs_service_intro_point_t *
+helper_create_service_ip(void)
+{
+  hs_desc_link_specifier_t *ls;
+  hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+  tor_assert(ip);
+  /* Add a first unused link specifier. */
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = LS_IPV4;
+  smartlist_add(ip->base.link_specifiers, ls);
+  /* Add a second link specifier used by a test. */
+  ls = tor_malloc_zero(sizeof(*ls));
+  ls->type = LS_LEGACY_ID;
+  memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id));
+  smartlist_add(ip->base.link_specifiers, ls);
+
+  return ip;
+}
+
+static void
+test_load_keys(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  char *hsdir_v2 = tor_strdup(get_fname("hs2"));
+  char *hsdir_v3 = tor_strdup(get_fname("hs3"));
+  char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  (void) arg;
+
+  /* We'll register two services, a v2 and a v3, then we'll load keys and
+   * validate that both are in a correct state. */
+
+  hs_init();
+
+#define conf_fmt \
+  "HiddenServiceDir %s\n" \
+  "HiddenServiceVersion %d\n" \
+  "HiddenServicePort 65535\n"
+
+  /* v2 service. */
+  tor_asprintf(&conf, conf_fmt, hsdir_v2, HS_VERSION_TWO);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* This one should now be registered into the v2 list. */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 0);
+  tt_int_op(rend_num_services(), OP_EQ, 1);
+
+  /* v3 service. */
+  tor_asprintf(&conf, conf_fmt, hsdir_v3, HS_VERSION_THREE);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* It's in staging? */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+
+  /* Load the keys for these. After that, the v3 service should be registered
+   * in the global map. */
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  hs_service_t *s = get_first_service();
+  tt_assert(s);
+
+  /* Ok we have the service object. Validate few things. */
+  tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
+  tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1);
+  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey,
+                             ED25519_SECKEY_LEN));
+  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey,
+                             ED25519_PUBKEY_LEN));
+  /* Check onion address from identity key. */
+  hs_build_address(&s->keys.identity_pk, s->config.version, addr);
+  tt_int_op(hs_address_is_valid(addr), OP_EQ, 1);
+  tt_str_op(addr, OP_EQ, s->onion_address);
+
+ done:
+  tor_free(hsdir_v2);
+  tor_free(hsdir_v3);
+  hs_free_all();
+}
+
+static void
+test_access_service(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  char *hsdir_v3 = tor_strdup(get_fname("hs3"));
+  hs_service_ht *global_map;
+  hs_service_t *s = NULL;
+
+  (void) arg;
+
+  /* We'll register two services, a v2 and a v3, then we'll load keys and
+   * validate that both are in a correct state. */
+
+  hs_init();
+
+#define conf_fmt \
+  "HiddenServiceDir %s\n" \
+  "HiddenServiceVersion %d\n" \
+  "HiddenServicePort 65535\n"
+
+  /* v3 service. */
+  tor_asprintf(&conf, conf_fmt, hsdir_v3, HS_VERSION_THREE);
+  ret = helper_config_service(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+  /* It's in staging? */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+
+  /* Load the keys for these. After that, the v3 service should be registered
+   * in the global map. */
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  s = get_first_service();
+  tt_assert(s);
+  global_map = get_hs_service_map();
+  tt_assert(global_map);
+
+  /* From here, we'll try the service accessors. */
+  hs_service_t *query = find_service(global_map, &s->keys.identity_pk);
+  tt_assert(query);
+  tt_mem_op(query, OP_EQ, s, sizeof(hs_service_t));
+  /* Remove service, check if it actually works and then put it back. */
+  remove_service(global_map, s);
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 0);
+  query = find_service(global_map, &s->keys.identity_pk);
+  tt_ptr_op(query, OP_EQ, NULL);
+
+  /* Register back the service in the map. */
+  ret = register_service(global_map, s);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  /* Twice should fail. */
+  ret = register_service(global_map, s);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Remove service from map so we don't double free on cleanup. */
+  remove_service(global_map, s);
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 0);
+  query = find_service(global_map, &s->keys.identity_pk);
+  tt_ptr_op(query, OP_EQ, NULL);
+  /* Let's try to remove twice for fun. */
+  setup_full_capture_of_logs(LOG_WARN);
+  remove_service(global_map, s);
+  expect_log_msg_containing("Could not find service in the global map");
+  teardown_capture_of_logs();
+
+ done:
+  hs_service_free(s);
+  tor_free(hsdir_v3);
+  hs_free_all();
+}
+
+/** Test that we can create intro point objects, index them and find them */
+static void
+test_service_intro_point(void *arg)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+
+  (void) arg;
+
+  /* Test simple creation of an object. */
+  {
+    time_t now = time(NULL);
+    ip = helper_create_service_ip();
+    tt_assert(ip);
+    /* Make sure the authentication keypair is not zeroes. */
+    tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp,
+                              sizeof(ed25519_keypair_t)), OP_EQ, 0);
+    /* The introduce2_max MUST be in that range. */
+    tt_u64_op(ip->introduce2_max, OP_GE,
+              INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS);
+    tt_u64_op(ip->introduce2_max, OP_LE,
+              INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS);
+    /* Time to expire MUST also be in that range. We subtract 500 seconds
+     * because there could be a gap between setting now and the time taken in
+     * service_intro_point_new. On ARM and other older CPUs, it can be
+     * surprisingly slow... */
+    tt_u64_op(ip->time_to_expire, OP_GE,
+              now + INTRO_POINT_LIFETIME_MIN_SECONDS - 500);
+    /* We add 500 seconds, because this time we're testing against the
+     * maximum allowed time. */
+    tt_u64_op(ip->time_to_expire, OP_LE,
+              now + INTRO_POINT_LIFETIME_MAX_SECONDS + 500);
+    tt_assert(ip->replay_cache);
+    tt_assert(ip->base.link_specifiers);
+    /* By default, this is NOT a legacy object. */
+    tt_int_op(ip->base.is_only_legacy, OP_EQ, 0);
+  }
+
+  /* Test functions that uses a service intropoints map with that previously
+   * created object (non legacy). */
+  {
+    ed25519_public_key_t garbage = { {0} };
+    hs_service_intro_point_t *query;
+
+    service = hs_service_new(get_options());
+    tt_assert(service);
+    service->desc_current = service_descriptor_new();
+    tt_assert(service->desc_current);
+    /* Add intropoint to descriptor map. */
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    query = service_intro_point_find(service, &ip->auth_key_kp.pubkey);
+    tt_mem_op(query, OP_EQ, ip, sizeof(hs_service_intro_point_t));
+    query = service_intro_point_find(service, &garbage);
+    tt_ptr_op(query, OP_EQ, NULL);
+
+    /* While at it, can I find the descriptor with the intro point? */
+    hs_service_descriptor_t *desc_lookup =
+      service_desc_find_by_intro(service, ip);
+    tt_mem_op(service->desc_current, OP_EQ, desc_lookup,
+              sizeof(hs_service_descriptor_t));
+
+    /* Remove object from service descriptor and make sure it is out. */
+    service_intro_point_remove(service, ip);
+    query = service_intro_point_find(service, &ip->auth_key_kp.pubkey);
+    tt_ptr_op(query, OP_EQ, NULL);
+  }
+
+ done:
+  /* If the test succeed, this object is no longer referenced in the service
+   * so we can free it without use after free. Else, it might explode because
+   * it's still in the service descriptor map. */
+  service_intro_point_free(ip);
+  hs_service_free(service);
+}
+
+static node_t mock_node;
+static const node_t *
+mock_node_get_by_id(const char *digest)
+{
+  (void) digest;
+  memset(mock_node.identity, 'A', DIGEST_LEN);
+  /* Only return the matchin identity of As */
+  if (!tor_memcmp(mock_node.identity, digest, DIGEST_LEN)) {
+    return &mock_node;
+  }
+  return NULL;
+}
+
+static void
+test_helper_functions(void *arg)
+{
+  int ret;
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL;
+  hs_ident_circuit_t ident;
+
+  (void) arg;
+
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  hs_service_init();
+  time_t now = time(NULL);
+  update_approx_time(now);
+
+  service = helper_create_service();
+
+  ip = helper_create_service_ip();
+  /* Immediately add the intro point to the service so the free service at the
+   * end cleans it as well. */
+  service_intro_point_add(service->desc_current->intro_points.map, ip);
+
+  /* Setup the circuit identifier. */
+  ed25519_pubkey_copy(&ident.intro_auth_pk, &ip->auth_key_kp.pubkey);
+  ed25519_pubkey_copy(&ident.identity_pk, &service->keys.identity_pk);
+
+  /* Testing get_objects_from_ident(). */
+  {
+    hs_service_t *s_lookup = NULL;
+    hs_service_intro_point_t *ip_lookup = NULL;
+    hs_service_descriptor_t *desc_lookup = NULL;
+
+    get_objects_from_ident(&ident, &s_lookup, &ip_lookup, &desc_lookup);
+    tt_mem_op(s_lookup, OP_EQ, service, sizeof(hs_service_t));
+    tt_mem_op(ip_lookup, OP_EQ, ip, sizeof(hs_service_intro_point_t));
+    tt_mem_op(desc_lookup, OP_EQ, service->desc_current,
+              sizeof(hs_service_descriptor_t));
+    /* Reset */
+    s_lookup = NULL; ip_lookup = NULL; desc_lookup = NULL;
+
+    /* NULL parameter should work. */
+    get_objects_from_ident(&ident, NULL, &ip_lookup, &desc_lookup);
+    tt_mem_op(ip_lookup, OP_EQ, ip, sizeof(hs_service_intro_point_t));
+    tt_mem_op(desc_lookup, OP_EQ, service->desc_current,
+              sizeof(hs_service_descriptor_t));
+    /* Reset. */
+    s_lookup = NULL; ip_lookup = NULL; desc_lookup = NULL;
+
+    /* Break the ident and we should find nothing. */
+    memset(&ident, 0, sizeof(ident));
+    get_objects_from_ident(&ident, &s_lookup, &ip_lookup, &desc_lookup);
+    tt_ptr_op(s_lookup, OP_EQ, NULL);
+    tt_ptr_op(ip_lookup, OP_EQ, NULL);
+    tt_ptr_op(desc_lookup, OP_EQ, NULL);
+  }
+
+  /* Testing get_node_from_intro_point() */
+  {
+    const node_t *node = get_node_from_intro_point(ip);
+    tt_ptr_op(node, OP_EQ, &mock_node);
+    SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
+                            hs_desc_link_specifier_t *, ls) {
+      if (ls->type == LS_LEGACY_ID) {
+        /* Change legacy id in link specifier which is not the mock node. */
+        memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id));
+      }
+    } SMARTLIST_FOREACH_END(ls);
+    node = get_node_from_intro_point(ip);
+    tt_ptr_op(node, OP_EQ, NULL);
+  }
+
+  /* Testing can_service_launch_intro_circuit() */
+  {
+    /* Put the start of the retry period back in time, we should be allowed.
+     * to launch intro circuit. */
+    service->state.num_intro_circ_launched = 2;
+    service->state.intro_circ_retry_started_time =
+      (now - INTRO_CIRC_RETRY_PERIOD - 1);
+    ret = can_service_launch_intro_circuit(service, now);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_u64_op(service->state.intro_circ_retry_started_time, OP_EQ, now);
+    tt_u64_op(service->state.num_intro_circ_launched, OP_EQ, 0);
+    /* Call it again, we should still be allowed because we are under
+     * MAX_INTRO_CIRCS_PER_PERIOD which been set to 0 previously. */
+    ret = can_service_launch_intro_circuit(service, now);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_u64_op(service->state.intro_circ_retry_started_time, OP_EQ, now);
+    tt_u64_op(service->state.num_intro_circ_launched, OP_EQ, 0);
+    /* Too many intro circuit launched means we are not allowed. */
+    service->state.num_intro_circ_launched = 20;
+    ret = can_service_launch_intro_circuit(service, now);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Testing intro_point_should_expire(). */
+  {
+    /* Just some basic test of the current state. */
+    tt_u64_op(ip->introduce2_max, OP_GE,
+              INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS);
+    tt_u64_op(ip->introduce2_max, OP_LE,
+              INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS);
+    tt_u64_op(ip->time_to_expire, OP_GE,
+              now + INTRO_POINT_LIFETIME_MIN_SECONDS);
+    tt_u64_op(ip->time_to_expire, OP_LE,
+              now + INTRO_POINT_LIFETIME_MAX_SECONDS);
+
+    /* This newly created IP from above shouldn't expire now. */
+    ret = intro_point_should_expire(ip, now);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Maximum number of INTRODUCE2 cell reached, it should expire. */
+    ip->introduce2_count = INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS + 1;
+    ret = intro_point_should_expire(ip, now);
+    tt_int_op(ret, OP_EQ, 1);
+    ip->introduce2_count = 0;
+    /* It should expire if time to expire has been reached. */
+    ip->time_to_expire = now - 1000;
+    ret = intro_point_should_expire(ip, now);
+    tt_int_op(ret, OP_EQ, 1);
+  }
+
+ done:
+  /* This will free the service and all objects associated to it. */
+  hs_service_free_all();
+  UNMOCK(node_get_by_id);
+}
+
+/** Test that we do the right operations when an intro circuit opens */
+static void
+test_intro_circuit_opened(void *arg)
+{
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  hs_service_t *service;
+  origin_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                      flags);
+
+  /* No service associated with this circuit. */
+  setup_full_capture_of_logs(LOG_WARN);
+  hs_service_circuit_has_opened(circ);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+
+  /* Set a service for this circuit. */
+  {
+    service = helper_create_service();
+    ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                        &service->keys.identity_pk);
+
+    /* No intro point associated with this circuit. */
+    setup_full_capture_of_logs(LOG_WARN);
+    hs_service_circuit_has_opened(circ);
+    expect_log_msg_containing("Unknown introduction point auth key");
+    teardown_capture_of_logs();
+  }
+
+  /* Set an IP object now for this circuit. */
+  {
+    hs_service_intro_point_t *ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* Update ident to contain the intro point auth key. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* This one should go all the way. */
+  setup_full_capture_of_logs(LOG_INFO);
+  hs_service_circuit_has_opened(circ);
+  expect_log_msg_containing("Introduction circuit 0 established for service");
+  teardown_capture_of_logs();
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+/** Test the operations we do on a circuit after we learn that we successfully
+ *  established an intro point on it */
+static void
+test_intro_established(void *arg)
+{
+  int ret;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  origin_circuit_t *circ = NULL;
+  hs_service_t *service;
+  hs_service_intro_point_t *ip = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO,
+                                      flags);
+  tt_assert(circ);
+
+  /* Test a wrong purpose. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_INTRO;
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Received an INTRO_ESTABLISHED cell on a "
+                            "non introduction circuit of purpose");
+  teardown_capture_of_logs();
+
+  /* Back to normal. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+
+  /* No service associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+  /* No introduction point associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Introduction circuit established without an "
+                            "intro point object on circuit");
+  teardown_capture_of_logs();
+
+  /* Set an IP object now for this circuit. */
+  {
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* Update ident to contain the intro point auth key. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* Send an empty payload. INTRO_ESTABLISHED cells are basically zeroes. */
+  ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(ip->circuit_established, OP_EQ, 1);
+  tt_int_op(TO_CIRCUIT(circ)->purpose, OP_EQ, CIRCUIT_PURPOSE_S_INTRO);
+
+ done:
+  if (circ)
+    circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/** Check the operations we do on a rendezvous circuit after we learn it's
+ *  open */
+static void
+test_rdv_circuit_opened(void *arg)
+{
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  origin_circuit_t *circ = NULL;
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_CONNECT_REND, flags);
+  crypto_rand((char *) circ->hs_ident->rendezvous_cookie, REND_COOKIE_LEN);
+  crypto_rand((char *) circ->hs_ident->rendezvous_handshake_info,
+              sizeof(circ->hs_ident->rendezvous_handshake_info));
+
+  /* No service associated with this circuit. */
+  setup_full_capture_of_logs(LOG_WARN);
+  hs_service_circuit_has_opened(circ);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+  /* This should be set to a non zero timestamp. */
+  tt_u64_op(TO_CIRCUIT(circ)->timestamp_dirty, OP_NE, 0);
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+  /* Should be all good. */
+  hs_service_circuit_has_opened(circ);
+  tt_int_op(TO_CIRCUIT(circ)->purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void
+mock_assert_circuit_ok(const circuit_t *c)
+{
+  (void) c;
+  return;
+}
+
+/** Test for the general mechanism for closing intro circs.
+ *  Also a way to identify that #23603 has been fixed. */
+static void
+test_closing_intro_circs(void *arg)
+{
+  hs_service_t *service = NULL;
+  hs_service_intro_point_t *ip = NULL, *entry = NULL;
+  origin_circuit_t *intro_circ = NULL, *tmp_circ;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+
+  (void) arg;
+
+  MOCK(assert_circuit_ok, mock_assert_circuit_ok);
+
+  hs_init();
+
+  /* Initialize service */
+  service = helper_create_service();
+  /* Initialize intro point */
+  ip = helper_create_service_ip();
+  tt_assert(ip);
+  service_intro_point_add(service->desc_current->intro_points.map, ip);
+
+  /* Initialize intro circuit */
+  intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                                              HS_IDENT_CIRCUIT_INTRO);
+  /* Register circuit in the circuitmap . */
+  hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
+                                                    &ip->auth_key_kp.pubkey);
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_ptr_op(tmp_circ, OP_EQ, intro_circ);
+
+  /* Pretend that intro point has failed too much */
+  ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES+1;
+
+  /* Now pretend we are freeing this intro circuit. We want to see that our
+   * destructor is not gonna kill our intro point structure since that's the
+   * job of the cleanup routine. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = NULL;
+  entry = service_intro_point_find(service, &ip->auth_key_kp.pubkey);
+  tt_assert(entry);
+  /* The free should also remove the circuit from the circuitmap. */
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_assert(!tmp_circ);
+
+  /* Now pretend that a new intro point circ was launched and opened. Check
+   * that the intro point will be established correctly. */
+  intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
+                                              HS_IDENT_CIRCUIT_INTRO);
+  ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk,
+                      &ip->auth_key_kp.pubkey);
+  /* Register circuit in the circuitmap . */
+  hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
+                                                    &ip->auth_key_kp.pubkey);
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_ptr_op(tmp_circ, OP_EQ, intro_circ);
+  tt_int_op(TO_CIRCUIT(intro_circ)->marked_for_close, OP_EQ, 0);
+  circuit_mark_for_close(TO_CIRCUIT(intro_circ), END_CIRC_REASON_INTERNAL);
+  tt_int_op(TO_CIRCUIT(intro_circ)->marked_for_close, OP_NE, 0);
+  /* At this point, we should not be able to find it in the circuitmap. */
+  tmp_circ =
+    hs_circuitmap_get_intro_circ_v3_service_side(&ip->auth_key_kp.pubkey);
+  tt_assert(!tmp_circ);
+
+ done:
+  if (intro_circ) {
+    circuit_free_(TO_CIRCUIT(intro_circ));
+  }
+  /* Frees the service object. */
+  hs_free_all();
+  UNMOCK(assert_circuit_ok);
+}
+
+/** Test sending and receiving introduce2 cells */
+static void
+test_introduce2(void *arg)
+{
+  int ret;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
+  origin_circuit_t *circ = NULL;
+  hs_service_t *service;
+  hs_service_intro_point_t *ip = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(get_or_state,
+       get_or_state_replacement);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
+  tt_assert(circ);
+
+  /* Test a wrong purpose. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Received an INTRODUCE2 cell on a "
+                            "non introduction circuit of purpose");
+  teardown_capture_of_logs();
+
+  /* Back to normal. */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_S_INTRO;
+
+  /* No service associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Unknown service identity key");
+  teardown_capture_of_logs();
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+  /* No introduction point associated to it. */
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Unknown introduction auth key when handling "
+                            "an INTRODUCE2 cell on circuit");
+  teardown_capture_of_logs();
+
+  /* Set an IP object now for this circuit. */
+  {
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* Update ident to contain the intro point auth key. */
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* This will fail because receiving an INTRODUCE2 cell implies a valid cell
+   * and then launching circuits so let's not do that and instead test that
+   * behaviour differently. */
+  ret = hs_service_receive_introduce2(circ, payload, sizeof(payload));
+  tt_int_op(ret, OP_EQ, -1);
+  tt_u64_op(ip->introduce2_count, OP_EQ, 0);
+
+ done:
+  or_state_free(dummy_state);
+  dummy_state = NULL;
+  if (circ)
+    circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/** Test basic hidden service housekeeping operations (maintaining intro
+ *  points, etc) */
+static void
+test_service_event(void *arg)
+{
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+  time_t now = time(NULL);
+  hs_service_t *service;
+  origin_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
+
+  /* Set a service for this circuit. */
+  service = helper_create_service();
+  ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
+                      &service->keys.identity_pk);
+
+  /* Currently this consists of cleaning invalid intro points. So adding IPs
+   * here that should get cleaned up. */
+  {
+    hs_service_intro_point_t *ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    /* This run will remove the IP because we have no circuits nor node_t
+     * associated with it. */
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+    /* We'll trigger a removal because we've reached our maximum amount of
+     * times we should retry a circuit. For this, we need to have a node_t
+     * that matches the identity of this IP. */
+    routerinfo_t ri;
+    memset(&ri, 0, sizeof(ri));
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    memset(ri.cache_info.identity_digest, 'A', DIGEST_LEN);
+    /* This triggers a node_t creation. */
+    tt_assert(nodelist_set_routerinfo(&ri, NULL));
+    ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+    /* No removal but no circuit so this means the IP object will stay in the
+     * descriptor map so we can retry it. */
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    ip->circuit_established = 1;  /* We'll test that, it MUST be 0 after. */
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 1);
+    /* Remove the IP object at once for the next test. */
+    ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+    /* Now, we'll create an IP with a registered circuit. The IP object
+     * shouldn't go away. */
+    ip = helper_create_service_ip();
+    service_intro_point_add(service->desc_current->intro_points.map, ip);
+    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                        &ip->auth_key_kp.pubkey);
+    hs_circuitmap_register_intro_circ_v3_service_side(
+                                         circ, &ip->auth_key_kp.pubkey);
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 1);
+    /* We'll mangle the IP object to expire. */
+    ip->time_to_expire = now;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 0);
+  }
+
+ done:
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
+  circuit_free_(TO_CIRCUIT(circ));
+  hs_free_all();
+  UNMOCK(circuit_mark_for_close_);
+}
+
+/** Test that we rotate descriptors correctly. */
+static void
+test_rotate_descriptors(void *arg)
+{
+  int ret;
+  time_t next_rotation_time, now = time(NULL);
+  hs_service_t *service;
+  hs_service_descriptor_t *desc_next;
+
+  (void) arg;
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  hs_init();
+  MOCK(get_or_state, get_or_state_replacement);
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Descriptor rotation happens with a consensus with a new SRV. */
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  /* Create a service with a default descriptor and state. It's added to the
+   * global map. */
+  service = helper_create_service();
+  service_descriptor_free(service->desc_current);
+  service->desc_current = NULL;
+  /* This triggers a build for both descriptors. The time now is only used in
+   * the descriptor certificate which is important to be now else the decoding
+   * will complain that the cert has expired if we use valid_after. */
+  build_all_descriptors(now);
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_next);
+
+  /* Tweak our service next rotation time so we can use a custom time. */
+  service->state.next_rotation_time = next_rotation_time =
+    mock_ns.valid_after + (11 * 60 * 60);
+
+  /* Nothing should happen, we are not at a new SRV. Our next rotation time
+   * should be untouched. */
+  rotate_all_descriptors(mock_ns.valid_after);
+  tt_u64_op(service->state.next_rotation_time, OP_EQ, next_rotation_time);
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_next);
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_previous_time_period_num(0));
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  /* Keep a reference so we can compare it after rotation to the current. */
+  desc_next = service->desc_next;
+
+  /* Going right after a new SRV. */
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  /* Note down what to expect for the next rotation time which is 01:00 + 23h
+   * meaning 00:00:00. */
+  next_rotation_time = mock_ns.valid_after + (23 * 60 * 60);
+  /* We should have our next rotation time modified, our current descriptor
+   * cleaned up and the next descriptor becoming the current. */
+  rotate_all_descriptors(mock_ns.valid_after);
+  tt_u64_op(service->state.next_rotation_time, OP_EQ, next_rotation_time);
+  tt_mem_op(service->desc_current, OP_EQ, desc_next, sizeof(*desc_next));
+  tt_assert(service->desc_next == NULL);
+
+  /* A second time should do nothing. */
+  rotate_all_descriptors(mock_ns.valid_after);
+  tt_u64_op(service->state.next_rotation_time, OP_EQ, next_rotation_time);
+  tt_mem_op(service->desc_current, OP_EQ, desc_next, sizeof(*desc_next));
+  tt_assert(service->desc_next == NULL);
+
+  build_all_descriptors(now);
+  tt_mem_op(service->desc_current, OP_EQ, desc_next, sizeof(*desc_next));
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_next_time_period_num(0));
+  tt_assert(service->desc_next);
+
+ done:
+  hs_free_all();
+  UNMOCK(get_or_state);
+  UNMOCK(circuit_mark_for_close_);
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+/** Test building descriptors: picking intro points, setting up their link
+ *  specifiers, etc. */
+static void
+test_build_update_descriptors(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+  node_t *node;
+  hs_service_t *service;
+  hs_service_intro_point_t *ip_cur, *ip_next;
+  routerinfo_t ri;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  /* Create a service without a current descriptor to trigger a build. */
+  service = helper_create_service();
+  tt_assert(service);
+  /* Unfortunately, the helper creates a dummy descriptor so get rid of it. */
+  service_descriptor_free(service->desc_current);
+  service->desc_current = NULL;
+
+  /* We have a fresh service so this should trigger a build for both
+   * descriptors for specific time period that we'll test. */
+  build_all_descriptors(now);
+  /* Check *current* descriptor. */
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_current->desc);
+  tt_assert(service->desc_current->intro_points.map);
+  /* The current time period is the one expected when starting at 03:00. */
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, 0);
+
+  /* Check *next* descriptor. */
+  tt_assert(service->desc_next);
+  tt_assert(service->desc_next->desc);
+  tt_assert(service->desc_next->intro_points.map);
+  tt_assert(service->desc_current != service->desc_next);
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_next_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
+
+  /* Time to test the update of those descriptors. At first, we have no node
+   * in the routerlist so this will find NO suitable node for the IPs. */
+  setup_full_capture_of_logs(LOG_INFO);
+  update_all_descriptors(now);
+  expect_log_msg_containing("Unable to find a suitable node to be an "
+                            "introduction point for service");
+  teardown_capture_of_logs();
+  tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+            OP_EQ, 0);
+  tt_int_op(digest256map_size(service->desc_next->intro_points.map),
+            OP_EQ, 0);
+
+  /* Now, we'll setup a node_t. */
+  {
+    tor_addr_t ipv4_addr;
+    curve25519_secret_key_t curve25519_secret_key;
+
+    memset(&ri, 0, sizeof(routerinfo_t));
+
+    tor_addr_parse(&ipv4_addr, "127.0.0.1");
+    ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+    ri.or_port = 1337;
+    ri.purpose = ROUTER_PURPOSE_GENERAL;
+    /* Ugly yes but we never free the "ri" object so this just makes things
+     * easier. */
+    ri.protocol_list = (char *) "HSDir=1-2 LinkAuth=3";
+    summarize_protover_flags(&ri.pv, ri.protocol_list, NULL);
+    ret = curve25519_secret_key_generate(&curve25519_secret_key, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    ri.onion_curve25519_pkey =
+      tor_malloc_zero(sizeof(curve25519_public_key_t));
+    ri.onion_pkey = crypto_pk_new();
+    curve25519_public_key_generate(ri.onion_curve25519_pkey,
+                                   &curve25519_secret_key);
+    memset(ri.cache_info.identity_digest, 'A', DIGEST_LEN);
+    /* Setup ed25519 identity */
+    ed25519_keypair_t kp1;
+    ed25519_keypair_generate(&kp1, 0);
+    ri.cache_info.signing_key_cert = tor_malloc_zero(sizeof(tor_cert_t));
+    tt_assert(ri.cache_info.signing_key_cert);
+    ed25519_pubkey_copy(&ri.cache_info.signing_key_cert->signing_key,
+                        &kp1.pubkey);
+    nodelist_set_routerinfo(&ri, NULL);
+    node = node_get_mutable_by_id(ri.cache_info.identity_digest);
+    tt_assert(node);
+    node->is_running = node->is_valid = node->is_fast = node->is_stable = 1;
+  }
+
+  /* We have to set this, or the lack of microdescriptors for these
+   * nodes will make them unusable. */
+  get_options_mutable()->UseMicrodescriptors = 0;
+
+  /* We expect to pick only one intro point from the node above. */
+  setup_full_capture_of_logs(LOG_INFO);
+  update_all_descriptors(now);
+  tor_free(node->ri->onion_curve25519_pkey); /* Avoid memleak. */
+  tor_free(node->ri->cache_info.signing_key_cert);
+  crypto_pk_free(node->ri->onion_pkey);
+  expect_log_msg_containing("just picked 1 intro points and wanted 3 for next "
+                            "descriptor. It currently has 0 intro points. "
+                            "Launching ESTABLISH_INTRO circuit shortly.");
+  teardown_capture_of_logs();
+  tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+            OP_EQ, 1);
+  tt_int_op(digest256map_size(service->desc_next->intro_points.map),
+            OP_EQ, 1);
+  /* Get the IP object. Because we don't have the auth key of the IP, we can't
+   * query it so get the first element in the map. */
+  {
+    void *obj = NULL;
+    const uint8_t *key;
+    digest256map_iter_t *iter =
+      digest256map_iter_init(service->desc_current->intro_points.map);
+    digest256map_iter_get(iter, &key, &obj);
+    tt_assert(obj);
+    ip_cur = obj;
+    /* Get also the IP from the next descriptor. We'll make sure it's not the
+     * same object as in the current descriptor. */
+    iter = digest256map_iter_init(service->desc_next->intro_points.map);
+    digest256map_iter_get(iter, &key, &obj);
+    tt_assert(obj);
+    ip_next = obj;
+  }
+  tt_mem_op(ip_cur, OP_NE, ip_next, sizeof(hs_desc_intro_point_t));
+
+  /* We won't test the service IP object because there is a specific test
+   * already for this but we'll make sure that the state is coherent.*/
+
+  /* Three link specifiers are mandatoy so make sure we do have them. */
+  tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3);
+  /* Make sure we have a valid encryption keypair generated when we pick an
+   * intro point in the update process. */
+  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
+                             CURVE25519_SECKEY_LEN));
+  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
+                             CURVE25519_PUBKEY_LEN));
+  tt_u64_op(ip_cur->time_to_expire, OP_GE, now +
+            INTRO_POINT_LIFETIME_MIN_SECONDS);
+  tt_u64_op(ip_cur->time_to_expire, OP_LE, now +
+            INTRO_POINT_LIFETIME_MAX_SECONDS);
+
+  /* Now, we will try to set up a service after a new time period has started
+   * and see if it behaves as expected. */
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Create a service without a current descriptor to trigger a build. */
+  service = helper_create_service();
+  tt_assert(service);
+  /* Unfortunately, the helper creates a dummy descriptor so get rid of it. */
+  service_descriptor_free(service->desc_current);
+  service->desc_current = NULL;
+
+  /* We have a fresh service so this should trigger a build for both
+   * descriptors for specific time period that we'll test. */
+  build_all_descriptors(now);
+  /* Check *current* descriptor. */
+  tt_assert(service->desc_current);
+  tt_assert(service->desc_current->desc);
+  tt_assert(service->desc_current->intro_points.map);
+  /* This should be for the previous time period. */
+  tt_u64_op(service->desc_current->time_period_num, OP_EQ,
+            hs_get_previous_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, 0);
+
+  /* Check *next* descriptor. */
+  tt_assert(service->desc_next);
+  tt_assert(service->desc_next->desc);
+  tt_assert(service->desc_next->intro_points.map);
+  tt_assert(service->desc_current != service->desc_next);
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
+
+  /* Let's remove the next descriptor to simulate a rotation. */
+  service_descriptor_free(service->desc_next);
+  service->desc_next = NULL;
+
+  build_all_descriptors(now);
+  /* Check *next* descriptor. */
+  tt_assert(service->desc_next);
+  tt_assert(service->desc_next->desc);
+  tt_assert(service->desc_next->intro_points.map);
+  tt_assert(service->desc_current != service->desc_next);
+  tt_u64_op(service->desc_next->time_period_num, OP_EQ,
+            hs_get_next_time_period_num(0));
+  /* This should be untouched, the update descriptor process changes it. */
+  tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
+
+ done:
+  hs_free_all();
+  nodelist_free_all();
+}
+
+static void
+test_upload_descriptors(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Create a service with no descriptor. It's added to the global map. */
+  service = hs_service_new(get_options());
+  tt_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  /* Register service to global map. */
+  ret = register_service(get_hs_service_map(), service);
+  tt_int_op(ret, OP_EQ, 0);
+  /* But first, build our descriptor. */
+  build_all_descriptors(now);
+
+  /* Nothing should happen because we have 0 introduction circuit established
+   * and we want (by default) 3 intro points. */
+  run_upload_descriptor_event(now);
+  /* If no upload happened, this should be untouched. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, 0);
+  /* We'll simulate that we've opened our intro point circuit and that we only
+   * want one intro point. */
+  service->config.num_intro_points = 1;
+
+  /* Set our next upload time after now which will skip the upload. */
+  service->desc_current->next_upload_time = now + 1000;
+  run_upload_descriptor_event(now);
+  /* If no upload happened, this should be untouched. */
+  tt_u64_op(service->desc_current->next_upload_time, OP_EQ, now + 1000);
+
+ done:
+  hs_free_all();
+  UNMOCK(get_or_state);
+}
+
+/** Test the functions that save and load HS revision counters to state. */
+static void
+test_revision_counter_state(void *arg)
+{
+  char *state_line_one = NULL;
+  char *state_line_two = NULL;
+
+  hs_service_descriptor_t *desc_one = service_descriptor_new();
+  hs_service_descriptor_t *desc_two = service_descriptor_new();
+
+  (void) arg;
+
+  /* Prepare both descriptors */
+  desc_one->desc->plaintext_data.revision_counter = 42;
+  desc_two->desc->plaintext_data.revision_counter = 240;
+  memset(&desc_one->blinded_kp.pubkey.pubkey, 66,
+         sizeof(desc_one->blinded_kp.pubkey.pubkey));
+  memset(&desc_two->blinded_kp.pubkey.pubkey, 240,
+         sizeof(desc_one->blinded_kp.pubkey.pubkey));
+
+  /* Turn the descriptor rev counters into state lines */
+  state_line_one = encode_desc_rev_counter_for_state(desc_one);
+  tt_str_op(state_line_one, OP_EQ,
+            "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI 42");
+
+  state_line_two = encode_desc_rev_counter_for_state(desc_two);
+  tt_str_op(state_line_two, OP_EQ,
+            "8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PA 240");
+
+  /* Now let's test our state parsing function: */
+  int service_found;
+  uint64_t cached_rev_counter;
+
+  /* First's try with wrong pubkey and check that no service was found */
+  cached_rev_counter =check_state_line_for_service_rev_counter(state_line_one,
+                                                 &desc_two->blinded_kp.pubkey,
+                                                               &service_found);
+  tt_int_op(service_found, OP_EQ, 0);
+  tt_u64_op(cached_rev_counter, OP_EQ, 0);
+
+  /* Now let's try with the right pubkeys */
+  cached_rev_counter =check_state_line_for_service_rev_counter(state_line_one,
+                                                 &desc_one->blinded_kp.pubkey,
+                                                               &service_found);
+  tt_int_op(service_found, OP_EQ, 1);
+  tt_u64_op(cached_rev_counter, OP_EQ, 42);
+
+  cached_rev_counter =check_state_line_for_service_rev_counter(state_line_two,
+                                                 &desc_two->blinded_kp.pubkey,
+                                                               &service_found);
+  tt_int_op(service_found, OP_EQ, 1);
+  tt_u64_op(cached_rev_counter, OP_EQ, 240);
+
+ done:
+  tor_free(state_line_one);
+  tor_free(state_line_two);
+  service_descriptor_free(desc_one);
+  service_descriptor_free(desc_two);
+}
+
+/** Global vars used by test_rendezvous1_parsing() */
+static char rend1_payload[RELAY_PAYLOAD_SIZE];
+static size_t rend1_payload_len = 0;
+
+/** Mock for relay_send_command_from_edge() to send a RENDEZVOUS1 cell. Instead
+ *  of sending it to the network, instead save it to the global `rend1_payload`
+ *  variable so that we can inspect it in the test_rendezvous1_parsing()
+ *  test. */
+static int
+mock_relay_send_rendezvous1(streamid_t stream_id, circuit_t *circ,
+                            uint8_t relay_command, const char *payload,
+                            size_t payload_len,
+                            crypt_path_t *cpath_layer,
+                            const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+
+  memcpy(rend1_payload, payload, payload_len);
+  rend1_payload_len = payload_len;
+
+  return 0;
+}
+
+/** Send a RENDEZVOUS1 as a service, and parse it as a client. */
+static void
+test_rendezvous1_parsing(void *arg)
+{
+  int retval;
+  static const char *test_addr =
+    "4acth47i6kxnvkewtm6q7ib2s3ufpo5sqbsnzjpbi7utijcltosqemad.onion";
+  hs_service_t *service = NULL;
+  origin_circuit_t *service_circ = NULL;
+  origin_circuit_t *client_circ = NULL;
+  ed25519_keypair_t ip_auth_kp;
+  curve25519_keypair_t ephemeral_kp;
+  curve25519_keypair_t client_kp;
+  curve25519_keypair_t ip_enc_kp;
+  int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_rendezvous1);
+
+  {
+    /* Let's start by setting up the service that will start the rend */
+    service = tor_malloc_zero(sizeof(hs_service_t));
+    ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+    ed25519_public_key_generate(&service->keys.identity_pk,
+                                &service->keys.identity_sk);
+    memcpy(service->onion_address, test_addr, sizeof(service->onion_address));
+    tt_assert(service);
+  }
+
+  {
+    /* Now let's set up the service rendezvous circuit and its keys. */
+    service_circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_CONNECT_REND,
+                                                flags);
+    tor_free(service_circ->hs_ident);
+    hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+    uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
+    curve25519_keypair_generate(&ip_enc_kp, 0);
+    curve25519_keypair_generate(&ephemeral_kp, 0);
+    curve25519_keypair_generate(&client_kp, 0);
+    ed25519_keypair_generate(&ip_auth_kp, 0);
+    retval = hs_ntor_service_get_rendezvous1_keys(&ip_auth_kp.pubkey,
+                                                  &ip_enc_kp,
+                                                  &ephemeral_kp,
+                                                  &client_kp.pubkey,
+                                                  &hs_ntor_rend_cell_keys);
+    tt_int_op(retval, OP_EQ, 0);
+
+    memset(rendezvous_cookie, 2, sizeof(rendezvous_cookie));
+    service_circ->hs_ident =
+      create_rp_circuit_identifier(service, rendezvous_cookie,
+                                   &ephemeral_kp.pubkey,
+                                   &hs_ntor_rend_cell_keys);
+  }
+
+  /* Send out the RENDEZVOUS1 and make sure that our mock func worked */
+  tt_assert(tor_mem_is_zero(rend1_payload, 32));
+  hs_circ_service_rp_has_opened(service, service_circ);
+  tt_assert(!tor_mem_is_zero(rend1_payload, 32));
+  tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE);
+
+  /******************************/
+
+  /** Now let's create the client rendezvous circuit */
+  client_circ =
+    helper_create_origin_circuit(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED,
+                                 flags);
+  /* fix up its circ ident */
+  ed25519_pubkey_copy(&client_circ->hs_ident->intro_auth_pk,
+                      &ip_auth_kp.pubkey);
+  memcpy(&client_circ->hs_ident->rendezvous_client_kp,
+         &client_kp, sizeof(client_circ->hs_ident->rendezvous_client_kp));
+  memcpy(&client_circ->hs_ident->intro_enc_pk.public_key,
+         &ip_enc_kp.pubkey.public_key,
+         sizeof(client_circ->hs_ident->intro_enc_pk.public_key));
+
+  /* Now parse the rendezvous2 circuit and make sure it was fine. We are
+   * skipping 20 bytes off its payload, since that's the rendezvous cookie
+   * which is only present in REND1. */
+  retval = handle_rendezvous2(client_circ,
+                              (uint8_t*)rend1_payload+20,
+                              rend1_payload_len-20);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* TODO: We are only simulating client/service here. We could also simulate
+   * the rendezvous point by plugging in rend_mid_establish_rendezvous(). We
+   * would need an extra circuit and some more stuff but it's doable. */
+
+ done:
+  circuit_free_(TO_CIRCUIT(service_circ));
+  circuit_free_(TO_CIRCUIT(client_circ));
+  hs_service_free(service);
+  hs_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+struct testcase_t hs_service_tests[] = {
+  { "e2e_rend_circuit_setup", test_e2e_rend_circuit_setup, TT_FORK,
+    NULL, NULL },
+  { "load_keys", test_load_keys, TT_FORK,
+    NULL, NULL },
+  { "access_service", test_access_service, TT_FORK,
+    NULL, NULL },
+  { "service_intro_point", test_service_intro_point, TT_FORK,
+    NULL, NULL },
+  { "helper_functions", test_helper_functions, TT_FORK,
+    NULL, NULL },
+  { "intro_circuit_opened", test_intro_circuit_opened, TT_FORK,
+    NULL, NULL },
+  { "intro_established", test_intro_established, TT_FORK,
+    NULL, NULL },
+  { "closing_intro_circs", test_closing_intro_circs, TT_FORK,
+    NULL, NULL },
+  { "rdv_circuit_opened", test_rdv_circuit_opened, TT_FORK,
+    NULL, NULL },
+  { "introduce2", test_introduce2, TT_FORK,
+    NULL, NULL },
+  { "service_event", test_service_event, TT_FORK,
+    NULL, NULL },
+  { "rotate_descriptors", test_rotate_descriptors, TT_FORK,
+    NULL, NULL },
+  { "build_update_descriptors", test_build_update_descriptors, TT_FORK,
+    NULL, NULL },
+  { "upload_descriptors", test_upload_descriptors, TT_FORK,
+    NULL, NULL },
+  { "revision_counter_state", test_revision_counter_state, TT_FORK,
+    NULL, NULL },
+  { "rendezvous1_parsing", test_rendezvous1_parsing, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 810b03c93d..d502bdddb1 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -307,7 +307,7 @@ do_parse_test(uint8_t *plaintext, size_t plaintext_len, int phase)
   /* Do early parsing */
   parsed_req = rend_service_begin_parse_intro(cell, cell_len, 2, &err_msg);
   tt_assert(parsed_req);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_mem_op(parsed_req->pk,OP_EQ, digest, DIGEST_LEN);
   tt_assert(parsed_req->ciphertext);
   tt_assert(parsed_req->ciphertext_len > 0);
@@ -318,7 +318,7 @@ do_parse_test(uint8_t *plaintext, size_t plaintext_len, int phase)
   /* Do decryption */
   r = rend_service_decrypt_intro(parsed_req, k, &err_msg);
   tt_assert(!r);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_assert(parsed_req->plaintext);
   tt_assert(parsed_req->plaintext_len > 0);
 
@@ -328,7 +328,7 @@ do_parse_test(uint8_t *plaintext, size_t plaintext_len, int phase)
   /* Do late parsing */
   r = rend_service_parse_intro_plaintext(parsed_req, &err_msg);
   tt_assert(!r);
-  tt_assert(!err_msg);
+  tt_ptr_op(err_msg, OP_EQ, NULL);
   tt_assert(parsed_req->parsed);
 
  done:
@@ -355,7 +355,7 @@ make_intro_from_plaintext(
 
   /*
    * Figure out an upper bound on how big the ciphertext will be
-   * (see crypto_pk_public_hybrid_encrypt())
+   * (see crypto_pk_obsolete_public_hybrid_encrypt())
    */
   ciphertext_size = PKCS1_OAEP_PADDING_OVERHEAD;
   ciphertext_size += crypto_pk_keysize(key);
@@ -372,7 +372,7 @@ make_intro_from_plaintext(
   tt_assert(r >= 0);
 
   /* Do encryption */
-  r = crypto_pk_public_hybrid_encrypt(
+  r = crypto_pk_obsolete_public_hybrid_encrypt(
       key, cell + DIGEST_LEN, ciphertext_size,
       buf, len,
       PK_PKCS1_OAEP_PADDING, 0);
diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh
new file mode 100755
index 0000000000..cf6608634d
--- /dev/null
+++ b/src/test/test_key_expiration.sh
@@ -0,0 +1,137 @@
+#!/bin/sh
+
+# Note: some of this code is lifted from zero_length_keys.sh and
+# test_keygen.sh, and could be unified.
+
+umask 077
+set -e
+
+if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+  if [ "$TESTING_TOR_BINARY" = "" ] ; then
+    echo "Usage: ${0} PATH_TO_TOR [case-number]"
+    exit 1
+  fi
+fi
+
+UNAME_OS=`uname -s | cut -d_ -f1`
+if test "$UNAME_OS" = 'CYGWIN' || \
+   test "$UNAME_OS" = 'MSYS' || \
+   test "$UNAME_OS" = 'MINGW'; then
+  echo "This test is unreliable on Windows. See trac #26076. Skipping." >&2
+  exit 77
+fi
+
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY}"
+fi
+
+if [ $# -ge 1 ]; then
+  dflt=0
+else
+  dflt=1
+fi
+
+CASE1=$dflt
+CASE2=$dflt
+CASE3=$dflt
+
+if [ $# -ge 1 ]; then
+  eval "CASE${1}"=1
+fi
+
+
+dump() { xxd -p "$1" | tr -d '\n '; }
+die() { echo "$1" >&2 ; exit 5; }
+check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
+check_file() { [ -e "$1" ] || die "$1 did not exist"; }
+check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
+
+DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX`
+if [ -z "$DATA_DIR" ]; then
+  echo "Failure: mktemp invocation returned empty string" >&2
+  exit 3
+fi
+if [ ! -d "$DATA_DIR" ]; then
+  echo "Failure: mktemp invocation result doesn't point to directory" >&2
+  exit 3
+fi
+trap "rm -rf '$DATA_DIR'" 0
+
+# Use an absolute path for this or Tor will complain
+DATA_DIR=`cd "${DATA_DIR}" && pwd`
+
+touch "${DATA_DIR}/empty_torrc"
+
+QUIETLY="--hush"
+SILENTLY="--quiet"
+TOR="${TOR_BINARY} --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --ExitRelay 0 -f ${DATA_DIR}/empty_torrc --DataDirectory ${DATA_DIR}"
+
+##### SETUP
+#
+# Here we create a set of keys.
+
+# Step 1: Start Tor with --list-fingerprint --quiet.  Make sure everything is there.
+echo "Setup step #1"
+${TOR} --list-fingerprint ${SILENTLY} > /dev/null
+
+check_dir "${DATA_DIR}/keys"
+check_file "${DATA_DIR}/keys/ed25519_master_id_public_key"
+check_file "${DATA_DIR}/keys/ed25519_master_id_secret_key"
+check_file "${DATA_DIR}/keys/ed25519_signing_cert"
+check_file "${DATA_DIR}/keys/ed25519_signing_secret_key"
+check_file "${DATA_DIR}/keys/secret_id_key"
+check_file "${DATA_DIR}/keys/secret_onion_key"
+check_file "${DATA_DIR}/keys/secret_onion_key_ntor"
+
+##### TEST CASES
+
+echo "=== Starting key expiration tests."
+
+FN="${DATA_DIR}/stderr"
+
+if [ "$CASE1" = 1 ]; then
+  echo "==== Case 1: Test --key-expiration without argument and ensure usage"
+  echo "             instructions are printed."
+
+  ${TOR} ${QUIETLY} --key-expiration 2>"$FN" || true
+  grep "No valid argument to --key-expiration found!" "$FN" >/dev/null || \
+    die "Tor didn't mention supported --key-expiration argmuents"
+
+  echo "==== Case 1: ok"
+fi
+
+if [ "$CASE2" = 1 ]; then
+  echo "==== Case 2: Start Tor with --key-expiration 'sign' and make sure it prints an expiration."
+
+  ${TOR} ${QUIETLY} --key-expiration sign 2>"$FN"
+  grep "signing-cert-expiry:" "$FN" >/dev/null || \
+    die "Tor didn't print an expiration"
+
+  echo "==== Case 2: ok"
+fi
+
+if [ "$CASE3" = 1 ]; then
+  echo "==== Case 3: Start Tor with --key-expiration 'sign', when there is no"
+  echo "             signing key, and make sure that Tor generates a new key"
+  echo "             and prints its certificate's expiration."
+
+  mv "${DATA_DIR}/keys/ed25519_signing_cert" \
+     "${DATA_DIR}/keys/ed25519_signing_cert.bak"
+
+  ${TOR} --key-expiration sign > "$FN" 2>&1
+  grep "It looks like I need to generate and sign a new medium-term signing key" "$FN" >/dev/null || \
+    die "Tor didn't create a new signing key"
+  check_file "${DATA_DIR}/keys/ed25519_signing_cert"
+  grep "signing-cert-expiry:" "$FN" >/dev/null || \
+    die "Tor didn't print an expiration"
+
+  mv "${DATA_DIR}/keys/ed25519_signing_cert.bak" \
+     "${DATA_DIR}/keys/ed25519_signing_cert"
+
+  echo "==== Case 3: ok"
+fi
diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh
index 4106425cb3..455f9e7d42 100755
--- a/src/test/test_keygen.sh
+++ b/src/test/test_keygen.sh
@@ -48,6 +48,12 @@ fi
   CASE8=$dflt
   CASE9=$dflt
   CASE10=$dflt
+  CASE11A=$dflt
+  CASE11B=$dflt
+  CASE11C=$dflt
+  CASE11D=$dflt
+  CASE11E=$dflt
+  CASE11F=$dflt
 
   if [ $# -ge 1 ]; then
      eval "CASE${1}"=1
@@ -371,6 +377,109 @@ echo "==== Case 10 ok"
 
 fi
 
+# Case 11a: -passphrase-fd without --keygen
+
+if [ "$CASE11A" = 1 ]; then
+
+ME="${DATA_DIR}/case11a"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true
+
+grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11A ok"
+
+fi
+
+# Case 11b: --no-passphrase without --keygen
+
+if [ "$CASE11B" = 1 ]; then
+
+ME="${DATA_DIR}/case11b"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true
+
+grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11B ok"
+
+fi
+
+# Case 11c: --newpass without --keygen
+
+if [ "$CASE11C" = 1 ]; then
+
+ME="${DATA_DIR}/case11C"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true
+
+grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11C ok"
+
+fi
+
+######## --master-key does not work yet, but this will test the error case
+########  when it does.
+#
+# Case 11d: --master-key without --keygen
+#
+if [ "$CASE11D" = 1 ]; then
+#
+# ME="${DATA_DIR}/case11d"
+#
+# mkdir -p "${ME}/keys"
+#
+# ${TOR} --DataDirectory "${ME}" --master-key "${ME}/foobar" > "${ME}/stdout" && die "Successfully started with master-key but no keygen?" || true
+#
+# cat "${ME}/stdout"
+#
+# grep "master-key without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+    echo "==== Case 11D skipped"
+
+fi
+
+
+# Case 11E: Silly passphrase-fd
+
+if [ "$CASE11E" = 1 ]; then
+
+ME="${DATA_DIR}/case11E"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true
+
+grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11E ok"
+
+fi
+
+
+# Case 11F: --no-passphrase with --passphrase-fd
+
+if [ "$CASE11F" = 1 ]; then
+
+ME="${DATA_DIR}/case11F"
+
+mkdir -p "${ME}/keys"
+
+${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true
+
+grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
+
+echo "==== Case 11F ok"
+
+fi
+
 
 # Check cert-only.
 
diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c
index 95657349c6..79d7bac902 100644
--- a/src/test/test_keypin.c
+++ b/src/test/test_keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -20,8 +20,8 @@ test_keypin_parse_line(void *arg)
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4");
   tt_assert(ent);
-  tt_mem_op(ent->rsa_id, ==, "here is a good sha1!", 20);
-  tt_mem_op(ent->ed25519_key, ==, "This ed25519 scoffs at the sha1.", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "here is a good sha1!", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "This ed25519 scoffs at the sha1.", 32);
   tor_free(ent); ent = NULL;
 
   /* Good line with extra stuff we will ignore. */
@@ -29,27 +29,27 @@ test_keypin_parse_line(void *arg)
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4helloworld");
   tt_assert(ent);
-  tt_mem_op(ent->rsa_id, ==, "here is a good sha1!", 20);
-  tt_mem_op(ent->ed25519_key, ==, "This ed25519 scoffs at the sha1.", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "here is a good sha1!", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "This ed25519 scoffs at the sha1.", 32);
   tor_free(ent); ent = NULL;
 
   /* Bad line: no space in the middle. */
   ent = keypin_parse_journal_line(
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE?"
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4");
-  tt_assert(! ent);
+  tt_ptr_op(ent, OP_EQ, NULL);
 
   /* Bad line: bad base64 in RSA ID */
   ent = keypin_parse_journal_line(
                 "aGVyZSBpcyBhIGdv!2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyBhdCB0aGUgc2hhMS4");
-  tt_assert(! ent);
+  tt_ptr_op(ent, OP_EQ, NULL);
 
   /* Bad line: bad base64 in Ed25519 */
   ent = keypin_parse_journal_line(
                 "aGVyZSBpcyBhIGdvb2Qgc2hhMSE "
                 "VGhpcyBlZDI1NTE5IHNjb2ZmcyB!dCB0aGUgc2hhMS4");
-  tt_assert(! ent);
+  tt_ptr_op(ent, OP_EQ, NULL);
 
  done:
   tor_free(ent);
@@ -82,11 +82,11 @@ test_keypin_parse_file(void *arg)
 "Z2dsZSBpbiBzd29tZWVzd2FucyA aW4gdm9sdXB0YXRlIGF4ZS1oYWNrZXIgZXNzZSByaXA\n"
 "cHVsdXMgY3J1bW1paSBldSBtb28 ZiBudWxsYSBzbnV2di5QTFVHSFBMT1ZFUlhZWlpZLi4\n";
 
-  tt_int_op(0, ==, keypin_load_journal_impl(data1, strlen(data1)));
-  tt_int_op(8, ==, smartlist_len(mock_addent_got));
+  tt_int_op(0, OP_EQ, keypin_load_journal_impl(data1, strlen(data1)));
+  tt_int_op(8, OP_EQ, smartlist_len(mock_addent_got));
   keypin_ent_t *ent = smartlist_get(mock_addent_got, 2);
-  tt_mem_op(ent->rsa_id, ==, "r lerkim, sed do bar", 20);
-  tt_mem_op(ent->ed25519_key, ==, "baloot tempor gluppitus ut labor", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "r lerkim, sed do bar", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "baloot tempor gluppitus ut labor", 32);
 
   /* More complex example: weird lines, bogus lines,
      duplicate/conflicting lines */
@@ -107,24 +107,25 @@ test_keypin_parse_file(void *arg)
     "ZHMgc3BlYWsgdHJ1dGgsIGFuZCA aXQgd2FzIHRydaUgdGhhdCBhbGwgdGhlIG1hc3Rlcgo\n"
     ;
 
-  tt_int_op(0, ==, keypin_load_journal_impl(data2, strlen(data2)));
-  tt_int_op(13, ==, smartlist_len(mock_addent_got));
+  tt_int_op(0, OP_EQ, keypin_load_journal_impl(data2, strlen(data2)));
+  tt_int_op(13, OP_EQ, smartlist_len(mock_addent_got));
   ent = smartlist_get(mock_addent_got, 9);
-  tt_mem_op(ent->rsa_id, ==, "\"You have made a goo", 20);
-  tt_mem_op(ent->ed25519_key, ==, "d beginning.\" But no more. Wizar", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "\"You have made a goo", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, "d beginning.\" But no more. Wizar", 32);
 
   ent = smartlist_get(mock_addent_got, 12);
-  tt_mem_op(ent->rsa_id, ==, "ds speak truth, and ", 20);
-  tt_mem_op(ent->ed25519_key, ==, "it was tru\xa5 that all the master\n", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "ds speak truth, and ", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ,
+            "it was tru\xa5 that all the master\n", 32);
 
   /* File truncated before NL */
   const char data3[] =
     "Tm8gZHJhZ29uIGNhbiByZXNpc3Q IHRoZSBmYXNjaW5hdGlvbiBvZiByaWRkbGluZyB0YWw";
-  tt_int_op(0, ==, keypin_load_journal_impl(data3, strlen(data3)));
-  tt_int_op(14, ==, smartlist_len(mock_addent_got));
+  tt_int_op(0, OP_EQ, keypin_load_journal_impl(data3, strlen(data3)));
+  tt_int_op(14, OP_EQ, smartlist_len(mock_addent_got));
   ent = smartlist_get(mock_addent_got, 13);
-  tt_mem_op(ent->rsa_id, ==, "No dragon can resist", 20);
-  tt_mem_op(ent->ed25519_key, ==, " the fascination of riddling tal", 32);
+  tt_mem_op(ent->rsa_id, OP_EQ, "No dragon can resist", 20);
+  tt_mem_op(ent->ed25519_key, OP_EQ, " the fascination of riddling tal", 32);
 
  done:
   keypin_clear();
@@ -141,32 +142,32 @@ test_keypin_add_entry(void *arg)
   (void)arg;
   keypin_clear();
 
-  tt_int_op(KEYPIN_ADDED, ==, ADD("ambassadors-at-large",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("ambassadors-at-large",
                                   "bread-and-butter thing-in-itself"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("gentleman-adventurer",
                                   "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_FOUND, ==, ADD("ambassadors-at-large",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("ambassadors-at-large",
                                   "bread-and-butter thing-in-itself"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("ambassadors-at-large",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("ambassadors-at-large",
                                   "bread-and-butter thing-in-itself"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("gentleman-adventurer",
                                   "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_ADDED, ==, ADD("Johnnies-come-lately",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("Johnnies-come-lately",
                                   "run-of-the-mill root-mean-square"));
 
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("gentleman-adventurer",
                                      "hypersentimental closefistedness"));
 
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("disestablismentarian",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("disestablismentarian",
                                      "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_FOUND, ==, ADD("gentleman-adventurer",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("gentleman-adventurer",
                                   "cloak-and-dagger what's-his-face"));
 
-  tt_int_op(KEYPIN_NOT_FOUND, ==, LONE_RSA("Llanfairpwllgwyngyll"));
-  tt_int_op(KEYPIN_MISMATCH, ==, LONE_RSA("Johnnies-come-lately"));
+  tt_int_op(KEYPIN_NOT_FOUND, OP_EQ, LONE_RSA("Llanfairpwllgwyngyll"));
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, LONE_RSA("Johnnies-come-lately"));
 
  done:
   keypin_clear();
@@ -179,51 +180,51 @@ test_keypin_journal(void *arg)
   char *contents = NULL;
   const char *fname = get_fname("keypin-journal");
 
-  tt_int_op(0, ==, keypin_load_journal(fname)); /* ENOENT is okay */
+  tt_int_op(0, OP_EQ, keypin_load_journal(fname)); /* ENOENT is okay */
   update_approx_time(1217709000);
-  tt_int_op(0, ==, keypin_open_journal(fname));
+  tt_int_op(0, OP_EQ, keypin_open_journal(fname));
 
-  tt_int_op(KEYPIN_ADDED, ==, ADD("king-of-the-herrings",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("king-of-the-herrings",
                                   "good-for-nothing attorney-at-law"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("yellowish-red-yellow",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("yellowish-red-yellow",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
   keypin_close_journal();
   keypin_clear();
 
-  tt_int_op(0, ==, keypin_load_journal(fname));
+  tt_int_op(0, OP_EQ, keypin_load_journal(fname));
   update_approx_time(1231041600);
-  tt_int_op(0, ==, keypin_open_journal(fname));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("yellowish-red-yellow",
+  tt_int_op(0, OP_EQ, keypin_open_journal(fname));
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("theatre-in-the-round",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("theatre-in-the-round",
                                   "holier-than-thou jack-in-the-box"));
-  tt_int_op(KEYPIN_ADDED, ==, ADD("no-deposit-no-return",
+  tt_int_op(KEYPIN_ADDED, OP_EQ, ADD("no-deposit-no-return",
                                   "across-the-board will-o-the-wisp"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("intellectualizations",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("intellectualizations",
                                      "salt-and-pepper high-muck-a-muck"));
   keypin_close_journal();
   keypin_clear();
 
-  tt_int_op(0, ==, keypin_load_journal(fname));
+  tt_int_op(0, OP_EQ, keypin_load_journal(fname));
   update_approx_time(1412278354);
-  tt_int_op(0, ==, keypin_open_journal(fname));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("yellowish-red-yellow",
+  tt_int_op(0, OP_EQ, keypin_open_journal(fname));
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("yellowish-red-yellow",
                                   "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("intellectualizations",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("intellectualizations",
                                      "salt-and-pepper high-muck-a-muck"));
-  tt_int_op(KEYPIN_FOUND, ==, ADD("theatre-in-the-round",
+  tt_int_op(KEYPIN_FOUND, OP_EQ, ADD("theatre-in-the-round",
                                   "holier-than-thou jack-in-the-box"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("counterrevolutionary",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("counterrevolutionary",
                                      "holier-than-thou jack-in-the-box"));
-  tt_int_op(KEYPIN_MISMATCH, ==, ADD("no-deposit-no-return",
+  tt_int_op(KEYPIN_MISMATCH, OP_EQ, ADD("no-deposit-no-return",
                                      "floccinaucinihilipilificationism"));
   keypin_close_journal();
 
   contents = read_file_to_str(fname, RFTS_BIN, NULL);
   tt_assert(contents);
-  tt_str_op(contents,==,
+  tt_str_op(contents,OP_EQ,
     "\n"
     "@opened-at 2008-08-02 20:30:00\n"
     "a2luZy1vZi10aGUtaGVycmluZ3M Z29vZC1mb3Itbm90aGluZyBhdHRvcm5leS1hdC1sYXc\n"
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index ddf66f4d34..6840072d76 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,13 +6,20 @@
 #define CHANNELTLS_PRIVATE
 #define CONNECTION_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
+#define TORTLS_PRIVATE
+
+#include "compat.h"
+
 #include "or.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_or.h"
 #include "channeltls.h"
 #include "link_handshake.h"
+#include "router.h"
+#include "routerkeys.h"
 #include "scheduler.h"
+#include "torcert.h"
 
 #include "test.h"
 #include "log_test_helpers.h"
@@ -37,6 +44,16 @@ mock_tls_cert_matches_key(const tor_tls_t *tls, const tor_x509_cert_t *cert)
   (void) cert; // XXXX look at this.
   return 1;
 }
+static tor_tls_t *mock_peer_cert_expect_tortls = NULL;
+static tor_x509_cert_t *mock_peer_cert = NULL;
+static tor_x509_cert_t *
+mock_get_peer_cert(tor_tls_t *tls)
+{
+  if (mock_peer_cert_expect_tortls &&
+      mock_peer_cert_expect_tortls != tls)
+    return NULL;
+  return tor_x509_cert_dup(mock_peer_cert);
+}
 
 static int mock_send_netinfo_called = 0;
 static int
@@ -57,14 +74,29 @@ mock_close_for_err(or_connection_t *orconn, int flush)
 }
 
 static int mock_send_authenticate_called = 0;
+static int mock_send_authenticate_called_with_type = 0;
 static int
 mock_send_authenticate(or_connection_t *conn, int type)
 {
   (void) conn;
-  (void) type;
+  mock_send_authenticate_called_with_type = type;
   ++mock_send_authenticate_called;// XXX check_this
   return 0;
 }
+static int
+mock_export_key_material(tor_tls_t *tls, uint8_t *secrets_out,
+                         const uint8_t *context,
+                         size_t context_len,
+                         const char *label)
+{
+  (void) tls;
+  (void)secrets_out;
+  (void)context;
+  (void)context_len;
+  (void)label;
+  memcpy(secrets_out, "int getRandomNumber(){return 4;}", 32);
+  return 0;
+}
 
 static tor_x509_cert_t *mock_own_cert = NULL;
 static tor_x509_cert_t *
@@ -78,20 +110,23 @@ mock_get_own_cert(tor_tls_t *tls)
 static void
 test_link_handshake_certs_ok(void *arg)
 {
-  (void) arg;
-
   or_connection_t *c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   or_connection_t *c2 = or_connection_new(CONN_TYPE_OR, AF_INET);
   var_cell_t *cell1 = NULL, *cell2 = NULL;
   certs_cell_t *cc1 = NULL, *cc2 = NULL;
   channel_tls_t *chan1 = NULL, *chan2 = NULL;
   crypto_pk_t *key1 = NULL, *key2 = NULL;
+  const int with_ed = !strcmp((const char *)arg, "Ed25519");
+
+  tor_addr_from_ipv4h(&c1->base_.addr, 0x7f000001);
+  tor_addr_from_ipv4h(&c2->base_.addr, 0x7f000001);
 
   scheduler_init();
 
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
+  MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);
   MOCK(tor_tls_get_own_cert, mock_get_own_cert);
 
   key1 = pk_generate(2);
@@ -101,8 +136,17 @@ test_link_handshake_certs_ok(void *arg)
    * actually generate a CERTS cell.
    */
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 key1, key2, 86400), ==, 0);
+                                 key1, key2, 86400), OP_EQ, 0);
+
+  if (with_ed) {
+    /* If we're making a CERTS cell for an ed handshake, let's make sure we
+     * have some Ed25519 certificates and keys. */
+    init_mock_ed_keys(key2);
+  } else {
+    certs_cell_ed25519_disabled_for_testing = 1;
+  }
 
+  /* c1 has started_here == 1 */
   {
     const tor_x509_cert_t *link_cert = NULL;
     tt_assert(!tor_tls_get_my_certs(1, &link_cert, NULL));
@@ -111,44 +155,66 @@ test_link_handshake_certs_ok(void *arg)
 
   c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   c1->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(c1, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(c1, 1), OP_EQ, 0);
 
+  /* c2 has started_here == 0 */
   c2->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   c2->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(c2, 0), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(c2, 0), OP_EQ, 0);
 
-  tt_int_op(0, ==, connection_or_send_certs_cell(c1));
+  tt_int_op(0, OP_EQ, connection_or_send_certs_cell(c1));
   tt_assert(mock_got_var_cell);
   cell1 = mock_got_var_cell;
 
-  tt_int_op(0, ==, connection_or_send_certs_cell(c2));
+  tt_int_op(0, OP_EQ, connection_or_send_certs_cell(c2));
   tt_assert(mock_got_var_cell);
   cell2 = mock_got_var_cell;
 
-  tt_int_op(cell1->command, ==, CELL_CERTS);
-  tt_int_op(cell1->payload_len, >, 1);
+  tt_int_op(cell1->command, OP_EQ, CELL_CERTS);
+  tt_int_op(cell1->payload_len, OP_GT, 1);
 
-  tt_int_op(cell2->command, ==, CELL_CERTS);
-  tt_int_op(cell2->payload_len, >, 1);
+  tt_int_op(cell2->command, OP_EQ, CELL_CERTS);
+  tt_int_op(cell2->payload_len, OP_GT, 1);
 
-  tt_int_op(cell1->payload_len, ==,
+  tt_int_op(cell1->payload_len, OP_EQ,
             certs_cell_parse(&cc1, cell1->payload, cell1->payload_len));
-  tt_int_op(cell2->payload_len, ==,
+  tt_int_op(cell2->payload_len, OP_EQ,
             certs_cell_parse(&cc2, cell2->payload, cell2->payload_len));
 
-  tt_int_op(2, ==, cc1->n_certs);
-  tt_int_op(2, ==, cc2->n_certs);
+  if (with_ed) {
+    tt_int_op(5, OP_EQ, cc1->n_certs);
+    tt_int_op(5, OP_EQ, cc2->n_certs);
+  } else {
+    tt_int_op(2, OP_EQ, cc1->n_certs);
+    tt_int_op(2, OP_EQ, cc2->n_certs);
+  }
 
-  tt_int_op(certs_cell_get_certs(cc1, 0)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc1, 0)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_AUTH);
-  tt_int_op(certs_cell_get_certs(cc1, 1)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc1, 1)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_ID);
 
-  tt_int_op(certs_cell_get_certs(cc2, 0)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc2, 0)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_LINK);
-  tt_int_op(certs_cell_get_certs(cc2, 1)->cert_type, ==,
+  tt_int_op(certs_cell_get_certs(cc2, 1)->cert_type, OP_EQ,
             CERTTYPE_RSA1024_ID_ID);
 
+  if (with_ed) {
+    tt_int_op(certs_cell_get_certs(cc1, 2)->cert_type, OP_EQ,
+              CERTTYPE_ED_ID_SIGN);
+    tt_int_op(certs_cell_get_certs(cc1, 3)->cert_type, OP_EQ,
+              CERTTYPE_ED_SIGN_AUTH);
+    tt_int_op(certs_cell_get_certs(cc1, 4)->cert_type, OP_EQ,
+              CERTTYPE_RSA1024_ID_EDID);
+
+    tt_int_op(certs_cell_get_certs(cc2, 2)->cert_type, OP_EQ,
+              CERTTYPE_ED_ID_SIGN);
+    tt_int_op(certs_cell_get_certs(cc2, 3)->cert_type, OP_EQ,
+              CERTTYPE_ED_SIGN_LINK);
+    tt_int_op(certs_cell_get_certs(cc2, 4)->cert_type, OP_EQ,
+              CERTTYPE_RSA1024_ID_EDID);
+  }
+
   chan1 = tor_malloc_zero(sizeof(*chan1));
   channel_tls_common_init(chan1);
   c1->chan = chan1;
@@ -159,13 +225,39 @@ test_link_handshake_certs_ok(void *arg)
   c1->base_.conn_array_index = -1;
   crypto_pk_get_digest(key2, c1->identity_digest);
 
+  if (with_ed) {
+    const tor_x509_cert_t *linkc, *idc;
+    tor_tls_get_my_certs(1, &linkc, &idc);
+    mock_peer_cert_expect_tortls = c1->tls; /* We should see this tls... */
+    mock_peer_cert = tor_x509_cert_dup(linkc); /* and when we do, the peer's
+                                                *  cert is this... */
+  }
   channel_tls_process_certs_cell(cell2, chan1);
+  mock_peer_cert_expect_tortls = NULL;
+  tor_x509_cert_free(mock_peer_cert);
+  mock_peer_cert = NULL;
+
+  tor_assert(c1->handshake_state->authenticated);
 
   tt_assert(c1->handshake_state->received_certs_cell);
-  tt_assert(c1->handshake_state->auth_cert == NULL);
-  tt_assert(c1->handshake_state->id_cert);
+  tt_ptr_op(c1->handshake_state->certs->auth_cert, OP_EQ, NULL);
+  tt_ptr_op(c1->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+  tt_assert(c1->handshake_state->certs->id_cert);
+  if (with_ed) {
+    tt_assert(c1->handshake_state->certs->ed_sign_link);
+    tt_assert(c1->handshake_state->certs->ed_rsa_crosscert);
+    tt_assert(c1->handshake_state->certs->ed_id_sign);
+    tt_assert(c1->handshake_state->authenticated_rsa);
+    tt_assert(c1->handshake_state->authenticated_ed25519);
+  } else {
+    tt_ptr_op(c1->handshake_state->certs->ed_sign_link, OP_EQ, NULL);
+    tt_ptr_op(c1->handshake_state->certs->ed_rsa_crosscert, OP_EQ, NULL);
+    tt_ptr_op(c1->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
+    tt_assert(c1->handshake_state->authenticated_rsa);
+    tt_assert(! c1->handshake_state->authenticated_ed25519);
+  }
   tt_assert(! tor_mem_is_zero(
-                  (char*)c1->handshake_state->authenticated_peer_id, 20));
+                (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
   channel_tls_common_init(chan2);
@@ -180,22 +272,38 @@ test_link_handshake_certs_ok(void *arg)
   channel_tls_process_certs_cell(cell1, chan2);
 
   tt_assert(c2->handshake_state->received_certs_cell);
-  tt_assert(c2->handshake_state->auth_cert);
-  tt_assert(c2->handshake_state->id_cert);
+  if (with_ed) {
+    tt_assert(c2->handshake_state->certs->ed_sign_auth);
+    tt_assert(c2->handshake_state->certs->ed_rsa_crosscert);
+    tt_assert(c2->handshake_state->certs->ed_id_sign);
+  } else {
+    tt_assert(c2->handshake_state->certs->auth_cert);
+    tt_ptr_op(c2->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(c2->handshake_state->certs->ed_rsa_crosscert, OP_EQ, NULL);
+    tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
+  }
+  tt_assert(c2->handshake_state->certs->id_cert);
   tt_assert(tor_mem_is_zero(
-                (char*)c2->handshake_state->authenticated_peer_id, 20));
+              (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
+  /* no authentication has happened yet, since we haen't gotten an AUTH cell.
+   */
+  tt_assert(! c2->handshake_state->authenticated);
+  tt_assert(! c2->handshake_state->authenticated_rsa);
+  tt_assert(! c2->handshake_state->authenticated_ed25519);
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
   UNMOCK(connection_or_write_var_cell_to_buf);
   UNMOCK(connection_or_send_netinfo);
+  UNMOCK(tor_tls_get_peer_cert);
   UNMOCK(tor_tls_get_own_cert);
   tor_x509_cert_free(mock_own_cert);
-  mock_own_cert = NULL;
+  tor_x509_cert_free(mock_peer_cert);
+  mock_own_cert = mock_peer_cert = NULL;
   memset(c1->identity_digest, 0, sizeof(c1->identity_digest));
   memset(c2->identity_digest, 0, sizeof(c2->identity_digest));
-  connection_free_(TO_CONN(c1));
-  connection_free_(TO_CONN(c2));
+  connection_free_minimal(TO_CONN(c1));
+  connection_free_minimal(TO_CONN(c2));
   tor_free(cell1);
   tor_free(cell2);
   certs_cell_free(cc1);
@@ -211,6 +319,8 @@ test_link_handshake_certs_ok(void *arg)
 }
 
 typedef struct certs_data_s {
+  int is_ed;
+  int is_link_cert;
   or_connection_t *c;
   channel_tls_t *chan;
   certs_cell_t *ccell;
@@ -226,18 +336,21 @@ recv_certs_cleanup(const struct testcase_t *test, void *obj)
   UNMOCK(tor_tls_cert_matches_key);
   UNMOCK(connection_or_send_netinfo);
   UNMOCK(connection_or_close_for_error);
+  UNMOCK(tor_tls_get_peer_cert);
+  UNMOCK(tor_tls_get_own_cert);
 
   if (d) {
     tor_free(d->cell);
     certs_cell_free(d->ccell);
-    connection_or_remove_from_identity_map(d->c);
-    connection_free_(TO_CONN(d->c));
+    connection_or_clear_identity(d->c);
+    connection_free_minimal(TO_CONN(d->c));
     circuitmux_free(d->chan->base_.cmux);
     tor_free(d->chan);
     crypto_pk_free(d->key1);
     crypto_pk_free(d->key2);
     tor_free(d);
   }
+  routerkeys_free_all();
   return 1;
 }
 
@@ -249,34 +362,47 @@ recv_certs_setup(const struct testcase_t *test)
   certs_cell_cert_t *ccc1 = NULL;
   certs_cell_cert_t *ccc2 = NULL;
   ssize_t n;
+  int is_ed = d->is_ed = !strcmpstart(test->setup_data, "Ed25519");
+  int is_rsa = !strcmpstart(test->setup_data, "RSA");
+  int is_link = d->is_link_cert = !strcmpend(test->setup_data, "-Link");
+  int is_auth = !strcmpend(test->setup_data, "-Auth");
+  tor_assert(is_ed != is_rsa);
+  tor_assert(is_link != is_auth);
 
   d->c = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->chan = tor_malloc_zero(sizeof(*d->chan));
   d->c->chan = d->chan;
   d->c->base_.address = tor_strdup("HaveAnAddress");
+  tor_addr_from_ipv4h(&d->c->base_.addr, 0x801f0127);
   d->c->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->chan->conn = d->c;
-  tt_int_op(connection_init_or_handshake_state(d->c, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c, 1), OP_EQ, 0);
   d->c->link_proto = 4;
 
   d->key1 = pk_generate(2);
   d->key2 = pk_generate(3);
 
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 d->key1, d->key2, 86400), ==, 0);
+                                 d->key1, d->key2, 86400), OP_EQ, 0);
+  if (is_ed) {
+    init_mock_ed_keys(d->key2);
+  } else {
+    routerkeys_free_all();
+  }
+
   d->ccell = certs_cell_new();
   ccc1 = certs_cell_cert_new();
   certs_cell_add_certs(d->ccell, ccc1);
   ccc2 = certs_cell_cert_new();
   certs_cell_add_certs(d->ccell, ccc2);
   d->ccell->n_certs = 2;
-  ccc1->cert_type = 1;
+  ccc1->cert_type = is_link ? 1 : 3;
   ccc2->cert_type = 2;
 
   const tor_x509_cert_t *a,*b;
   const uint8_t *enca, *encb;
   size_t lena, lenb;
-  tor_tls_get_my_certs(1, &a, &b);
+  tor_tls_get_my_certs(is_link ? 1 : 0, &a, &b);
   tor_x509_cert_get_der(a, &enca, &lena);
   tor_x509_cert_get_der(b, &encb, &lenb);
   certs_cell_cert_setlen_body(ccc1, lena);
@@ -287,20 +413,61 @@ recv_certs_setup(const struct testcase_t *test)
   memcpy(certs_cell_cert_getarray_body(ccc1), enca, lena);
   memcpy(certs_cell_cert_getarray_body(ccc2), encb, lenb);
 
+  if (is_ed) {
+    certs_cell_cert_t *ccc3 = NULL; /* Id->Sign */
+    certs_cell_cert_t *ccc4 = NULL; /* Sign->Link or Sign->Auth. */
+    certs_cell_cert_t *ccc5 = NULL; /* RSAId->Ed Id. */
+    const tor_cert_t *id_sign = get_master_signing_key_cert();
+    const tor_cert_t *secondary =
+      is_link ? get_current_link_cert_cert() : get_current_auth_key_cert();
+    const uint8_t *cc = NULL;
+    size_t cc_sz;
+    get_master_rsa_crosscert(&cc, &cc_sz);
+
+    ccc3 = certs_cell_cert_new();
+    ccc4 = certs_cell_cert_new();
+    ccc5 = certs_cell_cert_new();
+    certs_cell_add_certs(d->ccell, ccc3);
+    certs_cell_add_certs(d->ccell, ccc4);
+    certs_cell_add_certs(d->ccell, ccc5);
+    ccc3->cert_len = id_sign->encoded_len;
+    ccc4->cert_len = secondary->encoded_len;
+    ccc5->cert_len = cc_sz;
+    certs_cell_cert_setlen_body(ccc3, ccc3->cert_len);
+    certs_cell_cert_setlen_body(ccc4, ccc4->cert_len);
+    certs_cell_cert_setlen_body(ccc5, ccc5->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc3), id_sign->encoded,
+           ccc3->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc4), secondary->encoded,
+           ccc4->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc5), cc, ccc5->cert_len);
+    ccc3->cert_type = 4;
+    ccc4->cert_type = is_link ? 5 : 6;
+    ccc5->cert_type = 7;
+
+    d->ccell->n_certs = 5;
+  }
+
   d->cell = var_cell_new(4096);
   d->cell->command = CELL_CERTS;
 
   n = certs_cell_encode(d->cell->payload, 4096, d->ccell);
-  tt_int_op(n, >, 0);
+  tt_int_op(n, OP_GT, 0);
   d->cell->payload_len = n;
 
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
   MOCK(connection_or_close_for_error, mock_close_for_err);
+  MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);
+
+  if (is_link) {
+    /* Say that this is the peer's certificate */
+    mock_peer_cert = tor_x509_cert_dup(a);
+  }
 
-  tt_int_op(0, ==, d->c->handshake_state->received_certs_cell);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(0, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, d->c->handshake_state->received_certs_cell);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(0, OP_EQ, mock_send_netinfo_called);
 
   return d;
  done:
@@ -318,11 +485,26 @@ test_link_handshake_recv_certs_ok(void *arg)
 {
   certs_data_t *d = arg;
   channel_tls_process_certs_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(d->c->handshake_state->authenticated, ==, 1);
-  tt_int_op(d->c->handshake_state->received_certs_cell, ==, 1);
-  tt_assert(d->c->handshake_state->id_cert != NULL);
-  tt_assert(d->c->handshake_state->auth_cert == NULL);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(d->c->handshake_state->authenticated, OP_EQ, 1);
+  tt_int_op(d->c->handshake_state->authenticated_rsa, OP_EQ, 1);
+  tt_int_op(d->c->handshake_state->received_certs_cell, OP_EQ, 1);
+  tt_ptr_op(d->c->handshake_state->certs->id_cert, OP_NE, NULL);
+  tt_ptr_op(d->c->handshake_state->certs->auth_cert, OP_EQ, NULL);
+
+  if (d->is_ed) {
+    tt_ptr_op(d->c->handshake_state->certs->ed_id_sign, OP_NE, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_link, OP_NE, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_rsa_crosscert, OP_NE, NULL);
+    tt_int_op(d->c->handshake_state->authenticated_ed25519, OP_EQ, 1);
+  } else {
+    tt_ptr_op(d->c->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_link, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->ed_rsa_crosscert, OP_EQ, NULL);
+    tt_int_op(d->c->handshake_state->authenticated_ed25519, OP_EQ, 0);
+  }
 
  done:
   ;
@@ -333,17 +515,20 @@ test_link_handshake_recv_certs_ok_server(void *arg)
 {
   certs_data_t *d = arg;
   d->c->handshake_state->started_here = 0;
-  certs_cell_get_certs(d->ccell, 0)->cert_type = 3;
-  certs_cell_get_certs(d->ccell, 1)->cert_type = 2;
-  ssize_t n = certs_cell_encode(d->cell->payload, 2048, d->ccell);
-  tt_int_op(n, >, 0);
-  d->cell->payload_len = n;
+  d->c->handshake_state->certs->started_here = 0;
   channel_tls_process_certs_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(d->c->handshake_state->authenticated, ==, 0);
-  tt_int_op(d->c->handshake_state->received_certs_cell, ==, 1);
-  tt_assert(d->c->handshake_state->id_cert != NULL);
-  tt_assert(d->c->handshake_state->auth_cert != NULL);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(d->c->handshake_state->authenticated, OP_EQ, 0);
+  tt_int_op(d->c->handshake_state->received_certs_cell, OP_EQ, 1);
+  tt_ptr_op(d->c->handshake_state->certs->id_cert, OP_NE, NULL);
+  tt_ptr_op(d->c->handshake_state->certs->link_cert, OP_EQ, NULL);
+  if (d->is_ed) {
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_NE, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->auth_cert, OP_EQ, NULL);
+  } else {
+    tt_ptr_op(d->c->handshake_state->certs->ed_sign_auth, OP_EQ, NULL);
+    tt_ptr_op(d->c->handshake_state->certs->auth_cert, OP_NE, NULL);
+  }
 
  done:
   ;
@@ -358,9 +543,11 @@ test_link_handshake_recv_certs_ok_server(void *arg)
     setup_capture_of_logs(LOG_INFO);                                    \
     { code ; }                                                          \
     channel_tls_process_certs_cell(d->cell, d->chan);                   \
-    tt_int_op(1, ==, mock_close_called);                                \
-    tt_int_op(0, ==, mock_send_authenticate_called);                    \
-    tt_int_op(0, ==, mock_send_netinfo_called);                         \
+    tt_int_op(1, OP_EQ, mock_close_called);                                \
+    tt_int_op(0, OP_EQ, mock_send_authenticate_called);                    \
+    tt_int_op(0, OP_EQ, mock_send_netinfo_called);                         \
+    tt_int_op(0, OP_EQ, d->c->handshake_state->authenticated_rsa);         \
+    tt_int_op(0, OP_EQ, d->c->handshake_state->authenticated_ed25519);     \
     if (require_failure_message) {                                      \
       expect_log_msg_containing(require_failure_message);               \
     }                                                                   \
@@ -401,12 +588,41 @@ CERTS_FAIL(truncated_3,
              d->cell->payload_len = 7;
              memcpy(d->cell->payload, "\x01\x01\x00\x05""abc", 7);
            })
+CERTS_FAIL(truncated_4, /* ed25519 */
+           {
+             require_failure_message = "It couldn't be parsed";
+             d->cell->payload_len -= 10;
+           })
+CERTS_FAIL(truncated_5, /* ed25519 */
+           {
+             require_failure_message = "It couldn't be parsed";
+             d->cell->payload_len -= 100;
+           })
+
 #define REENCODE() do {                                                 \
+    const char *msg = certs_cell_check(d->ccell);                       \
+    if (msg) puts(msg);                                                 \
     ssize_t n = certs_cell_encode(d->cell->payload, 4096, d->ccell);    \
-    tt_int_op(n, >, 0);                                                 \
+    tt_int_op(n, OP_GT, 0);                                                 \
     d->cell->payload_len = n;                                           \
   } while (0)
 
+CERTS_FAIL(truncated_6, /* ed25519 */
+   {
+     /* truncate the link certificate */
+     require_failure_message = "undecodable Ed certificate";
+     certs_cell_cert_setlen_body(certs_cell_get_certs(d->ccell, 3), 7);
+     certs_cell_get_certs(d->ccell, 3)->cert_len = 7;
+     REENCODE();
+   })
+CERTS_FAIL(truncated_7, /* ed25519 */
+   {
+     /* truncate the crosscert */
+     require_failure_message = "Unparseable or overlong crosscert";
+     certs_cell_cert_setlen_body(certs_cell_get_certs(d->ccell, 4), 7);
+     certs_cell_get_certs(d->ccell, 4)->cert_len = 7;
+     REENCODE();
+   })
 CERTS_FAIL(not_x509,
   {
     require_failure_message = "Received undecodable certificate";
@@ -435,6 +651,201 @@ CERTS_FAIL(both_auth,
     certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
     REENCODE();
   })
+CERTS_FAIL(duplicate_id, /* ed25519 */
+  {
+    require_failure_message = "Duplicate Ed25519 certificate";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 4;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 4;
+    REENCODE();
+  })
+CERTS_FAIL(duplicate_link, /* ed25519 */
+  {
+    require_failure_message = "Duplicate Ed25519 certificate";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 5;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 5;
+    REENCODE();
+  })
+CERTS_FAIL(duplicate_crosscert, /* ed25519 */
+  {
+    require_failure_message = "Duplicate RSA->Ed25519 crosscert";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 7;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 7;
+    REENCODE();
+  })
+static void
+test_link_handshake_recv_certs_missing_id(void *arg) /* ed25519 */
+{
+  certs_data_t *d = arg;
+  tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+  certs_cell_set_certs(d->ccell, 2, certs_cell_get_certs(d->ccell, 4));
+  certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+  certs_cell_setlen_certs(d->ccell, 4);
+  d->ccell->n_certs = 4;
+  REENCODE();
+
+  /* This handshake succeeds, but since we have no ID cert, we will
+   * just do the RSA handshake. */
+  channel_tls_process_certs_cell(d->cell, d->chan);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(0, OP_EQ, d->c->handshake_state->authenticated_ed25519);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->authenticated_rsa);
+ done:
+  ;
+}
+CERTS_FAIL(missing_signing_key, /* ed25519 */
+  {
+    require_failure_message = "No Ed25519 signing key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 2);
+    tt_int_op(cert->cert_type, OP_EQ, CERTTYPE_ED_ID_SIGN);
+    /* replace this with a valid master->signing cert, but with no
+     * signing key. */
+    const ed25519_keypair_t *mk = get_master_identity_keypair();
+    const ed25519_keypair_t *sk = get_master_signing_keypair();
+    tor_cert_t *bad_cert = tor_cert_create(mk, CERT_TYPE_ID_SIGNING,
+                                           &sk->pubkey, time(NULL), 86400,
+                                           0 /* don't include signer */);
+    certs_cell_cert_setlen_body(cert, bad_cert->encoded_len);
+    memcpy(certs_cell_cert_getarray_body(cert),
+           bad_cert->encoded, bad_cert->encoded_len);
+    cert->cert_len = bad_cert->encoded_len;
+    tor_cert_free(bad_cert);
+    REENCODE();
+  })
+CERTS_FAIL(missing_link, /* ed25519 */
+  {
+    require_failure_message = "No Ed25519 link key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 3, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_auth, /* ed25519 */
+  {
+    d->c->handshake_state->started_here = 0;
+    d->c->handshake_state->certs->started_here = 0;
+    require_failure_message = "No Ed25519 link authentication key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 3, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_crosscert, /* ed25519 */
+  {
+    require_failure_message = "Missing RSA->Ed25519 crosscert";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_rsa_id, /* ed25519 */
+  {
+    require_failure_message = "Missing legacy RSA ID cert";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 1, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(link_mismatch, /* ed25519 */
+  {
+    require_failure_message = "Link certificate does not match "
+      "TLS certificate";
+    const tor_x509_cert_t *idc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    tor_x509_cert_free(mock_peer_cert);
+    /* Pretend that the peer cert was something else. */
+    mock_peer_cert = tor_x509_cert_dup(idc);
+    /* No reencode needed. */
+  })
+CERTS_FAIL(bad_ed_sig, /* ed25519 */
+  {
+    require_failure_message = "At least one Ed25519 certificate was "
+      "badly signed";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 3);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(bad_crosscert, /*ed25519*/
+  {
+    require_failure_message = "Invalid RSA->Ed25519 crosscert";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 4);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(bad_rsa_id_cert, /*ed25519*/
+  {
+    require_failure_message = "legacy RSA ID certificate was not valid";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 1);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(expired_rsa_id, /* both */
+  {
+    require_failure_message = "Certificate already expired";
+    /* we're going to replace the identity cert with an expired one. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 1);
+    const tor_x509_cert_t *idc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    tor_x509_cert_t *newc;
+    time_t new_end = time(NULL) - 86400 * 10;
+    newc = tor_x509_cert_replace_expiration(idc, new_end, d->key2);
+    certs_cell_cert_setlen_body(cert, newc->encoded_len);
+    memcpy(certs_cell_cert_getarray_body(cert),
+           newc->encoded, newc->encoded_len);
+    REENCODE();
+    tor_x509_cert_free(newc);
+  })
+CERTS_FAIL(expired_ed_id, /* ed25519 */
+  {
+    /* we're going to replace the Ed Id->sign cert with an expired one. */
+    require_failure_message = "At least one certificate expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 2);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [2..5].  It is in HOURS since the
+     * epoch. */
+    set_uint32(body+2, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+CERTS_FAIL(expired_ed_link, /* ed25519 */
+  {
+    /* we're going to replace the Ed Sign->link cert with an expired one. */
+    require_failure_message = "At least one certificate expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 3);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [2..5].  It is in HOURS since the
+     * epoch. */
+    set_uint32(body+2, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+CERTS_FAIL(expired_crosscert, /* ed25519 */
+  {
+    /* we're going to replace the Ed Sign->link cert with an expired one. */
+    require_failure_message = "Crosscert is expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 4);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [32..35]. once again, HOURS. */
+    set_uint32(body+32, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+
 CERTS_FAIL(wrong_labels_1,
   {
     require_failure_message = "The link certificate was not valid";
@@ -459,21 +870,26 @@ CERTS_FAIL(wrong_labels_2,
   })
 CERTS_FAIL(wrong_labels_3,
            {
-             require_failure_message = "The certs we wanted were missing";
+             require_failure_message =
+               "The certs we wanted (ID, Link) were missing";
              certs_cell_get_certs(d->ccell, 0)->cert_type = 2;
              certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
              REENCODE();
            })
 CERTS_FAIL(server_missing_certs,
            {
-             require_failure_message = "The certs we wanted were missing";
+             require_failure_message =
+               "The certs we wanted (ID, Auth) were missing";
              d->c->handshake_state->started_here = 0;
+             d->c->handshake_state->certs->started_here = 0;
+
            })
 CERTS_FAIL(server_wrong_labels_1,
            {
              require_failure_message =
                "The authentication certificate was not valid";
              d->c->handshake_state->started_here = 0;
+             d->c->handshake_state->certs->started_here = 0;
              certs_cell_get_certs(d->ccell, 0)->cert_type = 2;
              certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
              REENCODE();
@@ -487,31 +903,38 @@ test_link_handshake_send_authchallenge(void *arg)
   or_connection_t *c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   var_cell_t *cell1=NULL, *cell2=NULL;
 
+  crypto_pk_t *rsa0 = pk_generate(0), *rsa1 = pk_generate(1);
+  tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                                 rsa0, rsa1, 86400), OP_EQ, 0);
+  init_mock_ed_keys(rsa0);
+
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
 
-  tt_int_op(connection_init_or_handshake_state(c1, 0), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(c1, 0), OP_EQ, 0);
   c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
-  tt_assert(! mock_got_var_cell);
-  tt_int_op(0, ==, connection_or_send_auth_challenge_cell(c1));
+  tt_ptr_op(mock_got_var_cell, OP_EQ, NULL);
+  tt_int_op(0, OP_EQ, connection_or_send_auth_challenge_cell(c1));
   cell1 = mock_got_var_cell;
-  tt_int_op(0, ==, connection_or_send_auth_challenge_cell(c1));
+  tt_int_op(0, OP_EQ, connection_or_send_auth_challenge_cell(c1));
   cell2 = mock_got_var_cell;
-  tt_int_op(36, ==, cell1->payload_len);
-  tt_int_op(36, ==, cell2->payload_len);
-  tt_int_op(0, ==, cell1->circ_id);
-  tt_int_op(0, ==, cell2->circ_id);
-  tt_int_op(CELL_AUTH_CHALLENGE, ==, cell1->command);
-  tt_int_op(CELL_AUTH_CHALLENGE, ==, cell2->command);
+  tt_int_op(38, OP_EQ, cell1->payload_len);
+  tt_int_op(38, OP_EQ, cell2->payload_len);
+  tt_int_op(0, OP_EQ, cell1->circ_id);
+  tt_int_op(0, OP_EQ, cell2->circ_id);
+  tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command);
+  tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell2->command);
 
-  tt_mem_op("\x00\x01\x00\x01", ==, cell1->payload + 32, 4);
-  tt_mem_op("\x00\x01\x00\x01", ==, cell2->payload + 32, 4);
-  tt_mem_op(cell1->payload, !=, cell2->payload, 32);
+  tt_mem_op("\x00\x02\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 6);
+  tt_mem_op("\x00\x02\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 6);
+  tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32);
 
  done:
   UNMOCK(connection_or_write_var_cell_to_buf);
-  connection_free_(TO_CONN(c1));
+  connection_free_minimal(TO_CONN(c1));
   tor_free(cell1);
   tor_free(cell2);
+  crypto_pk_free(rsa0);
+  crypto_pk_free(rsa1);
 }
 
 typedef struct authchallenge_data_s {
@@ -532,7 +955,7 @@ recv_authchallenge_cleanup(const struct testcase_t *test, void *obj)
 
   if (d) {
     tor_free(d->cell);
-    connection_free_(TO_CONN(d->c));
+    connection_free_minimal(TO_CONN(d->c));
     circuitmux_free(d->chan->base_.cmux);
     tor_free(d->chan);
     tor_free(d);
@@ -551,14 +974,14 @@ recv_authchallenge_setup(const struct testcase_t *test)
   d->c->base_.address = tor_strdup("HaveAnAddress");
   d->c->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->chan->conn = d->c;
-  tt_int_op(connection_init_or_handshake_state(d->c, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c, 1), OP_EQ, 0);
   d->c->link_proto = 4;
   d->c->handshake_state->received_certs_cell = 1;
   d->cell = var_cell_new(128);
   d->cell->payload_len = 38;
-  d->cell->payload[33] = 2;
-  d->cell->payload[35] = 7;
-  d->cell->payload[37] = 1;
+  d->cell->payload[33] = 2; /* 2 methods */
+  d->cell->payload[35] = 7; /* This one isn't real */
+  d->cell->payload[37] = 1; /* This is the old RSA one. */
   d->cell->command = CELL_AUTH_CHALLENGE;
 
   get_options_mutable()->ORPort_set = 1;
@@ -566,10 +989,9 @@ recv_authchallenge_setup(const struct testcase_t *test)
   MOCK(connection_or_close_for_error, mock_close_for_err);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
   MOCK(connection_or_send_authenticate_cell, mock_send_authenticate);
-
-  tt_int_op(0, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(0, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(0, OP_EQ, mock_send_netinfo_called);
 
   return d;
  done:
@@ -588,10 +1010,30 @@ test_link_handshake_recv_authchallenge_ok(void *arg)
   authchallenge_data_t *d = arg;
 
   channel_tls_process_auth_challenge_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(1, ==, mock_send_authenticate_called);
-  tt_int_op(1, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(1, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(1, OP_EQ, mock_send_netinfo_called);
+  tt_int_op(1, OP_EQ, mock_send_authenticate_called_with_type); /* RSA */
+ done:
+  ;
+}
+
+static void
+test_link_handshake_recv_authchallenge_ok_ed25519(void *arg)
+{
+  authchallenge_data_t *d = arg;
+
+  /* Add the ed25519 authentication mechanism here. */
+  d->cell->payload[33] = 3; /* 3 types are supported now. */
+  d->cell->payload[39] = 3;
+  d->cell->payload_len += 2;
+  channel_tls_process_auth_challenge_cell(d->cell, d->chan);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(1, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(1, OP_EQ, mock_send_netinfo_called);
+  tt_int_op(3, OP_EQ, mock_send_authenticate_called_with_type); /* Ed25519 */
  done:
   ;
 }
@@ -603,10 +1045,10 @@ test_link_handshake_recv_authchallenge_ok_noserver(void *arg)
   get_options_mutable()->ORPort_set = 0;
 
   channel_tls_process_auth_challenge_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(0, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(0, OP_EQ, mock_send_netinfo_called);
  done:
   ;
 }
@@ -618,10 +1060,10 @@ test_link_handshake_recv_authchallenge_ok_unrecognized(void *arg)
   d->cell->payload[37] = 99;
 
   channel_tls_process_auth_challenge_cell(d->cell, d->chan);
-  tt_int_op(0, ==, mock_close_called);
-  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
-  tt_int_op(0, ==, mock_send_authenticate_called);
-  tt_int_op(1, ==, mock_send_netinfo_called);
+  tt_int_op(0, OP_EQ, mock_close_called);
+  tt_int_op(1, OP_EQ, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(0, OP_EQ, mock_send_authenticate_called);
+  tt_int_op(1, OP_EQ, mock_send_netinfo_called);
  done:
   ;
 }
@@ -635,9 +1077,9 @@ test_link_handshake_recv_authchallenge_ok_unrecognized(void *arg)
     setup_capture_of_logs(LOG_INFO);                                    \
     { code ; }                                                          \
     channel_tls_process_auth_challenge_cell(d->cell, d->chan);          \
-    tt_int_op(1, ==, mock_close_called);                                \
-    tt_int_op(0, ==, mock_send_authenticate_called);                    \
-    tt_int_op(0, ==, mock_send_netinfo_called);                         \
+    tt_int_op(1, OP_EQ, mock_close_called);                                \
+    tt_int_op(0, OP_EQ, mock_send_authenticate_called);                    \
+    tt_int_op(0, OP_EQ, mock_send_netinfo_called);                         \
     if (require_failure_message) {                                      \
       expect_log_msg_containing(require_failure_message);               \
     }                                                                   \
@@ -655,7 +1097,8 @@ AUTHCHALLENGE_FAIL(badproto,
 AUTHCHALLENGE_FAIL(as_server,
                    require_failure_message = "We didn't originate this "
                      "connection";
-                   d->c->handshake_state->started_here = 0;)
+                   d->c->handshake_state->started_here = 0;
+                   d->c->handshake_state->certs->started_here = 0;)
 AUTHCHALLENGE_FAIL(duplicate,
                    require_failure_message = "We already received one";
                    d->c->handshake_state->received_auth_challenge = 1)
@@ -673,15 +1116,6 @@ AUTHCHALLENGE_FAIL(nonzero_circid,
                    require_failure_message = "It had a nonzero circuit ID";
                    d->cell->circ_id = 1337)
 
-static tor_x509_cert_t *mock_peer_cert = NULL;
-
-static tor_x509_cert_t *
-mock_get_peer_cert(tor_tls_t *tls)
-{
-  (void)tls;
-  return tor_x509_cert_dup(mock_peer_cert);
-}
-
 static int
 mock_get_tlssecrets(tor_tls_t *tls, uint8_t *secrets_out)
 {
@@ -701,6 +1135,7 @@ mock_set_circid_type(channel_t *chan,
 }
 
 typedef struct authenticate_data_s {
+  int is_ed;
   or_connection_t *c1, *c2;
   channel_tls_t *chan2;
   var_cell_t *cell;
@@ -717,13 +1152,14 @@ authenticate_data_cleanup(const struct testcase_t *test, void *arg)
   UNMOCK(tor_tls_get_tlssecrets);
   UNMOCK(connection_or_close_for_error);
   UNMOCK(channel_set_circid_type);
+  UNMOCK(tor_tls_export_key_material);
   authenticate_data_t *d = arg;
   if (d) {
     tor_free(d->cell);
-    connection_or_remove_from_identity_map(d->c1);
-    connection_or_remove_from_identity_map(d->c2);
-    connection_free_(TO_CONN(d->c1));
-    connection_free_(TO_CONN(d->c2));
+    connection_or_clear_identity(d->c1);
+    connection_or_clear_identity(d->c2);
+    connection_free_minimal(TO_CONN(d->c1));
+    connection_free_minimal(TO_CONN(d->c2));
     circuitmux_free(d->chan2->base_.cmux);
     tor_free(d->chan2);
     crypto_pk_free(d->key1);
@@ -742,6 +1178,7 @@ static void *
 authenticate_data_setup(const struct testcase_t *test)
 {
   authenticate_data_t *d = tor_malloc_zero(sizeof(*d));
+  int is_ed = d->is_ed = (test->setup_data == (void*)3);
 
   scheduler_init();
 
@@ -751,6 +1188,7 @@ authenticate_data_setup(const struct testcase_t *test)
   MOCK(tor_tls_get_tlssecrets, mock_get_tlssecrets);
   MOCK(connection_or_close_for_error, mock_close_for_err);
   MOCK(channel_set_circid_type, mock_set_circid_type);
+  MOCK(tor_tls_export_key_material, mock_export_key_material);
   d->c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->c2 = or_connection_new(CONN_TYPE_OR, AF_INET);
   tor_addr_from_ipv4h(&d->c1->base_.addr, 0x01020304);
@@ -759,15 +1197,17 @@ authenticate_data_setup(const struct testcase_t *test)
   d->key1 = pk_generate(2);
   d->key2 = pk_generate(3);
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
-                                 d->key1, d->key2, 86400), ==, 0);
+                                 d->key1, d->key2, 86400), OP_EQ, 0);
+
+  init_mock_ed_keys(d->key2);
 
   d->c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->c1->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(d->c1, 1), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c1, 1), OP_EQ, 0);
 
   d->c2->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->c2->link_proto = 3;
-  tt_int_op(connection_init_or_handshake_state(d->c2, 0), ==, 0);
+  tt_int_op(connection_init_or_handshake_state(d->c2, 0), OP_EQ, 0);
   var_cell_t *cell = var_cell_new(16);
   cell->command = CELL_CERTS;
   or_handshake_state_record_var_cell(d->c1, d->c1->handshake_state, cell, 1);
@@ -791,21 +1231,37 @@ authenticate_data_setup(const struct testcase_t *test)
   const uint8_t *der;
   size_t sz;
   tor_x509_cert_get_der(id_cert, &der, &sz);
-  d->c1->handshake_state->id_cert = tor_x509_cert_decode(der, sz);
-  d->c2->handshake_state->id_cert = tor_x509_cert_decode(der, sz);
+  d->c1->handshake_state->certs->id_cert = tor_x509_cert_decode(der, sz);
+  d->c2->handshake_state->certs->id_cert = tor_x509_cert_decode(der, sz);
+
+  if (is_ed) {
+    d->c1->handshake_state->certs->ed_id_sign =
+      tor_cert_dup(get_master_signing_key_cert());
+    d->c2->handshake_state->certs->ed_id_sign =
+      tor_cert_dup(get_master_signing_key_cert());
+    d->c2->handshake_state->certs->ed_sign_auth =
+      tor_cert_dup(get_current_auth_key_cert());
+  } else {
+    tt_assert(! tor_tls_get_my_certs(0, &auth_cert, &id_cert));
+    tor_x509_cert_get_der(auth_cert, &der, &sz);
+    d->c2->handshake_state->certs->auth_cert = tor_x509_cert_decode(der, sz);
+  }
 
   tor_x509_cert_get_der(link_cert, &der, &sz);
   mock_peer_cert = tor_x509_cert_decode(der, sz);
   tt_assert(mock_peer_cert);
+
   mock_own_cert = tor_x509_cert_decode(der, sz);
   tt_assert(mock_own_cert);
-  tt_assert(! tor_tls_get_my_certs(0, &auth_cert, &id_cert));
-  tor_x509_cert_get_der(auth_cert, &der, &sz);
-  d->c2->handshake_state->auth_cert = tor_x509_cert_decode(der, sz);
 
   /* Make an authenticate cell ... */
-  tt_int_op(0, ==, connection_or_send_authenticate_cell(d->c1,
-                                             AUTHTYPE_RSA_SHA256_TLSSECRET));
+  int authtype;
+  if (is_ed)
+    authtype = AUTHTYPE_ED25519_SHA256_RFC5705;
+  else
+    authtype = AUTHTYPE_RSA_SHA256_TLSSECRET;
+  tt_int_op(0, OP_EQ, connection_or_send_authenticate_cell(d->c1, authtype));
+
   tt_assert(mock_got_var_cell);
   d->cell = mock_got_var_cell;
   mock_got_var_cell = NULL;
@@ -829,44 +1285,66 @@ test_link_handshake_auth_cell(void *arg)
   crypto_pk_t *auth_pubkey = NULL;
 
   /* Is the cell well-formed on the outer layer? */
-  tt_int_op(d->cell->command, ==, CELL_AUTHENTICATE);
-  tt_int_op(d->cell->payload[0], ==, 0);
-  tt_int_op(d->cell->payload[1], ==, 1);
-  tt_int_op(ntohs(get_uint16(d->cell->payload + 2)), ==,
+  tt_int_op(d->cell->command, OP_EQ, CELL_AUTHENTICATE);
+  tt_int_op(d->cell->payload[0], OP_EQ, 0);
+  if (d->is_ed)
+    tt_int_op(d->cell->payload[1], OP_EQ, 3);
+  else
+    tt_int_op(d->cell->payload[1], OP_EQ, 1);
+  tt_int_op(ntohs(get_uint16(d->cell->payload + 2)), OP_EQ,
             d->cell->payload_len - 4);
 
   /* Check it out for plausibility... */
   auth_ctx_t ctx;
-  ctx.is_ed = 0;
-  tt_int_op(d->cell->payload_len-4, ==, auth1_parse(&auth1,
+  ctx.is_ed = d->is_ed;
+  tt_int_op(d->cell->payload_len-4, OP_EQ, auth1_parse(&auth1,
                                              d->cell->payload+4,
                                              d->cell->payload_len - 4, &ctx));
   tt_assert(auth1);
 
-  tt_mem_op(auth1->type, ==, "AUTH0001", 8);
-  tt_mem_op(auth1->tlssecrets, ==, "int getRandomNumber(){return 4;}", 32);
-  tt_int_op(auth1_getlen_sig(auth1), >, 120);
+  if (d->is_ed) {
+    tt_mem_op(auth1->type, OP_EQ, "AUTH0003", 8);
+  } else {
+    tt_mem_op(auth1->type, OP_EQ, "AUTH0001", 8);
+  }
+  tt_mem_op(auth1->tlssecrets, OP_EQ, "int getRandomNumber(){return 4;}", 32);
 
   /* Is the signature okay? */
-  uint8_t sig[128];
-  uint8_t digest[32];
-
-  auth_pubkey = tor_tls_cert_get_key(d->c2->handshake_state->auth_cert);
-  int n = crypto_pk_public_checksig(
+  const uint8_t *start = d->cell->payload+4, *end = auth1->end_of_signed;
+  if (d->is_ed) {
+    ed25519_signature_t sig;
+    tt_int_op(auth1_getlen_sig(auth1), OP_EQ, ED25519_SIG_LEN);
+    memcpy(&sig.sig, auth1_getarray_sig(auth1), ED25519_SIG_LEN);
+    tt_assert(!ed25519_checksig(&sig, start, end-start,
+                                &get_current_auth_keypair()->pubkey));
+  } else {
+    uint8_t sig[128];
+    uint8_t digest[32];
+    tt_int_op(auth1_getlen_sig(auth1), OP_GT, 120);
+    auth_pubkey = tor_tls_cert_get_key(
+                                d->c2->handshake_state->certs->auth_cert);
+    int n = crypto_pk_public_checksig(
               auth_pubkey,
               (char*)sig, sizeof(sig), (char*)auth1_getarray_sig(auth1),
               auth1_getlen_sig(auth1));
-  tt_int_op(n, ==, 32);
-  const uint8_t *start = d->cell->payload+4, *end = auth1->end_of_signed;
-  crypto_digest256((char*)digest,
-                   (const char*)start, end-start, DIGEST_SHA256);
-  tt_mem_op(sig, ==, digest, 32);
+    tt_int_op(n, OP_EQ, 32);
+    crypto_digest256((char*)digest,
+                     (const char*)start, end-start, DIGEST_SHA256);
+    tt_mem_op(sig, OP_EQ, digest, 32);
+  }
 
   /* Then feed it to c2. */
-  tt_int_op(d->c2->handshake_state->authenticated, ==, 0);
+  tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 0);
   channel_tls_process_authenticate_cell(d->cell, d->chan2);
-  tt_int_op(mock_close_called, ==, 0);
-  tt_int_op(d->c2->handshake_state->authenticated, ==, 1);
+  tt_int_op(mock_close_called, OP_EQ, 0);
+  tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 1);
+  if (d->is_ed) {
+    tt_int_op(d->c2->handshake_state->authenticated_ed25519, OP_EQ, 1);
+    tt_int_op(d->c2->handshake_state->authenticated_rsa, OP_EQ, 1);
+  } else {
+    tt_int_op(d->c2->handshake_state->authenticated_ed25519, OP_EQ, 0);
+    tt_int_op(d->c2->handshake_state->authenticated_rsa, OP_EQ, 1);
+  }
 
  done:
   auth1_free(auth1);
@@ -881,10 +1359,10 @@ test_link_handshake_auth_cell(void *arg)
     const char *require_failure_message = NULL;                 \
     setup_capture_of_logs(LOG_INFO);                            \
     { code ; }                                                  \
-    tt_int_op(d->c2->handshake_state->authenticated, ==, 0);    \
+    tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 0);    \
     channel_tls_process_authenticate_cell(d->cell, d->chan2);   \
-    tt_int_op(mock_close_called, ==, 1);                        \
-    tt_int_op(d->c2->handshake_state->authenticated, ==, 0);    \
+    tt_int_op(mock_close_called, OP_EQ, 1);                        \
+    tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 0);    \
     if (require_failure_message) {                              \
       expect_log_msg_containing(require_failure_message);       \
     }                                                           \
@@ -900,7 +1378,8 @@ AUTHENTICATE_FAIL(badproto,
                   d->c2->link_proto = 2)
 AUTHENTICATE_FAIL(atclient,
                   require_failure_message = "We originated this connection";
-                  d->c2->handshake_state->started_here = 1)
+                  d->c2->handshake_state->started_here = 1;
+                  d->c2->handshake_state->certs->started_here = 1;)
 AUTHENTICATE_FAIL(duplicate,
                   require_failure_message = "We already got one";
                   d->c2->handshake_state->received_authenticate = 1)
@@ -911,8 +1390,8 @@ test_link_handshake_auth_already_authenticated(void *arg)
   setup_capture_of_logs(LOG_INFO);
   d->c2->handshake_state->authenticated = 1;
   channel_tls_process_authenticate_cell(d->cell, d->chan2);
-  tt_int_op(mock_close_called, ==, 1);
-  tt_int_op(d->c2->handshake_state->authenticated, ==, 1);
+  tt_int_op(mock_close_called, OP_EQ, 1);
+  tt_int_op(d->c2->handshake_state->authenticated, OP_EQ, 1);
   expect_log_msg_containing("The peer is already authenticated");
  done:
   teardown_capture_of_logs();
@@ -924,13 +1403,13 @@ AUTHENTICATE_FAIL(nocerts,
 AUTHENTICATE_FAIL(noidcert,
                   require_failure_message = "We never got an identity "
                     "certificate";
-                  tor_x509_cert_free(d->c2->handshake_state->id_cert);
-                  d->c2->handshake_state->id_cert = NULL)
+                  tor_x509_cert_free(d->c2->handshake_state->certs->id_cert);
+                  d->c2->handshake_state->certs->id_cert = NULL)
 AUTHENTICATE_FAIL(noauthcert,
-                  require_failure_message = "We never got an authentication "
-                    "certificate";
-                  tor_x509_cert_free(d->c2->handshake_state->auth_cert);
-                  d->c2->handshake_state->auth_cert = NULL)
+                  require_failure_message = "We never got an RSA "
+                    "authentication certificate";
+                  tor_x509_cert_free(d->c2->handshake_state->certs->auth_cert);
+                  d->c2->handshake_state->certs->auth_cert = NULL)
 AUTHENTICATE_FAIL(tooshort,
                   require_failure_message = "Cell was way too short";
                   d->cell->payload_len = 3)
@@ -946,7 +1425,7 @@ AUTHENTICATE_FAIL(truncated_2,
                   d->cell->payload[3]++)
 AUTHENTICATE_FAIL(tooshort_1,
                   require_failure_message = "Authenticator was too short";
-                  tt_int_op(d->cell->payload_len, >=, 260);
+                  tt_int_op(d->cell->payload_len, OP_GE, 260);
                   d->cell->payload[2] -= 1;
                   d->cell->payload_len -= 256;)
 AUTHENTICATE_FAIL(badcontent,
@@ -954,11 +1433,33 @@ AUTHENTICATE_FAIL(badcontent,
                     "cell body was not as expected";
                   d->cell->payload[10] ^= 0xff)
 AUTHENTICATE_FAIL(badsig_1,
-                  require_failure_message = "Signature wasn't valid";
+                  if (d->is_ed)
+                    require_failure_message = "Ed25519 signature wasn't valid";
+                  else
+                    require_failure_message = "RSA signature wasn't valid";
                   d->cell->payload[d->cell->payload_len - 5] ^= 0xff)
-
-#define TEST(name, flags)                                       \
-  { #name , test_link_handshake_ ## name, (flags), NULL, NULL }
+AUTHENTICATE_FAIL(missing_ed_id,
+                {
+                  tor_cert_free(d->c2->handshake_state->certs->ed_id_sign);
+                  d->c2->handshake_state->certs->ed_id_sign = NULL;
+                  require_failure_message = "Ed authenticate without Ed ID "
+                    "cert from peer";
+                })
+AUTHENTICATE_FAIL(missing_ed_auth,
+                {
+                  tor_cert_free(d->c2->handshake_state->certs->ed_sign_auth);
+                  d->c2->handshake_state->certs->ed_sign_auth = NULL;
+                  require_failure_message = "We never got an Ed25519 "
+                    "authentication certificate";
+                })
+
+#define TEST_RSA(name, flags)                                           \
+  { #name , test_link_handshake_ ## name, (flags),                      \
+      &passthrough_setup, (void*)"RSA" }
+
+#define TEST_ED(name, flags)                                            \
+  { #name "_ed25519" , test_link_handshake_ ## name, (flags),           \
+      &passthrough_setup, (void*)"Ed25519" }
 
 #define TEST_RCV_AUTHCHALLENGE(name)                            \
   { "recv_authchallenge/" #name ,                               \
@@ -968,17 +1469,34 @@ AUTHENTICATE_FAIL(badsig_1,
 #define TEST_RCV_CERTS(name)                                    \
   { "recv_certs/" #name ,                                       \
       test_link_handshake_recv_certs_ ## name, TT_FORK,         \
-      &setup_recv_certs, NULL }
+      &setup_recv_certs, (void*)"RSA-Link" }
+
+#define TEST_RCV_CERTS_RSA(name,type)                           \
+  { "recv_certs/" #name ,                                       \
+      test_link_handshake_recv_certs_ ## name, TT_FORK,         \
+      &setup_recv_certs, (void*)type }
+
+#define TEST_RCV_CERTS_ED(name, type)                           \
+  { "recv_certs/" #name "_ed25519",                             \
+      test_link_handshake_recv_certs_ ## name, TT_FORK,         \
+      &setup_recv_certs, (void*)type }
 
 #define TEST_AUTHENTICATE(name)                                         \
   { "authenticate/" #name , test_link_handshake_auth_ ## name, TT_FORK, \
       &setup_authenticate, NULL }
 
+#define TEST_AUTHENTICATE_ED(name)                                      \
+  { "authenticate/" #name "_ed25519" , test_link_handshake_auth_ ## name, \
+      TT_FORK, &setup_authenticate, (void*)3 }
+
 struct testcase_t link_handshake_tests[] = {
-  TEST(certs_ok, TT_FORK),
-  //TEST(certs_bad, TT_FORK),
+  TEST_RSA(certs_ok, TT_FORK),
+  TEST_ED(certs_ok, TT_FORK),
+
   TEST_RCV_CERTS(ok),
-  TEST_RCV_CERTS(ok_server),
+  TEST_RCV_CERTS_ED(ok, "Ed25519-Link"),
+  TEST_RCV_CERTS_RSA(ok_server, "RSA-Auth"),
+  TEST_RCV_CERTS_ED(ok_server, "Ed25519-Auth"),
   TEST_RCV_CERTS(badstate),
   TEST_RCV_CERTS(badproto),
   TEST_RCV_CERTS(duplicate),
@@ -988,18 +1506,41 @@ struct testcase_t link_handshake_tests[] = {
   TEST_RCV_CERTS(truncated_1),
   TEST_RCV_CERTS(truncated_2),
   TEST_RCV_CERTS(truncated_3),
+  TEST_RCV_CERTS_ED(truncated_4, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_5, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_6, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_7, "Ed25519-Link"),
   TEST_RCV_CERTS(not_x509),
   TEST_RCV_CERTS(both_link),
   TEST_RCV_CERTS(both_id_rsa),
   TEST_RCV_CERTS(both_auth),
+  TEST_RCV_CERTS_ED(duplicate_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(duplicate_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(duplicate_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_signing_key, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_auth, "Ed25519-Auth"),
+  TEST_RCV_CERTS_ED(missing_rsa_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(link_mismatch, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_ed_sig, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_rsa_id_cert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_RSA(expired_rsa_id, "RSA-Link"),
+  TEST_RCV_CERTS_ED(expired_rsa_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_ed_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_ed_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_crosscert, "Ed25519-Link"),
   TEST_RCV_CERTS(wrong_labels_1),
   TEST_RCV_CERTS(wrong_labels_2),
   TEST_RCV_CERTS(wrong_labels_3),
   TEST_RCV_CERTS(server_missing_certs),
   TEST_RCV_CERTS(server_wrong_labels_1),
 
-  TEST(send_authchallenge, TT_FORK),
+  TEST_RSA(send_authchallenge, TT_FORK),
   TEST_RCV_AUTHCHALLENGE(ok),
+  TEST_RCV_AUTHCHALLENGE(ok_ed25519),
   TEST_RCV_AUTHCHALLENGE(ok_noserver),
   TEST_RCV_AUTHCHALLENGE(ok_unrecognized),
   TEST_RCV_AUTHCHALLENGE(badstate),
@@ -1012,6 +1553,7 @@ struct testcase_t link_handshake_tests[] = {
   TEST_RCV_AUTHCHALLENGE(nonzero_circid),
 
   TEST_AUTHENTICATE(cell),
+  TEST_AUTHENTICATE_ED(cell),
   TEST_AUTHENTICATE(badstate),
   TEST_AUTHENTICATE(badproto),
   TEST_AUTHENTICATE(atclient),
@@ -1027,6 +1569,9 @@ struct testcase_t link_handshake_tests[] = {
   TEST_AUTHENTICATE(tooshort_1),
   TEST_AUTHENTICATE(badcontent),
   TEST_AUTHENTICATE(badsig_1),
+  TEST_AUTHENTICATE_ED(badsig_1),
+  TEST_AUTHENTICATE_ED(missing_ed_id),
+  TEST_AUTHENTICATE_ED(missing_ed_auth),
   //TEST_AUTHENTICATE(),
 
   END_OF_TESTCASES
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 15471e46d0..e373158e34 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -107,7 +107,7 @@ test_sigsafe_err(void *arg)
   close(STDERR_FILENO);
   content = read_file_to_str(fn, 0, NULL);
 
-  tt_assert(content != NULL);
+  tt_ptr_op(content, OP_NE, NULL);
   tor_split_lines(lines, content, (int)strlen(content));
   tt_int_op(smartlist_len(lines), OP_GE, 5);
 
@@ -140,7 +140,7 @@ test_ratelim(void *arg)
   char *msg = NULL;
 
   msg = rate_limit_log(&ten_min, now);
-  tt_assert(msg != NULL);
+  tt_ptr_op(msg, OP_NE, NULL);
   tt_str_op(msg, OP_EQ, ""); /* nothing was suppressed. */
 
   tt_int_op(ten_min.last_allowed, OP_EQ, now);
@@ -150,14 +150,14 @@ test_ratelim(void *arg)
   for (i = 0; i < 9; ++i) {
     now += 60; /* one minute has passed. */
     msg = rate_limit_log(&ten_min, now);
-    tt_assert(msg == NULL);
+    tt_ptr_op(msg, OP_EQ, NULL);
     tt_int_op(ten_min.last_allowed, OP_EQ, start);
     tt_int_op(ten_min.n_calls_since_last_time, OP_EQ, i + 1);
   }
 
   now += 240; /* Okay, we can be done. */
   msg = rate_limit_log(&ten_min, now);
-  tt_assert(msg != NULL);
+  tt_ptr_op(msg, OP_NE, NULL);
   tt_str_op(msg, OP_EQ,
             " [9 similar message(s) suppressed in last 600 seconds]");
  done:
diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c
new file mode 100644
index 0000000000..9da8a039dd
--- /dev/null
+++ b/src/test/test_mainloop.c
@@ -0,0 +1,142 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_mainloop.c
+ * \brief Tests for functions closely related to the Tor main loop
+ */
+
+#include "test.h"
+#include "log_test_helpers.h"
+
+#include "or.h"
+#include "main.h"
+
+static const uint64_t BILLION = 1000000000;
+
+static void
+test_mainloop_update_time_normal(void *arg)
+{
+  (void)arg;
+
+  monotime_enable_test_mocking();
+  /* This is arbitrary */
+  uint64_t mt_now = U64_LITERAL(7493289274986);
+  /* This time is in the past as of when this test was written. */
+  time_t now = 1525272090;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  reset_uptime();
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 0);
+
+  update_current_time(now); // Same time as before is a no-op.
+  tt_int_op(get_uptime(), OP_EQ, 0);
+
+  now += 1;
+  mt_now += BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 1);
+
+  now += 2; // two-second jump is unremarkable.
+  mt_now += 2*BILLION;
+  update_current_time(now);
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3);
+
+  now -= 1; // a one-second hop backwards is also unremarkable.
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now); // it changes the approx time...
+  tt_int_op(get_uptime(), OP_EQ, 3); // but it doesn't roll back our uptime
+
+ done:
+  monotime_disable_test_mocking();
+}
+
+static void
+test_mainloop_update_time_jumps(void *arg)
+{
+  (void)arg;
+
+  monotime_enable_test_mocking();
+  /* This is arbitrary */
+  uint64_t mt_now = U64_LITERAL(7493289274986);
+  /* This time is in the past as of when this test was written. */
+  time_t now = 220897152;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  reset_uptime();
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 0);
+
+  /* Put some uptime on the clock.. */
+  now += 3;
+  mt_now += 3*BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3);
+
+  /* Now try jumping forward and backward, without updating the monotonic
+   * clock.  */
+  setup_capture_of_logs(LOG_NOTICE);
+  now += 1800;
+  update_current_time(now);
+  expect_single_log_msg_containing(
+               "Your system clock just jumped 1800 seconds forward");
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change.
+  mock_clean_saved_logs();
+
+  now -= 600;
+  update_current_time(now);
+  expect_single_log_msg_containing(
+               "Your system clock just jumped 600 seconds backward");
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change.
+  mock_clean_saved_logs();
+
+  /* uptime tracking should go normally now if the clock moves sensibly. */
+  now += 2;
+  mt_now += 2*BILLION;
+  update_current_time(now);
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 5);
+
+  /* If we skip forward by a few minutes but the monotonic clock agrees,
+   * we've just been idle: that counts as not worth warning about. */
+  now += 1800;
+  mt_now += 1800*BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  expect_no_log_entry();
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 5); // this doesn't count to uptime, though.
+
+  /* If we skip forward by a long time, even if the clock agrees, it's
+   * idnless that counts. */
+  now += 4000;
+  mt_now += 4000*BILLION;
+  monotime_coarse_set_mock_time_nsec(mt_now);
+  update_current_time(now);
+  expect_single_log_msg_containing("Tor has been idle for 4000 seconds");
+  tt_int_op(approx_time(), OP_EQ, now);
+  tt_int_op(get_uptime(), OP_EQ, 5);
+
+ done:
+  teardown_capture_of_logs();
+  monotime_disable_test_mocking();
+}
+
+#define MAINLOOP_TEST(name) \
+  { #name, test_mainloop_## name , TT_FORK, NULL, NULL }
+
+struct testcase_t mainloop_tests[] = {
+  MAINLOOP_TEST(update_time_normal),
+  MAINLOOP_TEST(update_time_jumps),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 2ae605b8db..4b168f49ed 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -1,11 +1,12 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include "or.h"
 
 #include "config.h"
-#include "dirvote.h"
+#define DIRVOTE_PRIVATE
+#include "dirauth/dirvote.h"
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "routerlist.h"
@@ -14,18 +15,12 @@
 
 #include "test.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
-#include <openssl/rsa.h>
-#include <openssl/bn.h>
-#include <openssl/pem.h>
-ENABLE_GCC_WARNING(redundant-decls)
-
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
+#endif /* defined(_WIN32) */
 
 static const char test_md1[] =
   "onion-key\n"
@@ -75,12 +70,12 @@ test_md_cache(void *data)
   time3 = time(NULL) - 15*24*60*60;
 
   /* Possibly, turn this into a test setup/cleanup pair */
-  tor_free(options->DataDirectory);
-  options->DataDirectory = tor_strdup(get_fname("md_datadir_test"));
+  tor_free(options->CacheDirectory);
+  options->CacheDirectory = tor_strdup(get_fname("md_datadir_test"));
 #ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory));
 #else
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory, 0700));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory, 0700));
 #endif
 
   tt_assert(!strcmpstart(test_md3_noannotation, "onion-key"));
@@ -158,7 +153,7 @@ test_md_cache(void *data)
               strlen(test_md3_noannotation));
 
   tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs.new",
-               options->DataDirectory);
+               options->CacheDirectory);
   s = read_file_to_str(fn, RFTS_BIN, NULL);
   tt_assert(s);
   tt_mem_op(md1->body, OP_EQ, s + md1->off, md1->bodylen);
@@ -186,7 +181,7 @@ test_md_cache(void *data)
 
   /* read the cache. */
   tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs",
-               options->DataDirectory);
+               options->CacheDirectory);
   s = read_file_to_str(fn, RFTS_BIN, NULL);
   tt_mem_op(md1->body, OP_EQ, s + md1->off, strlen(test_md1));
   tt_mem_op(md2->body, OP_EQ, s + md2->off, strlen(test_md2));
@@ -240,7 +235,7 @@ test_md_cache(void *data)
 
  done:
   if (options)
-    tor_free(options->DataDirectory);
+    tor_free(options->CacheDirectory);
   microdesc_free_all();
 
   smartlist_free(added);
@@ -272,17 +267,17 @@ test_md_cache_broken(void *data)
 
   options = get_options_mutable();
   tt_assert(options);
-  tor_free(options->DataDirectory);
-  options->DataDirectory = tor_strdup(get_fname("md_datadir_test2"));
+  tor_free(options->CacheDirectory);
+  options->CacheDirectory = tor_strdup(get_fname("md_datadir_test2"));
 
 #ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory));
 #else
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory, 0700));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory, 0700));
 #endif
 
   tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-microdescs",
-               options->DataDirectory);
+               options->CacheDirectory);
 
   write_str_to_file(fn, truncated_md, 1);
 
@@ -291,7 +286,7 @@ test_md_cache_broken(void *data)
 
  done:
   if (options)
-    tor_free(options->DataDirectory);
+    tor_free(options->CacheDirectory);
   tor_free(fn);
   microdesc_free_all();
 }
@@ -391,25 +386,6 @@ static const char test_ri2[] =
   "cf34GXHv61XReJF3AlzNHFpbrPOYmowmhrTULKyMqow=\n"
   "-----END SIGNATURE-----\n";
 
-static const char test_md_8[] =
-  "onion-key\n"
-  "-----BEGIN RSA PUBLIC KEY-----\n"
-  "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n"
-  "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n"
-  "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n"
-  "-----END RSA PUBLIC KEY-----\n"
-  "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n";
-
-static const char test_md_16[] =
-  "onion-key\n"
-  "-----BEGIN RSA PUBLIC KEY-----\n"
-  "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n"
-  "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n"
-  "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n"
-  "-----END RSA PUBLIC KEY-----\n"
-  "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n"
-  "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n";
-
 static const char test_md_18[] =
   "onion-key\n"
   "-----BEGIN RSA PUBLIC KEY-----\n"
@@ -421,16 +397,6 @@ static const char test_md_18[] =
   "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"
   "id rsa1024 Cd47okjCHD83YGzThGBDptXs9Z4\n";
 
-static const char test_md2_18[] =
-  "onion-key\n"
-  "-----BEGIN RSA PUBLIC KEY-----\n"
-  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
-  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
-  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
-  "-----END RSA PUBLIC KEY-----\n"
-  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
-  "id rsa1024 t+J/EEITw28T5+mCkYKEXklZl6A\n";
-
 static const char test_md2_21[] =
   "onion-key\n"
   "-----BEGIN RSA PUBLIC KEY-----\n"
@@ -450,17 +416,6 @@ test_md_generate(void *arg)
 
   ri = router_parse_entry_from_string(test_ri, NULL, 0, 0, NULL, NULL);
   tt_assert(ri);
-  md = dirvote_create_microdescriptor(ri, 8);
-  tt_str_op(md->body, OP_EQ, test_md_8);
-
-  /* XXXX test family lines. */
-  /* XXXX test method 14 for A lines. */
-  /* XXXX test method 15 for P6 lines. */
-
-  microdesc_free(md);
-  md = NULL;
-  md = dirvote_create_microdescriptor(ri, 16);
-  tt_str_op(md->body, OP_EQ, test_md_16);
 
   microdesc_free(md);
   md = NULL;
@@ -470,20 +425,15 @@ test_md_generate(void *arg)
   microdesc_free(md);
   md = NULL;
   md = dirvote_create_microdescriptor(ri, 21);
-  tt_str_op(md->body, ==, test_md_18);
+  tt_str_op(md->body, OP_EQ, test_md_18);
 
   routerinfo_free(ri);
   ri = router_parse_entry_from_string(test_ri2, NULL, 0, 0, NULL, NULL);
 
   microdesc_free(md);
   md = NULL;
-  md = dirvote_create_microdescriptor(ri, 18);
-  tt_str_op(md->body, ==, test_md2_18);
-
-  microdesc_free(md);
-  md = NULL;
   md = dirvote_create_microdescriptor(ri, 21);
-  tt_str_op(md->body, ==, test_md2_21);
+  tt_str_op(md->body, OP_EQ, test_md2_21);
   tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey,
                               &ri->cache_info.signing_key_cert->signing_key));
 
@@ -760,8 +710,8 @@ test_md_reject_cache(void *arg)
   or_options_t *options = get_options_mutable();
   char buf[DIGEST256_LEN];
 
-  tor_free(options->DataDirectory);
-  options->DataDirectory = tor_strdup(get_fname("md_datadir_test_rej"));
+  tor_free(options->CacheDirectory);
+  options->CacheDirectory = tor_strdup(get_fname("md_datadir_test_rej"));
   mock_rgsbd_val_a = tor_malloc_zero(sizeof(routerstatus_t));
   mock_rgsbd_val_b = tor_malloc_zero(sizeof(routerstatus_t));
   mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
@@ -771,9 +721,9 @@ test_md_reject_cache(void *arg)
   mock_ns_val->flavor = FLAV_MICRODESC;
 
 #ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory));
 #else
-  tt_int_op(0, OP_EQ, mkdir(options->DataDirectory, 0700));
+  tt_int_op(0, OP_EQ, mkdir(options->CacheDirectory, 0700));
 #endif
 
   MOCK(router_get_mutable_consensus_status_by_descriptor_digest,
@@ -808,7 +758,7 @@ test_md_reject_cache(void *arg)
  done:
   UNMOCK(networkstatus_get_latest_consensus_by_flavor);
   UNMOCK(router_get_mutable_consensus_status_by_descriptor_digest);
-  tor_free(options->DataDirectory);
+  tor_free(options->CacheDirectory);
   microdesc_free_all();
   smartlist_free(added);
   SMARTLIST_FOREACH(wanted, char *, cp, tor_free(cp));
@@ -829,14 +779,14 @@ test_md_corrupt_desc(void *arg)
                                "@last-listed 2015-06-22 10:00:00\n"
                                "onion-k\n",
                                NULL, SAVED_IN_JOURNAL, 0, time(NULL), NULL);
-  tt_int_op(smartlist_len(sl), ==, 0);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
   smartlist_free(sl);
 
   sl = microdescs_add_to_cache(get_microdesc_cache(),
                                "@last-listed 2015-06-22 10:00:00\n"
                                "wiggly\n",
                                NULL, SAVED_IN_JOURNAL, 0, time(NULL), NULL);
-  tt_int_op(smartlist_len(sl), ==, 0);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
   smartlist_free(sl);
 
   tor_asprintf(&cp, "%s\n%s", test_md1, "@foobar\nonion-wobble\n");
@@ -844,7 +794,7 @@ test_md_corrupt_desc(void *arg)
   sl = microdescs_add_to_cache(get_microdesc_cache(),
                                cp, cp+strlen(cp),
                                SAVED_IN_JOURNAL, 0, time(NULL), NULL);
-  tt_int_op(smartlist_len(sl), ==, 0);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
 
  done:
   tor_free(cp);
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index d58f8a7fca..7d8e57543c 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,8 +7,12 @@
  **/
 
 #include "or.h"
+#include "crypto_rand.h"
+#include "networkstatus.h"
 #include "nodelist.h"
+#include "torcert.h"
 #include "test.h"
+#include "log_test_helpers.h"
 
 /** Test the case when node_get_by_id() returns NULL,
  * node_get_verbose_nickname_by_id should return the base 16 encoding
@@ -100,6 +104,126 @@ test_nodelist_node_is_dir(void *arg)
   return;
 }
 
+static networkstatus_t *dummy_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return dummy_ns;
+}
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  tor_assert(f == FLAV_MICRODESC);
+  return dummy_ns;
+}
+
+static void
+test_nodelist_ed_id(void *arg)
+{
+#define N_NODES 5
+  routerstatus_t *rs[N_NODES];
+  microdesc_t *md[N_NODES];
+  routerinfo_t *ri[N_NODES];
+  networkstatus_t *ns;
+  int i;
+  (void)arg;
+
+  ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->flavor = FLAV_MICRODESC;
+  ns->routerstatus_list = smartlist_new();
+  dummy_ns = ns;
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+
+  /* Make a bunch of dummy objects that we can play around with.  Only set the
+     necessary fields */
+
+  for (i = 0; i < N_NODES; ++i) {
+    rs[i] = tor_malloc_zero(sizeof(*rs[i]));
+    md[i] = tor_malloc_zero(sizeof(*md[i]));
+    ri[i] = tor_malloc_zero(sizeof(*ri[i]));
+
+    crypto_rand(md[i]->digest, sizeof(md[i]->digest));
+    md[i]->ed25519_identity_pkey = tor_malloc(sizeof(ed25519_public_key_t));
+    crypto_rand((char*)md[i]->ed25519_identity_pkey,
+                sizeof(ed25519_public_key_t));
+    crypto_rand(rs[i]->identity_digest, sizeof(rs[i]->identity_digest));
+    memcpy(ri[i]->cache_info.identity_digest, rs[i]->identity_digest,
+           DIGEST_LEN);
+    memcpy(rs[i]->descriptor_digest, md[i]->digest, DIGEST256_LEN);
+    ri[i]->cache_info.signing_key_cert = tor_malloc_zero(sizeof(tor_cert_t));
+    memcpy(&ri[i]->cache_info.signing_key_cert->signing_key,
+           md[i]->ed25519_identity_pkey, sizeof(ed25519_public_key_t));
+
+    if (i < 3)
+      smartlist_add(ns->routerstatus_list, rs[i]);
+  }
+
+  tt_int_op(0, OP_EQ, smartlist_len(nodelist_get_list()));
+
+  nodelist_set_consensus(ns);
+
+  tt_int_op(3, OP_EQ, smartlist_len(nodelist_get_list()));
+
+  /* No Ed25519 info yet, so nothing has an ED id. */
+  tt_ptr_op(NULL, OP_EQ, node_get_by_ed25519_id(md[0]->ed25519_identity_pkey));
+
+  /* Register the first one by md, then look it up. */
+  node_t *n = nodelist_add_microdesc(md[0]);
+  tt_ptr_op(n, OP_EQ, node_get_by_ed25519_id(md[0]->ed25519_identity_pkey));
+
+  /* Register the second by ri, then look it up. */
+  routerinfo_t *ri_old = NULL;
+  n = nodelist_set_routerinfo(ri[1], &ri_old);
+  tt_ptr_op(n, OP_EQ, node_get_by_ed25519_id(md[1]->ed25519_identity_pkey));
+  tt_ptr_op(ri_old, OP_EQ, NULL);
+
+  /* Register it by md too. */
+  node_t *n2 = nodelist_add_microdesc(md[1]);
+  tt_ptr_op(n2, OP_EQ, n);
+  tt_ptr_op(n, OP_EQ, node_get_by_ed25519_id(md[1]->ed25519_identity_pkey));
+
+  /* Register the 4th by ri only -- we never put it into the networkstatus,
+   * so it has to be independent */
+  node_t *n3 = nodelist_set_routerinfo(ri[3], &ri_old);
+  tt_ptr_op(n3, OP_EQ, node_get_by_ed25519_id(md[3]->ed25519_identity_pkey));
+  tt_ptr_op(ri_old, OP_EQ, NULL);
+  tt_int_op(4, OP_EQ, smartlist_len(nodelist_get_list()));
+
+  /* Register the 5th by ri only, and rewrite its ed25519 pubkey to be
+   * the same as the 4th, to test the duplicate ed25519 key logging in
+   * nodelist.c */
+  memcpy(md[4]->ed25519_identity_pkey, md[3]->ed25519_identity_pkey,
+         sizeof(ed25519_public_key_t));
+  memcpy(&ri[4]->cache_info.signing_key_cert->signing_key,
+         md[3]->ed25519_identity_pkey, sizeof(ed25519_public_key_t));
+
+  setup_capture_of_logs(LOG_NOTICE);
+  node_t *n4 = nodelist_set_routerinfo(ri[4], &ri_old);
+  tt_ptr_op(ri_old, OP_EQ, NULL);
+  tt_int_op(5, OP_EQ, smartlist_len(nodelist_get_list()));
+  tt_ptr_op(n4, OP_NE, node_get_by_ed25519_id(md[3]->ed25519_identity_pkey));
+  tt_ptr_op(n3, OP_EQ, node_get_by_ed25519_id(md[3]->ed25519_identity_pkey));
+  expect_log_msg_containing("Reused ed25519_id");
+
+ done:
+  teardown_capture_of_logs();
+  for (i = 0; i < N_NODES; ++i) {
+    tor_free(rs[i]);
+    tor_free(md[i]->ed25519_identity_pkey);
+    tor_free(md[i]);
+    tor_free(ri[i]->cache_info.signing_key_cert);
+    tor_free(ri[i]);
+  }
+  smartlist_clear(ns->routerstatus_list);
+  networkstatus_vote_free(ns);
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+#undef N_NODES
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
@@ -107,6 +231,7 @@ struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_by_id_null_node, TT_FORK),
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
   NODE(node_is_dir, TT_FORK),
+  NODE(ed_id, TT_FORK),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c
index a560e5fc5e..d0eea85d6f 100644
--- a/src/test/test_ntor_cl.c
+++ b/src/test/test_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index 6102af01f5..abf8896452 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOM handling logic */
@@ -13,8 +13,10 @@
 #include "compat_libevent.h"
 #include "connection.h"
 #include "config.h"
+#include "crypto_rand.h"
 #include "relay.h"
 #include "test.h"
+#include "test_helpers.h"
 
 /* small replacement mock for circuit_mark_for_close_ to avoid doing all
  * the other bookkeeping that comes with marking circuits. */
@@ -58,24 +60,6 @@ dummy_or_circuit_new(int n_p_cells, int n_n_cells)
   return TO_CIRCUIT(circ);
 }
 
-static circuit_t *
-dummy_origin_circuit_new(int n_cells)
-{
-  origin_circuit_t *circ = origin_circuit_new();
-  int i;
-  cell_t cell;
-
-  for (i=0; i < n_cells; ++i) {
-    crypto_rand((void*)&cell, sizeof(cell));
-    cell_queue_append_packed_copy(TO_CIRCUIT(circ),
-                                  &TO_CIRCUIT(circ)->n_chan_cells,
-                                  1, &cell, 1, 0);
-  }
-
-  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  return TO_CIRCUIT(circ);
-}
-
 static void
 add_bytes_to_buf(buf_t *buf, size_t n_bytes)
 {
@@ -84,7 +68,7 @@ add_bytes_to_buf(buf_t *buf, size_t n_bytes)
   while (n_bytes) {
     size_t this_add = n_bytes > sizeof(b) ? sizeof(b) : n_bytes;
     crypto_rand(b, this_add);
-    write_to_buf(b, this_add, buf);
+    buf_add(buf, b, this_add);
     n_bytes -= this_add;
   }
 }
@@ -219,7 +203,7 @@ test_oom_streambuf(void *arg)
 {
   or_options_t *options = get_options_mutable();
   circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL, *c5 = NULL;
-  uint32_t tvms;
+  uint32_t tvts;
   int i;
   smartlist_t *edgeconns = smartlist_new();
   const uint64_t start_ns = 1389641159 * (uint64_t)1000000000;
@@ -287,22 +271,28 @@ test_oom_streambuf(void *arg)
   now_ns -= now_ns % 1000000000;
   now_ns += 1000000000;
   monotime_coarse_set_mock_time_nsec(now_ns);
-  tvms = (uint32_t) monotime_coarse_absolute_msec();
+  tvts = monotime_coarse_get_stamp();
+
+#define ts_is_approx(ts, val) do {                                   \
+    uint32_t x_ = (uint32_t) monotime_coarse_stamp_units_to_approx_msec(ts); \
+    tt_int_op(x_, OP_GE, val - 5);                                      \
+    tt_int_op(x_, OP_LE, val + 5);                                      \
+  } while (0)
 
-  tt_int_op(circuit_max_queued_cell_age(c1, tvms), OP_EQ, 500);
-  tt_int_op(circuit_max_queued_cell_age(c2, tvms), OP_EQ, 490);
-  tt_int_op(circuit_max_queued_cell_age(c3, tvms), OP_EQ, 480);
-  tt_int_op(circuit_max_queued_cell_age(c4, tvms), OP_EQ, 0);
+  ts_is_approx(circuit_max_queued_cell_age(c1, tvts), 500);
+  ts_is_approx(circuit_max_queued_cell_age(c2, tvts), 490);
+  ts_is_approx(circuit_max_queued_cell_age(c3, tvts), 480);
+  ts_is_approx(circuit_max_queued_cell_age(c4, tvts), 0);
 
-  tt_int_op(circuit_max_queued_data_age(c1, tvms), OP_EQ, 390);
-  tt_int_op(circuit_max_queued_data_age(c2, tvms), OP_EQ, 380);
-  tt_int_op(circuit_max_queued_data_age(c3, tvms), OP_EQ, 0);
-  tt_int_op(circuit_max_queued_data_age(c4, tvms), OP_EQ, 370);
+  ts_is_approx(circuit_max_queued_data_age(c1, tvts), 390);
+  ts_is_approx(circuit_max_queued_data_age(c2, tvts), 380);
+  ts_is_approx(circuit_max_queued_data_age(c3, tvts), 0);
+  ts_is_approx(circuit_max_queued_data_age(c4, tvts), 370);
 
-  tt_int_op(circuit_max_queued_item_age(c1, tvms), OP_EQ, 500);
-  tt_int_op(circuit_max_queued_item_age(c2, tvms), OP_EQ, 490);
-  tt_int_op(circuit_max_queued_item_age(c3, tvms), OP_EQ, 480);
-  tt_int_op(circuit_max_queued_item_age(c4, tvms), OP_EQ, 370);
+  ts_is_approx(circuit_max_queued_item_age(c1, tvts), 500);
+  ts_is_approx(circuit_max_queued_item_age(c2, tvts), 490);
+  ts_is_approx(circuit_max_queued_item_age(c3, tvts), 480);
+  ts_is_approx(circuit_max_queued_item_age(c4, tvts), 370);
 
   tt_int_op(cell_queues_get_total_allocation(), OP_EQ,
             packed_cell_mem_cost() * 80);
@@ -318,7 +308,7 @@ test_oom_streambuf(void *arg)
     smartlist_add(edgeconns, ec);
   }
   tt_int_op(buf_get_total_allocation(), OP_EQ, 4096*17*2);
-  tt_int_op(circuit_max_queued_item_age(c4, tvms), OP_EQ, 1000);
+  ts_is_approx(circuit_max_queued_item_age(c4, tvts), 1000);
 
   tt_int_op(cell_queues_check_size(), OP_EQ, 0);
 
@@ -352,7 +342,7 @@ test_oom_streambuf(void *arg)
   circuit_free(c5);
 
   SMARTLIST_FOREACH(edgeconns, edge_connection_t *, ec,
-                    connection_free_(TO_CONN(ec)));
+                    connection_free_minimal(TO_CONN(ec)));
   smartlist_free(edgeconns);
 
   UNMOCK(circuit_mark_for_close_);
diff --git a/src/test/test_oos.c b/src/test/test_oos.c
index db06625116..e72fcf5de9 100644
--- a/src/test/test_oos.c
+++ b/src/test/test_oos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOS handler */
@@ -52,7 +52,7 @@ kill_conn_list_mock(smartlist_t *conns)
 {
   ++kill_conn_list_calls;
 
-  tt_assert(conns != NULL);
+  tt_ptr_op(conns, OP_NE, NULL);
 
   kill_conn_list_killed += smartlist_len(conns);
 
@@ -248,7 +248,7 @@ close_for_error_mock(or_connection_t *orconn, int flush)
 {
   (void)flush;
 
-  tt_assert(orconn != NULL);
+  tt_ptr_op(orconn, OP_NE, NULL);
   ++cfe_calls;
 
  done:
@@ -264,7 +264,7 @@ mark_for_close_oos_mock(connection_t *conn,
   (void)line;
   (void)file;
 
-  tt_assert(conn != NULL);
+  tt_ptr_op(conn, OP_NE, NULL);
   ++mark_calls;
 
  done:
@@ -298,8 +298,8 @@ test_oos_kill_conn_list(void *arg)
   dir_c2->base_.purpose = DIR_PURPOSE_MIN_;
   c2 = TO_CONN(dir_c2);
 
-  tt_assert(c1 != NULL);
-  tt_assert(c2 != NULL);
+  tt_ptr_op(c1, OP_NE, NULL);
+  tt_ptr_op(c2, OP_NE, NULL);
 
   /* Make list */
   l = smartlist_new();
@@ -345,7 +345,7 @@ get_num_circuits_mock(or_connection_t *conn)
 {
   int circs = 0;
 
-  tt_assert(conn != NULL);
+  tt_ptr_op(conn, OP_NE, NULL);
 
   if (conns_with_circs &&
       smartlist_contains(conns_with_circs, TO_CONN(conn))) {
@@ -397,7 +397,7 @@ test_oos_pick_oos_victims(void *arg)
   /* Try picking one */
   picked = pick_oos_victims(1);
   /* It should be the one with circuits */
-  tt_assert(picked != NULL);
+  tt_ptr_op(picked, OP_NE, NULL);
   tt_int_op(smartlist_len(picked), OP_EQ, 1);
   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
   smartlist_free(picked);
@@ -405,14 +405,14 @@ test_oos_pick_oos_victims(void *arg)
   /* Try picking none */
   picked = pick_oos_victims(0);
   /* We should get an empty list */
-  tt_assert(picked != NULL);
+  tt_ptr_op(picked, OP_NE, NULL);
   tt_int_op(smartlist_len(picked), OP_EQ, 0);
   smartlist_free(picked);
 
   /* Try picking two */
   picked = pick_oos_victims(2);
   /* We should get both active orconns */
-  tt_assert(picked != NULL);
+  tt_ptr_op(picked, OP_NE, NULL);
   tt_int_op(smartlist_len(picked), OP_EQ, 2);
   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
   tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
diff --git a/src/test/test_options.c b/src/test/test_options.c
index e85e11805b..65564f324c 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
@@ -18,6 +18,7 @@
 #include "sandbox.h"
 #include "memarea.h"
 #include "policies.h"
+#include "test_helpers.h"
 
 #define NS_MODULE test_options
 
@@ -104,11 +105,71 @@ clear_log_messages(void)
   "EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212:80 7EA6 EAD6 FD83" \
   " 083C 538F 4403 8BBF A077 587D D755\n"
 
+static int
+test_options_checklog(const char *configuration, int expect_log_severity,
+                      const char *expect_log)
+{
+  int found = 0, ret = -1;
+  char *actual_log = NULL;
+
+  if (messages) {
+    SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) {
+      if (m->severity == expect_log_severity &&
+          strstr(m->msg, expect_log)) {
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+  if (!found) {
+    actual_log = dump_logs();
+    TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.",
+            log_level_to_string(expect_log_severity), expect_log,
+            configuration, actual_log));
+  }
+  ret = 0;
+
+ done:
+  tor_free(actual_log);
+  return ret;
+}
+
+static int
+test_options_checkmsgs(const char *configuration,
+                       const char *expect_errmsg,
+                       int expect_log_severity,
+                       const char *expect_log,
+                       char *msg)
+{
+  if (expect_errmsg && !msg) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got none.",
+            expect_errmsg, configuration));
+  } else if (expect_errmsg && !strstr(msg, expect_errmsg)) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
+            expect_errmsg, configuration, msg));
+  } else if (!expect_errmsg && msg) {
+    TT_DIE(("Expected no error message from <%s> but got <%s>.",
+            configuration, msg));
+  }
+  if (expect_log) {
+    return test_options_checklog(configuration, expect_log_severity,
+                                 expect_log);
+  }
+  return 0;
+
+ done:
+  return -1;
+}
+
+/* Which phases of config parsing/validation to check for messages/logs */
+enum { PH_GETLINES, PH_ASSIGN, PH_VALIDATE };
+
 static void
 test_options_validate_impl(const char *configuration,
                            const char *expect_errmsg,
                            int expect_log_severity,
-                           const char *expect_log)
+                           const char *expect_log,
+                           int phase)
 {
   or_options_t *opt=NULL;
   or_options_t *dflt;
@@ -119,43 +180,34 @@ test_options_validate_impl(const char *configuration,
   setup_options(opt, dflt);
 
   r = config_get_lines(configuration, &cl, 1);
-  tt_int_op(r, OP_EQ, 0);
+  if (phase == PH_GETLINES) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
+  }
+  if (r)
+    goto done;
 
   r = config_assign(&options_format, opt, cl, 0, &msg);
-  tt_int_op(r, OP_EQ, 0);
-
-  r = options_validate(NULL, opt, dflt, 0, &msg);
-  if (expect_errmsg && !msg) {
-    TT_DIE(("Expected error message <%s> from <%s>, but got none.",
-            expect_errmsg, configuration));
-  } else if (expect_errmsg && !strstr(msg, expect_errmsg)) {
-    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
-            expect_errmsg, configuration, msg));
-  } else if (!expect_errmsg && msg) {
-    TT_DIE(("Expected no error message from <%s> but got <%s>.",
-            configuration, msg));
+  if (phase == PH_ASSIGN) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
   }
   tt_int_op((r == 0), OP_EQ, (msg == NULL));
+  if (r)
+    goto done;
 
-  if (expect_log) {
-    int found = 0;
-    if (messages) {
-      SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) {
-        if (m->severity == expect_log_severity &&
-            strstr(m->msg, expect_log)) {
-          found = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(m);
-    }
-    if (!found) {
-      tor_free(msg);
-      msg = dump_logs();
-      TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.",
-              log_level_to_string(expect_log_severity), expect_log,
-              configuration, msg));
-    }
+  r = options_validate(NULL, opt, dflt, 0, &msg);
+  if (phase == PH_VALIDATE) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
   }
+  tt_int_op((r == 0), OP_EQ, (msg == NULL));
 
  done:
   escaped(NULL);
@@ -167,14 +219,14 @@ test_options_validate_impl(const char *configuration,
   clear_log_messages();
 }
 
-#define WANT_ERR(config, msg)                           \
-  test_options_validate_impl((config), (msg), 0, NULL)
-#define WANT_LOG(config, severity, msg)                         \
-  test_options_validate_impl((config), NULL, (severity), (msg))
-#define WANT_ERR_LOG(config, msg, severity, logmsg)                     \
-  test_options_validate_impl((config), (msg), (severity), (logmsg))
-#define OK(config)                                      \
-  test_options_validate_impl((config), NULL, 0, NULL)
+#define WANT_ERR(config, msg, ph)                               \
+  test_options_validate_impl((config), (msg), 0, NULL, (ph))
+#define WANT_LOG(config, severity, msg, ph)                             \
+  test_options_validate_impl((config), NULL, (severity), (msg), (ph))
+#define WANT_ERR_LOG(config, msg, severity, logmsg, ph)                 \
+  test_options_validate_impl((config), (msg), (severity), (logmsg), (ph))
+#define OK(config, ph)                                          \
+  test_options_validate_impl((config), NULL, 0, NULL, (ph))
 
 static void
 test_options_validate(void *arg)
@@ -183,21 +235,39 @@ test_options_validate(void *arg)
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
 
-  WANT_ERR("ExtORPort 500000", "Invalid ExtORPort");
+  WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
   WANT_ERR_LOG("ServerTransportOptions trebuchet",
                "ServerTransportOptions did not parse",
-               LOG_WARN, "Too few arguments");
-  OK("ServerTransportOptions trebuchet sling=snappy");
-  OK("ServerTransportOptions trebuchet sling=");
+               LOG_WARN, "Too few arguments", PH_VALIDATE);
+  OK("ServerTransportOptions trebuchet sling=snappy", PH_VALIDATE);
+  OK("ServerTransportOptions trebuchet sling=", PH_VALIDATE);
   WANT_ERR_LOG("ServerTransportOptions trebuchet slingsnappy",
                "ServerTransportOptions did not parse",
-               LOG_WARN, "\"slingsnappy\" is not a k=v");
+               LOG_WARN, "\"slingsnappy\" is not a k=v", PH_VALIDATE);
 
   WANT_ERR("DirPort 8080\nDirCache 0",
-           "DirPort configured but DirCache disabled.");
+           "DirPort configured but DirCache disabled.", PH_VALIDATE);
   WANT_ERR("BridgeRelay 1\nDirCache 0",
-           "We're a bridge but DirCache is disabled.");
+           "We're a bridge but DirCache is disabled.", PH_VALIDATE);
+
+  WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
+               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
+               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               PH_ASSIGN);
+  WANT_ERR_LOG("LogTimeGranularity 21 snarks",
+               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
+               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               PH_ASSIGN);
+  OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
+  OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
+
+  WANT_LOG("ControlSocket \"string with trailing garbage\" bogus", LOG_WARN,
+           "Error while parsing configuration: "
+           "Excess data after quoted string", PH_GETLINES);
+  WANT_LOG("ControlSocket \"bogus escape \\@\"", LOG_WARN,
+           "Error while parsing configuration: "
+           "Invalid escape sequence in quoted string", PH_GETLINES);
 
   close_temp_logs();
   clear_log_messages();
@@ -212,7 +282,7 @@ test_have_enough_mem_for_dircache(void *arg)
   or_options_t *opt=NULL;
   or_options_t *dflt=NULL;
   config_line_t *cl=NULL;
-  char *msg=NULL;;
+  char *msg=NULL;
   int r;
   const char *configuration = "ORPort 8080\nDirCache 1", *expect_errmsg;
 
@@ -229,7 +299,7 @@ test_have_enough_mem_for_dircache(void *arg)
   /* 300 MB RAM available, DirCache enabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
   tt_int_op(r, OP_EQ, 0);
-  tt_assert(!msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   /* 200 MB RAM available, DirCache enabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
@@ -252,7 +322,7 @@ test_have_enough_mem_for_dircache(void *arg)
   /* 300 MB RAM available, DirCache enabled, Bridge */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
   tt_int_op(r, OP_EQ, 0);
-  tt_assert(!msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   /* 200 MB RAM available, DirCache enabled, Bridge */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
@@ -275,7 +345,7 @@ test_have_enough_mem_for_dircache(void *arg)
   /* 200 MB RAM available, DirCache disabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(200), &msg);
   tt_int_op(r, OP_EQ, 0);
-  tt_assert(!msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   /* 300 MB RAM available, DirCache disabled */
   r = have_enough_mem_for_dircache(opt, MEGABYTEIFY(300), &msg);
@@ -307,17 +377,11 @@ fixed_get_uname(void)
 }
 
 #define TEST_OPTIONS_OLD_VALUES   "TestingV3AuthInitialVotingInterval 1800\n" \
-  "ClientBootstrapConsensusMaxDownloadTries 7\n" \
-  "ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries 4\n" \
   "ClientBootstrapConsensusMaxInProgressTries 3\n" \
   "TestingV3AuthInitialVoteDelay 300\n"   \
   "TestingV3AuthInitialDistDelay 300\n" \
   "TestingClientMaxIntervalWithoutRequest 600\n" \
   "TestingDirConnectionMaxStall 600\n" \
-  "TestingConsensusMaxDownloadTries 8\n" \
-  "TestingDescriptorMaxDownloadTries 8\n" \
-  "TestingMicrodescMaxDownloadTries 8\n" \
-  "TestingCertMaxDownloadTries 8\n"
 
 #define TEST_OPTIONS_DEFAULT_VALUES TEST_OPTIONS_OLD_VALUES \
   "MaxClientCircuitsPending 1\n"                                        \
@@ -328,11 +392,12 @@ fixed_get_uname(void)
   "V3AuthVoteDelay 20\n"                                                \
   "V3AuthDistDelay 20\n"                                                \
   "V3AuthNIntervalsValid 3\n"                                           \
-  "ClientUseIPv4 1\n"                                                     \
+  "ClientUseIPv4 1\n"                                                   \
   "VirtualAddrNetworkIPv4 127.192.0.0/10\n"                             \
   "VirtualAddrNetworkIPv6 [FE80::]/10\n"                                \
-  "SchedulerHighWaterMark__ 42\n"                                       \
-  "SchedulerLowWaterMark__ 10\n"
+  "UseEntryGuards 1\n"                                                  \
+  "Schedulers Vanilla\n"                                                \
+  "ClientDNSRejectInternalAddresses 1\n"
 
 typedef struct {
   or_options_t *old_opt;
@@ -352,25 +417,31 @@ get_options_test_data(const char *conf)
   result->opt = options_new();
   result->old_opt = options_new();
   result->def_opt = options_new();
+
+  // XXX: Really, all of these options should be set to defaults
+  // with options_init(), but about a dozen tests break when I do that.
+  // Being kinda lame and just fixing the immedate breakage for now..
+  result->opt->ConnectionPadding = -1; // default must be "auto"
+
   rv = config_get_lines(conf, &cl, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   rv = config_assign(&options_format, result->opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
   }
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   config_free_lines(cl);
   result->opt->LogTimeGranularity = 1;
   result->opt->TokenBucketRefillInterval = 1;
   rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1);
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
   rv = config_assign(&options_format, result->def_opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
   }
-  tt_assert(rv == 0);
+  tt_int_op(rv, OP_EQ, 0);
 
  done:
   config_free_lines(cl);
@@ -400,7 +471,7 @@ test_options_validate__uname_for_server(void *ignored)
   (void)ignored;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                      "ORListenAddress 127.0.0.1:5555");
+                                      "ORPort 127.0.0.1:5555");
   setup_capture_of_logs(LOG_WARN);
 
   MOCK(get_uname, fixed_get_uname);
@@ -430,7 +501,7 @@ test_options_validate__uname_for_server(void *ignored)
   fixed_get_uname_result = "Windows 2000";
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  expect_log_entry();
+  expect_no_log_entry();
   tor_free(msg);
 
  done:
@@ -451,6 +522,8 @@ test_options_validate__outbound_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "Multiple outbound bind addresses configured: "
+                        "xxyy!!!sdfaf");
 
  done:
   free_options_test_data(tdata);
@@ -514,13 +587,14 @@ test_options_validate__nickname(void *ignored)
   tdata = get_options_test_data("Nickname AMoreValidNick");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_assert(!msg);
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("DataDirectory /tmp/somewhere");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_assert(!msg);
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
 
  done:
   free_options_test_data(tdata);
@@ -534,7 +608,7 @@ test_options_validate__contactinfo(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                "ORListenAddress 127.0.0.1:5555\nORPort 955");
+                                "ORPort 127.0.0.1:5555");
   setup_capture_of_logs(LOG_DEBUG);
   tdata->opt->ContactInfo = NULL;
 
@@ -547,7 +621,7 @@ test_options_validate__contactinfo(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ORListenAddress 127.0.0.1:5555\nORPort 955\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "ContactInfo hella@example.org");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -580,6 +654,7 @@ test_options_validate__logs(void *ignored)
   tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
   tt_str_op(tdata->opt->Logs->value, OP_EQ, "notice stdout");
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -590,6 +665,7 @@ test_options_validate__logs(void *ignored)
   tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
   tt_str_op(tdata->opt->Logs->value, OP_EQ, "warn stdout");
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -599,6 +675,7 @@ test_options_validate__logs(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -607,6 +684,7 @@ test_options_validate__logs(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 1, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -615,6 +693,7 @@ test_options_validate__logs(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+  tt_int_op(ret, OP_EQ, -1);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -624,6 +703,7 @@ test_options_validate__logs(void *ignored)
   tdata->opt->Logs = cl;
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op((intptr_t)tdata->opt->Logs, OP_EQ, (intptr_t)cl);
+  tt_int_op(ret, OP_EQ, -1);
 
  done:
   quiet_level = orig_quiet_level;
@@ -650,16 +730,18 @@ test_options_validate__authdir(void *ignored)
   setup_capture_of_logs(LOG_INFO);
   options_test_data_t *tdata = get_options_test_data(
                                  "AuthoritativeDirectory 1\n"
-                                 "Address this.should.not_exist.example.org");
+                                 "Address this.should.not!exist!.example.org");
 
   sandbox_disable_getaddrinfo_cache();
 
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to resolve/guess local address. See logs for"
             " details.");
   expect_log_msg("Could not resolve local Address "
-            "'this.should.not_exist.example.org'. Failing.\n");
+            "'this.should.not!exist!.example.org'. Failing.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
@@ -668,13 +750,13 @@ test_options_validate__authdir(void *ignored)
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_assert(!msg);
+  tt_str_op(msg, OP_EQ, "Authoritative directory servers must set "
+                        "ContactInfo");
+  tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "Address 100.200.10.1\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -685,9 +767,7 @@ test_options_validate__authdir(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
-                                "TestingTorNetwork 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "TestingTorNetwork 1\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -698,9 +778,7 @@ test_options_validate__authdir(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -712,9 +790,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "RecommendedVersions 1.2, 3.14\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "1.2, 3.14");
@@ -727,9 +803,7 @@ test_options_validate__authdir(void *ignored)
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
                                 "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "25");
@@ -743,9 +817,7 @@ test_options_validate__authdir(void *ignored)
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
                                 "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
@@ -757,9 +829,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "VersioningAuthoritativeDirectory 1\n"
                                 "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
@@ -771,9 +841,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "VersioningAuthoritativeDirectory 1\n"
                                 "RecommendedClientVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
@@ -784,9 +852,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "UseEntryGuards 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   expect_log_msg("Authoritative directory servers "
@@ -798,9 +864,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   expect_log_msg("Authoritative directories always try"
@@ -813,9 +877,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "DownloadExtraInfo 1\n"
                                 "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   expect_no_log_msg("Authoritative directories always try"
@@ -826,9 +888,7 @@ test_options_validate__authdir(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
@@ -840,9 +900,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "V3BandwidthsFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -854,9 +912,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "V3BandwidthsFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -868,9 +924,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "GuardfractionFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -882,9 +936,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
                                 "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existant-file\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "GuardfractionFile non-existent-file\n");
   mock_clean_saved_logs();
   options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
   tt_str_op(msg, OP_EQ,
@@ -895,9 +947,7 @@ test_options_validate__authdir(void *ignored)
   tdata = get_options_test_data("AuthoritativeDirectory 1\n"
                                 "Address 100.200.10.1\n"
                                 "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -910,9 +960,7 @@ test_options_validate__authdir(void *ignored)
                                 "Address 100.200.10.1\n"
                                 "DirPort 999\n"
                                 "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -929,9 +977,7 @@ test_options_validate__authdir(void *ignored)
   /*                               "ORPort 888\n" */
   /*                               "ClientOnly 1\n" */
   /*                               "BridgeAuthoritativeDir 1\n" */
-  /*                               "ContactInfo hello@hello.com\n" */
-  /*                               "SchedulerHighWaterMark__ 42\n" */
-  /*                               "SchedulerLowWaterMark__ 10\n"); */
+  /*                               "ContactInfo hello@hello.com\n" ); */
   /* mock_clean_saved_logs(); */
   /* ret = options_validate(tdata->old_opt, tdata->opt, */
   /*                        tdata->def_opt, 0, &msg); */
@@ -953,8 +999,7 @@ test_options_validate__relay_with_hidden_services(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_DEBUG);
   options_test_data_t *tdata = get_options_test_data(
-                                  "ORListenAddress 127.0.0.1:5555\n"
-                                  "ORPort 955\n"
+                                  "ORPort 127.0.0.1:5555\n"
                                   "HiddenServiceDir "
                                   "/Library/Tor/var/lib/tor/hidden_service/\n"
                                   "HiddenServicePort 80 127.0.0.1:8080\n"
@@ -1018,14 +1063,14 @@ test_options_validate__transproxy(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
-#if !defined(__OpenBSD__) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined( DARWIN )
   tt_str_op(msg, OP_EQ,
           "pf-divert is a OpenBSD-specific and OS X/Darwin-specific feature.");
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_PF_DIVERT);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without "
-            "any valid TransPort or TransListenAddress.");
-#endif
+            "any valid TransPort.");
+#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
   tor_free(msg);
 
   // Test tproxy trans proxy
@@ -1039,8 +1084,8 @@ test_options_validate__transproxy(void *ignored)
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_TPROXY);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without any valid "
-            "TransPort or TransListenAddress.");
-#endif
+            "TransPort.");
+#endif /* !defined(__linux__) */
   tor_free(msg);
 
   // Test ipfw trans proxy
@@ -1055,13 +1100,13 @@ test_options_validate__transproxy(void *ignored)
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_IPFW);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without any valid "
-            "TransPort or TransListenAddress.");
-#endif
+            "TransPort.");
+#endif /* !defined(KERNEL_MAY_SUPPORT_IPFW) */
   tor_free(msg);
 
   // Test unknown trans proxy
   free_options_test_data(tdata);
-  tdata = get_options_test_data("TransProxyType non-existant\n");
+  tdata = get_options_test_data("TransProxyType non-existent\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value for TransProxyType");
@@ -1076,47 +1121,41 @@ test_options_validate__transproxy(void *ignored)
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
 #elif defined(KERNEL_MAY_SUPPORT_IPFW)
   tdata = get_options_test_data("TransProxyType ipfw\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
-#elif defined(__OpenBSD__)
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
+#elif defined(OpenBSD)
   tdata = get_options_test_data("TransProxyType pf-divert\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
 #elif defined(__NetBSD__)
   tdata = get_options_test_data("TransProxyType default\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  if (msg) {
-    TT_DIE(("Expected NULL but got '%s'", msg));
-  }
-#endif
+  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  tor_free(msg);
+#endif /* defined(__linux__) || ... */
 
   // Assert that a test has run for some TransProxyType
   tt_assert(tdata);
 
-#else
+#else /* !(defined(USE_TRANSPARENT)) */
   tdata = get_options_test_data("TransPort 127.0.0.1:555\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "TransPort and TransListenAddress are disabled in "
-            "this build.");
+  tt_str_op(msg, OP_EQ, "TransPort is disabled in this build.");
   tor_free(msg);
-#endif
+#endif /* defined(USE_TRANSPARENT) */
 
  done:
   free_options_test_data(tdata);
@@ -1181,9 +1220,7 @@ test_options_validate__exclude_nodes(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n"
-                                "StrictNodes 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "StrictNodes 1\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1194,9 +1231,7 @@ test_options_validate__exclude_nodes(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ExcludeNodes {cn}\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("ExcludeNodes {cn}\n");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1214,49 +1249,6 @@ test_options_validate__exclude_nodes(void *ignored)
 }
 
 static void
-test_options_validate__scheduler(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_DEBUG);
-  options_test_data_t *tdata = get_options_test_data(
-                                            "SchedulerLowWaterMark__ 0\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Bad SchedulerLowWaterMark__ option\n");
-  tor_free(msg);
-
-  // TODO: this test cannot run on platforms where UINT32_MAX == UINT64_MAX.
-  // I suspect it's unlikely this branch can actually happen
-  /* free_options_test_data(tdata); */
-  /* tdata = get_options_test_data( */
-  /*                      "SchedulerLowWaterMark 10000000000000000000\n"); */
-  /* tdata->opt->SchedulerLowWaterMark__ = (uint64_t)UINT32_MAX; */
-  /* tdata->opt->SchedulerLowWaterMark__++; */
-  /* mock_clean_saved_logs(); */
-  /* ret = options_validate(tdata->old_opt, tdata->opt, */
-  /*                        tdata->def_opt, 0, &msg); */
-  /* tt_int_op(ret, OP_EQ, -1); */
-  /* expect_log_msg("Bad SchedulerLowWaterMark__ option\n"); */
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("SchedulerLowWaterMark__ 42\n"
-                                "SchedulerHighWaterMark__ 42\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Bad SchedulerHighWaterMark option\n");
-  tor_free(msg);
-
- done:
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__node_families(void *ignored)
 {
   (void)ignored;
@@ -1264,9 +1256,7 @@ test_options_validate__node_families(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                      "NodeFamily flux, flax\n"
-                                     "NodeFamily somewhere\n"
-                                     "SchedulerHighWaterMark__ 42\n"
-                                     "SchedulerLowWaterMark__ 10\n");
+                                     "NodeFamily somewhere\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1284,8 +1274,7 @@ test_options_validate__node_families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1293,9 +1282,7 @@ test_options_validate__node_families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("NodeFamily !flux\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("NodeFamily !flux\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1309,54 +1296,6 @@ test_options_validate__node_families(void *ignored)
 }
 
 static void
-test_options_validate__tlsec(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_DEBUG);
-  options_test_data_t *tdata = get_options_test_data(
-                                 "TLSECGroup ed25519\n"
-                                 "SchedulerHighWaterMark__ 42\n"
-                                 "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(!tdata->opt->TLSECGroup);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("TLSECGroup P224\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg(
-            "Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(tdata->opt->TLSECGroup);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("TLSECGroup P256\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg(
-            "Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(tdata->opt->TLSECGroup);
-  tor_free(msg);
-
- done:
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__token_bucket(void *ignored)
 {
   (void)ignored;
@@ -1392,9 +1331,7 @@ test_options_validate__recommended_packages(void *ignored)
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
             "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n"
-            "RecommendedPackages invalid-package-line\n"
-            "SchedulerHighWaterMark__ 42\n"
-            "SchedulerLowWaterMark__ 10\n");
+            "RecommendedPackages invalid-package-line\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1416,9 +1353,7 @@ test_options_validate__fetch_dir(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                             "FetchDirInfoExtraEarly 1\n"
-                                            "FetchDirInfoEarly 0\n"
-                                            "SchedulerHighWaterMark__ 42\n"
-                                            "SchedulerLowWaterMark__ 10\n");
+                                            "FetchDirInfoEarly 0\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1428,9 +1363,7 @@ test_options_validate__fetch_dir(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("FetchDirInfoExtraEarly 1\n"
-                                "FetchDirInfoEarly 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "FetchDirInfoEarly 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1450,9 +1383,7 @@ test_options_validate__conn_limit(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                            "ConnLimit 0\n"
-                                            "SchedulerHighWaterMark__ 42\n"
-                                            "SchedulerLowWaterMark__ 10\n");
+                                            "ConnLimit 0\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1460,9 +1391,7 @@ test_options_validate__conn_limit(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+  tdata = get_options_test_data("ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1484,9 +1413,7 @@ test_options_validate__paths_needed(void *ignored)
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
                                       "PathsNeededToBuildCircuits 0.1\n"
-                                      "ConnLimit 1\n"
-                                      "SchedulerHighWaterMark__ 42\n"
-                                      "SchedulerLowWaterMark__ 10\n");
+                                      "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1499,9 +1426,7 @@ test_options_validate__paths_needed(void *ignored)
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("PathsNeededToBuildCircuits 0.99\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1514,9 +1439,7 @@ test_options_validate__paths_needed(void *ignored)
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("PathsNeededToBuildCircuits 0.91\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1539,9 +1462,7 @@ test_options_validate__max_client_circuits(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                            "MaxClientCircuitsPending 0\n"
-                                           "ConnLimit 1\n"
-                                           "SchedulerHighWaterMark__ 42\n"
-                                           "SchedulerLowWaterMark__ 10\n");
+                                           "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1551,9 +1472,7 @@ test_options_validate__max_client_circuits(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("MaxClientCircuitsPending 1025\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1563,9 +1482,7 @@ test_options_validate__max_client_circuits(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1586,9 +1503,7 @@ test_options_validate__ports(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                       "FirewallPorts 65537\n"
                                       "MaxClientCircuitsPending 1\n"
-                                      "ConnLimit 1\n"
-                                      "SchedulerHighWaterMark__ 42\n"
-                                      "SchedulerLowWaterMark__ 10\n");
+                                      "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1599,9 +1514,7 @@ test_options_validate__ports(void *ignored)
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 124444\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1613,9 +1526,7 @@ test_options_validate__ports(void *ignored)
                                 "LongLivedPorts 2\n"
                                 "RejectPlaintextPorts 112233\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1628,9 +1539,7 @@ test_options_validate__ports(void *ignored)
                                 "RejectPlaintextPorts 3\n"
                                 "WarnPlaintextPorts 65536\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1643,9 +1552,7 @@ test_options_validate__ports(void *ignored)
                                 "RejectPlaintextPorts 3\n"
                                 "WarnPlaintextPorts 4\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1667,9 +1574,7 @@ test_options_validate__reachable_addresses(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                      "FascistFirewall 1\n"
                                      "MaxClientCircuitsPending 1\n"
-                                     "ConnLimit 1\n"
-                                     "SchedulerHighWaterMark__ 42\n"
-                                     "SchedulerLowWaterMark__ 10\n");
+                                     "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1687,9 +1592,7 @@ test_options_validate__reachable_addresses(void *ignored)
                                 "ReachableDirAddresses *:81\n"
                                 "ReachableORAddresses *:444\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
   tdata->opt->FirewallPorts = smartlist_new();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1703,9 +1606,7 @@ test_options_validate__reachable_addresses(void *ignored)
   tdata = get_options_test_data("FascistFirewall 1\n"
                                 "FirewallPort 123\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1722,9 +1623,7 @@ test_options_validate__reachable_addresses(void *ignored)
                                 "ReachableAddresses *:83\n"
                                 "ReachableAddresses reject *:*\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1738,12 +1637,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1752,12 +1648,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableORAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1766,12 +1659,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableDirAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1780,12 +1670,9 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ClientUseIPv4 0\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1794,14 +1681,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   /* Test IPv4-only clients setting IPv6 preferences */
 
-#define WARN_PLEASE_USE_IPV6_OR_LOG_MSG \
-        "ClientPreferIPv6ORPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges.\n"
-
-#define WARN_PLEASE_USE_IPV6_DIR_LOG_MSG \
-        "ClientPreferIPv6DirPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges.\n"
-
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ClientUseIPv4 1\n"
@@ -1811,7 +1690,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg(WARN_PLEASE_USE_IPV6_OR_LOG_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
@@ -1823,7 +1701,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg(WARN_PLEASE_USE_IPV6_DIR_LOG_MSG);
   tor_free(msg);
 
   /* Now test an IPv4/IPv6 client setting IPv6 preferences */
@@ -1891,12 +1768,9 @@ test_options_validate__use_bridges(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                    "UseBridges 1\n"
                                    "ClientUseIPv4 1\n"
-                                   "ORListenAddress 127.0.0.1:5555\n"
-                                   "ORPort 955\n"
+                                   "ORPort 127.0.0.1:5555\n"
                                    "MaxClientCircuitsPending 1\n"
-                                   "ConnLimit 1\n"
-                                   "SchedulerHighWaterMark__ 42\n"
-                                   "SchedulerLowWaterMark__ 10\n");
+                                   "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1907,9 +1781,7 @@ test_options_validate__use_bridges(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("UseBridges 1\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1922,9 +1794,7 @@ test_options_validate__use_bridges(void *ignored)
   tdata = get_options_test_data("UseBridges 1\n"
                                 "EntryNodes {cn}\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1945,6 +1815,19 @@ test_options_validate__use_bridges(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
+                                "UseEntryGuards 0\n"
+                                );
+
+  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "Setting UseBridges requires also setting UseEntryGuards.");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+                                "UseBridges 1\n"
+                                "Bridge 10.0.0.1\n"
                                 "Bridge !!!\n"
                                 );
 
@@ -1971,9 +1854,7 @@ test_options_validate__entry_nodes(void *ignored)
                                          "EntryNodes {cn}\n"
                                          "UseEntryGuards 0\n"
                                          "MaxClientCircuitsPending 1\n"
-                                         "ConnLimit 1\n"
-                                         "SchedulerHighWaterMark__ 42\n"
-                                         "SchedulerLowWaterMark__ 10\n");
+                                         "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -1985,9 +1866,7 @@ test_options_validate__entry_nodes(void *ignored)
   tdata = get_options_test_data("EntryNodes {cn}\n"
                                 "UseEntryGuards 1\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2001,56 +1880,6 @@ test_options_validate__entry_nodes(void *ignored)
 }
 
 static void
-test_options_validate__invalid_nodes(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                  "AllowInvalidNodes something_stupid\n"
-                                  "MaxClientCircuitsPending 1\n"
-                                  "ConnLimit 1\n"
-                                  "SchedulerHighWaterMark__ 42\n"
-                                  "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Unrecognized value 'something_stupid' in AllowInvalidNodes");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("AllowInvalidNodes entry, middle, exit\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_int_op(tdata->opt->AllowInvalid_, OP_EQ, ALLOW_INVALID_ENTRY |
-            ALLOW_INVALID_EXIT | ALLOW_INVALID_MIDDLE);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("AllowInvalidNodes introduction, rendezvous\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_int_op(tdata->opt->AllowInvalid_, OP_EQ, ALLOW_INVALID_INTRODUCTION |
-            ALLOW_INVALID_RENDEZVOUS);
-  tor_free(msg);
-
- done:
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__safe_logging(void *ignored)
 {
   (void)ignored;
@@ -2058,9 +1887,7 @@ test_options_validate__safe_logging(void *ignored)
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
                                             "MaxClientCircuitsPending 1\n"
-                                            "ConnLimit 1\n"
-                                            "SchedulerHighWaterMark__ 42\n"
-                                            "SchedulerLowWaterMark__ 10\n");
+                                            "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2070,9 +1897,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging 0\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2082,9 +1907,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging Relay\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2094,9 +1917,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging 1\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2106,9 +1927,7 @@ test_options_validate__safe_logging(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("SafeLogging stuffy\n"
                                 "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
+                                "ConnLimit 1\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
@@ -2248,17 +2067,14 @@ test_options_validate__testing(void *ignored)
   ENSURE_DEFAULT(TestingV3AuthVotingStartOffset, 3000);
   ENSURE_DEFAULT(TestingAuthDirTimeToLearnReachability, 3000);
   ENSURE_DEFAULT(TestingEstimatedDescriptorPropagationTime, 3000);
-  ENSURE_DEFAULT(TestingServerDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingClientDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingServerConsensusDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingClientConsensusDownloadSchedule, 3000);
-  ENSURE_DEFAULT(TestingBridgeDownloadSchedule, 3000);
+  ENSURE_DEFAULT(TestingServerDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingClientDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingServerConsensusDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingClientConsensusDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingBridgeDownloadInitialDelay, 3000);
+  ENSURE_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay, 3000);
   ENSURE_DEFAULT(TestingClientMaxIntervalWithoutRequest, 3000);
   ENSURE_DEFAULT(TestingDirConnectionMaxStall, 3000);
-  ENSURE_DEFAULT(TestingConsensusMaxDownloadTries, 3000);
-  ENSURE_DEFAULT(TestingDescriptorMaxDownloadTries, 3000);
-  ENSURE_DEFAULT(TestingMicrodescMaxDownloadTries, 3000);
-  ENSURE_DEFAULT(TestingCertMaxDownloadTries, 3000);
   ENSURE_DEFAULT(TestingAuthKeyLifetime, 3000);
   ENSURE_DEFAULT(TestingLinkCertLifetime, 3000);
   ENSURE_DEFAULT(TestingSigningKeySlop, 3000);
@@ -2320,30 +2136,6 @@ test_options_validate__hidserv(void *ignored)
 }
 
 static void
-test_options_validate__predicted_ports(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-
-  options_test_data_t *tdata = get_options_test_data(
-                                     "PredictedPortsRelevanceTime 100000000\n"
-                                     TEST_OPTIONS_DEFAULT_VALUES);
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("PredictedPortsRelevanceTime is too "
-            "large; clipping to 3600s.\n");
-  tt_int_op(tdata->opt->PredictedPortsRelevanceTime, OP_EQ, 3600);
-
- done:
-  teardown_capture_of_logs();
-  policies_free_all();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__path_bias(void *ignored)
 {
   (void)ignored;
@@ -2489,8 +2281,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 1\n"
                                 );
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -2501,8 +2292,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 30000\n"
                                 );
@@ -2514,8 +2304,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "RelayBandwidthRate 1\n"
                                 "MaxAdvertisedBandwidth 38400\n"
@@ -2528,8 +2317,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "BandwidthBurst 76800\n"
                                 "RelayBandwidthRate 76800\n"
@@ -2634,37 +2422,6 @@ test_options_validate__circuits(void *ignored)
 }
 
 static void
-test_options_validate__port_forwarding(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = NULL;
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PortForwarding 1\nSandbox 1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "PortForwarding is not compatible with Sandbox;"
-            " at most one can be set");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PortForwarding 1\nSandbox 0\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_assert(!msg);
-  tor_free(msg);
-
- done:
-  free_options_test_data(tdata);
-  policies_free_all();
-  tor_free(msg);
-}
-
-static void
 test_options_validate__tor2web(void *ignored)
 {
   (void)ignored;
@@ -2819,8 +2576,8 @@ test_options_validate__single_onion(void *ignored)
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode is incompatible with "
             "using Tor as an anonymous client. Please set "
-            "Socks/Trans/NATD/DNSPort to 0, or HiddenServiceNonAnonymousMode "
-            "to 0, or use the non-anonymous Tor2webMode.");
+            "Socks/Trans/NATD/DNSPort to 0, or revert "
+            "HiddenServiceNonAnonymousMode to 0.");
   tor_free(msg);
   free_options_test_data(tdata);
 
@@ -2967,8 +2724,7 @@ test_options_validate__accounting(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(
            TEST_OPTIONS_DEFAULT_VALUES
-           "ORListenAddress 127.0.0.1:5555\n"
-           "ORPort 955\n"
+           "ORPort 127.0.0.1:5555\n"
            "BandwidthRate 76800\n"
            "BandwidthBurst 76800\n"
            "MaxAdvertisedBandwidth 38400\n"
@@ -3037,6 +2793,7 @@ test_options_validate__proxy(void *ignored)
   options_test_data_t *tdata = NULL;
   sandbox_disable_getaddrinfo_cache();
   setup_capture_of_logs(LOG_WARN);
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -3057,6 +2814,7 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
+
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "HttpProxy not_so_valid!\n"
                                 );
@@ -3357,6 +3115,7 @@ test_options_validate__proxy(void *ignored)
   policies_free_all();
   // sandbox_free_getaddrinfo_cache();
   tor_free(msg);
+  UNMOCK(tor_addr_lookup);
 }
 
 static void
@@ -3550,7 +3309,7 @@ test_options_validate__control(void *ignored)
             "can reconfigure your Tor.  That's bad!  You should upgrade your "
             "Tor controller as soon as possible.\n");
   tor_free(msg);
-#endif
+#endif /* defined(HAVE_SYS_UN_H) */
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -3599,8 +3358,7 @@ test_options_validate__families(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "MyFamily home\n"
                                 "BridgeRelay 1\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 51300\n"
                                 "BandwidthBurst 51300\n"
                                 "MaxAdvertisedBandwidth 25700\n"
@@ -3829,8 +3587,7 @@ test_options_validate__transport(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ServerTransportPlugin foo exec bar\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
@@ -3872,8 +3629,7 @@ test_options_validate__transport(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 "ServerTransportPlugin foo exec bar\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
@@ -4230,48 +3986,6 @@ test_options_validate__virtual_addr(void *ignored)
 }
 
 static void
-test_options_validate__exits(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = NULL;
-  setup_capture_of_logs(LOG_WARN);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AllowSingleHopExits 1"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("You have set AllowSingleHopExits; "
-            "now your relay will allow others to make one-hop exits. However,"
-            " since by default most clients avoid relays that set this option,"
-            " most clients will ignore you.\n");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AllowSingleHopExits 1\n"
-                                VALID_DIR_AUTH
-                                );
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_no_log_msg("You have set AllowSingleHopExits; "
-            "now your relay will allow others to make one-hop exits. However,"
-            " since by default most clients avoid relays that set this option,"
-            " most clients will ignore you.\n");
-  tor_free(msg);
-
- done:
-  policies_free_all();
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__testing_options(void *ignored)
 {
   (void)ignored;
@@ -4314,15 +4028,6 @@ test_options_validate__testing_options(void *ignored)
                       "is way too low.");
   TEST_TESTING_OPTION(TestingDirConnectionMaxStall, 1, 3601,
                       "is way too low.");
-  // TODO: I think this points to a bug/regression in options_validate
-  TEST_TESTING_OPTION(TestingConsensusMaxDownloadTries, 1, 801,
-                      "must be greater than 2.");
-  TEST_TESTING_OPTION(TestingDescriptorMaxDownloadTries, 1, 801,
-                      "must be greater than 1.");
-  TEST_TESTING_OPTION(TestingMicrodescMaxDownloadTries, 1, 801,
-                      "must be greater than 1.");
-  TEST_TESTING_OPTION(TestingCertMaxDownloadTries, 1, 801,
-                      "must be greater than 1.");
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -4399,16 +4104,6 @@ test_options_validate__testing_options(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "TestingEnableTbEmptyEvent 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "TestingEnableTbEmptyEvent may only be changed "
-            "in testing Tor networks!");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableTbEmptyEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
@@ -4507,9 +4202,7 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(relay_with_hidden_services),
   LOCAL_VALIDATE_TEST(transproxy),
   LOCAL_VALIDATE_TEST(exclude_nodes),
-  LOCAL_VALIDATE_TEST(scheduler),
   LOCAL_VALIDATE_TEST(node_families),
-  LOCAL_VALIDATE_TEST(tlsec),
   LOCAL_VALIDATE_TEST(token_bucket),
   LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
@@ -4520,16 +4213,13 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(reachable_addresses),
   LOCAL_VALIDATE_TEST(use_bridges),
   LOCAL_VALIDATE_TEST(entry_nodes),
-  LOCAL_VALIDATE_TEST(invalid_nodes),
   LOCAL_VALIDATE_TEST(safe_logging),
   LOCAL_VALIDATE_TEST(publish_server_descriptor),
   LOCAL_VALIDATE_TEST(testing),
   LOCAL_VALIDATE_TEST(hidserv),
-  LOCAL_VALIDATE_TEST(predicted_ports),
   LOCAL_VALIDATE_TEST(path_bias),
   LOCAL_VALIDATE_TEST(bandwidth),
   LOCAL_VALIDATE_TEST(circuits),
-  LOCAL_VALIDATE_TEST(port_forwarding),
   LOCAL_VALIDATE_TEST(tor2web),
   LOCAL_VALIDATE_TEST(rend),
   LOCAL_VALIDATE_TEST(single_onion),
@@ -4543,7 +4233,6 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(constrained_sockets),
   LOCAL_VALIDATE_TEST(v3_auth),
   LOCAL_VALIDATE_TEST(virtual_addr),
-  LOCAL_VALIDATE_TEST(exits),
   LOCAL_VALIDATE_TEST(testing_options),
   LOCAL_VALIDATE_TEST(accel),
   END_OF_TESTCASES              /*  */
diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c
new file mode 100644
index 0000000000..f159c4f83a
--- /dev/null
+++ b/src/test/test_periodic_event.c
@@ -0,0 +1,329 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_periodic_event.c
+ * \brief Test the periodic events that Tor uses for different roles. They are
+ *        part of the libevent mainloop
+ */
+
+#define CONFIG_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define MAIN_PRIVATE
+
+#include "test.h"
+#include "test_helpers.h"
+
+#include "or.h"
+#include "config.h"
+#include "hibernate.h"
+#include "hs_service.h"
+#include "main.h"
+#include "periodic.h"
+
+/** Helper function: This is replaced in some tests for the event callbacks so
+ * we don't actually go into the code path of those callbacks. */
+static int
+dumb_event_fn(time_t now, const or_options_t *options)
+{
+  (void) now;
+  (void) options;
+
+  /* Will get rescheduled in 300 seconds. It just can't be 0. */
+  return 300;
+}
+
+static void
+register_dummy_hidden_service(hs_service_t *service)
+{
+  memset(service, 0, sizeof(hs_service_t));
+  memset(&service->keys.identity_pk, 'A', sizeof(service->keys.identity_pk));
+  (void) register_service(get_hs_service_map(), service);
+}
+
+static void
+test_pe_initialize(void *arg)
+{
+  (void) arg;
+
+  /* Initialize the events but the callback won't get called since we would
+   * need to run the main loop and then wait for a second delaying the unit
+   * tests. Instead, we'll test the callback work indepedently elsewhere. */
+  initialize_periodic_events();
+
+  /* Validate that all events have been set up. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_assert(item->ev);
+    tt_assert(item->fn);
+    tt_u64_op(item->last_action_time, OP_EQ, 0);
+    /* Every event must have role(s) assign to it. This is done statically. */
+    tt_u64_op(item->roles, OP_NE, 0);
+    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  }
+
+ done:
+  teardown_periodic_events();
+}
+
+static void
+test_pe_launch(void *arg)
+{
+  hs_service_t service, *to_remove = NULL;
+  or_options_t *options;
+
+  (void) arg;
+
+  hs_init();
+  /* We need to put tor in hibernation live state so the events requiring
+   * network gets enabled. */
+  consider_hibernation(time(NULL));
+
+  /* Hack: We'll set a dumb fn() of each events so they don't get called when
+   * dispatching them. We just want to test the state of the callbacks, not
+   * the whole code path. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    item->fn = dumb_event_fn;
+  }
+
+  /* Lets make sure that before intialization, we can't scan the periodic
+   * events list and launch them. Lets try by being a Client. */
+  options = get_options_mutable();
+  options->SocksPort_set = 1;
+  periodic_events_on_new_options(options);
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  }
+
+  initialize_periodic_events();
+
+  /* Now that we've initialized, rescan the list to launch. */
+  periodic_events_on_new_options(options);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+    } else {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+    // enabled or not, the event has not yet been run.
+    tt_u64_op(item->last_action_time, OP_EQ, 0);
+  }
+
+  /* Remove Client but become a Relay. */
+  options->SocksPort_set = 0;
+  options->ORPort_set = 1;
+  periodic_events_on_new_options(options);
+
+  unsigned roles = get_my_roles(options);
+  tt_uint_op(roles, OP_EQ,
+             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    /* Only Client role should be disabled. */
+    if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+    if (item->roles & PERIODIC_EVENT_ROLE_RELAY) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+    }
+    /* Non Relay role should be disabled, except for Dirserver. */
+    if (!(item->roles & roles)) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+  }
+
+  /* Disable everything and we'll enable them ALL. */
+  options->SocksPort_set = 0;
+  options->ORPort_set = 0;
+  periodic_events_on_new_options(options);
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  }
+
+  /* Enable everything. */
+  options->SocksPort_set = 1; options->ORPort_set = 1;
+  options->BridgeRelay = 1; options->AuthoritativeDir = 1;
+  options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1;
+  register_dummy_hidden_service(&service);
+  periodic_events_on_new_options(options);
+  /* Note down the reference because we need to remove this service from the
+   * global list before the hs_free_all() call so it doesn't try to free
+   * memory on the stack. Furthermore, we can't remove it now else it will
+   * trigger a rescan of the event disabling the HS service event. */
+  to_remove = &service;
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+  }
+
+ done:
+  if (to_remove) {
+    remove_service(get_hs_service_map(), to_remove);
+  }
+  hs_free_all();
+}
+
+static void
+test_pe_get_roles(void *arg)
+{
+  int roles;
+
+  (void) arg;
+
+  /* Just so the HS global map exists. */
+  hs_init();
+
+  or_options_t *options = get_options_mutable();
+  tt_assert(options);
+
+  /* Nothing configured, should be no roles. */
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ, 0);
+
+  /* Indicate we have a SocksPort, roles should be come Client. */
+  options->SocksPort_set = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT);
+
+  /* Now, we'll add a ORPort so should now be a Relay + Client. */
+  options->ORPort_set = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
+             PERIODIC_EVENT_ROLE_DIRSERVER));
+
+  /* Now add a Bridge. */
+  options->BridgeRelay = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
+             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER));
+  tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER);
+  /* Unset client so we can solely test Router role. */
+  options->SocksPort_set = 0;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER);
+
+  /* Reset options so we can test authorities. */
+  options->SocksPort_set = 0;
+  options->ORPort_set = 0;
+  options->BridgeRelay = 0;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ, 0);
+
+  /* Now upgrade to Dirauth. */
+  options->DirPort_set = 1;
+  options->AuthoritativeDir = 1;
+  options->V3AuthoritativeDir = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+  /* Now Bridge Authority. */
+  options->V3AuthoritativeDir = 0;
+  options->BridgeAuthoritativeDir = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+  /* Move that bridge auth to become a relay. */
+  options->ORPort_set = 1;
+  roles = get_my_roles(options);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY
+             | PERIODIC_EVENT_ROLE_DIRSERVER));
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+  /* And now an Hidden service. */
+  hs_service_t service;
+  register_dummy_hidden_service(&service);
+  roles = get_my_roles(options);
+  /* Remove it now so the hs_free_all() doesn't try to free stack memory. */
+  remove_service(get_hs_service_map(), &service);
+  tt_int_op(roles, OP_EQ,
+            (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY |
+             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER));
+  tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
+
+ done:
+  hs_free_all();
+}
+
+static void
+test_pe_hs_service(void *arg)
+{
+  hs_service_t service, *to_remove = NULL;
+
+  (void) arg;
+
+  hs_init();
+  /* We need to put tor in hibernation live state so the events requiring
+   * network gets enabled. */
+  consider_hibernation(time(NULL));
+  /* Initialize the events so we can enable them */
+  initialize_periodic_events();
+
+  /* Hack: We'll set a dumb fn() of each events so they don't get called when
+   * dispatching them. We just want to test the state of the callbacks, not
+   * the whole code path. */
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    item->fn = dumb_event_fn;
+  }
+
+  /* This should trigger a rescan of the list and enable the HS service
+   * events. */
+  register_dummy_hidden_service(&service);
+  /* Note down the reference because we need to remove this service from the
+   * global list before the hs_free_all() call so it doesn't try to free
+   * memory on the stack. Furthermore, we can't remove it now else it will
+   * trigger a rescan of the event disabling the HS service event. */
+  to_remove = &service;
+
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+    }
+  }
+  to_remove = NULL;
+
+  /* Remove the service from the global map, it should trigger a rescan and
+   * disable the HS service events. */
+  remove_service(get_hs_service_map(), &service);
+  for (int i = 0; periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &periodic_events[i];
+    if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
+      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    }
+  }
+
+ done:
+  if (to_remove) {
+    remove_service(get_hs_service_map(), to_remove);
+  }
+  hs_free_all();
+}
+
+#define PE_TEST(name) \
+  { #name, test_pe_## name , TT_FORK, NULL, NULL }
+
+struct testcase_t periodic_event_tests[] = {
+  PE_TEST(initialize),
+  PE_TEST(launch),
+  PE_TEST(get_roles),
+  PE_TEST(hs_service),
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 1ffdc2cd51..e89d49aaf5 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -59,7 +59,7 @@ test_policy_summary_helper_family_flags(const char *policy_str,
 
   summary = policy_summarize(policy, family);
 
-  tt_assert(summary != NULL);
+  tt_ptr_op(summary, OP_NE, NULL);
   tt_str_op(summary,OP_EQ, expected_summary);
 
   short_policy = parse_short_policy(summary);
@@ -147,7 +147,7 @@ test_policies_general(void *arg)
 
   p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   tt_int_op(ADDR_POLICY_REJECT,OP_EQ, p->policy_type);
   tor_addr_from_ipv4h(&tar, 0xc0a80000u);
   tt_int_op(0,OP_EQ, tor_addr_compare(&p->addr, &tar, CMP_EXACT));
@@ -192,75 +192,75 @@ test_policies_general(void *arg)
   policy3 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("reject *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy3, p);
   p = router_parse_addr_policy_item_from_string("accept *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy3, p);
 
   policy4 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept *:443", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy4, p);
   p = router_parse_addr_policy_item_from_string("accept *:443", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy4, p);
 
   policy5 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",
                                                 -1, &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject *:1-65534", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("reject *:65535", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
   p = router_parse_addr_policy_item_from_string("accept *:1-65535", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy5, p);
 
   policy6 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy6, p);
 
   policy7 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy7, p);
 
   tt_int_op(0, OP_EQ, policies_parse_exit_policy(NULL, &policy8,
@@ -282,13 +282,13 @@ test_policies_general(void *arg)
   policy10 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("accept6 *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy10, p);
 
   policy11 = smartlist_new();
   p = router_parse_addr_policy_item_from_string("reject6 *:*", -1,
                                                 &malformed_list);
-  tt_assert(p != NULL);
+  tt_ptr_op(p, OP_NE, NULL);
   smartlist_add(policy11, p);
 
   tt_assert(!exit_policy_is_general_exit(policy));
@@ -392,21 +392,21 @@ test_policies_general(void *arg)
   p = router_parse_addr_policy_item_from_string("acce::abcd",
                                                 ADDR_POLICY_ACCEPT,
                                                 &malformed_list);
-  tt_assert(!p);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
   malformed_list = 0;
 
   p = router_parse_addr_policy_item_from_string("7:1234",
                                                 ADDR_POLICY_ACCEPT,
                                                 &malformed_list);
-  tt_assert(!p);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
   malformed_list = 0;
 
   p = router_parse_addr_policy_item_from_string("::",
                                                 ADDR_POLICY_ACCEPT,
                                                 &malformed_list);
-  tt_assert(!p);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
   malformed_list = 0;
 
@@ -968,73 +968,73 @@ test_policies_general(void *arg)
   /* Make sure that IPv4 addresses are ignored in accept6/reject6 lines. */
   p = router_parse_addr_policy_item_from_string("accept6 1.2.3.4:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(!malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject6 2.4.6.0/24:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(!malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept6 *4:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(!malformed_list);
 
   /* Make sure malformed policies are detected as such. */
   p = router_parse_addr_policy_item_from_string("bad_token *4:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept6 **:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept */15:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject6 */:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept 127.0.0.1/33:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("accept6 [::1]/129:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject 8.8.8.8/-1:*", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject 8.8.4.4:10-5", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   p = router_parse_addr_policy_item_from_string("reject 1.2.3.4:-1", -1,
                                                 &malformed_list);
-  tt_assert(p == NULL);
+  tt_ptr_op(p, OP_EQ, NULL);
   tt_assert(malformed_list);
 
   /* Test a too-long policy. */
   {
     char *policy_strng = NULL;
     smartlist_t *chunks = smartlist_new();
-    smartlist_add(chunks, tor_strdup("accept "));
+    smartlist_add_strdup(chunks, "accept ");
     for (i=1; i<10000; ++i)
       smartlist_add_asprintf(chunks, "%d,", i);
-    smartlist_add(chunks, tor_strdup("20000"));
+    smartlist_add_strdup(chunks, "20000");
     policy_strng = smartlist_join_strings(chunks, "", 0, NULL);
     SMARTLIST_FOREACH(chunks, char *, ch, tor_free(ch));
     smartlist_free(chunks);
@@ -1048,9 +1048,9 @@ test_policies_general(void *arg)
   for (i=1; i<2000; i+=2) {
     char buf[POLICY_BUF_LEN];
     tor_snprintf(buf, sizeof(buf), "reject *:%d", i);
-    smartlist_add(sm, tor_strdup(buf));
+    smartlist_add_strdup(sm, buf);
   }
-  smartlist_add(sm, tor_strdup("accept *:*"));
+  smartlist_add_strdup(sm, "accept *:*");
   policy_str = smartlist_join_strings(sm, ",", 0, NULL);
   test_policy_summary_helper( policy_str,
     "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,"
@@ -1148,7 +1148,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv4 addresses are rejected on an IPv4-only exit */
   policies_parse_exit_policy_reject_private(&policy, 0, ipv4_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1158,12 +1158,12 @@ test_policies_reject_exit_address(void *arg)
    * on IPv4-only exits, so policies_parse_exit_policy_reject_private doesn't
    * need to do anything) */
   policies_parse_exit_policy_reject_private(&policy, 0, ipv6_list, 0, 0);
-  tt_assert(policy == NULL);
+  tt_ptr_op(policy, OP_EQ, NULL);
 
   /* test that only IPv4 addresses are rejected on an IPv4-only exit */
   policies_parse_exit_policy_reject_private(&policy, 0, both_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1171,7 +1171,7 @@ test_policies_reject_exit_address(void *arg)
   /* Test that lists with duplicate entries produce the same results */
   policies_parse_exit_policy_reject_private(&policy, 0, dupl_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1181,7 +1181,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv4 addresses are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, ipv4_list, 0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1189,7 +1189,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv6 addresses are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, ipv6_list,  0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv6_addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1197,7 +1197,7 @@ test_policies_reject_exit_address(void *arg)
   /* test that IPv4 and IPv6 addresses are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, both_list,  0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 2);
+  tt_int_op(smartlist_len(policy), OP_EQ, 2);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   tt_assert(test_policy_has_address_helper(policy, &ipv6_addr));
   addr_policy_list_free(policy);
@@ -1206,7 +1206,7 @@ test_policies_reject_exit_address(void *arg)
   /* Test that lists with duplicate entries produce the same results */
   policies_parse_exit_policy_reject_private(&policy, 1, dupl_list,  0, 0);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 2);
+  tt_int_op(smartlist_len(policy), OP_EQ, 2);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_addr));
   tt_assert(test_policy_has_address_helper(policy, &ipv6_addr));
   addr_policy_list_free(policy);
@@ -1258,7 +1258,7 @@ test_policies_reject_port_address(void *arg)
    * with IPv6 addresses on IPv4-only exits) */
   policies_parse_exit_policy_reject_private(&policy, 0, NULL, 0, 1);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 1);
+  tt_int_op(smartlist_len(policy), OP_EQ, 1);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_port->addr));
   addr_policy_list_free(policy);
   policy = NULL;
@@ -1266,7 +1266,7 @@ test_policies_reject_port_address(void *arg)
   /* test that IPv4 and IPv6 ports are rejected on an IPv4/IPv6 exit */
   policies_parse_exit_policy_reject_private(&policy, 1, NULL, 0, 1);
   tt_assert(policy);
-  tt_assert(smartlist_len(policy) == 2);
+  tt_int_op(smartlist_len(policy), OP_EQ, 2);
   tt_assert(test_policy_has_address_helper(policy, &ipv4_port->addr));
   tt_assert(test_policy_has_address_helper(policy, &ipv6_port->addr));
   addr_policy_list_free(policy);
@@ -1318,7 +1318,7 @@ mock_get_interface_address6_list(int severity,
   return clone_list;
 
  done:
-  free_interface_address6_list(clone_list);
+  interface_address6_list_free(clone_list);
   return NULL;
 }
 
@@ -1337,7 +1337,7 @@ test_policies_reject_interface_address(void *arg)
 
   /* test that no addresses are rejected when none are supplied/requested */
   policies_parse_exit_policy_reject_private(&policy, 0, NULL, 0, 0);
-  tt_assert(policy == NULL);
+  tt_ptr_op(policy, OP_EQ, NULL);
 
   /* test that only IPv4 interface addresses are rejected on an IPv4-only exit
    * (and allow for duplicates)
@@ -1372,7 +1372,7 @@ test_policies_reject_interface_address(void *arg)
 
   /* test that no addresses are rejected when none are supplied/requested */
   policies_parse_exit_policy_reject_private(&policy, 0, NULL, 0, 0);
-  tt_assert(policy == NULL);
+  tt_ptr_op(policy, OP_EQ, NULL);
 
   /* test that only IPv4 interface addresses are rejected on an IPv4-only exit
    */
@@ -1393,11 +1393,11 @@ test_policies_reject_interface_address(void *arg)
 
  done:
   addr_policy_list_free(policy);
-  free_interface_address6_list(public_ipv4_addrs);
-  free_interface_address6_list(public_ipv6_addrs);
+  interface_address6_list_free(public_ipv4_addrs);
+  interface_address6_list_free(public_ipv6_addrs);
 
   UNMOCK(get_interface_address6_list);
-  /* we don't use free_interface_address6_list on these lists because their
+  /* we don't use interface_address6_list_free on these lists because their
    * address pointers are stack-based */
   smartlist_free(mock_ipv4_addrs);
   smartlist_free(mock_ipv6_addrs);
@@ -1496,9 +1496,21 @@ test_dump_exit_policy_to_string(void *arg)
 }
 
 static routerinfo_t *mock_desc_routerinfo = NULL;
+static int routerinfo_err;
+
 static const routerinfo_t *
-mock_router_get_my_routerinfo(void)
+mock_router_get_my_routerinfo_with_err(int *err)
 {
+  if (routerinfo_err) {
+    if (err)
+      *err = routerinfo_err;
+
+    return NULL;
+  }
+
+  if (err)
+    *err = 0;
+
   return mock_desc_routerinfo;
 }
 
@@ -1528,20 +1540,21 @@ test_policies_getinfo_helper_policies(void *arg)
   memset(&mock_my_routerinfo, 0, sizeof(mock_my_routerinfo));
 
   rv = getinfo_helper_policies(NULL, "exit-policy/default", &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/default",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
   memset(&mock_my_routerinfo, 0, sizeof(routerinfo_t));
-  MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo_with_err,
+       mock_router_get_my_routerinfo_with_err);
   mock_my_routerinfo.exit_policy = smartlist_new();
   mock_desc_routerinfo = &mock_my_routerinfo;
 
@@ -1550,15 +1563,15 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) == 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv4", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv4_len = strlen(answer);
   tt_assert(ipv4_len == 0 || ipv4_len == strlen(DEFAULT_POLICY_STRING));
   tt_assert(ipv4_len == 0 || !strcasecmp(answer, DEFAULT_POLICY_STRING));
@@ -1566,8 +1579,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv6", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv6_len = strlen(answer);
   tt_assert(ipv6_len == 0 || ipv6_len == strlen(DEFAULT_POLICY_STRING));
   tt_assert(ipv6_len == 0 || !strcasecmp(answer, DEFAULT_POLICY_STRING));
@@ -1575,8 +1588,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   /* It's either empty or it's the default */
   tt_assert(strlen(answer) == 0 || !strcasecmp(answer, DEFAULT_POLICY_STRING));
   tor_free(answer);
@@ -1587,16 +1600,20 @@ test_policies_getinfo_helper_policies(void *arg)
   append_exit_policy_string(&mock_my_routerinfo.exit_policy, "reject *6:*");
 
   mock_options.IPv6Exit = 1;
-  tor_addr_from_ipv4h(&mock_options.OutboundBindAddressIPv4_, TEST_IPV4_ADDR);
-  tor_addr_parse(&mock_options.OutboundBindAddressIPv6_, TEST_IPV6_ADDR);
+  tor_addr_from_ipv4h(
+      &mock_options.OutboundBindAddresses[OUTBOUND_ADDR_EXIT][0],
+      TEST_IPV4_ADDR);
+  tor_addr_parse(
+      &mock_options.OutboundBindAddresses[OUTBOUND_ADDR_EXIT][1],
+      TEST_IPV6_ADDR);
 
   mock_options.ExitPolicyRejectPrivate = 1;
   mock_options.ExitPolicyRejectLocalInterfaces = 1;
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
@@ -1605,8 +1622,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
@@ -1615,8 +1632,8 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tor_free(answer);
 
@@ -1625,35 +1642,84 @@ test_policies_getinfo_helper_policies(void *arg)
 
   rv = getinfo_helper_policies(NULL, "exit-policy/reject-private/relay",
                                &answer, &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) == 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv4", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv4_len = strlen(answer);
   tt_assert(ipv4_len > 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/ipv6", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   ipv6_len = strlen(answer);
   tt_assert(ipv6_len > 0);
   tor_free(answer);
 
   rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
                                &errmsg);
-  tt_assert(rv == 0);
-  tt_assert(answer != NULL);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_NE, NULL);
   tt_assert(strlen(answer) > 0);
   tt_assert(strlen(answer) == ipv4_len + ipv6_len + 1);
   tor_free(answer);
 
+  routerinfo_err = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "No known exit address yet");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_PARSE;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Cannot parse descriptor");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Not running in server mode");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Key digest failed");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Cannot generate descriptor");
+
+  routerinfo_err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+  rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer,
+                               &errmsg);
+  tt_int_op(rv, OP_EQ, -1);
+  tt_ptr_op(answer, OP_EQ, NULL);
+  tt_ptr_op(errmsg, OP_NE, NULL);
+  tt_str_op(errmsg, OP_EQ, "Descriptor still rebuilding - not ready yet");
+
  done:
   tor_free(answer);
   UNMOCK(get_options);
@@ -1742,34 +1808,34 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 1;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv4 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv6 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 0);
 
   /* Test the function's address matching with UseBridges on */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1777,46 +1843,46 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 1;
   mock_options.UseBridges = 1;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv4 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 0),
+            OP_EQ, 0);
 
   /* Preferring IPv6 */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 1, 1),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 1, 1),
+            OP_EQ, 0);
 
   /* bridge clients always use IPv6, regardless of ClientUseIPv6 */
   mock_options.ClientUseIPv4 = 1;
   mock_options.ClientUseIPv6 = 0;
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching with IPv4 on */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1824,14 +1890,14 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 0;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching with IPv6 on */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1839,14 +1905,14 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 1;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching with ClientUseIPv4 0.
    * This means "use IPv6" regardless of the other settings. */
@@ -1855,14 +1921,14 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.ClientUseIPv6 = 0;
   mock_options.UseBridges = 0;
 
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&r_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* Test the function's address matching for unusual inputs */
   memset(&mock_options, 0, sizeof(or_options_t));
@@ -1871,27 +1937,28 @@ test_policies_fascist_firewall_allows_address(void *arg)
   mock_options.UseBridges = 1;
 
   /* NULL and tor_addr_is_null addresses are rejected */
-  tt_assert(fascist_firewall_allows_address(NULL, port, policy, 0, 0) == 0);
-  tt_assert(fascist_firewall_allows_address(&n_ipv4_addr, port, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&n_ipv6_addr, port, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(NULL, port, policy, 0, 0), OP_EQ,
+            0);
+  tt_int_op(fascist_firewall_allows_address(&n_ipv4_addr, port, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&n_ipv6_addr, port, policy, 0, 0),
+            OP_EQ, 0);
 
   /* zero ports are rejected */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, 0, policy, 0, 0)
-            == 0);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, 0, policy, 0, 0)
-            == 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, 0, policy, 0, 0),
+            OP_EQ, 0);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, 0, policy, 0, 0),
+            OP_EQ, 0);
 
   /* NULL and empty policies accept everything */
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, NULL, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, NULL, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv4_addr, port, e_policy, 0, 0)
-            == 1);
-  tt_assert(fascist_firewall_allows_address(&ipv6_addr, port, e_policy, 0, 0)
-            == 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, NULL, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, NULL, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv4_addr, port, e_policy, 0, 0),
+            OP_EQ, 1);
+  tt_int_op(fascist_firewall_allows_address(&ipv6_addr, port, e_policy, 0, 0),
+            OP_EQ, 1);
 
  done:
   addr_policy_free(item);
@@ -1918,11 +1985,8 @@ test_policies_fascist_firewall_allows_address(void *arg)
     tor_addr_port_t chosen_rs_ap; \
     tor_addr_make_null(&chosen_rs_ap.addr, AF_INET); \
     chosen_rs_ap.port = 0; \
-    tt_int_op(fascist_firewall_choose_address_rs(&(fake_rs), \
-                                                 (fw_connection), \
-                                                 (pref_only), \
-                                                 &chosen_rs_ap), \
-              OP_EQ, (expect_rv)); \
+    fascist_firewall_choose_address_rs(&(fake_rs), (fw_connection), \
+                                       (pref_only), &chosen_rs_ap); \
     tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_rs_ap.addr)); \
     tt_int_op((expect_ap).port, OP_EQ, chosen_rs_ap.port); \
   STMT_END
@@ -1935,11 +1999,8 @@ test_policies_fascist_firewall_allows_address(void *arg)
     tor_addr_port_t chosen_node_ap; \
     tor_addr_make_null(&chosen_node_ap.addr, AF_INET); \
     chosen_node_ap.port = 0; \
-    tt_int_op(fascist_firewall_choose_address_node(&(fake_node), \
-                                                   (fw_connection), \
-                                                   (pref_only), \
-                                                   &chosen_node_ap), \
-              OP_EQ, (expect_rv)); \
+    fascist_firewall_choose_address_node(&(fake_node),(fw_connection), \
+                                         (pref_only), &chosen_node_ap); \
     tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_node_ap.addr)); \
     tt_int_op((expect_ap).port, OP_EQ, chosen_node_ap.port); \
   STMT_END
@@ -2028,12 +2089,12 @@ test_policies_fascist_firewall_choose_address(void *arg)
             == &ipv6_or_ap);
 
   /* null both OR addresses */
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
-                                            FIREWALL_OR_CONNECTION, 0, 1)
-            == NULL);
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
-                                            FIREWALL_OR_CONNECTION, 0, 0)
-            == NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
+                                            FIREWALL_OR_CONNECTION, 0, 1),
+            OP_EQ, NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
+                                            FIREWALL_OR_CONNECTION, 0, 0),
+            OP_EQ, NULL);
 
   /* null preferred Dir addresses */
   tt_assert(fascist_firewall_choose_address(&ipv4_dir_ap, &n_ipv6_ap, 0,
@@ -2044,12 +2105,12 @@ test_policies_fascist_firewall_choose_address(void *arg)
             == &ipv6_dir_ap);
 
   /* null both Dir addresses */
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
-                                            FIREWALL_DIR_CONNECTION, 0, 1)
-            == NULL);
-  tt_assert(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
-                                            FIREWALL_DIR_CONNECTION, 0, 0)
-            == NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 0,
+                                            FIREWALL_DIR_CONNECTION, 0, 1),
+            OP_EQ, NULL);
+  tt_ptr_op(fascist_firewall_choose_address(&n_ipv4_ap, &n_ipv6_ap, 1,
+                                            FIREWALL_DIR_CONNECTION, 0, 0),
+            OP_EQ, NULL);
 
   /* Prefer IPv4 but want IPv6 (contradictory) */
   tt_assert(fascist_firewall_choose_address(&ipv4_or_ap, &ipv6_or_ap, 0,
diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c
index 9e63fc006d..5c52af8693 100644
--- a/src/test/test_procmon.c
+++ b/src/test/test_procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROCMON_PRIVATE
diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c
new file mode 100644
index 0000000000..2f36fbccd7
--- /dev/null
+++ b/src/test/test_proto_http.c
@@ -0,0 +1,213 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_proto_http.c
+ * \brief Tests for our HTTP protocol parser code
+ */
+
+#include "or.h"
+#include "test.h"
+#include "buffers.h"
+#include "proto_http.h"
+#include "log_test_helpers.h"
+
+#define S(str) str, sizeof(str)-1
+
+static void
+test_proto_http_peek(void *arg)
+{
+  (void) arg;
+  const struct {
+    int is_http;
+    const char *message;
+    size_t len;
+  } cases[] = {
+    { 1, S("GET /index HTTP/1.0\r\n") },
+    { 1, S("GET /index HTTP/1.1\r\n") },
+    { 1, S("GET ") },
+    { 0, S("GIT ") },
+    { 0, S("GET") },
+    { 0, S("get ") },
+    { 0, S("GETAWAY") },
+  };
+  unsigned i;
+  buf_t *buf = buf_new();
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    TT_BLATHER(("Trying case %u", i));
+    buf_add(buf, cases[i].message, cases[i].len);
+    tt_int_op(cases[i].is_http, OP_EQ, peek_buf_has_http_command(buf));
+    buf_clear(buf);
+  }
+ done:
+  buf_free(buf);
+}
+
+static void
+test_proto_http_valid(void *arg)
+{
+  (void) arg;
+  const struct {
+    const char *message;
+    size_t len;
+    const char *headers;
+    const char *body;
+    size_t bodylen;
+    int should_detect_truncated;
+    int bytes_left_over;
+  } cases[] = {
+    { S("GET /index.html HTTP/1.0\r\n\r\n"),
+      "GET /index.html HTTP/1.0\r\n\r\n",
+      S(""),
+      1, 0,
+    },
+    { S("PUT /tor/foo HTTP/1.1\r\n"
+        "Content-Length: 51\r\n\r\n"
+        "this is a test of the http parsing system . test te"),
+      "PUT /tor/foo HTTP/1.1\r\n" "Content-Length: 51\r\n\r\n",
+      S("this is a test of the http parsing system . test te"),
+      1, 0,
+    },
+    { S("PUT /tor/foo HTTP/1.1\r\n"
+        "Content-Length: 5\r\n\r\n"
+        "there are more than 5 characters in this body."),
+      "PUT /tor/foo HTTP/1.1\r\n" "Content-Length: 5\r\n\r\n",
+      S("there"),
+      0, 41,
+    },
+    { S("PUT /tor/bar HTTP/1.1\r\n\r\n"
+        "this is another \x00test"),
+      "PUT /tor/bar HTTP/1.1\r\n\r\n",
+      S("this is another \x00test"),
+      0, 0,
+    }
+  };
+  unsigned i;
+  buf_t *buf = buf_new();
+  char *h = NULL, *b = NULL;
+
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    TT_BLATHER(("Trying case %u", i));
+    size_t bl = 0;
+    // truncate by 2 chars
+    buf_add(buf, cases[i].message, cases[i].len - 2);
+
+    if (cases[i].should_detect_truncated) {
+      tt_int_op(0, OP_EQ, fetch_from_buf_http(buf, &h, 1024*16,
+                                              &b, &bl, 1024*16, 0));
+      tt_ptr_op(h, OP_EQ, NULL);
+      tt_ptr_op(b, OP_EQ, NULL);
+      tt_u64_op(bl, OP_EQ, 0);
+      tt_int_op(buf_datalen(buf), OP_EQ, cases[i].len - 2);
+    }
+
+    // add the rest.
+    buf_add(buf, cases[i].message+cases[i].len-2, 2);
+    tt_int_op(1, OP_EQ, fetch_from_buf_http(buf, &h, 1024*16,
+                                            &b, &bl, 1024*16, 0));
+    tt_str_op(h, OP_EQ, cases[i].headers);
+    tt_u64_op(bl, OP_EQ, cases[i].bodylen);
+    tt_mem_op(b, OP_EQ, cases[i].body, bl);
+    tt_int_op(buf_datalen(buf), OP_EQ, cases[i].bytes_left_over);
+
+    buf_clear(buf);
+    tor_free(h);
+    tor_free(b);
+  }
+ done:
+  tor_free(h);
+  tor_free(b);
+  buf_free(buf);
+}
+
+static void
+test_proto_http_invalid(void *arg)
+{
+  (void) arg;
+  const struct {
+    const char *message;
+    size_t len;
+    const char *expect;
+  } cases[] = {
+    /* Overlong headers, headers not finished. */
+    { S("GET /index.xhml HTTP/1.0\r\n"
+        "X-My-headers-are-too-long: yes indeed they are. They might be\r\n"
+        "X-My-headers-are-too-long: normal under other circumstances, but\r\n"
+        "X-My-headers-are-too-long: the 128-byte limit makes them bad\r\n"),
+      "headers too long." },
+    /* Overlong finished headers. */
+    { S("GET /index.xhml HTTP/1.0\r\n"
+        "X-My-headers-are-too-long: yes indeed they are. They might be\r\n"
+        "X-My-headers-are-too-long: normal under other circumstances, but\r\n"
+        "X-My-headers-are-too-long: the 128-byte limit makes them bad\r\n"
+        "\r\n"),
+        "headers too long." },
+    /* Exactly too long finished headers. */
+    { S("GET /index.xhml HTTP/1.0\r\n"
+        "X-My-headers-are-too-long: yes indeed they are. They might be\r\n"
+        "X-My-headers-are-too-long: normal un\r\n\r\n"),
+      "headerlen 129 larger than 127. Failing." },
+    /* Body too long, with content-length */
+    { S("GET /index.html HTTP/1.0\r\n"
+        "Content-Length: 129\r\n\r\n"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxx"),
+      "bodylen 129 larger than 127" },
+    /* Body too long, with content-length lying */
+    { S("GET /index.html HTTP/1.0\r\n"
+        "Content-Length: 99999\r\n\r\n"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxx"),
+      "bodylen 138 larger than 127" },
+    /* Body too long, no content-length. */
+    { S("GET /index.html HTTP/1.0\r\n\r\n"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+        "xxxxxxxxxxxxxxxxxxxxxxxxxxxxz"),
+      "bodylen 139 larger than 127" },
+    /* Content-Length is junk. */
+    { S("GET /index.html HTTP/1.0\r\n"
+        "Content-Length: Cheese\r\n\r\n"
+        "foo"),
+      "Content-Length is bogus; maybe someone is trying to crash us." },
+    };
+  unsigned i;
+  buf_t *buf = buf_new();
+  char *h = NULL, *b = NULL;
+  setup_capture_of_logs(LOG_DEBUG);
+
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    TT_BLATHER(("Trying case %u", i));
+    size_t bl = 0;
+    buf_add(buf, cases[i].message, cases[i].len);
+
+    /* Use low body limits here so we can force over-sized object warnings */
+    tt_int_op(-1, OP_EQ, fetch_from_buf_http(buf, &h, 128,
+                                             &b, &bl, 128, 0));
+    tt_ptr_op(h, OP_EQ, NULL);
+    tt_ptr_op(b, OP_EQ, NULL);
+    tt_u64_op(bl, OP_EQ, 0);
+    expect_log_msg_containing(cases[i].expect);
+
+    buf_clear(buf);
+    tor_free(h);
+    tor_free(b);
+    mock_clean_saved_logs();
+  }
+ done:
+  tor_free(h);
+  tor_free(b);
+  buf_free(buf);
+  teardown_capture_of_logs();
+}
+
+struct testcase_t proto_http_tests[] = {
+  { "peek", test_proto_http_peek, 0, NULL, NULL },
+  { "valid", test_proto_http_valid, 0, NULL, NULL },
+  { "invalid", test_proto_http_invalid, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c
new file mode 100644
index 0000000000..263ca47447
--- /dev/null
+++ b/src/test/test_proto_misc.c
@@ -0,0 +1,263 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_proto_misc.c
+ * \brief Test our smaller buffer-based protocol functions
+ */
+
+#include "or.h"
+#include "test.h"
+#include "buffers.h"
+#include "connection_or.h"
+#include "ext_orport.h"
+#include "proto_cell.h"
+#include "proto_control0.h"
+#include "proto_ext_or.h"
+
+static void
+test_proto_var_cell(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  char tmp[1024];
+  buf_t *buf = NULL;
+  var_cell_t *cell = NULL;
+
+  buf = buf_new();
+  memset(tmp, 0xf0, sizeof(tmp));
+
+  /* Short little commands will make us say "no cell yet." */
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  tt_ptr_op(cell, OP_EQ, NULL);
+  buf_add(buf, "\x01\x02\x02\0x2", 4);
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  /* An incomplete fixed-length cell makes us say "no cell yet". */
+  buf_add(buf, "\x03", 1);
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  /* A complete fixed length-cell makes us say "not a variable-length cell" */
+  buf_add(buf, tmp, 509);
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  buf_clear(buf);
+
+  /* An incomplete versions cell is a variable-length cell that isn't ready
+   * yet. */
+  buf_add(buf,
+          "\x01\x02\x03\x04" /* circid */
+          "\x07" /* VERSIONS */
+          "\x00\x04" /* 4 bytes long */
+          "\x00" /* incomplete */, 8);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  tt_ptr_op(cell, OP_EQ, NULL);
+  /* Complete it, and it's a variable-length cell. Leave a byte on the end for
+   * fun. */
+  buf_add(buf, "\x09\x00\x25\ff", 4);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 4));
+  tt_ptr_op(cell, OP_NE, NULL);
+  tt_int_op(cell->command, OP_EQ, CELL_VERSIONS);
+  tt_uint_op(cell->circ_id, OP_EQ, 0x01020304);
+  tt_int_op(cell->payload_len, OP_EQ, 4);
+  test_mem_op_hex(cell->payload, OP_EQ, "00090025");
+  var_cell_free(cell);
+  cell = NULL;
+  tt_int_op(buf_datalen(buf), OP_EQ, 1);
+  buf_clear(buf);
+
+  /* In link protocol 3 and earlier, circid fields were two bytes long. Let's
+   * ensure that gets handled correctly. */
+  buf_add(buf,
+          "\x23\x45\x81\x00\x06" /* command 81; 6 bytes long */
+          "coraje", 11);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 3));
+  tt_ptr_op(cell, OP_NE, NULL);
+  tt_int_op(cell->command, OP_EQ, 129);
+  tt_uint_op(cell->circ_id, OP_EQ, 0x2345);
+  tt_int_op(cell->payload_len, OP_EQ, 6);
+  tt_mem_op(cell->payload, OP_EQ, "coraje", 6);
+  var_cell_free(cell);
+  cell = NULL;
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* In link protocol 2, only VERSIONS cells counted as variable-length */
+  buf_add(buf,
+          "\x23\x45\x81\x00\x06"
+          "coraje", 11); /* As above */
+  tt_int_op(0, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 2));
+  buf_clear(buf);
+  buf_add(buf,
+          "\x23\x45\x07\x00\x06"
+          "futuro", 11);
+  tt_int_op(1, OP_EQ, fetch_var_cell_from_buf(buf, &cell, 2));
+  tt_ptr_op(cell, OP_NE, NULL);
+  tt_int_op(cell->command, OP_EQ, 7);
+  tt_uint_op(cell->circ_id, OP_EQ, 0x2345);
+  tt_int_op(cell->payload_len, OP_EQ, 6);
+  tt_mem_op(cell->payload, OP_EQ, "futuro", 6);
+  var_cell_free(cell);
+  cell = NULL;
+
+ done:
+  buf_free(buf);
+  var_cell_free(cell);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_proto_control0(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+
+  /* The only remaining function for the v0 control protocol is the function
+     that detects whether the user has stumbled across an old controller
+     that's using it.  The format was:
+        u16 length;
+        u16 command;
+        u8 body[length];
+  */
+
+  /* Empty buffer -- nothing to do. */
+  tt_int_op(0, OP_EQ, peek_buf_has_control0_command(buf));
+  /* 3 chars in buf -- can't tell */
+  buf_add(buf, "AUT", 3);
+  tt_int_op(0, OP_EQ, peek_buf_has_control0_command(buf));
+  /* command in buf -- easy to tell */
+  buf_add(buf, "HENTICATE ", 10);
+  tt_int_op(0, OP_EQ, peek_buf_has_control0_command(buf));
+
+  /* Control0 command header in buf: make sure we detect it. */
+  buf_clear(buf);
+  buf_add(buf, "\x09\x05" "\x00\x05" "blah", 8);
+  tt_int_op(1, OP_EQ, peek_buf_has_control0_command(buf));
+
+ done:
+  buf_free(buf);
+}
+
+static void
+test_proto_ext_or_cmd(void *arg)
+{
+  ext_or_cmd_t *cmd = NULL;
+  buf_t *buf = buf_new();
+  char *tmp = NULL;
+  (void) arg;
+
+  /* Empty -- should give "not there. */
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_EQ, cmd);
+
+  /* Three bytes: shouldn't work. */
+  buf_add(buf, "\x00\x20\x00", 3);
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_EQ, cmd);
+  tt_int_op(3, OP_EQ, buf_datalen(buf));
+
+  /* 0020 0000: That's a nil command. It should work. */
+  buf_add(buf, "\x00", 1);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0x20, OP_EQ, cmd->cmd);
+  tt_int_op(0, OP_EQ, cmd->len);
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+  /* Now try a length-6 command with one byte missing. */
+  buf_add(buf, "\x10\x21\x00\x06""abcde", 9);
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_EQ, cmd);
+  buf_add(buf, "f", 1);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0x1021, OP_EQ, cmd->cmd);
+  tt_int_op(6, OP_EQ, cmd->len);
+  tt_mem_op("abcdef", OP_EQ, cmd->body, 6);
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+  /* Now try a length-10 command with 4 extra bytes. */
+  buf_add(buf, "\xff\xff\x00\x0aloremipsum\x10\x00\xff\xff", 18);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0xffff, OP_EQ, cmd->cmd);
+  tt_int_op(10, OP_EQ, cmd->len);
+  tt_mem_op("loremipsum", OP_EQ, cmd->body, 10);
+  tt_int_op(4, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+  /* Finally, let's try a maximum-length command. We already have the header
+   * waiting. */
+  tt_int_op(0, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tmp = tor_malloc_zero(65535);
+  buf_add(buf, tmp, 65535);
+  tt_int_op(1, OP_EQ, fetch_ext_or_command_from_buf(buf, &cmd));
+  tt_ptr_op(NULL, OP_NE, cmd);
+  tt_int_op(0x1000, OP_EQ, cmd->cmd);
+  tt_int_op(0xffff, OP_EQ, cmd->len);
+  tt_mem_op(tmp, OP_EQ, cmd->body, 65535);
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+  ext_or_cmd_free(cmd);
+  cmd = NULL;
+
+ done:
+  ext_or_cmd_free(cmd);
+  buf_free(buf);
+  tor_free(tmp);
+}
+
+static void
+test_proto_line(void *arg)
+{
+  (void)arg;
+  char tmp[60];
+  buf_t *buf = buf_new();
+#define S(str) str, sizeof(str)-1
+  const struct {
+    const char *input;
+    size_t input_len;
+    size_t line_len;
+    const char *output;
+    int returnval;
+  } cases[] = {
+    { S("Hello world"), 0, NULL, 0 },
+    { S("Hello world\n"), 12, "Hello world\n", 1 },
+    { S("Hello world\nMore"), 12, "Hello world\n", 1 },
+    { S("\n oh hello world\nMore"), 1, "\n", 1 },
+    { S("Hello worpd\n\nMore"), 12, "Hello worpd\n", 1 },
+    { S("------------------------------------------------------------\n"), 0,
+      NULL, -1 },
+  };
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(cases); ++i) {
+    buf_add(buf, cases[i].input, cases[i].input_len);
+    memset(tmp, 0xfe, sizeof(tmp));
+    size_t sz = sizeof(tmp);
+    int rv = buf_get_line(buf, tmp, &sz);
+    tt_int_op(rv, OP_EQ, cases[i].returnval);
+    if (rv == 1) {
+      tt_int_op(sz, OP_LT, sizeof(tmp));
+      tt_mem_op(cases[i].output, OP_EQ, tmp, sz+1);
+      tt_int_op(buf_datalen(buf), OP_EQ, cases[i].input_len - strlen(tmp));
+      tt_int_op(sz, OP_EQ, cases[i].line_len);
+    } else {
+      tt_int_op(buf_datalen(buf), OP_EQ, cases[i].input_len);
+      // tt_int_op(sz, OP_EQ, sizeof(tmp));
+    }
+    buf_clear(buf);
+  }
+
+ done:
+  buf_free(buf);
+}
+
+struct testcase_t proto_misc_tests[] = {
+  { "var_cell", test_proto_var_cell, 0, NULL, NULL },
+  { "control0", test_proto_control0, 0, NULL, NULL },
+  { "ext_or_cmd", test_proto_ext_or_cmd, TT_FORK, NULL, NULL },
+  { "line", test_proto_line, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index c4379a15e1..06fd575631 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROTOVER_PRIVATE
@@ -8,10 +8,20 @@
 
 #include "protover.h"
 
+#include "or.h"
+#include "connection_or.h"
+
 static void
 test_protover_parse(void *arg)
 {
   (void) arg;
+#ifdef HAVE_RUST
+  /** This test is disabled on rust builds, because it only exists to test
+   * internal C functions. */
+  tt_skip();
+ done:
+  ;
+#else
   char *re_encoded = NULL;
 
   const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900";
@@ -78,38 +88,51 @@ test_protover_parse(void *arg)
     SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(elts);
   tor_free(re_encoded);
+#endif
 }
 
 static void
 test_protover_parse_fail(void *arg)
 {
   (void)arg;
+#ifdef HAVE_RUST
+  /** This test is disabled on rust builds, because it only exists to test
+   * internal C functions. */
+  tt_skip();
+#else
   smartlist_t *elts;
 
   /* random junk */
   elts = parse_protocol_list("!!3@*");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Missing equals sign in an entry */
   elts = parse_protocol_list("Link=4 Haprauxymatyve Desc=9");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Missing word. */
   elts = parse_protocol_list("Link=4 =3 Desc=9");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Broken numbers */
   elts = parse_protocol_list("Link=fred");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
   elts = parse_protocol_list("Link=1,fred");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
   elts = parse_protocol_list("Link=1,fred,3");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
 
   /* Broken range */
   elts = parse_protocol_list("Link=1,9-8,3");
-  tt_assert(elts == NULL);
+  tt_ptr_op(elts, OP_EQ, NULL);
+
+  /* Protocol name too long */
+  elts = parse_protocol_list("DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                           "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                           "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+  tt_ptr_op(elts, OP_EQ, NULL);
 
+#endif
  done:
   ;
 }
@@ -203,6 +226,15 @@ test_protover_vote(void *arg)
   tt_str_op(result, OP_EQ, "");
   tor_free(result);
 
+  /* Protocol name too long */
+  smartlist_clear(lst);
+  smartlist_add(lst, (void*) "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+  result = protover_compute_vote(lst, 1);
+  tt_str_op(result, OP_EQ, "");
+  tor_free(result);
+
  done:
   tor_free(result);
   smartlist_free(lst);
@@ -215,18 +247,19 @@ test_protover_all_supported(void *arg)
   char *msg = NULL;
 
   tt_assert(protover_all_supported(NULL, &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   tt_assert(protover_all_supported("", &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   // Some things that we do support
   tt_assert(protover_all_supported("Link=3-4", &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
   tt_assert(protover_all_supported("Link=3-4 Desc=2", &msg));
-  tt_assert(msg == NULL);
+  tt_ptr_op(msg, OP_EQ, NULL);
 
   // Some things we don't support
+  tt_assert(! protover_all_supported("Wombat=9", NULL));
   tt_assert(! protover_all_supported("Wombat=9", &msg));
   tt_str_op(msg, OP_EQ, "Wombat=9");
   tor_free(msg);
@@ -238,35 +271,60 @@ test_protover_all_supported(void *arg)
   tt_assert(! protover_all_supported("Link=3-4 Wombat=9", &msg));
   tt_str_op(msg, OP_EQ, "Wombat=9");
   tor_free(msg);
+
+  /* Mix of things we support and don't support within a single protocol
+   * which we do support */
   tt_assert(! protover_all_supported("Link=3-999", &msg));
-  tt_str_op(msg, OP_EQ, "Link=3-999");
+  tt_str_op(msg, OP_EQ, "Link=6-999");
+  tor_free(msg);
+  tt_assert(! protover_all_supported("Link=1-3,345-666", &msg));
+  tt_str_op(msg, OP_EQ, "Link=345-666");
+  tor_free(msg);
+  tt_assert(! protover_all_supported("Link=1-3,5-12", &msg));
+  tt_str_op(msg, OP_EQ, "Link=6-12");
   tor_free(msg);
 
-  /* CPU/RAM DoS loop: Rust only */
-  tt_assert(! protover_all_supported("Sleen=0-2147483648", &msg));
-  tt_str_op(msg, OP_EQ, "Sleen=0-2147483648");
+  /* Mix of protocols we do support and some we don't, where the protocols
+   * we do support have some versions we don't support. */
+  tt_assert(! protover_all_supported("Link=1-3,5-12 Quokka=9000-9001", &msg));
+  tt_str_op(msg, OP_EQ, "Link=6-12 Quokka=9000-9001");
+  tor_free(msg);
+
+  /* We shouldn't be able to DoS ourselves parsing a large range. */
+  tt_assert(! protover_all_supported("Sleen=1-2147483648", &msg));
+  tt_str_op(msg, OP_EQ, "Sleen=1-2147483648");
   tor_free(msg);
 
   /* This case is allowed. */
-  tt_assert(! protover_all_supported("Sleen=0-4294967294", &msg));
-  tt_str_op(msg, OP_EQ, "Sleen=0-4294967294");
+  tt_assert(! protover_all_supported("Sleen=1-4294967294", &msg));
+  tt_str_op(msg, OP_EQ, "Sleen=1-4294967294");
   tor_free(msg);
 
-  /* If we get an unparseable list, we say "yes, that's supported." */
-#ifndef HAVE_RUST
-  // XXXX let's make this section unconditional: rust should behave the
-  // XXXX same as C here!
+  /* If we get a (barely) valid (but unsupported list, we say "yes, that's
+   * supported." */
+  tt_assert(protover_all_supported("Fribble=", &msg));
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  /* If we get a completely unparseable list, protover_all_supported should
+   * hit a fatal assertion for BUG(entries == NULL). */
   tor_capture_bugs_(1);
   tt_assert(protover_all_supported("Fribble", &msg));
-  tt_ptr_op(msg, OP_EQ, NULL);
   tor_end_capture_bugs_();
 
-  /* This case is forbidden. Since it came from a protover_all_supported,
-   * it can trigger a bug message.  */
+  /* If we get a completely unparseable list, protover_all_supported should
+   * hit a fatal assertion for BUG(entries == NULL). */
   tor_capture_bugs_(1);
-  tt_assert(protover_all_supported("Sleen=0-4294967295", &msg));
-  tt_ptr_op(msg, OP_EQ, NULL);
-  tor_free(msg);
+  tt_assert(protover_all_supported("Sleen=1-4294967295", &msg));
+  tor_end_capture_bugs_();
+
+  /* Protocol name too long */
+#ifndef HAVE_RUST // XXXXXX ?????
+  tor_capture_bugs_(1);
+  tt_assert(protover_all_supported(
+                 "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                 "aaaaaaaaaaaa=1-65536", &msg));
   tor_end_capture_bugs_();
 #endif
 
@@ -276,6 +334,209 @@ test_protover_all_supported(void *arg)
 }
 
 static void
+test_protover_list_supports_protocol_returns_true(void *arg)
+{
+  (void)arg;
+
+  const char *protocols = "Link=1";
+  int is_supported = protocol_list_supports_protocol(protocols, PRT_LINK, 1);
+  tt_int_op(is_supported, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_protover_list_supports_protocol_for_unsupported_returns_false(void *arg)
+{
+  (void)arg;
+
+  const char *protocols = "Link=1";
+  int is_supported = protocol_list_supports_protocol(protocols, PRT_LINK, 10);
+  tt_int_op(is_supported, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+static void
+test_protover_supports_version(void *arg)
+{
+  (void)arg;
+
+  tt_assert(protocol_list_supports_protocol("Link=3-6", PRT_LINK, 3));
+  tt_assert(protocol_list_supports_protocol("Link=3-6", PRT_LINK, 6));
+  tt_assert(!protocol_list_supports_protocol("Link=3-6", PRT_LINK, 7));
+  tt_assert(!protocol_list_supports_protocol("Link=3-6", PRT_LINKAUTH, 3));
+
+  tt_assert(!protocol_list_supports_protocol("Link=4-6 LinkAuth=3",
+                                            PRT_LINKAUTH, 2));
+  tt_assert(protocol_list_supports_protocol("Link=4-6 LinkAuth=3",
+                                            PRT_LINKAUTH, 3));
+  tt_assert(!protocol_list_supports_protocol("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 4));
+  tt_assert(!protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 4));
+  tt_assert(protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 3));
+  tt_assert(protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                             PRT_LINKAUTH, 2));
+
+  tt_assert(!protocol_list_supports_protocol_or_later("Link=4-6 LinkAuth=3",
+                                                      PRT_DESC, 2));
+ done:
+ ;
+}
+
+/* This could be MAX_PROTOCOLS_TO_EXPAND, but that's not exposed by protover */
+#define MAX_PROTOCOLS_TO_TEST 1024
+
+/* LinkAuth and Relay protocol versions.
+ * Hard-coded here, because they are not in the code, or not exposed in the
+ * headers. */
+#define PROTOVER_LINKAUTH_V1 1
+#define PROTOVER_LINKAUTH_V3 3
+
+#define PROTOVER_RELAY_V1 1
+#define PROTOVER_RELAY_V2 2
+
+/* Highest supported HSv2 introduce protocol version.
+ * Hard-coded here, because it does not appear anywhere in the code.
+ * It's not clear if we actually support version 2, see #25068. */
+#define PROTOVER_HSINTRO_V2 3
+
+/* HSv2 Rend and HSDir protocol versions.
+ * Hard-coded here, because they do not appear anywhere in the code. */
+#define PROTOVER_HS_RENDEZVOUS_POINT_V2 1
+#define PROTOVER_HSDIR_V2 1
+
+/* DirCache, Desc, Microdesc, and Cons protocol versions.
+ * Hard-coded here, because they do not appear anywhere in the code. */
+#define PROTOVER_DIRCACHE_V1 1
+#define PROTOVER_DIRCACHE_V2 2
+
+#define PROTOVER_DESC_V1 1
+#define PROTOVER_DESC_V2 2
+
+#define PROTOVER_MICRODESC_V1 1
+#define PROTOVER_MICRODESC_V2 2
+
+#define PROTOVER_CONS_V1 1
+#define PROTOVER_CONS_V2 2
+
+/* Make sure we haven't forgotten any supported protocols */
+static void
+test_protover_supported_protocols(void *arg)
+{
+  (void)arg;
+
+  const char *supported_protocols = protover_get_supported_protocols();
+
+  /* Test for new Link in the code, that hasn't been added to supported
+   * protocols */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_LINK,
+                                            MAX_LINK_PROTO));
+  for (uint16_t i = 0; i < MAX_PROTOCOLS_TO_TEST; i++) {
+    if (is_or_protocol_version_known(i)) {
+      tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                                PRT_LINK,
+                                                i));
+    }
+  }
+
+  /* Legacy LinkAuth does not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_LINKAUTH,
+                                            PROTOVER_LINKAUTH_V1));
+  /* Latest LinkAuth is not exposed in the headers. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_LINKAUTH,
+                                            PROTOVER_LINKAUTH_V3));
+  /* Is there any way to test for new LinkAuth? */
+
+  /* Relay protovers do not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_RELAY,
+                                            PROTOVER_RELAY_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_RELAY,
+                                            PROTOVER_RELAY_V2));
+  /* Is there any way to test for new Relay? */
+
+  /* We could test legacy HSIntro by calling rend_service_update_descriptor(),
+   * and checking the protocols field. But that's unlikely to change, so
+   * we just use a hard-coded value. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSINTRO,
+                                            PROTOVER_HSINTRO_V2));
+  /* Test for HSv3 HSIntro */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSINTRO,
+                                            PROTOVER_HS_INTRO_V3));
+  /* Is there any way to test for new HSIntro? */
+
+  /* Legacy HSRend does not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSREND,
+                                            PROTOVER_HS_RENDEZVOUS_POINT_V2));
+  /* Test for HSv3 HSRend */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSREND,
+                                            PROTOVER_HS_RENDEZVOUS_POINT_V3));
+  /* Is there any way to test for new HSRend? */
+
+  /* Legacy HSDir does not appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSDIR,
+                                            PROTOVER_HSDIR_V2));
+  /* Test for HSv3 HSDir */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_HSDIR,
+                                            PROTOVER_HSDIR_V3));
+  /* Is there any way to test for new HSDir? */
+
+  /* No DirCache versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DIRCACHE,
+                                            PROTOVER_DIRCACHE_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DIRCACHE,
+                                            PROTOVER_DIRCACHE_V2));
+  /* Is there any way to test for new DirCache? */
+
+  /* No Desc versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DESC,
+                                            PROTOVER_DESC_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_DESC,
+                                            PROTOVER_DESC_V2));
+  /* Is there any way to test for new Desc? */
+
+  /* No Microdesc versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_MICRODESC,
+                                            PROTOVER_MICRODESC_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_MICRODESC,
+                                            PROTOVER_MICRODESC_V2));
+  /* Is there any way to test for new Microdesc? */
+
+  /* No Cons versions appear anywhere in the code. */
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_CONS,
+                                            PROTOVER_CONS_V1));
+  tt_assert(protocol_list_supports_protocol(supported_protocols,
+                                            PRT_CONS,
+                                            PROTOVER_CONS_V2));
+  /* Is there any way to test for new Cons? */
+
+ done:
+ ;
+}
+
+static void
 test_protover_vote_roundtrip(void *args)
 {
   (void) args;
@@ -292,11 +553,11 @@ test_protover_vote_roundtrip(void *args)
     { "Zn=4294967295-1", NULL },
     { "Zn=4294967293-4294967295", NULL },
     /* Will fail because of 4294967295. */
-    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=0,4294967295",
+    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=1,4294967295",
        NULL },
-    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=0,4294967294",
-      "Bar=3 Foo=1,3 Quux=9-12,14-16,900 Zn=0,4294967294" },
-    { "Zu16=0,65536", "Zu16=0,65536" },
+    { "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900 Zn=1,4294967294",
+      "Bar=3 Foo=1,3 Quux=9-12,14-16,900 Zn=1,4294967294" },
+    { "Zu16=1,65536", "Zu16=1,65536" },
     { "N-1=1,2", "N-1=1-2" },
     { "-1=4294967295", NULL },
     { "-1=3", "-1=3" },
@@ -316,8 +577,6 @@ test_protover_vote_roundtrip(void *args)
     { "Link=1,9-8,3", NULL },
     { "Faux=-0", NULL },
     { "Faux=0--0", NULL },
-    // "These fail at the splitting stage in Rust, but the number parsing
-    // stage in C."
     { "Faux=-1", NULL },
     { "Faux=-1-3", NULL },
     { "Faux=1--1", NULL },
@@ -326,10 +585,10 @@ test_protover_vote_roundtrip(void *args)
     /* Large range */
     { "Sleen=1-501", "Sleen=1-501" },
     { "Sleen=1-65537", NULL },
-    /* CPU/RAM DoS Loop: Rust only. */
-    { "Sleen=0-2147483648", NULL },
-    /* Rust seems to experience an internal error here. */
-    { "Sleen=0-4294967295", NULL },
+    /* Both C/Rust implementations should be able to handle this mild DoS. */
+    { "Sleen=1-2147483648", NULL },
+    /* Rust tests are built in debug mode, so ints are bounds-checked. */
+    { "Sleen=1-4294967295", NULL },
   };
   unsigned u;
   smartlist_t *votes = smartlist_new();
@@ -364,6 +623,10 @@ struct testcase_t protover_tests[] = {
   PV_TEST(parse_fail, 0),
   PV_TEST(vote, 0),
   PV_TEST(all_supported, 0),
+  PV_TEST(list_supports_protocol_for_unsupported_returns_false, 0),
+  PV_TEST(list_supports_protocol_returns_true, 0),
+  PV_TEST(supports_version, 0),
+  PV_TEST(supported_protocols, 0),
   PV_TEST(vote_roundtrip, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index e5cdc5f3cd..07b6712ff9 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -40,34 +40,34 @@ test_pt_parsing(void *arg)
 
   /* incomplete cmethod */
   strlcpy(line,"CMETHOD trebuchet",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) < 0);
+  tt_int_op(parse_cmethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* wrong proxy type */
   strlcpy(line,"CMETHOD trebuchet dog 127.0.0.1:1999",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) < 0);
+  tt_int_op(parse_cmethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* wrong addrport */
   strlcpy(line,"CMETHOD trebuchet socks4 abcd",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) < 0);
+  tt_int_op(parse_cmethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* correct line */
   strlcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999",sizeof(line));
-  tt_assert(parse_cmethod_line(line, mp) == 0);
-  tt_assert(smartlist_len(mp->transports) == 1);
+  tt_int_op(parse_cmethod_line(line, mp), OP_EQ, 0);
+  tt_int_op(smartlist_len(mp->transports), OP_EQ, 1);
   transport = smartlist_get(mp->transports, 0);
   /* test registered address of transport */
   tor_addr_parse(&test_addr, "127.0.0.1");
   tt_assert(tor_addr_eq(&test_addr, &transport->addr));
   /* test registered port of transport */
-  tt_assert(transport->port == 1999);
+  tt_uint_op(transport->port, OP_EQ, 1999);
   /* test registered SOCKS version of transport */
-  tt_assert(transport->socks_version == PROXY_SOCKS5);
+  tt_int_op(transport->socks_version, OP_EQ, PROXY_SOCKS5);
   /* test registered name of transport */
   tt_str_op(transport->name,OP_EQ, "trebuchet");
 
@@ -75,26 +75,26 @@ test_pt_parsing(void *arg)
 
   /* incomplete smethod */
   strlcpy(line,"SMETHOD trebuchet",sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) < 0);
+  tt_int_op(parse_smethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* wrong addr type */
   strlcpy(line,"SMETHOD trebuchet abcd",sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) < 0);
+  tt_int_op(parse_smethod_line(line, mp), OP_LT, 0);
 
   reset_mp(mp);
 
   /* cowwect */
   strlcpy(line,"SMETHOD trebuchy 127.0.0.2:2999",sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) == 0);
-  tt_assert(smartlist_len(mp->transports) == 1);
+  tt_int_op(parse_smethod_line(line, mp), OP_EQ, 0);
+  tt_int_op(smartlist_len(mp->transports), OP_EQ, 1);
   transport = smartlist_get(mp->transports, 0);
   /* test registered address of transport */
   tor_addr_parse(&test_addr, "127.0.0.2");
   tt_assert(tor_addr_eq(&test_addr, &transport->addr));
   /* test registered port of transport */
-  tt_assert(transport->port == 2999);
+  tt_uint_op(transport->port, OP_EQ, 2999);
   /* test registered name of transport */
   tt_str_op(transport->name,OP_EQ, "trebuchy");
 
@@ -104,7 +104,7 @@ test_pt_parsing(void *arg)
   strlcpy(line,"SMETHOD trebuchet 127.0.0.1:9999 "
           "ARGS:counterweight=3,sling=snappy",
           sizeof(line));
-  tt_assert(parse_smethod_line(line, mp) == 0);
+  tt_int_op(parse_smethod_line(line, mp), OP_EQ, 0);
   tt_int_op(1, OP_EQ, smartlist_len(mp->transports));
   {
     const transport_t *transport_ = smartlist_get(mp->transports, 0);
@@ -119,15 +119,15 @@ test_pt_parsing(void *arg)
 
   /* unsupported version */
   strlcpy(line,"VERSION 666",sizeof(line));
-  tt_assert(parse_version(line, mp) < 0);
+  tt_int_op(parse_version(line, mp), OP_LT, 0);
 
   /* incomplete VERSION */
   strlcpy(line,"VERSION ",sizeof(line));
-  tt_assert(parse_version(line, mp) < 0);
+  tt_int_op(parse_version(line, mp), OP_LT, 0);
 
   /* correct VERSION */
   strlcpy(line,"VERSION 1",sizeof(line));
-  tt_assert(parse_version(line, mp) == 0);
+  tt_int_op(parse_version(line, mp), OP_EQ, 0);
 
  done:
   reset_mp(mp);
@@ -155,9 +155,9 @@ test_pt_get_transport_options(void *arg)
   opt_str = get_transport_options_for_server_proxy(mp);
   tt_ptr_op(opt_str, OP_EQ, NULL);
 
-  smartlist_add(mp->transports_to_launch, tor_strdup("gruyere"));
-  smartlist_add(mp->transports_to_launch, tor_strdup("roquefort"));
-  smartlist_add(mp->transports_to_launch, tor_strdup("stnectaire"));
+  smartlist_add_strdup(mp->transports_to_launch, "gruyere");
+  smartlist_add_strdup(mp->transports_to_launch, "roquefort");
+  smartlist_add_strdup(mp->transports_to_launch, "stnectaire");
 
   tt_assert(options);
 
@@ -284,13 +284,13 @@ test_pt_get_extrainfo_string(void *arg)
 }
 
 #ifdef _WIN32
-#define STDIN_HANDLE HANDLE
+#define STDIN_HANDLE HANDLE*
 #else
-#define STDIN_HANDLE FILE
+#define STDIN_HANDLE int
 #endif
 
 static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle,
+tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
                                       enum stream_status *stream_status_out)
 {
   static int times_called = 0;
@@ -305,7 +305,7 @@ tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle,
     smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
                            times_called, times_called);
   } else {
-    smartlist_add(retval_sl, tor_strdup("SMETHODS DONE"));
+    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
   }
 
   return retval_sl;
@@ -461,7 +461,7 @@ test_get_pt_proxy_uri(void *arg)
 
   /* Test with no proxy. */
   uri = get_pt_proxy_uri();
-  tt_assert(uri == NULL);
+  tt_ptr_op(uri, OP_EQ, NULL);
 
   /* Test with a SOCKS4 proxy. */
   options->Socks4Proxy = tor_strdup("192.0.2.1:1080");
diff --git a/src/test/test_pubsub.c b/src/test/test_pubsub.c
index 547d6c6b32..2f047d9f2c 100644
--- a/src/test/test_pubsub.c
+++ b/src/test/test_pubsub.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/test/test_relay.c b/src/test/test_relay.c
index 57dcb2406a..a64d9af5b9 100644
--- a/src/test/test_relay.c
+++ b/src/test/test_relay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -48,12 +48,6 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan)
   circ->deliver_window = CIRCWINDOW_START_MAX;
   circ->n_chan_create_cell = NULL;
 
-  /* for assert_circ_ok */
-  orcirc->p_crypto = (void*)1;
-  orcirc->n_crypto = (void*)1;
-  orcirc->n_digest = (void*)1;
-  orcirc->p_digest = (void*)1;
-
   circuit_set_p_circid_chan(orcirc, get_unique_circ_id_by_chan(pchan), pchan);
   cell_queue_init(&(orcirc->p_chan_cells));
 
@@ -61,6 +55,13 @@ new_fake_orcirc(channel_t *nchan, channel_t *pchan)
 }
 
 static void
+assert_circuit_ok_mock(const circuit_t *c)
+{
+  (void) c;
+  return;
+}
+
+static void
 test_relay_close_circuit(void *arg)
 {
   channel_t *nchan = NULL, *pchan = NULL;
@@ -77,10 +78,6 @@ test_relay_close_circuit(void *arg)
   pchan = new_fake_channel();
   tt_assert(pchan);
 
-  /* We'll need chans with working cmuxes */
-  nchan->cmux = circuitmux_alloc();
-  pchan->cmux = circuitmux_alloc();
-
   /* Make a fake orcirc */
   orcirc = new_fake_orcirc(nchan, pchan);
   tt_assert(orcirc);
@@ -95,6 +92,8 @@ test_relay_close_circuit(void *arg)
 
   MOCK(scheduler_channel_has_waiting_cells,
        scheduler_channel_has_waiting_cells_mock);
+  MOCK(assert_circuit_ok,
+       assert_circuit_ok_mock);
 
   /* Append it */
   old_count = get_mock_scheduler_has_waiting_cells_count();
@@ -146,6 +145,7 @@ test_relay_close_circuit(void *arg)
   tor_free(orcirc);
   free_fake_channel(nchan);
   free_fake_channel(pchan);
+  UNMOCK(assert_circuit_ok);
 
   return;
 }
@@ -167,10 +167,6 @@ test_relay_append_cell_to_circuit_queue(void *arg)
   pchan = new_fake_channel();
   tt_assert(pchan);
 
-  /* We'll need chans with working cmuxes */
-  nchan->cmux = circuitmux_alloc();
-  pchan->cmux = circuitmux_alloc();
-
   /* Make a fake orcirc */
   orcirc = new_fake_orcirc(nchan, pchan);
   tt_assert(orcirc);
@@ -191,14 +187,14 @@ test_relay_append_cell_to_circuit_queue(void *arg)
   append_cell_to_circuit_queue(TO_CIRCUIT(orcirc), nchan, cell,
                                CELL_DIRECTION_OUT, 0);
   new_count = get_mock_scheduler_has_waiting_cells_count();
-  tt_int_op(new_count, ==, old_count + 1);
+  tt_int_op(new_count, OP_EQ, old_count + 1);
 
   /* Now try the reverse direction */
   old_count = get_mock_scheduler_has_waiting_cells_count();
   append_cell_to_circuit_queue(TO_CIRCUIT(orcirc), pchan, cell,
                                CELL_DIRECTION_IN, 0);
   new_count = get_mock_scheduler_has_waiting_cells_count();
-  tt_int_op(new_count, ==, old_count + 1);
+  tt_int_op(new_count, OP_EQ, old_count + 1);
 
   UNMOCK(scheduler_channel_has_waiting_cells);
 
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index fb6748965a..2fc0288f69 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -1,16 +1,31 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for handling different kinds of relay cell */
 
 #define RELAY_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CONNECTION_EDGE_PRIVATE
+#define CONNECTION_PRIVATE
+
 #include "or.h"
+#include "main.h"
 #include "config.h"
 #include "connection.h"
+#include "crypto.h"
+#include "crypto_rand.h"
+#include "circuitbuild.h"
+#include "circuitlist.h"
 #include "connection_edge.h"
+#include "log_test_helpers.h"
 #include "relay.h"
 #include "test.h"
 
+#include "log_test_helpers.h"
+
+#include "circpathbias.h"
+#include "connection_edge.h"
+
 static int srm_ncalls;
 static entry_connection_t *srm_conn;
 static int srm_atype;
@@ -60,6 +75,805 @@ mark_unattached_mock(entry_connection_t *conn, int endreason,
   (void) file;
 }
 
+/* Helper: Return a newly allocated and initialized origin circuit with
+ * purpose and flags. A default HS identifier is set to an ed25519
+ * authentication key for introduction point. */
+static origin_circuit_t *
+helper_create_origin_circuit(int purpose, int flags)
+{
+  origin_circuit_t *circ = NULL;
+
+  circ = origin_circuit_init(purpose, flags);
+  tor_assert(circ);
+  circ->cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  circ->cpath->magic = CRYPT_PATH_MAGIC;
+  circ->cpath->state = CPATH_STATE_OPEN;
+  circ->cpath->package_window = circuit_initial_package_window();
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+  circ->cpath->prev = circ->cpath;
+  /* Create a default HS identifier. */
+  circ->hs_ident = tor_malloc_zero(sizeof(hs_ident_circuit_t));
+
+  return circ;
+}
+
+static void
+mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
+                                    int line, const char *file)
+{
+  (void) line;
+  (void) file;
+  conn->edge_.end_reason = endreason;
+}
+
+static void
+mock_mark_circ_for_close(circuit_t *circ, int reason, int line,
+                          const char *file)
+{
+  (void)reason; (void)line; (void)file;
+
+  circ->marked_for_close = 1;
+  return;
+}
+
+static void
+mock_mark_for_close(connection_t *conn,
+                        int line, const char *file)
+{
+  (void)line;
+  (void)file;
+
+  conn->marked_for_close = 1;
+  return;
+}
+
+static void
+mock_start_reading(connection_t *conn)
+{
+  (void)conn;
+  return;
+}
+
+static int
+mock_send_command(streamid_t stream_id, circuit_t *circ,
+                               uint8_t relay_command, const char *payload,
+                               size_t payload_len, crypt_path_t *cpath_layer,
+                               const char *filename, int lineno)
+{
+ (void)stream_id; (void)circ;
+ (void)relay_command; (void)payload;
+ (void)payload_len; (void)cpath_layer;
+ (void)filename; (void)lineno;
+
+ return 0;
+}
+
+static entry_connection_t *
+fake_entry_conn(origin_circuit_t *oncirc, streamid_t id)
+{
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn;
+
+  entryconn = entry_connection_new(CONN_TYPE_AP, AF_INET);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn);
+  edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT;
+  edgeconn->deliver_window = STREAMWINDOW_START;
+  edgeconn->package_window = STREAMWINDOW_START;
+
+  edgeconn->stream_id = id;
+  edgeconn->on_circuit = TO_CIRCUIT(oncirc);
+  edgeconn->cpath_layer = oncirc->cpath;
+
+  return entryconn;
+}
+
+#define PACK_CELL(id, cmd, body_s) do {                                  \
+    memset(&cell, 0, sizeof(cell));                                     \
+    memset(&rh, 0, sizeof(rh));                                         \
+    memcpy(cell.payload+RELAY_HEADER_SIZE, (body_s), sizeof((body_s))-1); \
+    rh.length = sizeof((body_s))-1;                                     \
+    rh.command = (cmd);                                                 \
+    rh.stream_id = (id);                                                \
+    relay_header_pack((uint8_t*)&cell.payload, &rh);                    \
+  } while (0)
+#define ASSERT_COUNTED_BW() do { \
+    tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered+rh.length); \
+    tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ,                      \
+              overhead+RELAY_PAYLOAD_SIZE-rh.length);               \
+    delivered = circ->n_delivered_read_circ_bw;                          \
+    overhead = circ->n_overhead_read_circ_bw;                            \
+ } while (0)
+#define ASSERT_UNCOUNTED_BW() do { \
+    tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered); \
+    tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ, overhead); \
+ } while (0)
+
+static int
+subtest_circbw_halfclosed(origin_circuit_t *circ, streamid_t init_id)
+{
+  cell_t cell;
+  relay_header_t rh;
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn2=NULL;
+  entry_connection_t *entryconn3=NULL;
+  entry_connection_t *entryconn4=NULL;
+  int delivered = circ->n_delivered_read_circ_bw;
+  int overhead = circ->n_overhead_read_circ_bw;
+
+  /* Make new entryconns */
+  entryconn2 = fake_entry_conn(circ, init_id);
+  entryconn2->socks_request->has_finished = 1;
+  entryconn3 = fake_entry_conn(circ, init_id+1);
+  entryconn3->socks_request->has_finished = 1;
+  entryconn4 = fake_entry_conn(circ, init_id+2);
+  entryconn4->socks_request->has_finished = 1;
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn2);
+  edgeconn->package_window = 23;
+  edgeconn->base_.state = AP_CONN_STATE_OPEN;
+
+  int data_cells = edgeconn->deliver_window;
+  int sendme_cells = (STREAMWINDOW_START-edgeconn->package_window)
+                             /STREAMWINDOW_INCREMENT;
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  connection_edge_reached_eof(edgeconn);
+
+  /* Data cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                       circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_SENDME, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Connected cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Resolved cell not in the half-opened list */
+  PACK_CELL(4000, RELAY_COMMAND_RESOLVED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Connected cell: not counted -- we were open */
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn2);
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* DATA cells up to limit */
+  while (data_cells > 0) {
+    ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+    ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+    PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+    if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+      pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+    else
+      connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                       circ->cpath);
+    ASSERT_COUNTED_BW();
+    data_cells--;
+  }
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* SENDME cells up to limit */
+  while (sendme_cells > 0) {
+    ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+    ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+    PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+    if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+      pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+    else
+      connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                       circ->cpath);
+    ASSERT_COUNTED_BW();
+    sendme_cells--;
+  }
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Only one END cell */
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn2)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn3);
+  edgeconn->base_.state = AP_CONN_STATE_OPEN;
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  /* sendme cell on open entryconn with full window */
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+  int ret =
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  tt_int_op(ret, OP_EQ, -END_CIRC_REASON_TORPROTOCOL);
+  ASSERT_UNCOUNTED_BW();
+
+  /* connected cell on a after EOF */
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT;
+  connection_edge_reached_eof(edgeconn);
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ),  NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ),  NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* DATA and SENDME after END cell */
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ),  NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234");
+  ret =
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  tt_int_op(ret, OP_NE, -END_CIRC_REASON_TORPROTOCOL);
+  ASSERT_UNCOUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn3)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Resolved: 1 counted, more not */
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn4);
+  entryconn4->socks_request->command = SOCKS_COMMAND_RESOLVE;
+  edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
+  edgeconn->on_circuit = TO_CIRCUIT(circ);
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  connection_edge_reached_eof(edgeconn);
+
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_RESOLVED,
+            "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_RESOLVED,
+            "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Data not counted after resolved */
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* End not counted after resolved */
+  ENTRY_TO_CONN(entryconn4)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0;
+  PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234");
+  if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING)
+    pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  else
+    connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  connection_free_minimal(ENTRY_TO_CONN(entryconn2));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn3));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn4));
+  return 1;
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(entryconn2));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn3));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn4));
+  return 0;
+}
+
+static int
+halfstream_insert(origin_circuit_t *circ, edge_connection_t *edgeconn,
+                  streamid_t *streams, int num, int random)
+{
+  int inserted = 0;
+
+  /* Insert num random elements */
+  while (inserted < num) {
+    streamid_t id;
+
+    if (random)
+      id = (streamid_t)crypto_rand_int(65535)+1;
+    else
+      id = get_unique_stream_id_by_circ(circ);
+
+    edgeconn->stream_id = id;
+
+    /* Ensure it isn't there */
+    if (connection_half_edge_find_stream_id(circ->half_streams, id)) {
+      continue;
+    }
+
+    connection_half_edge_add(edgeconn, circ);
+    if (streams)
+      streams[inserted] = id;
+    inserted++;
+  }
+
+  return inserted;
+}
+
+static void
+subtest_halfstream_insertremove(int num)
+{
+  origin_circuit_t *circ =
+      helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0);
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn;
+  streamid_t *streams = tor_malloc_zero(num*sizeof(streamid_t));
+  int i = 0;
+
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+
+  entryconn = fake_entry_conn(circ, 23);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn);
+
+  /* Explicity test all operations on an absent stream list */
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 0);
+
+  /* Insert a duplicate element; verify that other elements absent;
+   * ensure removing it once works */
+  edgeconn->stream_id = 23;
+  connection_half_edge_add(edgeconn, circ);
+  connection_half_edge_add(edgeconn, circ);
+  connection_half_edge_add(edgeconn, circ);
+
+  /* Verify that other elements absent */
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            22), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            22), OP_EQ, 0);
+
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            24), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            24), OP_EQ, 0);
+
+  /* Verify we only remove it once */
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 1);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 0);
+
+  halfstream_insert(circ, edgeconn, streams, num, 1);
+
+  /* Remove half of them */
+  for (i = 0; i < num/2; i++) {
+    tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+                                                streams[i]),
+              OP_EQ, 1);
+  }
+
+  /* Verify first half of list is gone */
+  for (i = 0; i < num/2; i++) {
+    tt_ptr_op(connection_half_edge_find_stream_id(circ->half_streams,
+                                                  streams[i]),
+              OP_EQ, NULL);
+  }
+
+  /* Verify second half of list is present */
+  for (; i < num; i++) {
+    tt_ptr_op(connection_half_edge_find_stream_id(circ->half_streams,
+                                                  streams[i]),
+              OP_NE, NULL);
+  }
+
+  /* Remove other half. Verify list is empty. */
+  for (i = num/2; i < num; i++) {
+    tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+                                                streams[i]),
+              OP_EQ, 1);
+  }
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 0);
+
+  /* Explicity test all operations on an empty stream list */
+  tt_int_op(connection_half_edge_is_valid_data(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams,
+            23), OP_EQ, 0);
+  tt_int_op(connection_half_edge_is_valid_end(circ->half_streams,
+            23), OP_EQ, 0);
+
+  /* For valgrind, leave some around then free the circ */
+  halfstream_insert(circ, edgeconn, NULL, 10, 0);
+
+ done:
+  tor_free(streams);
+  circuit_free_(TO_CIRCUIT(circ));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn));
+}
+
+static void
+test_halfstream_insertremove(void *arg)
+{
+  (void)arg;
+
+  /* Suppress the WARN message we generate in this test */
+  setup_full_capture_of_logs(LOG_WARN);
+
+  /* Test insertion and removal with a few different sizes */
+  subtest_halfstream_insertremove(10);
+  subtest_halfstream_insertremove(100);
+  subtest_halfstream_insertremove(1000);
+}
+
+static void
+test_halfstream_wrap(void *arg)
+{
+  origin_circuit_t *circ =
+      helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0);
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn;
+
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+
+  entryconn = fake_entry_conn(circ, 23);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn);
+
+  (void)arg;
+
+  /* Suppress the WARN message we generate in this test */
+  setup_full_capture_of_logs(LOG_WARN);
+  MOCK(connection_mark_for_close_internal_, mock_mark_for_close);
+
+  /* Verify that get_unique_stream_id_by_circ() can wrap uint16_t */
+  circ->next_stream_id = 65530;
+  halfstream_insert(circ, edgeconn, NULL, 7, 0);
+  tt_int_op(circ->next_stream_id, OP_EQ, 2);
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 7);
+
+  /* Insert full-1 */
+  halfstream_insert(circ, edgeconn, NULL,
+                    65534-smartlist_len(circ->half_streams), 0);
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65534);
+
+  /* Verify that we can get_unique_stream_id_by_circ() successfully */
+  edgeconn->stream_id = get_unique_stream_id_by_circ(circ);
+  tt_int_op(edgeconn->stream_id, OP_NE, 0); /* 0 is failure */
+
+  /* Insert an opened stream on the circ with that id */
+  ENTRY_TO_CONN(entryconn)->marked_for_close = 0;
+  ENTRY_TO_CONN(entryconn)->outbuf_flushlen = 0;
+  edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT;
+  circ->p_streams = edgeconn;
+
+  /* Verify that get_unique_stream_id_by_circ() fails */
+  tt_int_op(get_unique_stream_id_by_circ(circ), OP_EQ, 0); /* 0 is failure */
+
+  /* eof the one opened stream. Verify it is now in half-closed */
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65534);
+  connection_edge_reached_eof(edgeconn);
+  tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65535);
+
+  /* Verify get_unique_stream_id_by_circ() fails due to full half-closed */
+  circ->p_streams = NULL;
+  tt_int_op(get_unique_stream_id_by_circ(circ), OP_EQ, 0); /* 0 is failure */
+
+ done:
+  circuit_free_(TO_CIRCUIT(circ));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn));
+  UNMOCK(connection_mark_for_close_internal_);
+}
+
+static void
+test_circbw_relay(void *arg)
+{
+  cell_t cell;
+  relay_header_t rh;
+  tor_addr_t addr;
+  edge_connection_t *edgeconn;
+  entry_connection_t *entryconn1=NULL;
+  origin_circuit_t *circ;
+  int delivered = 0;
+  int overhead = 0;
+
+  (void)arg;
+
+  MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_);
+  MOCK(connection_start_reading, mock_start_reading);
+  MOCK(connection_mark_for_close_internal_, mock_mark_for_close);
+  MOCK(relay_send_command_from_edge_, mock_send_command);
+  MOCK(circuit_mark_for_close_, mock_mark_circ_for_close);
+
+  circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0);
+  circ->cpath->state = CPATH_STATE_AWAITING_KEYS;
+  circ->cpath->deliver_window = CIRCWINDOW_START;
+
+  entryconn1 = fake_entry_conn(circ, 1);
+  edgeconn = ENTRY_TO_EDGE_CONN(entryconn1);
+
+  /* Stream id 0: Not counted */
+  PACK_CELL(0, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Stream id 1: Counted */
+  PACK_CELL(1, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Properly formatted connect cell: counted */
+  PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234");
+  tor_addr_parse(&addr, "30.40.50.60");
+  rh.length = connected_cell_format_payload(cell.payload+RELAY_HEADER_SIZE,
+                                            &addr, 1024);
+  relay_header_pack((uint8_t*)&cell.payload, &rh);                    \
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Properly formatted resolved cell in correct state: counted */
+  edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT;
+  entryconn1->socks_request->command = SOCKS_COMMAND_RESOLVE;
+  edgeconn->on_circuit = TO_CIRCUIT(circ);
+  PACK_CELL(1, RELAY_COMMAND_RESOLVED,
+            "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  edgeconn->base_.state = AP_CONN_STATE_OPEN;
+  entryconn1->socks_request->has_finished = 1;
+
+  /* Connected cell after open: not counted */
+  PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Resolved cell after open: not counted */
+  PACK_CELL(1, RELAY_COMMAND_RESOLVED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Drop cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_DROP, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Data cell on stream 0: not counted */
+  PACK_CELL(0, RELAY_COMMAND_DATA, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Data cell on open connection: counted */
+  ENTRY_TO_CONN(entryconn1)->marked_for_close = 0;
+  PACK_CELL(1, RELAY_COMMAND_DATA, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Empty Data cell on open connection: not counted */
+  ENTRY_TO_CONN(entryconn1)->marked_for_close = 0;
+  PACK_CELL(1, RELAY_COMMAND_DATA, "");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on valid stream: counted */
+  edgeconn->package_window -= STREAMWINDOW_INCREMENT;
+  ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0;
+  PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Sendme on valid stream with full window: not counted */
+  ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0;
+  PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234");
+  edgeconn->package_window = STREAMWINDOW_START;
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on unknown stream: not counted */
+  ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0;
+  PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on circuit with full window: not counted */
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Sendme on circuit with non-full window: counted */
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  circ->cpath->package_window = 900;
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* Invalid extended cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_EXTENDED2, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Invalid extended cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_EXTENDED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Invalid HS cell: not counted */
+  PACK_CELL(1, RELAY_COMMAND_ESTABLISH_INTRO, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* "Valid" HS cell in expected state: counted */
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
+  PACK_CELL(1, RELAY_COMMAND_RENDEZVOUS_ESTABLISHED, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* End cell on non-closed connection: counted */
+  PACK_CELL(1, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
+                                     circ->cpath);
+  ASSERT_COUNTED_BW();
+
+  /* End cell on connection that already got one: not counted */
+  PACK_CELL(1, RELAY_COMMAND_END, "Data1234");
+  connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL,
+                                     circ->cpath);
+  ASSERT_UNCOUNTED_BW();
+
+  /* Simulate closed stream on entryconn, then test: */
+  if (!subtest_circbw_halfclosed(circ, 2))
+    goto done;
+
+  circ->base_.purpose = CIRCUIT_PURPOSE_PATH_BIAS_TESTING;
+  if (!subtest_circbw_halfclosed(circ, 6))
+    goto done;
+
+  /* Path bias: truncated */
+  tt_int_op(circ->base_.marked_for_close, OP_EQ, 0);
+  PACK_CELL(0, RELAY_COMMAND_TRUNCATED, "Data1234");
+  pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell);
+  tt_int_op(circ->base_.marked_for_close, OP_EQ, 1);
+
+ done:
+  UNMOCK(connection_start_reading);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(connection_mark_for_close_internal_);
+  UNMOCK(relay_send_command_from_edge_);
+  UNMOCK(circuit_mark_for_close_);
+  circuit_free_(TO_CIRCUIT(circ));
+  connection_free_minimal(ENTRY_TO_CONN(entryconn1));
+}
+
 /* Tests for connection_edge_process_resolved_cell().
 
    The point of ..process_resolved_cell() is to handle an incoming cell
@@ -244,6 +1058,8 @@ test_relaycell_resolved(void *arg)
 
 struct testcase_t relaycell_tests[] = {
   { "resolved", test_relaycell_resolved, TT_FORK, NULL, NULL },
+  { "circbw", test_circbw_relay, TT_FORK, NULL, NULL },
+  { "halfstream", test_halfstream_insertremove, TT_FORK, NULL, NULL },
+  { "streamwrap", test_halfstream_wrap, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c
new file mode 100644
index 0000000000..60bd479719
--- /dev/null
+++ b/src/test/test_relaycrypt.c
@@ -0,0 +1,185 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "circuitbuild.h"
+#define CIRCUITLIST_PRIVATE
+#include "circuitlist.h"
+#include "crypto_rand.h"
+#include "relay.h"
+#include "relay_crypto.h"
+#include "test.h"
+
+static const char KEY_MATERIAL[3][CPATH_KEY_MATERIAL_LEN] = {
+  "    'My public key is in this signed x509 object', said Tom assertively.",
+  "'Let's chart the pedal phlanges in the tomb', said Tom cryptographically",
+  "     'Segmentation fault bugs don't _just happen_', said Tom seethingly.",
+};
+
+typedef struct testing_circuitset_t {
+  or_circuit_t *or_circ[3];
+  origin_circuit_t *origin_circ;
+} testing_circuitset_t;
+
+static int testing_circuitset_teardown(const struct testcase_t *testcase,
+                                       void *ptr);
+
+static void *
+testing_circuitset_setup(const struct testcase_t *testcase)
+{
+  testing_circuitset_t *cs = tor_malloc_zero(sizeof(testing_circuitset_t));
+  int i;
+
+  for (i=0; i<3; ++i) {
+    cs->or_circ[i] = or_circuit_new(0, NULL);
+    tt_int_op(0, OP_EQ,
+              relay_crypto_init(&cs->or_circ[i]->crypto,
+                                KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
+                                0, 0));
+  }
+
+  cs->origin_circ = origin_circuit_new();
+  cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  for (i=0; i<3; ++i) {
+    crypt_path_t *hop = tor_malloc_zero(sizeof(*hop));
+    relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
+                      0, 0);
+    hop->state = CPATH_STATE_OPEN;
+    onion_append_to_cpath(&cs->origin_circ->cpath, hop);
+    tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev);
+  }
+
+  return cs;
+ done:
+  testing_circuitset_teardown(testcase, cs);
+  return NULL;
+}
+
+static int
+testing_circuitset_teardown(const struct testcase_t *testcase, void *ptr)
+{
+  (void)testcase;
+  testing_circuitset_t *cs = ptr;
+  int i;
+  for (i=0; i<3; ++i) {
+    circuit_free_(TO_CIRCUIT(cs->or_circ[i]));
+  }
+  circuit_free_(TO_CIRCUIT(cs->origin_circ));
+  tor_free(cs);
+  return 1;
+}
+
+static const struct testcase_setup_t relaycrypt_setup = {
+  testing_circuitset_setup, testing_circuitset_teardown
+};
+
+/* Test encrypting a cell to the final hop on a circuit, decrypting it
+ * at each hop, and recognizing it at the other end.  Then do it again
+ * and again as the state evolves. */
+static void
+test_relaycrypt_outbound(void *arg)
+{
+  testing_circuitset_t *cs = arg;
+  tt_assert(cs);
+
+  relay_header_t rh;
+  cell_t orig;
+  cell_t encrypted;
+  int i, j;
+
+  for (i = 0; i < 50; ++i) {
+    crypto_rand((char *)&orig, sizeof(orig));
+
+    relay_header_unpack(&rh, orig.payload);
+    rh.recognized = 0;
+    memset(rh.integrity, 0, sizeof(rh.integrity));
+    relay_header_pack(orig.payload, &rh);
+
+    memcpy(&encrypted, &orig, sizeof(orig));
+
+    /* Encrypt the cell to the last hop */
+    relay_encrypt_cell_outbound(&encrypted, cs->origin_circ,
+                                cs->origin_circ->cpath->prev);
+
+    for (j = 0; j < 3; ++j) {
+      crypt_path_t *layer_hint = NULL;
+      char recognized = 0;
+      int r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]),
+                                 &encrypted,
+                                 CELL_DIRECTION_OUT,
+                                 &layer_hint, &recognized);
+      tt_int_op(r, OP_EQ, 0);
+      tt_ptr_op(layer_hint, OP_EQ, NULL);
+      tt_int_op(recognized != 0, OP_EQ, j == 2);
+    }
+
+    tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE);
+  }
+
+ done:
+  ;
+}
+
+/* As above, but simulate inbound cells from the last hop. */
+static void
+test_relaycrypt_inbound(void *arg)
+{
+  testing_circuitset_t *cs = arg;
+  tt_assert(cs);
+
+  relay_header_t rh;
+  cell_t orig;
+  cell_t encrypted;
+  int i, j;
+
+  for (i = 0; i < 50; ++i) {
+    crypto_rand((char *)&orig, sizeof(orig));
+
+    relay_header_unpack(&rh, orig.payload);
+    rh.recognized = 0;
+    memset(rh.integrity, 0, sizeof(rh.integrity));
+    relay_header_pack(orig.payload, &rh);
+
+    memcpy(&encrypted, &orig, sizeof(orig));
+
+    /* Encrypt the cell to the last hop */
+    relay_encrypt_cell_inbound(&encrypted, cs->or_circ[2]);
+
+    crypt_path_t *layer_hint = NULL;
+    char recognized = 0;
+    int r;
+    for (j = 1; j >= 0; --j) {
+      r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]),
+                             &encrypted,
+                             CELL_DIRECTION_IN,
+                             &layer_hint, &recognized);
+      tt_int_op(r, OP_EQ, 0);
+      tt_ptr_op(layer_hint, OP_EQ, NULL);
+      tt_int_op(recognized, OP_EQ, 0);
+    }
+
+    relay_decrypt_cell(TO_CIRCUIT(cs->origin_circ),
+                       &encrypted,
+                       CELL_DIRECTION_IN,
+                       &layer_hint, &recognized);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(recognized, OP_EQ, 1);
+    tt_ptr_op(layer_hint, OP_EQ, cs->origin_circ->cpath->prev);
+
+    tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE);
+  }
+ done:
+  ;
+}
+
+#define TEST(name) \
+  { # name, test_relaycrypt_ ## name, 0, &relaycrypt_setup, NULL }
+
+struct testcase_t relaycrypt_tests[] = {
+  TEST(outbound),
+  TEST(inbound),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c
index a5d3f351f8..9f6cfc4a22 100644
--- a/src/test/test_rendcache.c
+++ b/src/test/test_rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -10,7 +10,7 @@
 #include "router.h"
 #include "routerlist.h"
 #include "config.h"
-#include <openssl/rsa.h>
+#include "hs_common.h"
 #include "rend_test_helpers.h"
 #include "log_test_helpers.h"
 
@@ -21,21 +21,6 @@ static const int TIME_IN_THE_PAST = -(REND_CACHE_MAX_AGE + \
                                       REND_CACHE_MAX_SKEW + 60);
 static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 60;
 
-static rend_data_t *
-mock_rend_data(const char *onion_address)
-{
-  rend_data_t *rend_query = tor_malloc_zero(sizeof(rend_data_t));
-
-  strlcpy(rend_query->onion_address, onion_address,
-          sizeof(rend_query->onion_address));
-  rend_query->auth_type = REND_NO_AUTH;
-  rend_query->hsdirs_fp = smartlist_new();
-  smartlist_add(rend_query->hsdirs_fp, tor_memdup("aaaaaaaaaaaaaaaaaaaaaaaa",
-                                                 DIGEST_LEN));
-
-  return rend_query;
-}
-
 static void
 test_rend_cache_lookup_entry(void *data)
 {
@@ -73,6 +58,7 @@ test_rend_cache_lookup_entry(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
   ret = rend_cache_lookup_entry(service_id, 2, &entry);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(entry);
   tt_int_op(entry->len, OP_EQ, strlen(desc_holder->desc_str));
   tt_str_op(entry->desc, OP_EQ, desc_holder->desc_str);
@@ -144,7 +130,8 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   // Test mismatch between service ID and onion address
   rend_cache_init();
-  strncpy(mock_rend_query->onion_address, "abc", REND_SERVICE_ID_LEN_BASE32+1);
+  strncpy(TO_REND_DATA_V2(mock_rend_query)->onion_address, "abc",
+          REND_SERVICE_ID_LEN_BASE32+1);
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
                                            desc_id_base32,
                                            mock_rend_query, NULL);
@@ -155,12 +142,16 @@ test_rend_cache_store_v2_desc_as_client(void *data)
   // Test incorrect descriptor ID
   rend_cache_init();
   mock_rend_query = mock_rend_data(service_id);
-  desc_id_base32[0]++;
+  char orig = desc_id_base32[0];
+  if (desc_id_base32[0] == 'a')
+    desc_id_base32[0] = 'b';
+  else
+    desc_id_base32[0] = 'a';
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
                                            desc_id_base32, mock_rend_query,
                                            NULL);
   tt_int_op(ret, OP_EQ, -1);
-  desc_id_base32[0]--;
+  desc_id_base32[0] = orig;
   rend_cache_free_all();
 
   // Test too old descriptor
@@ -230,9 +221,9 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   generate_desc(RECENT_TIME, &desc_holder, &service_id, 3);
   mock_rend_query = mock_rend_data(service_id);
-  mock_rend_query->auth_type = REND_BASIC_AUTH;
+  TO_REND_DATA_V2(mock_rend_query)->auth_type = REND_BASIC_AUTH;
   client_cookie[0] = 'A';
-  memcpy(mock_rend_query->descriptor_cookie, client_cookie,
+  memcpy(TO_REND_DATA_V2(mock_rend_query)->descriptor_cookie, client_cookie,
          REND_DESC_COOKIE_LEN);
   base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_holder->desc_id,
                 DIGEST_LEN);
@@ -250,7 +241,7 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   generate_desc(RECENT_TIME, &desc_holder, &service_id, 3);
   mock_rend_query = mock_rend_data(service_id);
-  mock_rend_query->auth_type = REND_BASIC_AUTH;
+  TO_REND_DATA_V2(mock_rend_query)->auth_type = REND_BASIC_AUTH;
   base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_holder->desc_id,
                 DIGEST_LEN);
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
@@ -843,7 +834,7 @@ test_rend_cache_failure_entry_free(void *data)
   (void)data;
 
   // Test that it can deal with a NULL argument
-  rend_cache_failure_entry_free(NULL);
+  rend_cache_failure_entry_free_(NULL);
 
  /* done: */
  /*  (void)0; */
@@ -956,9 +947,9 @@ test_rend_cache_free_all(void *data)
 
   rend_cache_free_all();
 
-  tt_assert(!rend_cache);
-  tt_assert(!rend_cache_v2_dir);
-  tt_assert(!rend_cache_failure);
+  tt_ptr_op(rend_cache, OP_EQ, NULL);
+  tt_ptr_op(rend_cache_v2_dir, OP_EQ, NULL);
+  tt_ptr_op(rend_cache_failure, OP_EQ, NULL);
   tt_assert(!rend_cache_total_allocation);
 
  done:
@@ -972,7 +963,7 @@ test_rend_cache_entry_free(void *data)
   rend_cache_entry_t *e;
 
   // Handles NULL correctly
-  rend_cache_entry_free(NULL);
+  rend_cache_entry_free_(NULL);
 
   // Handles NULL descriptor correctly
   e = tor_malloc_zero(sizeof(rend_cache_entry_t));
@@ -1078,9 +1069,10 @@ static void
 test_rend_cache_clean_v2_descs_as_dir(void *data)
 {
   rend_cache_entry_t *e;
-  time_t now;
+  time_t now, cutoff;
   rend_service_descriptor_t *desc;
   now = time(NULL);
+  cutoff = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW);
   const char key[DIGEST_LEN] = "abcde";
 
   (void)data;
@@ -1088,7 +1080,7 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   rend_cache_init();
 
   // Test running with an empty cache
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  rend_cache_clean_v2_descs_as_dir(cutoff);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
 
   // Test with only one new entry
@@ -1100,38 +1092,15 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   e->parsed = desc;
   digestmap_set(rend_cache_v2_dir, key, e);
 
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  /* Set the cutoff to minus 10 seconds. */
+  rend_cache_clean_v2_descs_as_dir(cutoff - 10);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 1);
 
   // Test with one old entry
-  desc->timestamp = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW + 1000);
-  rend_cache_clean_v2_descs_as_dir(now, 0);
-  tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
-
-  // Test with one entry that has an old last served
-  e = tor_malloc_zero(sizeof(rend_cache_entry_t));
-  e->last_served = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW + 1000);
-  desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  desc->timestamp = now;
-  desc->pk = pk_generate(0);
-  e->parsed = desc;
-  digestmap_set(rend_cache_v2_dir, key, e);
-
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  desc->timestamp = cutoff - 1000;
+  rend_cache_clean_v2_descs_as_dir(cutoff);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
 
-  // Test a run through asking for a large force_remove
-  e = tor_malloc_zero(sizeof(rend_cache_entry_t));
-  e->last_served = now;
-  desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  desc->timestamp = now;
-  desc->pk = pk_generate(0);
-  e->parsed = desc;
-  digestmap_set(rend_cache_v2_dir, key, e);
-
-  rend_cache_clean_v2_descs_as_dir(now, 20000);
-  tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 1);
-
  done:
   rend_cache_free_all();
 }
@@ -1166,7 +1135,7 @@ test_rend_cache_failure_intro_entry_free(void *data)
   rend_cache_failure_intro_t *entry;
 
   // Handles a null argument
-  rend_cache_failure_intro_entry_free(NULL);
+  rend_cache_failure_intro_entry_free_(NULL);
 
   // Handles a non-null argument
   entry = rend_cache_failure_intro_entry_new(INTRO_POINT_FAILURE_TIMEOUT);
@@ -1179,7 +1148,7 @@ test_rend_cache_failure_purge(void *data)
   (void)data;
 
   // Handles a null failure cache
-  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_);
+  strmap_free(rend_cache_failure, rend_cache_failure_entry_free_void);
   rend_cache_failure = NULL;
 
   rend_cache_failure_purge();
diff --git a/src/test/test_replay.c b/src/test/test_replay.c
index e882bc6164..d8dcc7370c 100644
--- a/src/test/test_replay.c
+++ b/src/test/test_replay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define REPLAYCACHE_PRIVATE
@@ -38,7 +38,7 @@ test_replaycache_alloc(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
  done:
   if (r) replaycache_free(r);
@@ -54,15 +54,15 @@ test_replaycache_badalloc(void *arg)
   /* Negative horizon should fail */
   (void)arg;
   r = replaycache_new(-600, 300);
-  tt_assert(r == NULL);
+  tt_ptr_op(r, OP_EQ, NULL);
   /* Negative interval should get adjusted to zero */
   r = replaycache_new(600, -300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
   tt_int_op(r->scrub_interval,OP_EQ, 0);
   replaycache_free(r);
   /* Negative horizon and negative interval should still fail */
   r = replaycache_new(-600, -300);
-  tt_assert(r == NULL);
+  tt_ptr_op(r, OP_EQ, NULL);
 
  done:
   if (r) replaycache_free(r);
@@ -74,7 +74,7 @@ static void
 test_replaycache_free_null(void *arg)
 {
   (void)arg;
-  replaycache_free(NULL);
+  replaycache_free_(NULL);
   /* Assert that we're here without horrible death */
   tt_assert(1);
 
@@ -90,7 +90,7 @@ test_replaycache_miss(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -123,7 +123,7 @@ test_replaycache_hit(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -161,7 +161,7 @@ test_replaycache_age(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -193,7 +193,7 @@ test_replaycache_elapsed(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -220,7 +220,7 @@ test_replaycache_noexpire(void *arg)
 
   (void)arg;
   r = replaycache_new(0, 0);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   result =
     replaycache_add_and_test_internal(1200, r, test_buffer,
@@ -251,7 +251,7 @@ test_replaycache_scrub(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   /* Set up like in test_replaycache_hit() */
   result =
@@ -294,7 +294,7 @@ test_replaycache_future(void *arg)
 
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   /* Set up like in test_replaycache_hit() */
   result =
@@ -343,7 +343,7 @@ test_replaycache_realtime(void *arg)
   /* Test the realtime as well as *_internal() entry points */
   (void)arg;
   r = replaycache_new(600, 300);
-  tt_assert(r != NULL);
+  tt_ptr_op(r, OP_NE, NULL);
 
   /* This should miss */
   result =
diff --git a/src/test/test_router.c b/src/test/test_router.c
index 51055a3367..84473822a2 100644
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@ -1,4 +1,5 @@
-/* Copyright (c) 2018, The Tor Project, Inc. */
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* Copyright (c) 2017, isis agora lovecruft  */
 /* See LICENSE for licensing information     */
 
 /**
@@ -7,12 +8,103 @@
  **/
 
 #include "or.h"
+#include "config.h"
+#include "crypto_curve25519.h"
+#include "crypto_ed25519.h"
 #include "hibernate.h"
-#include "log_test_helpers.h"
 #include "main.h"
 #include "rephist.h"
 #include "router.h"
+#include "routerlist.h"
+
+/* Test suite stuff */
 #include "test.h"
+#include "log_test_helpers.h"
+
+NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+
+static routerinfo_t* mock_routerinfo;
+
+static const routerinfo_t*
+NS(router_get_my_routerinfo)(void)
+{
+  crypto_pk_t* ident_key;
+  crypto_pk_t* tap_key;
+  time_t now;
+
+  if (!mock_routerinfo) {
+    /* Mock the published timestamp, otherwise router_dump_router_to_string()
+     * will poop its pants. */
+    time(&now);
+
+    /* We'll need keys, or router_dump_router_to_string() would return NULL. */
+    ident_key = pk_generate(0);
+    tap_key = pk_generate(0);
+
+    tor_assert(ident_key != NULL);
+    tor_assert(tap_key != NULL);
+
+    mock_routerinfo = tor_malloc_zero(sizeof(routerinfo_t));
+    mock_routerinfo->nickname = tor_strdup("ConlonNancarrow");
+    mock_routerinfo->addr = 123456789;
+    mock_routerinfo->or_port = 443;
+    mock_routerinfo->platform = tor_strdup("unittest");
+    mock_routerinfo->cache_info.published_on = now;
+    mock_routerinfo->identity_pkey = crypto_pk_dup_key(ident_key);
+    mock_routerinfo->onion_pkey = crypto_pk_dup_key(tap_key);
+    mock_routerinfo->bandwidthrate = 9001;
+    mock_routerinfo->bandwidthburst = 9002;
+  }
+
+  return mock_routerinfo;
+}
+
+/* If no distribution option was set, then check_bridge_distribution_setting()
+ * should have set it to "any". */
+static void
+test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
+{
+  const char* needle = "bridge-distribution-request any";
+  or_options_t* options = get_options_mutable();
+  routerinfo_t* router = NULL;
+  curve25519_keypair_t ntor_keypair;
+  ed25519_keypair_t signing_keypair;
+  char* desc = NULL;
+  char* found = NULL;
+  (void)arg;
+
+  NS_MOCK(router_get_my_routerinfo);
+
+  options->ORPort_set = 1;
+  options->BridgeRelay = 1;
+
+  /* Generate keys which router_dump_router_to_string() expects to exist. */
+  tt_int_op(0, ==, curve25519_keypair_generate(&ntor_keypair, 0));
+  tt_int_op(0, ==, ed25519_keypair_generate(&signing_keypair, 0));
+
+  /* Set up part of our routerinfo_t so that we don't trigger any other
+   * assertions in router_dump_router_to_string(). */
+  router = (routerinfo_t*)router_get_my_routerinfo();
+  tt_ptr_op(router, !=, NULL);
+
+  router->onion_curve25519_pkey = &ntor_keypair.pubkey;
+
+  /* Generate our server descriptor and ensure that the substring
+   * "bridge-distribution-request any" occurs somewhere within it. */
+  desc = router_dump_router_to_string(router,
+                                      router->identity_pkey,
+                                      router->onion_pkey,
+                                      &ntor_keypair,
+                                      &signing_keypair);
+  tt_ptr_op(desc, !=, NULL);
+  found = strstr(desc, needle);
+  tt_ptr_op(found, !=, NULL);
+
+ done:
+  NS_UNMOCK(router_get_my_routerinfo);
+
+  tor_free(desc);
+}
 
 static routerinfo_t *mock_router_get_my_routerinfo_result = NULL;
 
@@ -136,7 +228,7 @@ test_router_check_descriptor_bandwidth_changed(void *arg)
   { #name, test_router_ ## name, flags, NULL, NULL }
 
 struct testcase_t router_tests[] = {
+  ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
   ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 24b0da1c46..e4abcdb92d 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -92,8 +92,8 @@ test_routerkeys_ed_certs(void *args)
   uint8_t *junk = NULL;
   char *base64 = NULL;
 
-  tt_int_op(0,==,ed25519_keypair_generate(&kp1, 0));
-  tt_int_op(0,==,ed25519_keypair_generate(&kp2, 0));
+  tt_int_op(0,OP_EQ,ed25519_keypair_generate(&kp1, 0));
+  tt_int_op(0,OP_EQ,ed25519_keypair_generate(&kp2, 0));
 
   for (int i = 0; i <= 1; ++i) {
     uint32_t flags = i ? CERT_FLAG_INCLUDE_SIGNING_KEY : 0;
@@ -101,45 +101,45 @@ test_routerkeys_ed_certs(void *args)
     cert[i] = tor_cert_create(&kp1, 5, &kp2.pubkey, now, 10000, flags);
     tt_assert(cert[i]);
 
-    tt_assert(cert[i]->sig_bad == 0);
-    tt_assert(cert[i]->sig_ok == 1);
-    tt_assert(cert[i]->cert_expired == 0);
-    tt_assert(cert[i]->cert_valid == 1);
-    tt_int_op(cert[i]->cert_type, ==, 5);
-    tt_mem_op(cert[i]->signed_key.pubkey, ==, &kp2.pubkey.pubkey, 32);
-    tt_mem_op(cert[i]->signing_key.pubkey, ==, &kp1.pubkey.pubkey, 32);
-    tt_int_op(cert[i]->signing_key_included, ==, i);
+    tt_uint_op(cert[i]->sig_bad, OP_EQ, 0);
+    tt_uint_op(cert[i]->sig_ok, OP_EQ, 1);
+    tt_uint_op(cert[i]->cert_expired, OP_EQ, 0);
+    tt_uint_op(cert[i]->cert_valid, OP_EQ, 1);
+    tt_int_op(cert[i]->cert_type, OP_EQ, 5);
+    tt_mem_op(cert[i]->signed_key.pubkey, OP_EQ, &kp2.pubkey.pubkey, 32);
+    tt_mem_op(cert[i]->signing_key.pubkey, OP_EQ, &kp1.pubkey.pubkey, 32);
+    tt_int_op(cert[i]->signing_key_included, OP_EQ, i);
 
     tt_assert(cert[i]->encoded);
-    tt_int_op(cert[i]->encoded_len, ==, 104 + 36 * i);
-    tt_int_op(cert[i]->encoded[0], ==, 1);
-    tt_int_op(cert[i]->encoded[1], ==, 5);
+    tt_int_op(cert[i]->encoded_len, OP_EQ, 104 + 36 * i);
+    tt_int_op(cert[i]->encoded[0], OP_EQ, 1);
+    tt_int_op(cert[i]->encoded[1], OP_EQ, 5);
 
     parsed_cert[i] = tor_cert_parse(cert[i]->encoded, cert[i]->encoded_len);
     tt_assert(parsed_cert[i]);
-    tt_int_op(cert[i]->encoded_len, ==, parsed_cert[i]->encoded_len);
-    tt_mem_op(cert[i]->encoded, ==, parsed_cert[i]->encoded,
+    tt_int_op(cert[i]->encoded_len, OP_EQ, parsed_cert[i]->encoded_len);
+    tt_mem_op(cert[i]->encoded, OP_EQ, parsed_cert[i]->encoded,
               cert[i]->encoded_len);
-    tt_assert(parsed_cert[i]->sig_bad == 0);
-    tt_assert(parsed_cert[i]->sig_ok == 0);
-    tt_assert(parsed_cert[i]->cert_expired == 0);
-    tt_assert(parsed_cert[i]->cert_valid == 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->sig_ok, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->cert_expired, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->cert_valid, OP_EQ, 0);
 
     /* Expired */
     tt_int_op(tor_cert_checksig(parsed_cert[i], &kp1.pubkey, now + 30000),
-              <, 0);
-    tt_assert(parsed_cert[i]->cert_expired == 1);
+              OP_LT, 0);
+    tt_uint_op(parsed_cert[i]->cert_expired, OP_EQ, 1);
     parsed_cert[i]->cert_expired = 0;
 
     /* Wrong key */
-    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp2.pubkey, now), <, 0);
-    tt_assert(parsed_cert[i]->sig_bad== 1);
+    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp2.pubkey, now), OP_LT, 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 1);
     parsed_cert[i]->sig_bad = 0;
 
     /* Missing key */
     int ok = tor_cert_checksig(parsed_cert[i], NULL, now);
-    tt_int_op(ok < 0, ==, i == 0);
-    tt_assert(parsed_cert[i]->sig_bad == 0);
+    tt_int_op(ok < 0, OP_EQ, i == 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 0);
     tt_assert(parsed_cert[i]->sig_ok == (i != 0));
     tt_assert(parsed_cert[i]->cert_valid == (i != 0));
     parsed_cert[i]->sig_bad = 0;
@@ -147,29 +147,29 @@ test_routerkeys_ed_certs(void *args)
     parsed_cert[i]->cert_valid = 0;
 
     /* Right key */
-    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp1.pubkey, now), ==, 0);
-    tt_assert(parsed_cert[i]->sig_bad == 0);
-    tt_assert(parsed_cert[i]->sig_ok == 1);
-    tt_assert(parsed_cert[i]->cert_expired == 0);
-    tt_assert(parsed_cert[i]->cert_valid == 1);
+    tt_int_op(tor_cert_checksig(parsed_cert[i], &kp1.pubkey, now), OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->sig_bad, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->sig_ok, OP_EQ, 1);
+    tt_uint_op(parsed_cert[i]->cert_expired, OP_EQ, 0);
+    tt_uint_op(parsed_cert[i]->cert_valid, OP_EQ, 1);
   }
 
   /* Now try some junky certs. */
   /* - Truncated */
   nocert = tor_cert_parse(cert[0]->encoded, cert[0]->encoded_len-1);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
 
   /* - First byte modified */
   cert[0]->encoded[0] = 99;
   nocert = tor_cert_parse(cert[0]->encoded, cert[0]->encoded_len);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
   cert[0]->encoded[0] = 1;
 
   /* - Extra byte at the end*/
   junk = tor_malloc_zero(cert[0]->encoded_len + 1);
   memcpy(junk, cert[0]->encoded, cert[0]->encoded_len);
   nocert = tor_cert_parse(junk, cert[0]->encoded_len+1);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
 
   /* - Multiple signing key instances */
   tor_free(junk);
@@ -183,7 +183,7 @@ test_routerkeys_ed_certs(void *args)
   junk[77] = 32; /* extlen */
   junk[78] = 4; /* exttype */
   nocert = tor_cert_parse(junk, 104 + 36 * 2);
-  tt_ptr_op(NULL, ==, nocert);
+  tt_ptr_op(NULL, OP_EQ, nocert);
 
  done:
   tor_cert_free(cert[0]);
@@ -211,11 +211,12 @@ test_routerkeys_ed_key_create(void *arg)
   kp2 = ed_key_new(kp1, INIT_ED_KEY_NEEDCERT, now, 3600, 4, &cert);
   tt_assert(kp2);
   tt_assert(cert);
-  tt_mem_op(&cert->signed_key, ==, &kp2->pubkey, sizeof(ed25519_public_key_t));
+  tt_mem_op(&cert->signed_key, OP_EQ, &kp2->pubkey,
+            sizeof(ed25519_public_key_t));
   tt_assert(! cert->signing_key_included);
 
-  tt_int_op(cert->valid_until, >=, now);
-  tt_int_op(cert->valid_until, <=, now+7200);
+  tt_int_op(cert->valid_until, OP_GE, now);
+  tt_int_op(cert->valid_until, OP_LE, now+7200);
 
   /* Create a new key-including certificate signed by kp1 */
   ed25519_keypair_free(kp2);
@@ -227,8 +228,10 @@ test_routerkeys_ed_key_create(void *arg)
   tt_assert(kp2);
   tt_assert(cert);
   tt_assert(cert->signing_key_included);
-  tt_mem_op(&cert->signed_key, ==, &kp2->pubkey, sizeof(ed25519_public_key_t));
-  tt_mem_op(&cert->signing_key, ==, &kp1->pubkey,sizeof(ed25519_public_key_t));
+  tt_mem_op(&cert->signed_key, OP_EQ, &kp2->pubkey,
+            sizeof(ed25519_public_key_t));
+  tt_mem_op(&cert->signing_key, OP_EQ, &kp1->pubkey,
+            sizeof(ed25519_public_key_t));
 
  done:
   ed25519_keypair_free(kp1);
@@ -261,8 +264,8 @@ test_routerkeys_ed_key_init_basic(void *arg)
                               NULL, now, 0, 7, &cert);
   tt_assert(kp1 != NULL);
   tt_assert(cert == NULL);
-  tt_int_op(stat(get_fname("test_ed_key_1_cert"), &st), <, 0);
-  tt_int_op(stat(get_fname("test_ed_key_1_secret_key"), &st), ==, 0);
+  tt_int_op(stat(get_fname("test_ed_key_1_cert"), &st), OP_LT, 0);
+  tt_int_op(stat(get_fname("test_ed_key_1_secret_key"), &st), OP_EQ, 0);
 
   /* Fail to load if we say we need a cert */
   kp2 = ed_key_init_from_file(fname1, INIT_ED_KEY_NEEDCERT, LOG_INFO,
@@ -279,14 +282,14 @@ test_routerkeys_ed_key_init_basic(void *arg)
                               NULL, now, 0, 7, &cert);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(kp1, ==, kp2, sizeof(*kp1));
+  tt_mem_op(kp1, OP_EQ, kp2, sizeof(*kp1));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   kp2 = ed_key_init_from_file(fname1, 0, LOG_INFO,
                               NULL, now, 0, 7, &cert);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(kp1, ==, kp2, sizeof(*kp1));
+  tt_mem_op(kp1, OP_EQ, kp2, sizeof(*kp1));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   /* Now create a key with a cert. */
@@ -295,34 +298,34 @@ test_routerkeys_ed_key_init_basic(void *arg)
                               LOG_INFO, kp1, now, 7200, 7, &cert);
   tt_assert(kp2 != NULL);
   tt_assert(cert != NULL);
-  tt_mem_op(kp1, !=, kp2, sizeof(*kp1));
-  tt_int_op(stat(get_fname("test_ed_key_2_cert"), &st), ==, 0);
-  tt_int_op(stat(get_fname("test_ed_key_2_secret_key"), &st), ==, 0);
+  tt_mem_op(kp1, OP_NE, kp2, sizeof(*kp1));
+  tt_int_op(stat(get_fname("test_ed_key_2_cert"), &st), OP_EQ, 0);
+  tt_int_op(stat(get_fname("test_ed_key_2_secret_key"), &st), OP_EQ, 0);
 
   tt_assert(cert->cert_valid == 1);
-  tt_mem_op(&cert->signed_key, ==, &kp2->pubkey, 32);
+  tt_mem_op(&cert->signed_key, OP_EQ, &kp2->pubkey, 32);
 
   /* Now verify we can load the cert... */
   kp3 = ed_key_init_from_file(fname2, (INIT_ED_KEY_CREATE|
                                        INIT_ED_KEY_NEEDCERT),
                               LOG_INFO, kp1, now, 7200, 7, &cert2);
-  tt_mem_op(kp2, ==, kp3, sizeof(*kp2));
-  tt_mem_op(cert2->encoded, ==, cert->encoded, cert->encoded_len);
+  tt_mem_op(kp2, OP_EQ, kp3, sizeof(*kp2));
+  tt_mem_op(cert2->encoded, OP_EQ, cert->encoded, cert->encoded_len);
   ed25519_keypair_free(kp3); kp3 = NULL;
   tor_cert_free(cert2); cert2 = NULL;
 
   /* ... even without create... */
   kp3 = ed_key_init_from_file(fname2, INIT_ED_KEY_NEEDCERT,
                               LOG_INFO, kp1, now, 7200, 7, &cert2);
-  tt_mem_op(kp2, ==, kp3, sizeof(*kp2));
-  tt_mem_op(cert2->encoded, ==, cert->encoded, cert->encoded_len);
+  tt_mem_op(kp2, OP_EQ, kp3, sizeof(*kp2));
+  tt_mem_op(cert2->encoded, OP_EQ, cert->encoded, cert->encoded_len);
   ed25519_keypair_free(kp3); kp3 = NULL;
   tor_cert_free(cert2); cert2 = NULL;
 
   /* ... but that we don't crash or anything if we say we don't want it. */
   kp3 = ed_key_init_from_file(fname2, INIT_ED_KEY_NEEDCERT,
                               LOG_INFO, kp1, now, 7200, 7, NULL);
-  tt_mem_op(kp2, ==, kp3, sizeof(*kp2));
+  tt_mem_op(kp2, OP_EQ, kp3, sizeof(*kp2));
   ed25519_keypair_free(kp3); kp3 = NULL;
 
   /* Fail if we're told the wrong signing key */
@@ -367,16 +370,16 @@ test_routerkeys_ed_key_init_split(void *arg)
                               LOG_INFO, NULL, now, 0, 7, &cert);
   tt_assert(kp1 != NULL);
   tt_assert(cert == NULL);
-  tt_int_op(stat(get_fname("test_ed_key_3_cert"), &st), <, 0);
-  tt_int_op(stat(get_fname("test_ed_key_3_secret_key"), &st), ==, 0);
-  tt_int_op(stat(get_fname("test_ed_key_3_public_key"), &st), ==, 0);
+  tt_int_op(stat(get_fname("test_ed_key_3_cert"), &st), OP_LT, 0);
+  tt_int_op(stat(get_fname("test_ed_key_3_secret_key"), &st), OP_EQ, 0);
+  tt_int_op(stat(get_fname("test_ed_key_3_public_key"), &st), OP_EQ, 0);
 
   /* Load it. */
   kp2 = ed_key_init_from_file(fname1, flags|INIT_ED_KEY_CREATE,
                               LOG_INFO, NULL, now, 0, 7, &cert);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(kp1, ==, kp2, sizeof(*kp2));
+  tt_mem_op(kp1, OP_EQ, kp2, sizeof(*kp2));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
   /* Okay, try killing the secret key and loading it. */
@@ -385,7 +388,7 @@ test_routerkeys_ed_key_init_split(void *arg)
                               LOG_INFO, NULL, now, 0, 7, &cert);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(&kp1->pubkey, ==, &kp2->pubkey, sizeof(kp2->pubkey));
+  tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
   tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
@@ -395,7 +398,7 @@ test_routerkeys_ed_key_init_split(void *arg)
                               LOG_INFO, NULL, now, 0, 7, &cert);
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
-  tt_mem_op(&kp1->pubkey, ==, &kp2->pubkey, sizeof(kp2->pubkey));
+  tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
   tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
@@ -418,6 +421,7 @@ test_routerkeys_ed_keys_init_all(void *arg)
 {
   (void)arg;
   char *dir = tor_strdup(get_fname("test_ed_keys_init_all"));
+  char *keydir = tor_strdup(get_fname("test_ed_keys_init_all/KEYS"));
   or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
   time_t now = time(NULL);
   ed25519_public_key_t id;
@@ -442,16 +446,17 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
 #ifdef _WIN32
   mkdir(dir);
-  mkdir(get_fname("test_ed_keys_init_all/keys"));
+  mkdir(keydir);
 #else
   mkdir(dir, 0700);
-  mkdir(get_fname("test_ed_keys_init_all/keys"), 0700);
-#endif
+  mkdir(keydir, 0700);
+#endif /* defined(_WIN32) */
 
   options->DataDirectory = dir;
+  options->KeyDirectory = keydir;
 
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
   tt_assert(get_master_identity_key());
   tt_assert(get_master_identity_key());
   tt_assert(get_master_signing_keypair());
@@ -465,21 +470,21 @@ test_routerkeys_ed_keys_init_all(void *arg)
   link_cert = tor_cert_dup(get_current_link_cert_cert());
 
   /* Call load_ed_keys again, but nothing has changed. */
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
-  tt_mem_op(&auth, ==, get_current_auth_keypair(), sizeof(auth));
+  tt_int_op(0, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
+  tt_mem_op(&auth, OP_EQ, get_current_auth_keypair(), sizeof(auth));
   tt_assert(tor_cert_eq(link_cert, get_current_link_cert_cert()));
 
   /* Force a reload: we make new link/auth keys. */
   routerkeys_free_all();
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
   tt_assert(tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -488,12 +493,12 @@ test_routerkeys_ed_keys_init_all(void *arg)
   memcpy(&auth, get_current_auth_keypair(), sizeof(auth));
 
   /* Force a link/auth-key regeneration by advancing time. */
-  tt_int_op(0, ==, load_ed_keys(options, now+3*86400));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now+3*86400));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(0, OP_EQ, load_ed_keys(options, now+3*86400));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now+3*86400, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -502,12 +507,12 @@ test_routerkeys_ed_keys_init_all(void *arg)
   memcpy(&auth, get_current_auth_keypair(), sizeof(auth));
 
   /* Force a signing-key regeneration by advancing time. */
-  tt_int_op(0, ==, load_ed_keys(options, now+100*86400));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now+100*86400));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, !=, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now+100*86400));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now+100*86400, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_NE, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -518,14 +523,14 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
   /* Demonstrate that we can start up with no secret identity key */
   routerkeys_free_all();
-  unlink(get_fname("test_ed_keys_init_all/keys/"
+  unlink(get_fname("test_ed_keys_init_all/KEYS/"
                    "ed25519_master_id_secret_key"));
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
-  tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
-  tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
+  tt_int_op(1, OP_EQ, load_ed_keys(options, now));
+  tt_int_op(0, OP_EQ, generate_ed_link_cert(options, now, 0));
+  tt_mem_op(&id, OP_EQ, get_master_identity_key(), sizeof(id));
+  tt_mem_op(&sign, OP_EQ, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
-  tt_mem_op(&auth, !=, get_current_auth_keypair(), sizeof(auth));
+  tt_mem_op(&auth, OP_NE, get_current_auth_keypair(), sizeof(auth));
   tt_assert(get_master_signing_key_cert());
   tt_assert(get_current_link_cert_cert());
   tt_assert(get_current_auth_key_cert());
@@ -535,10 +540,11 @@ test_routerkeys_ed_keys_init_all(void *arg)
   log_global_min_severity_ = LOG_ERR; /* Suppress warnings.
                                        * XXX (better way to do this)? */
   routerkeys_free_all();
-  tt_int_op(-1, ==, load_ed_keys(options, now+200*86400));
+  tt_int_op(-1, OP_EQ, load_ed_keys(options, now+200*86400));
 
  done:
   tor_free(dir);
+  tor_free(keydir);
   tor_free(options);
   tor_cert_free(link_cert);
   routerkeys_free_all();
@@ -556,20 +562,20 @@ test_routerkeys_cross_certify_ntor(void *args)
   time_t now = time(NULL);
   int sign;
 
-  tt_int_op(0, ==, ed25519_public_from_base64(&master_key,
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&master_key,
                                "IamwritingthesetestsOnARainyAfternoonin2014"));
-  tt_int_op(0, ==, curve25519_keypair_generate(&onion_keys, 0));
+  tt_int_op(0, OP_EQ, curve25519_keypair_generate(&onion_keys, 0));
   cert = make_ntor_onion_key_crosscert(&onion_keys,
                                        &master_key,
                                        now, 10000,
                                        &sign);
   tt_assert(cert);
   tt_assert(sign == 0 || sign == 1);
-  tt_int_op(cert->cert_type, ==, CERT_TYPE_ONION_ID);
-  tt_int_op(1, ==, ed25519_pubkey_eq(&cert->signed_key, &master_key));
-  tt_int_op(0, ==, ed25519_public_key_from_curve25519_public_key(
+  tt_int_op(cert->cert_type, OP_EQ, CERT_TYPE_ONION_ID);
+  tt_int_op(1, OP_EQ, ed25519_pubkey_eq(&cert->signed_key, &master_key));
+  tt_int_op(0, OP_EQ, ed25519_public_key_from_curve25519_public_key(
                                &onion_check_key, &onion_keys.pubkey, sign));
-  tt_int_op(0, ==, tor_cert_checksig(cert, &onion_check_key, now));
+  tt_int_op(0, OP_EQ, tor_cert_checksig(cert, &onion_check_key, now));
 
  done:
   tor_cert_free(cert);
@@ -587,7 +593,7 @@ test_routerkeys_cross_certify_tap(void *args)
   char buf[128];
   int n;
 
-  tt_int_op(0, ==, ed25519_public_from_base64(&master_key,
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&master_key,
                                "IAlreadyWroteTestsForRouterdescsUsingTheseX"));
 
   cc = make_tap_onion_key_crosscert(onion_key,
@@ -598,14 +604,14 @@ test_routerkeys_cross_certify_tap(void *args)
 
   n = crypto_pk_public_checksig(onion_key, buf, sizeof(buf),
                                 (char*)cc, cc_len);
-  tt_int_op(n,>,0);
-  tt_int_op(n,==,52);
+  tt_int_op(n,OP_GT,0);
+  tt_int_op(n,OP_EQ,52);
 
   crypto_pk_get_digest(id_key, digest);
-  tt_mem_op(buf,==,digest,20);
-  tt_mem_op(buf+20,==,master_key.pubkey,32);
+  tt_mem_op(buf,OP_EQ,digest,20);
+  tt_mem_op(buf+20,OP_EQ,master_key.pubkey,32);
 
-  tt_int_op(0, ==, check_tap_onion_key_crosscert(cc, cc_len,
+  tt_int_op(0, OP_EQ, check_tap_onion_key_crosscert(cc, cc_len,
                                     onion_key, &master_key, (uint8_t*)digest));
 
  done:
@@ -614,6 +620,66 @@ test_routerkeys_cross_certify_tap(void *args)
   crypto_pk_free(onion_key);
 }
 
+static void
+test_routerkeys_rsa_ed_crosscert(void *arg)
+{
+  (void)arg;
+  ed25519_public_key_t ed;
+  crypto_pk_t *rsa = pk_generate(2);
+
+  uint8_t *cc = NULL;
+  ssize_t cc_len;
+  time_t expires_in = 1470846177;
+
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&ed,
+                        "ThisStringCanContainAnythingSoNoKeyHereNowX"));
+  cc_len = tor_make_rsa_ed25519_crosscert(&ed, rsa, expires_in, &cc);
+
+  tt_int_op(cc_len, OP_GT, 0);
+  tt_int_op(cc_len, OP_GT, 37); /* key, expires, siglen */
+  tt_mem_op(cc, OP_EQ, ed.pubkey, 32);
+  time_t expires_out = 3600 * ntohl(get_uint32(cc+32));
+  tt_int_op(expires_out, OP_GE, expires_in);
+  tt_int_op(expires_out, OP_LE, expires_in + 3600);
+
+  tt_int_op(cc_len, OP_EQ, 37 + get_uint8(cc+36));
+
+  tt_int_op(0, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+
+  /* Now try after it has expired */
+  tt_int_op(-4, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_out + 1));
+
+  /* Truncated object */
+  tt_int_op(-2, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len - 2, rsa, &ed,
+                                                  expires_in - 10));
+
+  /* Key not as expected */
+  cc[0] ^= 3;
+  tt_int_op(-3, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+  cc[0] ^= 3;
+
+  /* Bad signature */
+  cc[40] ^= 3;
+  tt_int_op(-5, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                   expires_in - 10));
+  cc[40] ^= 3;
+
+  /* Signature of wrong data */
+  cc[0] ^= 3;
+  ed.pubkey[0] ^= 3;
+  tt_int_op(-6, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+  cc[0] ^= 3;
+  ed.pubkey[0] ^= 3;
+
+ done:
+  crypto_pk_free(rsa);
+  tor_free(cc);
+}
+
 #define TEST(name, flags)                                       \
   { #name , test_routerkeys_ ## name, (flags), NULL, NULL }
 
@@ -626,6 +692,7 @@ struct testcase_t routerkeys_tests[] = {
   TEST(ed_keys_init_all, TT_FORK),
   TEST(cross_certify_ntor, 0),
   TEST(cross_certify_tap, 0),
+  TEST(rsa_ed_crosscert, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 088bd257c3..701227c1c7 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -1,11 +1,15 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <math.h>
 #include <time.h>
 
+#define CONNECTION_PRIVATE
+#define DIRECTORY_PRIVATE
 #define DIRVOTE_PRIVATE
+#define ENTRYNODES_PRIVATE
+#define HIBERNATE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define ROUTERLIST_PRIVATE
 #define TOR_UNIT_TESTING
@@ -13,20 +17,27 @@
 #include "config.h"
 #include "connection.h"
 #include "container.h"
+#include "control.h"
+#include "crypto_rand.h"
 #include "directory.h"
-#include "dirvote.h"
+#include "dirauth/dirvote.h"
+#include "entrynodes.h"
+#include "hibernate.h"
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
 #include "router.h"
 #include "routerlist.h"
+#include "routerset.h"
 #include "routerparse.h"
-#include "shared_random.h"
+#include "dirauth/shared_random.h"
+#include "statefile.h"
 #include "test.h"
 #include "test_dir_common.h"
+#include "log_test_helpers.h"
 
-void construct_consensus(char **consensus_text_md);
+void construct_consensus(char **consensus_text_md, time_t now);
 
 static authority_cert_t *mock_cert;
 
@@ -116,7 +127,7 @@ test_routerlist_launch_descriptor_downloads(void *arg)
   MOCK(initiate_descriptor_downloads, mock_initiate_descriptor_downloads);
   launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC, downloadable,
                               NULL, now);
-  tt_int_op(3, ==, count);
+  tt_int_op(3, OP_EQ, count);
   UNMOCK(initiate_descriptor_downloads);
 
  done:
@@ -125,7 +136,7 @@ test_routerlist_launch_descriptor_downloads(void *arg)
 }
 
 void
-construct_consensus(char **consensus_text_md)
+construct_consensus(char **consensus_text_md, time_t now)
 {
   networkstatus_t *vote = NULL;
   networkstatus_t *v1 = NULL, *v2 = NULL, *v3 = NULL;
@@ -133,7 +144,6 @@ construct_consensus(char **consensus_text_md)
   authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL;
   crypto_pk_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL;
   crypto_pk_t *sign_skey_leg=NULL;
-  time_t now = time(NULL);
   smartlist_t *votes = NULL;
   int n_vrs;
 
@@ -147,24 +157,24 @@ construct_consensus(char **consensus_text_md)
                               &v1, &n_vrs, now, 1);
   networkstatus_vote_free(vote);
   tt_assert(v1);
-  tt_int_op(n_vrs, ==, 4);
-  tt_int_op(smartlist_len(v1->routerstatus_list), ==, 4);
+  tt_int_op(n_vrs, OP_EQ, 4);
+  tt_int_op(smartlist_len(v1->routerstatus_list), OP_EQ, 4);
 
   dir_common_construct_vote_2(&vote, cert2, sign_skey_2,
                               &dir_common_gen_routerstatus_for_v3ns,
                               &v2, &n_vrs, now, 1);
   networkstatus_vote_free(vote);
   tt_assert(v2);
-  tt_int_op(n_vrs, ==, 4);
-  tt_int_op(smartlist_len(v2->routerstatus_list), ==, 4);
+  tt_int_op(n_vrs, OP_EQ, 4);
+  tt_int_op(smartlist_len(v2->routerstatus_list), OP_EQ, 4);
 
   dir_common_construct_vote_3(&vote, cert3, sign_skey_3,
                               &dir_common_gen_routerstatus_for_v3ns,
                               &v3, &n_vrs, now, 1);
 
   tt_assert(v3);
-  tt_int_op(n_vrs, ==, 4);
-  tt_int_op(smartlist_len(v3->routerstatus_list), ==, 4);
+  tt_int_op(n_vrs, OP_EQ, 4);
+  tt_int_op(smartlist_len(v3->routerstatus_list), OP_EQ, 4);
   networkstatus_vote_free(vote);
   votes = smartlist_new();
   smartlist_add(votes, v1);
@@ -246,16 +256,16 @@ test_router_pick_directory_server_impl(void *arg)
 
   /* No consensus available, fail early */
   rs = router_pick_directory_server_impl(V3_DIRINFO, (const int) 0, NULL);
-  tt_assert(rs == NULL);
+  tt_ptr_op(rs, OP_EQ, NULL);
 
-  construct_consensus(&consensus_text_md);
+  construct_consensus(&consensus_text_md, now);
   tt_assert(consensus_text_md);
   con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
-  tt_int_op(con_md->flavor,==, FLAV_MICRODESC);
+  tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
   tt_assert(con_md->routerstatus_list);
-  tt_int_op(smartlist_len(con_md->routerstatus_list), ==, 3);
+  tt_int_op(smartlist_len(con_md->routerstatus_list), OP_EQ, 3);
   tt_assert(!networkstatus_set_current_consensus_from_ns(con_md,
                                                  "microdesc"));
 
@@ -286,7 +296,7 @@ test_router_pick_directory_server_impl(void *arg)
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
   /* We should not fail now we have a consensus and routerstatus_list
    * and nodelist are populated. */
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
 
   /* Manipulate the nodes so we get the dir server we expect */
   router1_id = tor_malloc(DIGEST_LEN);
@@ -305,7 +315,7 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->is_running = 0;
   node_router3->is_running = 0;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
   rs = NULL;
   node_router1->is_running = 1;
@@ -318,7 +328,7 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->rs->dir_port = 0;
   node_router3->rs->dir_port = 0;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
   rs = NULL;
   node_router1->rs->is_v2_dir = 1;
@@ -329,42 +339,18 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->is_valid = 0;
   node_router3->is_valid = 0;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router2_id, DIGEST_LEN));
   rs = NULL;
   node_router1->is_valid = 1;
   node_router3->is_valid = 1;
 
-  flags |= PDS_FOR_GUARD;
-  node_router1->using_as_guard = 1;
-  node_router2->using_as_guard = 1;
-  node_router3->using_as_guard = 1;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs == NULL);
-  node_router1->using_as_guard = 0;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
-  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
-  rs = NULL;
-  node_router2->using_as_guard = 0;
-  node_router3->using_as_guard = 0;
-
-  /* One not valid, one guard. This should leave one remaining */
-  node_router1->is_valid = 0;
-  node_router2->using_as_guard = 1;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
-  tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
-  rs = NULL;
-  node_router1->is_valid = 1;
-  node_router2->using_as_guard = 0;
-
   /* Manipulate overloaded */
 
   node_router2->rs->last_dir_503_at = now;
   node_router3->rs->last_dir_503_at = now;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
   node_router2->rs->last_dir_503_at = 0;
   node_router3->rs->last_dir_503_at = 0;
@@ -381,13 +367,13 @@ test_router_pick_directory_server_impl(void *arg)
   node_router2->rs->or_port = 443;
   node_router3->rs->or_port = 442;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
   node_router1->rs->or_port = 442;
   node_router2->rs->or_port = 443;
   node_router3->rs->or_port = 444;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
 
   /* Fascist firewall and overloaded */
@@ -396,7 +382,7 @@ test_router_pick_directory_server_impl(void *arg)
   node_router3->rs->or_port = 442;
   node_router3->rs->last_dir_503_at = now;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
   node_router3->rs->last_dir_503_at = 0;
 
@@ -414,12 +400,13 @@ test_router_pick_directory_server_impl(void *arg)
   node_router3->rs->dir_port = 81;
   node_router1->rs->last_dir_503_at = now;
   rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
+  tt_ptr_op(rs, OP_NE, NULL);
   tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
   node_router1->rs->last_dir_503_at = 0;
 
  done:
   UNMOCK(usable_consensus_flavor);
+
   if (router1_id)
     tor_free(router1_id);
   if (router2_id)
@@ -433,6 +420,112 @@ test_router_pick_directory_server_impl(void *arg)
   networkstatus_vote_free(con_md);
 }
 
+static or_state_t *dummy_state = NULL;
+static or_state_t *
+get_or_state_replacement(void)
+{
+  return dummy_state;
+}
+
+static void
+mock_directory_initiate_request(directory_request_t *req)
+{
+  (void)req;
+  return;
+}
+
+static circuit_guard_state_t *
+mock_circuit_guard_state_new(entry_guard_t *guard, unsigned state,
+                        entry_guard_restriction_t *rst)
+{
+  (void) guard;
+  (void) state;
+  (void) rst;
+  return NULL;
+}
+
+/** Test that we will use our directory guards to fetch mds even if we don't
+ *  have any dirinfo (tests bug #23862). */
+static void
+test_directory_guard_fetch_with_no_dirinfo(void *arg)
+{
+  int retval;
+  char *consensus_text_md = NULL;
+  or_options_t *options = get_options_mutable();
+  time_t now = time(NULL);
+
+  (void) arg;
+
+  hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
+
+  /* Initialize the SRV subsystem */
+  MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  sr_init(0);
+  UNMOCK(get_my_v3_authority_cert);
+
+  /* Initialize the entry node configuration from the ticket */
+  options->UseEntryGuards = 1;
+  options->StrictNodes = 1;
+  get_options_mutable()->EntryNodes = routerset_new();
+  routerset_parse(get_options_mutable()->EntryNodes,
+                  "2121212121212121212121212121212121212121", "foo");
+
+  /* Mock some functions */
+  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  MOCK(get_or_state, get_or_state_replacement);
+  MOCK(directory_initiate_request, mock_directory_initiate_request);
+  /* we need to mock this one to avoid memleaks */
+  MOCK(circuit_guard_state_new, mock_circuit_guard_state_new);
+
+  /* Call guards_update_all() to simulate loading our state file (see
+   * entry_guards_load_guards_from_state() and ticket #23989). */
+  guards_update_all();
+
+  /* Test logic: Simulate the arrival of a new consensus when we have no
+   * dirinfo at all. Tor will need to fetch the mds from the consensus. Make
+   * sure that Tor will use the specified entry guard instead of relying on the
+   * fallback directories. */
+
+  /* Fixup the dirconn that will deliver the consensus */
+  dir_connection_t *conn = dir_connection_new(AF_INET);
+  tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  conn->base_.port = 8800;
+  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
+  conn->base_.purpose = DIR_PURPOSE_FETCH_CONSENSUS;
+  conn->requested_resource = tor_strdup("ns");
+
+  /* Construct a consensus */
+  construct_consensus(&consensus_text_md, now);
+  tt_assert(consensus_text_md);
+
+  /* Place the consensus in the dirconn */
+  response_handler_args_t args;
+  memset(&args, 0, sizeof(response_handler_args_t));
+  args.status_code = 200;
+  args.body = consensus_text_md;
+  args.body_len = strlen(consensus_text_md);
+
+  /* Update approx time so that the consensus is considered live */
+  update_approx_time(now+1010);
+
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Now handle the consensus */
+  retval = handle_response_fetch_consensus(conn, &args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Make sure that our primary guard was chosen */
+  expect_log_msg_containing("Selected primary guard router3");
+
+ done:
+  tor_free(consensus_text_md);
+  tor_free(dummy_state);
+  connection_free_minimal(TO_CONN(conn));
+  entry_guards_free_all();
+  teardown_capture_of_logs();
+}
+
 static connection_t *mocked_connection = NULL;
 
 /* Mock connection_get_by_type_addr_port_purpose by returning
@@ -471,27 +564,27 @@ test_routerlist_router_is_already_dir_fetching(void *arg)
 
   /* Test that we never get 1 from a NULL connection */
   mocked_connection = NULL;
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 0) == 0);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 1) == 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 0), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 1), OP_EQ, 0);
   /* We always expect 0 in these cases */
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 0) == 0);
-  tt_assert(router_is_already_dir_fetching(NULL, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&null_addr_ap, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&zero_port_ap, 1, 1) == 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 0), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(NULL, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&null_addr_ap, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&zero_port_ap, 1, 1), OP_EQ, 0);
 
   /* Test that we get 1 with a connection in the appropriate circumstances */
   mocked_connection = connection_new(CONN_TYPE_DIR, AF_INET);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 1) == 1);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 1, 0) == 1);
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 1) == 1);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 1), OP_EQ, 1);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 1, 0), OP_EQ, 1);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 1), OP_EQ, 1);
 
   /* Test that we get 0 even with a connection in the appropriate
    * circumstances */
-  tt_assert(router_is_already_dir_fetching(&test_ap, 0, 0) == 0);
-  tt_assert(router_is_already_dir_fetching(NULL, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&null_addr_ap, 1, 1) == 0);
-  tt_assert(router_is_already_dir_fetching(&zero_port_ap, 1, 1) == 0);
+  tt_int_op(router_is_already_dir_fetching(&test_ap, 0, 0), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(NULL, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&null_addr_ap, 1, 1), OP_EQ, 0);
+  tt_int_op(router_is_already_dir_fetching(&zero_port_ap, 1, 1), OP_EQ, 0);
 
  done:
   /* If a connection is never set up, connection_free chokes on it. */
@@ -506,16 +599,181 @@ test_routerlist_router_is_already_dir_fetching(void *arg)
 #undef TEST_ADDR_STR
 #undef TEST_DIR_PORT
 
+static long mock_apparent_skew = 0;
+
+/** Store apparent_skew and assert that the other arguments are as
+ * expected. */
+static void
+mock_clock_skew_warning(const connection_t *conn, long apparent_skew,
+                        int trusted, log_domain_mask_t domain,
+                        const char *received, const char *source)
+{
+  (void)conn;
+  mock_apparent_skew = apparent_skew;
+  tt_int_op(trusted, OP_EQ, 1);
+  tt_int_op(domain, OP_EQ, LD_GENERAL);
+  tt_str_op(received, OP_EQ, "microdesc flavor consensus");
+  tt_str_op(source, OP_EQ, "CONSENSUS");
+ done:
+  ;
+}
+
+/** Do common setup for test_timely_consensus() and
+ * test_early_consensus().  Call networkstatus_set_current_consensus()
+ * on a constructed consensus and with an appropriately-modified
+ * approx_time.  Callers expect presence or absence of appropriate log
+ * messages and control events. */
+static int
+test_skew_common(void *arg, time_t now, unsigned long *offset)
+{
+  char *consensus = NULL;
+  int retval = 0;
+
+  *offset = strtoul(arg, NULL, 10);
+
+  /* Initialize the SRV subsystem */
+  MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  sr_init(0);
+  UNMOCK(get_my_v3_authority_cert);
+
+  construct_consensus(&consensus, now);
+  tt_assert(consensus);
+
+  update_approx_time(now + *offset);
+
+  mock_apparent_skew = 0;
+  /* Caller will call UNMOCK() */
+  MOCK(clock_skew_warning, mock_clock_skew_warning);
+  /* Caller will call teardown_capture_of_logs() */
+  setup_capture_of_logs(LOG_WARN);
+  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+                                               NULL);
+
+ done:
+  tor_free(consensus);
+  return retval;
+}
+
+/** Test non-early consensus */
+static void
+test_timely_consensus(void *arg)
+{
+  time_t now = time(NULL);
+  unsigned long offset = 0;
+  int retval = 0;
+
+  retval = test_skew_common(arg, now, &offset);
+  (void)offset;
+  expect_no_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(retval, OP_EQ, 0);
+  tt_int_op(mock_apparent_skew, OP_EQ, 0);
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(clock_skew_warning);
+}
+
+/** Test early consensus  */
+static void
+test_early_consensus(void *arg)
+{
+  time_t now = time(NULL);
+  unsigned long offset = 0;
+  int retval = 0;
+
+  retval = test_skew_common(arg, now, &offset);
+  /* Can't use expect_single_log_msg() because of unrecognized authorities */
+  expect_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(retval, OP_EQ, 0);
+  /* This depends on construct_consensus() setting valid_after=now+1000 */
+  tt_int_op(mock_apparent_skew, OP_EQ, offset - 1000);
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(clock_skew_warning);
+}
+
+/** Test warn_early_consensus(), expecting no warning  */
+static void
+test_warn_early_consensus_no(const networkstatus_t *c, time_t now,
+                             long offset)
+{
+  mock_apparent_skew = 0;
+  setup_capture_of_logs(LOG_WARN);
+  warn_early_consensus(c, "microdesc", now + offset);
+  expect_no_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(mock_apparent_skew, OP_EQ, 0);
+ done:
+  teardown_capture_of_logs();
+}
+
+/** Test warn_early_consensus(), expecting a warning */
+static void
+test_warn_early_consensus_yes(const networkstatus_t *c, time_t now,
+                              long offset)
+{
+  mock_apparent_skew = 0;
+  setup_capture_of_logs(LOG_WARN);
+  warn_early_consensus(c, "microdesc", now + offset);
+  /* Can't use expect_single_log_msg() because of unrecognized authorities */
+  expect_log_msg_containing("behind the time published in the consensus");
+  tt_int_op(mock_apparent_skew, OP_EQ, offset);
+ done:
+  teardown_capture_of_logs();
+}
+
+/**
+ * Test warn_early_consensus() directly, checking both the non-warning
+ * case (consensus is not early) and the warning case (consensus is
+ * early).  Depends on EARLY_CONSENSUS_NOTICE_SKEW=60.
+ */
+static void
+test_warn_early_consensus(void *arg)
+{
+  networkstatus_t *c = NULL;
+  time_t now = time(NULL);
+
+  (void)arg;
+  c = tor_malloc_zero(sizeof *c);
+  c->valid_after = now;
+  c->dist_seconds = 300;
+  mock_apparent_skew = 0;
+  MOCK(clock_skew_warning, mock_clock_skew_warning);
+  test_warn_early_consensus_no(c, now, 60);
+  test_warn_early_consensus_no(c, now, 0);
+  test_warn_early_consensus_no(c, now, -60);
+  test_warn_early_consensus_no(c, now, -360);
+  test_warn_early_consensus_yes(c, now, -361);
+  test_warn_early_consensus_yes(c, now, -600);
+  UNMOCK(clock_skew_warning);
+  tor_free(c);
+}
+
 #define NODE(name, flags) \
   { #name, test_routerlist_##name, (flags), NULL, NULL }
 #define ROUTER(name,flags) \
   { #name, test_router_##name, (flags), NULL, NULL }
 
+#define TIMELY(name, arg)                                     \
+  { name, test_timely_consensus, TT_FORK, &passthrough_setup, \
+    (char *)(arg) }
+#define EARLY(name, arg) \
+  { name, test_early_consensus, TT_FORK, &passthrough_setup, \
+    (char *)(arg) }
+
 struct testcase_t routerlist_tests[] = {
   NODE(initiate_descriptor_downloads, 0),
   NODE(launch_descriptor_downloads, 0),
   NODE(router_is_already_dir_fetching, TT_FORK),
   ROUTER(pick_directory_server_impl, TT_FORK),
+  { "directory_guard_fetch_with_no_dirinfo",
+    test_directory_guard_fetch_with_no_dirinfo, TT_FORK, NULL, NULL },
+  /* These depend on construct_consensus() setting
+   * valid_after=now+1000 and dist_seconds=250 */
+  TIMELY("timely_consensus1", "1010"),
+  TIMELY("timely_consensus2", "1000"),
+  TIMELY("timely_consensus3", "690"),
+  EARLY("early_consensus1", "689"),
+  { "warn_early_consensus", test_warn_early_consensus, 0, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index 1b526d430b..c541324674 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -1,3 +1,6 @@
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #define ROUTERSET_PRIVATE
 
 #include "or.h"
@@ -623,7 +626,7 @@ NS(test_main)(void *arg)
   (void)arg;
 
   tgt = routerset_new();
-  smartlist_add(src->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(src->list, "{xx}");
   routerset_union(tgt, src);
 
   tt_int_op(smartlist_len(tgt->list), OP_NE, 0);
@@ -696,7 +699,7 @@ NS(test_main)(void *arg)
 static void
 NS(test_main)(void *arg)
 {
-  const routerset_t *set;
+  routerset_t *set;
   int needs_geoip;
   (void)arg;
 
@@ -706,14 +709,14 @@ NS(test_main)(void *arg)
 
   set = routerset_new();
   needs_geoip = routerset_needs_geoip(set);
-  routerset_free((routerset_t *)set);
+  routerset_free(set);
   tt_int_op(needs_geoip, OP_EQ, 0);
   set = NULL;
 
   set = routerset_new();
   smartlist_add(set->country_names, tor_strndup("xx", 2));
   needs_geoip = routerset_needs_geoip(set);
-  routerset_free((routerset_t *)set);
+  routerset_free(set);
   set = NULL;
   tt_int_op(needs_geoip, OP_NE, 0);
 
@@ -745,7 +748,7 @@ NS(test_main)(void *arg)
   tt_int_op(is_empty, OP_NE, 0);
 
   set = routerset_new();
-  smartlist_add(set->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->list, "{xx}");
   is_empty = routerset_is_empty(set);
   routerset_free(set);
   set = NULL;
@@ -1602,7 +1605,7 @@ NS(test_main)(void *arg)
  */
 
 NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unused));
+    (const char *nickname, unsigned flags));
 static const char *NS(mock_nickname);
 
 static void
@@ -1616,7 +1619,7 @@ NS(test_main)(void *arg)
   NS_MOCK(node_get_by_nickname);
 
   NS(mock_nickname) = "foo";
-  smartlist_add(set->list, tor_strdup(NS(mock_nickname)));
+  smartlist_add_strdup(set->list, NS(mock_nickname));
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1632,11 +1635,11 @@ NS(test_main)(void *arg)
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
+NS(node_get_by_nickname)(const char *nickname, unsigned flags)
 {
   CALLED(node_get_by_nickname)++;
   tt_str_op(nickname, OP_EQ, NS(mock_nickname));
-  tt_int_op(warn_if_unused, OP_EQ, 1);
+  tt_uint_op(flags, OP_EQ, 0);
 
   done:
     return NULL;
@@ -1651,7 +1654,7 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
  */
 
 NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unused));
+    (const char *nickname, unsigned flags));
 static const char *NS(mock_nickname);
 static node_t NS(mock_node);
 
@@ -1667,7 +1670,7 @@ NS(test_main)(void *arg)
 
   NS(mock_node).is_running = 0;
   NS(mock_nickname) = "foo";
-  smartlist_add(set->list, tor_strdup(NS(mock_nickname)));
+  smartlist_add_strdup(set->list, NS(mock_nickname));
 
   routerset_get_all_nodes(out, set, NULL, 1);
   out_len = smartlist_len(out);
@@ -1683,11 +1686,11 @@ NS(test_main)(void *arg)
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
+NS(node_get_by_nickname)(const char *nickname, unsigned flags)
 {
   CALLED(node_get_by_nickname)++;
   tt_str_op(nickname, OP_EQ, NS(mock_nickname));
-  tt_int_op(warn_if_unused, OP_EQ, 1);
+  tt_int_op(flags, OP_EQ, 0);
 
   done:
     return &NS(mock_node);
@@ -1701,7 +1704,7 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
  */
 
 NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, int warn_if_unused));
+    (const char *nickname, unsigned flags));
 static char *NS(mock_nickname);
 static node_t NS(mock_node);
 
@@ -1735,11 +1738,11 @@ NS(test_main)(void *arg)
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, int warn_if_unused)
+NS(node_get_by_nickname)(const char *nickname, unsigned flags)
 {
   CALLED(node_get_by_nickname)++;
   tt_str_op(nickname, OP_EQ, NS(mock_nickname));
-  tt_int_op(warn_if_unused, OP_EQ, 1);
+  tt_int_op(flags, OP_EQ, 0);
 
   done:
     return &NS(mock_node);
@@ -1766,7 +1769,7 @@ NS(test_main)(void *arg)
 
   NS_MOCK(nodelist_get_list);
 
-  smartlist_add(set->country_names, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->country_names, "{xx}");
   NS(mock_smartlist) = smartlist_new();
 
   routerset_get_all_nodes(out, set, NULL, 1);
@@ -1813,7 +1816,7 @@ NS(test_main)(void *arg)
 
   NS_MOCK(nodelist_get_list);
 
-  smartlist_add(set->country_names, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->country_names, "{xx}");
   NS(mock_smartlist) = smartlist_new();
   NS(mock_node).is_running = 0;
   smartlist_add(NS(mock_smartlist), (void *)&NS(mock_node));
@@ -1944,7 +1947,7 @@ NS(test_main)(void *arg)
 
  done:
   tor_free(s);
-  routerset_free((routerset_t *)set);
+  routerset_free(set);
 }
 
 #undef NS_SUBMODULE
@@ -1985,7 +1988,7 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(b->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(b->list, "{xx}");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2010,9 +2013,9 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("{aa}"));
-  smartlist_add(b->list, tor_strdup("{b1}"));
-  smartlist_add(b->list, tor_strdup("{b2}"));
+  smartlist_add_strdup(a->list, "{aa}");
+  smartlist_add_strdup(b->list, "{b1}");
+  smartlist_add_strdup(b->list, "{b2}");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2037,8 +2040,8 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("foo"));
-  smartlist_add(b->list, tor_strdup("bar"));
+  smartlist_add_strdup(a->list, "foo");
+  smartlist_add_strdup(b->list, "bar");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2063,8 +2066,8 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("foo"));
-  smartlist_add(b->list, tor_strdup("foo"));
+  smartlist_add_strdup(a->list, "foo");
+  smartlist_add_strdup(b->list, "foo");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2081,28 +2084,28 @@ NS(test_main)(void *arg)
  * Structural test for routerset_free, where the routerset is NULL.
  */
 
-NS_DECL(void, smartlist_free, (smartlist_t *sl));
+NS_DECL(void, smartlist_free_, (smartlist_t *sl));
 
 static void
 NS(test_main)(void *arg)
 {
   (void)arg;
 
-  NS_MOCK(smartlist_free);
+  NS_MOCK(smartlist_free_);
 
-  routerset_free(NULL);
+  routerset_free_(NULL);
 
-  tt_int_op(CALLED(smartlist_free), OP_EQ, 0);
+  tt_int_op(CALLED(smartlist_free_), OP_EQ, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free)(smartlist_t *s)
+NS(smartlist_free_)(smartlist_t *s)
 {
   (void)s;
-  CALLED(smartlist_free)++;
+  CALLED(smartlist_free_)++;
 }
 
 #undef NS_SUBMODULE
@@ -2112,9 +2115,9 @@ NS(smartlist_free)(smartlist_t *s)
  * Structural test for routerset_free.
  */
 
-NS_DECL(void, smartlist_free, (smartlist_t *sl));
-NS_DECL(void, strmap_free,(strmap_t *map, void (*free_val)(void*)));
-NS_DECL(void, digestmap_free, (digestmap_t *map, void (*free_val)(void*)));
+NS_DECL(void, smartlist_free_, (smartlist_t *sl));
+NS_DECL(void, strmap_free_,(strmap_t *map, void (*free_val)(void*)));
+NS_DECL(void, digestmap_free_, (digestmap_t *map, void (*free_val)(void*)));
 
 static void
 NS(test_main)(void *arg)
@@ -2122,39 +2125,39 @@ NS(test_main)(void *arg)
   routerset_t *routerset = routerset_new();
   (void)arg;
 
-  NS_MOCK(smartlist_free);
-  NS_MOCK(strmap_free);
-  NS_MOCK(digestmap_free);
+  NS_MOCK(smartlist_free_);
+  NS_MOCK(strmap_free_);
+  NS_MOCK(digestmap_free_);
 
   routerset_free(routerset);
 
-  tt_int_op(CALLED(smartlist_free), OP_NE, 0);
-  tt_int_op(CALLED(strmap_free), OP_NE, 0);
-  tt_int_op(CALLED(digestmap_free), OP_NE, 0);
+  tt_int_op(CALLED(smartlist_free_), OP_NE, 0);
+  tt_int_op(CALLED(strmap_free_), OP_NE, 0);
+  tt_int_op(CALLED(digestmap_free_), OP_NE, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free)(smartlist_t *s)
+NS(smartlist_free_)(smartlist_t *s)
 {
-  CALLED(smartlist_free)++;
-  smartlist_free__real(s);
+  CALLED(smartlist_free_)++;
+  smartlist_free___real(s);
 }
 
 void
-NS(strmap_free)(strmap_t *map, void (*free_val)(void*))
+NS(strmap_free_)(strmap_t *map, void (*free_val)(void*))
 {
-  CALLED(strmap_free)++;
-  strmap_free__real(map, free_val);
+  CALLED(strmap_free_)++;
+  strmap_free___real(map, free_val);
 }
 
 void
-NS(digestmap_free)(digestmap_t *map, void (*free_val)(void*))
+NS(digestmap_free_)(digestmap_t *map, void (*free_val)(void*))
 {
-  CALLED(digestmap_free)++;
-  digestmap_free__real(map, free_val);
+  CALLED(digestmap_free_)++;
+  digestmap_free___real(map, free_val);
 }
 
 #undef NS_SUBMODULE
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
new file mode 100755
index 0000000000..5405af436b
--- /dev/null
+++ b/src/test/test_rust.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+# Test all Rust crates
+
+set -e
+
+export LSAN_OPTIONS=suppressions=${abs_top_srcdir:-../../..}/src/test/rust_supp.txt
+
+for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
+    if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
+	cd "${abs_top_builddir:-../../..}/src/rust" && \
+	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
+	    "${CARGO:-cargo}" test --all-features ${CARGO_ONLINE-"--frozen"} \
+	    ${EXTRA_CARGO_OPTIONS} \
+	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
+    fi
+done
+
+exit $exitcode
diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c
index 05ea8e86e8..841fc69456 100644
--- a/src/test/test_scheduler.c
+++ b/src/test/test_scheduler.c
@@ -1,16 +1,20 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #include <math.h>
-#include <event2/event.h>
 
+#define SCHEDULER_KIST_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define CHANNEL_PRIVATE_
 #include "or.h"
+#include "config.h"
 #include "compat_libevent.h"
 #include "channel.h"
+#include "channeltls.h"
+#include "connection.h"
+#include "networkstatus.h"
 #define SCHEDULER_PRIVATE_
 #include "scheduler.h"
 
@@ -18,99 +22,156 @@
 #include "test.h"
 #include "fakechans.h"
 
-/* Event base for scheduelr tests */
-static struct event_base *mock_event_base = NULL;
-
-/* Statics controlling mocks */
-static circuitmux_t *mock_ccm_tgt_1 = NULL;
-static circuitmux_t *mock_ccm_tgt_2 = NULL;
+/* Shamelessly stolen from compat_libevent.c */
+#define V(major, minor, patch) \
+  (((major) << 24) | ((minor) << 16) | ((patch) << 8))
 
-static circuitmux_t *mock_cgp_tgt_1 = NULL;
-static circuitmux_policy_t *mock_cgp_val_1 = NULL;
-static circuitmux_t *mock_cgp_tgt_2 = NULL;
-static circuitmux_policy_t *mock_cgp_val_2 = NULL;
+/******************************************************************************
+ * Statistical info
+ *****************************************************************************/
 static int scheduler_compare_channels_mock_ctr = 0;
 static int scheduler_run_mock_ctr = 0;
 
-static void channel_flush_some_cells_mock_free_all(void);
-static void channel_flush_some_cells_mock_set(channel_t *chan,
-                                              ssize_t num_cells);
-
-/* Setup for mock event stuff */
-static void mock_event_free_all(void);
-static void mock_event_init(void);
-
-/* Mocks used by scheduler tests */
-static ssize_t channel_flush_some_cells_mock(channel_t *chan,
-                                             ssize_t num_cells);
-static int circuitmux_compare_muxes_mock(circuitmux_t *cmux_1,
-                                         circuitmux_t *cmux_2);
-static const circuitmux_policy_t * circuitmux_get_policy_mock(
-    circuitmux_t *cmux);
-static int scheduler_compare_channels_mock(const void *c1_v,
-                                           const void *c2_v);
-static void scheduler_run_noop_mock(void);
-static struct event_base * tor_libevent_get_base_mock(void);
-
-/* Scheduler test cases */
-static void test_scheduler_channel_states(void *arg);
-static void test_scheduler_compare_channels(void *arg);
-static void test_scheduler_initfree(void *arg);
-static void test_scheduler_loop(void *arg);
-static void test_scheduler_queue_heuristic(void *arg);
-
-/* Mock event init/free */
+/******************************************************************************
+ * Utility functions and things we need to mock
+ *****************************************************************************/
+static or_options_t mocked_options;
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mocked_options;
+}
 
-/* Shamelessly stolen from compat_libevent.c */
-#define V(major, minor, patch) \
-  (((major) << 24) | ((minor) << 16) | ((patch) << 8))
+static void
+cleanup_scheduler_options(void)
+{
+  if (mocked_options.SchedulerTypes_) {
+    SMARTLIST_FOREACH(mocked_options.SchedulerTypes_, int *, i, tor_free(i));
+    smartlist_free(mocked_options.SchedulerTypes_);
+    mocked_options.SchedulerTypes_ = NULL;
+  }
+}
 
 static void
-mock_event_free_all(void)
+set_scheduler_options(int val)
 {
-  tt_assert(mock_event_base != NULL);
+  int *type;
 
-  if (mock_event_base) {
-    event_base_free(mock_event_base);
-    mock_event_base = NULL;
+  if (mocked_options.SchedulerTypes_ == NULL) {
+    mocked_options.SchedulerTypes_ = smartlist_new();
   }
+  type = tor_malloc_zero(sizeof(int));
+  *type = val;
+  smartlist_add(mocked_options.SchedulerTypes_, type);
+}
+
+static void
+clear_options(void)
+{
+  cleanup_scheduler_options();
+  memset(&mocked_options, 0, sizeof(mocked_options));
+}
 
-  tt_ptr_op(mock_event_base, ==, NULL);
+static int32_t
+mock_vanilla_networkstatus_get_param(
+    const networkstatus_t *ns, const char *param_name, int32_t default_val,
+    int32_t min_val, int32_t max_val)
+{
+  (void)ns;
+  (void)default_val;
+  (void)min_val;
+  (void)max_val;
+  // only support KISTSchedRunInterval right now
+  tor_assert(strcmp(param_name, "KISTSchedRunInterval")==0);
+  return 0;
+}
 
- done:
-  return;
+static int32_t
+mock_kist_networkstatus_get_param(
+    const networkstatus_t *ns, const char *param_name, int32_t default_val,
+    int32_t min_val, int32_t max_val)
+{
+  (void)ns;
+  (void)default_val;
+  (void)min_val;
+  (void)max_val;
+  // only support KISTSchedRunInterval right now
+  tor_assert(strcmp(param_name, "KISTSchedRunInterval")==0);
+  return 12;
+}
+
+static int
+scheduler_compare_channels_mock(const void *c1_v,
+                                const void *c2_v)
+{
+  uintptr_t p1, p2;
+
+  p1 = (uintptr_t)(c1_v);
+  p2 = (uintptr_t)(c2_v);
+
+  ++scheduler_compare_channels_mock_ctr;
+
+  if (p1 == p2) return 0;
+  else if (p1 < p2) return 1;
+  else return -1;
 }
 
 static void
-mock_event_init(void)
+scheduler_run_noop_mock(void)
 {
-  struct event_config *cfg = NULL;
+  ++scheduler_run_mock_ctr;
+}
 
-  tt_ptr_op(mock_event_base, ==, NULL);
+static circuitmux_t *mock_ccm_tgt_1 = NULL;
+static circuitmux_t *mock_ccm_tgt_2 = NULL;
+static circuitmux_t *mock_cgp_tgt_1 = NULL;
+static circuitmux_policy_t *mock_cgp_val_1 = NULL;
+static circuitmux_t *mock_cgp_tgt_2 = NULL;
+static circuitmux_policy_t *mock_cgp_val_2 = NULL;
 
-  /*
-   * Really cut down from tor_libevent_initialize of
-   * src/common/compat_libevent.c to kill config dependencies
-   */
+static const circuitmux_policy_t *
+circuitmux_get_policy_mock(circuitmux_t *cmux)
+{
+  const circuitmux_policy_t *result = NULL;
 
-  if (!mock_event_base) {
-    cfg = event_config_new();
-#if LIBEVENT_VERSION_NUMBER >= V(2,0,9)
-    /* We can enable changelist support with epoll, since we don't give
-     * Libevent any dup'd fds.  This lets us avoid some syscalls. */
-    event_config_set_flag(cfg, EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST);
-#endif
-    mock_event_base = event_base_new_with_config(cfg);
-    event_config_free(cfg);
+  tt_assert(cmux != NULL);
+  if (cmux) {
+    if (cmux == mock_cgp_tgt_1) result = mock_cgp_val_1;
+    else if (cmux == mock_cgp_tgt_2) result = mock_cgp_val_2;
+    else result = circuitmux_get_policy__real(cmux);
   }
 
-  tt_assert(mock_event_base != NULL);
-
  done:
-  return;
+  return result;
 }
 
-/* Mocks */
+static int
+circuitmux_compare_muxes_mock(circuitmux_t *cmux_1,
+                              circuitmux_t *cmux_2)
+{
+  int result = 0;
+
+  tt_assert(cmux_1 != NULL);
+  tt_assert(cmux_2 != NULL);
+
+  if (cmux_1 != cmux_2) {
+    if (cmux_1 == mock_ccm_tgt_1 && cmux_2 == mock_ccm_tgt_2) result = -1;
+    else if (cmux_1 == mock_ccm_tgt_2 && cmux_2 == mock_ccm_tgt_1) {
+      result = 1;
+    } else {
+      if (cmux_1 == mock_ccm_tgt_1 || cmux_1 == mock_ccm_tgt_2) result = -1;
+      else if (cmux_2 == mock_ccm_tgt_1 || cmux_2 == mock_ccm_tgt_2) {
+        result = 1;
+      } else {
+        result = circuitmux_compare_muxes__real(cmux_1, cmux_2);
+      }
+    }
+  }
+  /* else result = 0 always */
+
+ done:
+  return result;
+}
 
 typedef struct {
   const channel_t *chan;
@@ -174,6 +235,63 @@ channel_flush_some_cells_mock_set(channel_t *chan, ssize_t num_cells)
   }
 }
 
+static int
+channel_more_to_flush_mock(channel_t *chan)
+{
+  tor_assert(chan);
+
+  flush_mock_channel_t *found_mock_ch = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(chans_for_flush_mock,
+                          flush_mock_channel_t *,
+                          flush_mock_ch) {
+    if (flush_mock_ch != NULL && flush_mock_ch->chan != NULL) {
+      if (flush_mock_ch->chan == chan) {
+        /* Found it */
+        found_mock_ch = flush_mock_ch;
+        break;
+      }
+    } else {
+      /* That shouldn't be there... */
+      SMARTLIST_DEL_CURRENT(chans_for_flush_mock, flush_mock_ch);
+      tor_free(flush_mock_ch);
+    }
+  } SMARTLIST_FOREACH_END(flush_mock_ch);
+
+  tor_assert(found_mock_ch);
+
+  /* Check if any circuits would like to queue some */
+  /* special for the mock: return the number of cells (instead of 1), or zero
+   * if nothing to flush */
+  return (found_mock_ch->cells > 0 ? (int)found_mock_ch->cells : 0 );
+}
+
+static void
+channel_write_to_kernel_mock(channel_t *chan)
+{
+  (void)chan;
+  //log_debug(LD_SCHED, "chan=%d writing to kernel",
+  //    (int)chan->global_identifier);
+}
+
+static int
+channel_should_write_to_kernel_mock(outbuf_table_t *ot, channel_t *chan)
+{
+  (void)ot;
+  (void)chan;
+  return 1;
+  /* We could make this more complicated if we wanted. But I don't think doing
+   * so tests much of anything */
+  //static int called_counter = 0;
+  //if (++called_counter >= 3) {
+  //  called_counter -= 3;
+  //  log_debug(LD_SCHED, "chan=%d should write to kernel",
+  //      (int)chan->global_identifier);
+  //  return 1;
+  //}
+  //return 0;
+}
+
 static ssize_t
 channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
 {
@@ -181,7 +299,7 @@ channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
   char unlimited = 0;
   flush_mock_channel_t *found = NULL;
 
-  tt_assert(chan != NULL);
+  tt_ptr_op(chan, OP_NE, NULL);
   if (chan) {
     if (num_cells < 0) {
       num_cells = 0;
@@ -215,11 +333,6 @@ channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
 
         flushed += max;
         found->cells -= max;
-
-        if (found->cells <= 0) {
-          smartlist_remove(chans_for_flush_mock, found);
-          tor_free(found);
-        }
       }
     }
   }
@@ -228,92 +341,26 @@ channel_flush_some_cells_mock(channel_t *chan, ssize_t num_cells)
   return flushed;
 }
 
-static int
-circuitmux_compare_muxes_mock(circuitmux_t *cmux_1,
-                              circuitmux_t *cmux_2)
-{
-  int result = 0;
-
-  tt_assert(cmux_1 != NULL);
-  tt_assert(cmux_2 != NULL);
-
-  if (cmux_1 != cmux_2) {
-    if (cmux_1 == mock_ccm_tgt_1 && cmux_2 == mock_ccm_tgt_2) result = -1;
-    else if (cmux_1 == mock_ccm_tgt_2 && cmux_2 == mock_ccm_tgt_1) {
-      result = 1;
-    } else {
-      if (cmux_1 == mock_ccm_tgt_1 || cmux_1 == mock_ccm_tgt_2) result = -1;
-      else if (cmux_2 == mock_ccm_tgt_1 || cmux_2 == mock_ccm_tgt_2) {
-        result = 1;
-      } else {
-        result = circuitmux_compare_muxes__real(cmux_1, cmux_2);
-      }
-    }
-  }
-  /* else result = 0 always */
-
- done:
-  return result;
-}
-
-static const circuitmux_policy_t *
-circuitmux_get_policy_mock(circuitmux_t *cmux)
-{
-  const circuitmux_policy_t *result = NULL;
-
-  tt_assert(cmux != NULL);
-  if (cmux) {
-    if (cmux == mock_cgp_tgt_1) result = mock_cgp_val_1;
-    else if (cmux == mock_cgp_tgt_2) result = mock_cgp_val_2;
-    else result = circuitmux_get_policy__real(cmux);
-  }
-
- done:
-  return result;
-}
-
-static int
-scheduler_compare_channels_mock(const void *c1_v,
-                                const void *c2_v)
-{
-  uintptr_t p1, p2;
-
-  p1 = (uintptr_t)(c1_v);
-  p2 = (uintptr_t)(c2_v);
-
-  ++scheduler_compare_channels_mock_ctr;
-
-  if (p1 == p2) return 0;
-  else if (p1 < p2) return 1;
-  else return -1;
-}
-
 static void
-scheduler_run_noop_mock(void)
+update_socket_info_impl_mock(socket_table_ent_t *ent)
 {
-  ++scheduler_run_mock_ctr;
+  ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0;
+  ent->limit = INT_MAX;
 }
 
-static struct event_base *
-tor_libevent_get_base_mock(void)
-{
-  return mock_event_base;
-}
-
-/* Test cases */
-
 static void
-test_scheduler_channel_states(void *arg)
+perform_channel_state_tests(int KISTSchedRunInterval, int sched_type)
 {
   channel_t *ch1 = NULL, *ch2 = NULL;
   int old_count;
 
-  (void)arg;
-
-  /* Set up libevent and scheduler */
+  /* setup options so we're sure about what sched we are running */
+  MOCK(get_options, mock_get_options);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = KISTSchedRunInterval;
+  set_scheduler_options(sched_type);
 
-  mock_event_init();
-  MOCK(tor_libevent_get_base, tor_libevent_get_base_mock);
+  /* Set up scheduler */
   scheduler_init();
   /*
    * Install the compare channels mock so we can test
@@ -324,9 +371,9 @@ test_scheduler_channel_states(void *arg)
    * Disable scheduler_run so we can just check the state transitions
    * without having to make everything it might call work too.
    */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  ((scheduler_t *) the_scheduler)->run = scheduler_run_noop_mock;
 
-  tt_int_op(smartlist_len(channels_pending), ==, 0);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 0);
 
   /* Set up a fake channel */
   ch1 = new_fake_channel();
@@ -334,89 +381,97 @@ test_scheduler_channel_states(void *arg)
 
   /* Start it off in OPENING */
   ch1->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch1->cmux = circuitmux_alloc();
   /* Try to register it */
   channel_register(ch1);
   tt_assert(ch1->registered);
 
   /* It should start off in SCHED_CHAN_IDLE */
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* Now get another one */
   ch2 = new_fake_channel();
   tt_assert(ch2);
   ch2->state = CHANNEL_STATE_OPENING;
-  ch2->cmux = circuitmux_alloc();
   channel_register(ch2);
   tt_assert(ch2->registered);
 
-  /* Send it to SCHED_CHAN_WAITING_TO_WRITE */
+  /* Send ch1 to SCHED_CHAN_WAITING_TO_WRITE */
   scheduler_channel_has_waiting_cells(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_WAITING_TO_WRITE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
 
   /* This should send it to SCHED_CHAN_PENDING */
   scheduler_channel_wants_writes(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 1);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
 
   /* Now send ch2 to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* Drop ch2 back to idle */
   scheduler_channel_doesnt_want_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* ...and back to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* ...and this should kick ch2 into SCHED_CHAN_PENDING */
   scheduler_channel_has_waiting_cells(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /* This should send ch2 to SCHED_CHAN_WAITING_TO_WRITE */
   scheduler_channel_doesnt_want_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_TO_WRITE);
-  tt_int_op(smartlist_len(channels_pending), ==, 1);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
 
   /* ...and back to SCHED_CHAN_PENDING */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /* Now we exercise scheduler_touch_channel */
   old_count = scheduler_compare_channels_mock_ctr;
   scheduler_touch_channel(ch1);
   tt_assert(scheduler_compare_channels_mock_ctr > old_count);
 
+  /* Release the ch2 and then do it another time to make sure it doesn't blow
+   * up and we are still in a quiescent state. */
+  scheduler_release_channel(ch2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
+  /* Cheat a bit so make the release more confused but also will tells us if
+   * the release did put the channel in the right state. */
+  ch2->scheduler_state = SCHED_CHAN_PENDING;
+  scheduler_release_channel(ch2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
+
   /* Close */
   channel_mark_for_close(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_mark_for_close(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_closed(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch1 = NULL;
   channel_closed(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch2 = NULL;
 
   /* Shut things down */
 
   channel_free_all();
   scheduler_free_all();
-  mock_event_free_all();
 
  done:
   tor_free(ch1);
   tor_free(ch2);
 
   UNMOCK(scheduler_compare_channels);
-  UNMOCK(scheduler_run);
-  UNMOCK(tor_libevent_get_base);
+  UNMOCK(get_options);
+  cleanup_scheduler_options();
 
   return;
 }
@@ -464,21 +519,21 @@ test_scheduler_compare_channels(void *arg)
 
   /* Equal-channel case */
   result = scheduler_compare_channels(&c1, &c1);
-  tt_int_op(result, ==, 0);
+  tt_int_op(result, OP_EQ, 0);
 
   /* Distinct channels, distinct policies */
   result = scheduler_compare_channels(&c1, &c2);
-  tt_int_op(result, ==, -1);
+  tt_int_op(result, OP_EQ, -1);
   result = scheduler_compare_channels(&c2, &c1);
-  tt_int_op(result, ==, 1);
+  tt_int_op(result, OP_EQ, 1);
 
   /* Distinct channels, same policy */
   tor_free(mock_cgp_val_2);
   mock_cgp_val_2 = mock_cgp_val_1;
   result = scheduler_compare_channels(&c1, &c2);
-  tt_int_op(result, ==, -1);
+  tt_int_op(result, OP_EQ, -1);
   result = scheduler_compare_channels(&c2, &c1);
-  tt_int_op(result, ==, 1);
+  tt_int_op(result, OP_EQ, 1);
 
  done:
 
@@ -502,45 +557,24 @@ test_scheduler_compare_channels(void *arg)
   return;
 }
 
-static void
-test_scheduler_initfree(void *arg)
-{
-  (void)arg;
-
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
-
-  mock_event_init();
-  MOCK(tor_libevent_get_base, tor_libevent_get_base_mock);
-
-  scheduler_init();
-
-  tt_assert(channels_pending != NULL);
-  tt_assert(run_sched_ev != NULL);
-
-  scheduler_free_all();
-
-  UNMOCK(tor_libevent_get_base);
-  mock_event_free_all();
-
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
-
- done:
-  return;
-}
+/******************************************************************************
+ * The actual tests!
+ *****************************************************************************/
 
 static void
-test_scheduler_loop(void *arg)
+test_scheduler_loop_vanilla(void *arg)
 {
-  channel_t *ch1 = NULL, *ch2 = NULL;
-
   (void)arg;
+  channel_t *ch1 = NULL, *ch2 = NULL;
+  void (*run_func_ptr)(void);
 
-  /* Set up libevent and scheduler */
+  /* setup options so we're sure about what sched we are running */
+  MOCK(get_options, mock_get_options);
+  clear_options();
+  set_scheduler_options(SCHEDULER_VANILLA);
+  mocked_options.KISTSchedRunInterval = 0;
 
-  mock_event_init();
-  MOCK(tor_libevent_get_base, tor_libevent_get_base_mock);
+  /* Set up scheduler */
   scheduler_init();
   /*
    * Install the compare channels mock so we can test
@@ -551,32 +585,32 @@ test_scheduler_loop(void *arg)
    * Disable scheduler_run so we can just check the state transitions
    * without having to make everything it might call work too.
    */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  run_func_ptr = the_scheduler->run;
+  ((scheduler_t *) the_scheduler)->run = scheduler_run_noop_mock;
 
-  tt_int_op(smartlist_len(channels_pending), ==, 0);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 0);
 
   /* Set up a fake channel */
   ch1 = new_fake_channel();
+  ch1->magic = TLS_CHAN_MAGIC;
   tt_assert(ch1);
 
   /* Start it off in OPENING */
   ch1->state = CHANNEL_STATE_OPENING;
-  /* We'll need a cmux */
-  ch1->cmux = circuitmux_alloc();
   /* Try to register it */
   channel_register(ch1);
   tt_assert(ch1->registered);
   /* Finish opening it */
-  channel_change_state(ch1, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch1);
 
   /* It should start off in SCHED_CHAN_IDLE */
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* Now get another one */
   ch2 = new_fake_channel();
+  ch2->magic = TLS_CHAN_MAGIC;
   tt_assert(ch2);
   ch2->state = CHANNEL_STATE_OPENING;
-  ch2->cmux = circuitmux_alloc();
   channel_register(ch2);
   tt_assert(ch2->registered);
   /*
@@ -584,68 +618,62 @@ test_scheduler_loop(void *arg)
    * zero and we'll get coverage of that exception case in scheduler_run()
    */
 
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_OPENING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_OPENING);
 
   /* Send it to SCHED_CHAN_WAITING_TO_WRITE */
   scheduler_channel_has_waiting_cells(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_WAITING_TO_WRITE);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
 
   /* This should send it to SCHED_CHAN_PENDING */
   scheduler_channel_wants_writes(ch1);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 1);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 1);
 
   /* Now send ch2 to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* Drop ch2 back to idle */
   scheduler_channel_doesnt_want_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_IDLE);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
 
   /* ...and back to SCHED_CHAN_WAITING_FOR_CELLS */
   scheduler_channel_wants_writes(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* ...and this should kick ch2 into SCHED_CHAN_PENDING */
   scheduler_channel_has_waiting_cells(ch2);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /*
    * Now we've got two pending channels and need to fire off
-   * scheduler_run(); first, unmock it.
+   * the scheduler run() that we kept.
    */
-
-  UNMOCK(scheduler_run);
-
-  scheduler_run();
-
-  /* Now re-mock it */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  run_func_ptr();
 
   /*
    * Assert that they're still in the states we left and aren't still
    * pending
    */
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_OPENING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_OPENING);
   tt_assert(ch1->scheduler_state != SCHED_CHAN_PENDING);
   tt_assert(ch2->scheduler_state != SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 0);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 0);
 
   /* Now, finish opening ch2, and get both back to pending */
-  channel_change_state(ch2, CHANNEL_STATE_OPEN);
+  channel_change_state_open(ch2);
   scheduler_channel_wants_writes(ch1);
   scheduler_channel_wants_writes(ch2);
   scheduler_channel_has_waiting_cells(ch1);
   scheduler_channel_has_waiting_cells(ch2);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_OPEN);
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_PENDING);
-  tt_int_op(smartlist_len(channels_pending), ==, 2);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_OPEN);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(channels_pending), OP_EQ, 2);
 
   /* Now, set up the channel_flush_some_cells() mock */
   MOCK(channel_flush_some_cells, channel_flush_some_cells_mock);
@@ -661,98 +689,593 @@ test_scheduler_loop(void *arg)
   channel_flush_some_cells_mock_set(ch2, 48);
 
   /*
-   * And re-run the scheduler_run() loop with non-zero returns from
+   * And re-run the scheduler run() loop with non-zero returns from
    * channel_flush_some_cells() this time.
    */
-  UNMOCK(scheduler_run);
-
-  scheduler_run();
-
-  /* Now re-mock it */
-  MOCK(scheduler_run, scheduler_run_noop_mock);
+  run_func_ptr();
 
   /*
    * ch1 should have gone to SCHED_CHAN_WAITING_FOR_CELLS, with 16 flushed
    * and 32 writeable.
    */
-  tt_int_op(ch1->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
   /*
    * ...ch2 should also have gone to SCHED_CHAN_WAITING_FOR_CELLS, with
    * channel_more_to_flush() returning false and channel_num_cells_writeable()
    * > 0/
    */
-  tt_int_op(ch2->scheduler_state, ==, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(ch2->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
 
   /* Close */
   channel_mark_for_close(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_mark_for_close(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSING);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSING);
   channel_closed(ch1);
-  tt_int_op(ch1->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch1->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch1 = NULL;
   channel_closed(ch2);
-  tt_int_op(ch2->state, ==, CHANNEL_STATE_CLOSED);
+  tt_int_op(ch2->state, OP_EQ, CHANNEL_STATE_CLOSED);
   ch2 = NULL;
 
   /* Shut things down */
   channel_flush_some_cells_mock_free_all();
   channel_free_all();
   scheduler_free_all();
-  mock_event_free_all();
 
  done:
   tor_free(ch1);
   tor_free(ch2);
+  cleanup_scheduler_options();
 
   UNMOCK(channel_flush_some_cells);
   UNMOCK(scheduler_compare_channels);
-  UNMOCK(scheduler_run);
-  UNMOCK(tor_libevent_get_base);
+  UNMOCK(get_options);
 }
 
 static void
-test_scheduler_queue_heuristic(void *arg)
+test_scheduler_loop_kist(void *arg)
 {
-  time_t now = approx_time();
-  uint64_t qh;
+  (void) arg;
+
+#ifndef HAVE_KIST_SUPPORT
+  return;
+#endif
+
+  channel_t *ch1 = new_fake_channel(), *ch2 = new_fake_channel();
+  channel_t *ch3 = new_fake_channel();
+
+  /* setup options so we're sure about what sched we are running */
+  MOCK(get_options, mock_get_options);
+  MOCK(channel_flush_some_cells, channel_flush_some_cells_mock);
+  MOCK(channel_more_to_flush, channel_more_to_flush_mock);
+  MOCK(channel_write_to_kernel, channel_write_to_kernel_mock);
+  MOCK(channel_should_write_to_kernel, channel_should_write_to_kernel_mock);
+  MOCK(update_socket_info_impl, update_socket_info_impl_mock);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 11;
+  set_scheduler_options(SCHEDULER_KIST);
+  scheduler_init();
 
+  tt_assert(ch1);
+  ch1->magic = TLS_CHAN_MAGIC;
+  ch1->state = CHANNEL_STATE_OPENING;
+  channel_register(ch1);
+  tt_assert(ch1->registered);
+  channel_change_state_open(ch1);
+  scheduler_channel_has_waiting_cells(ch1);
+  scheduler_channel_wants_writes(ch1);
+  channel_flush_some_cells_mock_set(ch1, 5);
+
+  tt_assert(ch2);
+  ch2->magic = TLS_CHAN_MAGIC;
+  ch2->state = CHANNEL_STATE_OPENING;
+  channel_register(ch2);
+  tt_assert(ch2->registered);
+  channel_change_state_open(ch2);
+  scheduler_channel_has_waiting_cells(ch2);
+  scheduler_channel_wants_writes(ch2);
+  channel_flush_some_cells_mock_set(ch2, 5);
+
+  the_scheduler->run();
+
+  scheduler_channel_has_waiting_cells(ch1);
+  channel_flush_some_cells_mock_set(ch1, 5);
+
+  the_scheduler->run();
+
+  scheduler_channel_has_waiting_cells(ch1);
+  channel_flush_some_cells_mock_set(ch1, 5);
+  scheduler_channel_has_waiting_cells(ch2);
+  channel_flush_some_cells_mock_set(ch2, 5);
+
+  the_scheduler->run();
+
+  channel_flush_some_cells_mock_free_all();
+
+  /* We'll try to run this closed channel threw the scheduler loop and make
+   * sure it ends up in the right state. */
+  tt_assert(ch3);
+  ch3->magic = TLS_CHAN_MAGIC;
+  ch3->state = CHANNEL_STATE_OPEN;
+  circuitmux_free(ch3->cmux);
+  ch3->cmux = circuitmux_alloc();
+  channel_register(ch3);
+  tt_assert(ch3->registered);
+
+  ch3->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS;
+  scheduler_channel_has_waiting_cells(ch3);
+  /* Should be in the pending list now waiting to be handled. */
+  tt_int_op(ch3->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  /* By running the scheduler on a closed channel, it should end up in the
+   * IDLE state and not in the pending channel list. */
+  ch3->state = CHANNEL_STATE_CLOSED;
+  the_scheduler->run();
+  tt_int_op(ch3->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+
+ done:
+  /* Prep the channel so the free() function doesn't explode. */
+  ch1->state = ch2->state = ch3->state = CHANNEL_STATE_CLOSED;
+  ch1->registered = ch2->registered = ch3->registered = 0;
+  channel_free(ch1);
+  channel_free(ch2);
+  channel_free(ch3);
+  UNMOCK(update_socket_info_impl);
+  UNMOCK(channel_should_write_to_kernel);
+  UNMOCK(channel_write_to_kernel);
+  UNMOCK(channel_more_to_flush);
+  UNMOCK(channel_flush_some_cells);
+  UNMOCK(get_options);
+  scheduler_free_all();
+  return;
+}
+
+static void
+test_scheduler_channel_states(void *arg)
+{
+  (void)arg;
+  perform_channel_state_tests(-1, SCHEDULER_VANILLA);
+  perform_channel_state_tests(11, SCHEDULER_KIST_LITE);
+#ifdef HAVE_KIST_SUPPORT
+  perform_channel_state_tests(11, SCHEDULER_KIST);
+#endif
+}
+
+static void
+test_scheduler_initfree(void *arg)
+{
   (void)arg;
 
-  queue_heuristic = 0;
-  queue_heuristic_timestamp = 0;
+  tt_ptr_op(channels_pending, ==, NULL);
+  tt_ptr_op(run_sched_ev, ==, NULL);
+
+  MOCK(get_options, mock_get_options);
+  set_scheduler_options(SCHEDULER_KIST);
+  set_scheduler_options(SCHEDULER_KIST_LITE);
+  set_scheduler_options(SCHEDULER_VANILLA);
+
+  scheduler_init();
+
+  tt_ptr_op(channels_pending, !=, NULL);
+  tt_ptr_op(run_sched_ev, !=, NULL);
+  /* We have specified nothing in the torrc and there's no consensus so the
+   * KIST scheduler is what should be in use */
+  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+  tt_int_op(sched_run_interval, ==, 10);
+
+  scheduler_free_all();
+
+  tt_ptr_op(channels_pending, ==, NULL);
+  tt_ptr_op(run_sched_ev, ==, NULL);
+
+ done:
+  UNMOCK(get_options);
+  cleanup_scheduler_options();
+  return;
+}
+
+static void
+test_scheduler_can_use_kist(void *arg)
+{
+  (void)arg;
+
+  int res_should, res_freq;
+  MOCK(get_options, mock_get_options);
+
+  /* Test force enabling of KIST */
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 1234;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+#ifdef HAVE_KIST_SUPPORT
+  tt_int_op(res_should, ==, 1);
+#else /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_should, ==, 0);
+#endif /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_freq, ==, 1234);
+
+  /* Test defer to consensus, but no consensus available */
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 0;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+#ifdef HAVE_KIST_SUPPORT
+  tt_int_op(res_should, ==, 1);
+#else /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_should, ==, 0);
+#endif /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_freq, ==, 10);
+
+  /* Test defer to consensus, and kist consensus available */
+  MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 0;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+#ifdef HAVE_KIST_SUPPORT
+  tt_int_op(res_should, ==, 1);
+#else /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_should, ==, 0);
+#endif /* HAVE_KIST_SUPPORT */
+  tt_int_op(res_freq, ==, 12);
+  UNMOCK(networkstatus_get_param);
+
+  /* Test defer to consensus, and vanilla consensus available */
+  MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
+  clear_options();
+  mocked_options.KISTSchedRunInterval = 0;
+  res_should = scheduler_can_use_kist();
+  res_freq = kist_scheduler_run_interval();
+  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_freq, ==, 0);
+  UNMOCK(networkstatus_get_param);
+
+ done:
+  UNMOCK(get_options);
+  return;
+}
+
+static void
+test_scheduler_ns_changed(void *arg)
+{
+  (void) arg;
+
+  /*
+   * Currently no scheduler implementations use the old/new consensuses passed
+   * in scheduler_notify_networkstatus_changed, so it is okay to pass NULL.
+   *
+   * "But then what does test actually exercise???" It tests that
+   * scheduler_notify_networkstatus_changed fetches the correct value from the
+   * consensus, and then switches the scheduler if necessasry.
+   */
+
+  MOCK(get_options, mock_get_options);
+  clear_options();
+  set_scheduler_options(SCHEDULER_KIST);
+  set_scheduler_options(SCHEDULER_VANILLA);
+
+  tt_ptr_op(the_scheduler, ==, NULL);
+
+  /* Change from vanilla to kist via consensus */
+  the_scheduler = get_vanilla_scheduler();
+  MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+#ifdef HAVE_KIST_SUPPORT
+  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+#else
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+#endif
+
+  /* Change from kist to vanilla via consensus */
+  the_scheduler = get_kist_scheduler();
+  MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+
+  /* Doesn't change when using KIST */
+  the_scheduler = get_kist_scheduler();
+  MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+#ifdef HAVE_KIST_SUPPORT
+  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+#else
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+#endif
+
+  /* Doesn't change when using vanilla */
+  the_scheduler = get_vanilla_scheduler();
+  MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
+  scheduler_notify_networkstatus_changed();
+  UNMOCK(networkstatus_get_param);
+  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+
+ done:
+  UNMOCK(get_options);
+  cleanup_scheduler_options();
+  return;
+}
 
-  /* Not yet inited case */
-  scheduler_update_queue_heuristic(now - 180);
-  tt_u64_op(queue_heuristic, ==, 0);
-  tt_int_op(queue_heuristic_timestamp, ==, now - 180);
+/*
+ * Mocked functions for the kist_pending_list test.
+ */
 
-  queue_heuristic = 1000000000L;
-  queue_heuristic_timestamp = now - 120;
+static int mock_flush_some_cells_num = 1;
+static int mock_more_to_flush = 0;
+static int mock_update_socket_info_limit = 0;
 
-  scheduler_update_queue_heuristic(now - 119);
-  tt_u64_op(queue_heuristic, ==, 500000000L);
-  tt_int_op(queue_heuristic_timestamp, ==, now - 119);
+static ssize_t
+channel_flush_some_cells_mock_var(channel_t *chan, ssize_t num_cells)
+{
+  (void) chan;
+  (void) num_cells;
+  return mock_flush_some_cells_num;
+}
 
-  scheduler_update_queue_heuristic(now - 116);
-  tt_u64_op(queue_heuristic, ==, 62500000L);
-  tt_int_op(queue_heuristic_timestamp, ==, now - 116);
+/* Because when we flush cells, it is possible that the connection outbuf gets
+ * fully drained, the wants to write scheduler event is fired back while we
+ * are in the scheduler loop so this mock function does it for us.
+ * Furthermore, the socket limit is set to 0 so once this is triggered, it
+ * informs the scheduler that it can't write on the socket anymore. */
+static void
+channel_write_to_kernel_mock_trigger_24700(channel_t *chan)
+{
+  static int chan_id_seen[2] = {0};
+  if (++chan_id_seen[chan->global_identifier - 1] > 1) {
+    tt_assert(0);
+  }
 
-  qh = scheduler_get_queue_heuristic();
-  tt_u64_op(qh, ==, 0);
+  scheduler_channel_wants_writes(chan);
 
  done:
   return;
 }
 
+static int
+channel_more_to_flush_mock_var(channel_t *chan)
+{
+  (void) chan;
+  return mock_more_to_flush;
+}
+
+static void
+update_socket_info_impl_mock_var(socket_table_ent_t *ent)
+{
+  ent->cwnd = ent->unacked = ent->mss = ent->notsent = 0;
+  ent->limit = mock_update_socket_info_limit;
+}
+
+static void
+test_scheduler_kist_pending_list(void *arg)
+{
+  (void) arg;
+
+#ifndef HAVE_KIST_SUPPORT
+  return;
+#endif
+
+  /* This is for testing the channel flow with the pending list that is
+   * depending on the channel state, what will be the expected behavior of the
+   * scheduler with that list.
+   *
+   * For instance, we want to catch double channel add or removing a channel
+   * that doesn't exists, or putting a channel in the list in a wrong state.
+   * Essentially, this will articifically test cases of the KIST main loop and
+   * entry point in the channel subsystem.
+   *
+   * In part, this is to also catch things like #24700 and provide a test bed
+   * for more testing in the future like so. */
+
+  /* Mocking a series of scheduler function to control the flow of the
+   * scheduler loop to test every use cases and assess the pending list. */
+  MOCK(get_options, mock_get_options);
+  MOCK(channel_flush_some_cells, channel_flush_some_cells_mock_var);
+  MOCK(channel_more_to_flush, channel_more_to_flush_mock_var);
+  MOCK(update_socket_info_impl, update_socket_info_impl_mock_var);
+  MOCK(channel_write_to_kernel, channel_write_to_kernel_mock);
+  MOCK(channel_should_write_to_kernel, channel_should_write_to_kernel_mock);
+
+  /* Setup options so we're sure about what sched we are running */
+  mocked_options.KISTSchedRunInterval = 10;
+  set_scheduler_options(SCHEDULER_KIST);
+
+  /* Init scheduler. */
+  scheduler_init();
+
+  /* Initialize a channel. We'll need a second channel for the #24700 bug
+   * test. */
+  channel_t *chan1 = new_fake_channel();
+  channel_t *chan2 = new_fake_channel();
+  tt_assert(chan1);
+  tt_assert(chan2);
+  chan1->magic = chan2->magic = TLS_CHAN_MAGIC;
+  channel_register(chan1);
+  channel_register(chan2);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(chan1->sched_heap_idx, OP_EQ, -1);
+  tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+  tt_int_op(chan2->sched_heap_idx, OP_EQ, -1);
+
+  /* Once a channel becomes OPEN, it always have at least one cell in it so
+   * the scheduler is notified that the channel wants to write so this is the
+   * first step. Might not make sense to you but it is the way it is. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+
+  /* Signal the scheduler that it has waiting cells which means the channel
+   * will get scheduled. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  /* Subsequent call should not add it more times. It is possible we add many
+   * cells in rapid succession before the channel is scheduled. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+
+  /* We'll flush one cell and make it that the socket can write but no more to
+   * flush else we end up in an infinite loop. We expect the channel to be put
+   * in waiting for cells state and the pending list empty. */
+  mock_update_socket_info_limit = INT_MAX;
+  mock_more_to_flush = 0;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* Lets make believe that a cell is now in the channel but this time the
+   * channel can't write so obviously it has more to flush. We expect the
+   * channel to be back in the pending list. */
+  scheduler_channel_has_waiting_cells(chan1);
+  mock_update_socket_info_limit = 0;
+  mock_more_to_flush = 1;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* Channel is in the pending list now, during that time, we'll trigger a
+   * wants to write event because maybe the channel buffers were emptied in
+   * the meantime. This is possible because once the connection outbuf is
+   * flushed down the low watermark, the scheduler is notified.
+   *
+   * We expect the channel to NOT be added in the pending list again and stay
+   * in PENDING state. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* Make it that the channel can write now but has nothing else to flush. We
+   * expect that it is removed from the pending list and waiting for cells. */
+  mock_update_socket_info_limit = INT_MAX;
+  mock_more_to_flush = 0;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* While waiting for cells, lets say we were able to write more things on
+   * the connection outbuf (unlikely that this can happen but let say it
+   * does). We expect the channel to stay in waiting for cells. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* We'll not put it in the pending list and make the flush cell fail with 0
+   * cell flushed. We expect that it is put back in waiting for cells. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  mock_flush_some_cells_num = 0;
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_FOR_CELLS);
+
+  /* Set the channel to a state where it doesn't want to write more. We expect
+   * that the channel becomes idle. */
+  scheduler_channel_doesnt_want_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_IDLE);
+
+  /* Some cells arrive on the channel now. We expect it to go back in waiting
+   * to write. You might wonder why it is not put in the pending list? Because
+   * once the channel becomes OPEN again (the doesn't want to write event only
+   * occurs if the channel goes in MAINT mode), if there are cells in the
+   * channel, the wants to write event is triggered thus putting the channel
+   * in pending mode.
+   *
+   * Else, if no cells, it stays IDLE and then once a cell comes in, it should
+   * go in waiting to write which is a BUG itself because the channel can't be
+   * scheduled until a second cell comes in. Hopefully, #24554 will fix that
+   * for KIST. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
+
+  /* Second cell comes in, unfortunately, it won't get scheduled until a wants
+   * to write event occurs like described above. */
+  scheduler_channel_has_waiting_cells(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 0);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_WAITING_TO_WRITE);
+
+  /* Unblock everything putting the channel in the pending list. */
+  scheduler_channel_wants_writes(chan1);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 1);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* Testing bug #24700 which is the situation where we have at least two
+   * different channels in the pending list. The first one gets flushed and
+   * bytes are written on the wire which triggers a wants to write event
+   * because the outbuf is below the low watermark. The bug was that this
+   * exact channel was added back in the pending list because its state wasn't
+   * PENDING.
+   *
+   * The following does some ninja-tsu to try to make it happen. We need two
+   * different channels so we create a second one and add it to the pending
+   * list. Then, we have a custom function when we write to kernel that does
+   * two important things:
+   *
+   *  1) Calls scheduler_channel_wants_writes(chan) on the channel.
+   *  2) Keeps track of how many times it sees the channel going through. If
+   *     that limit goes > 1, it means we've added the channel twice in the
+   *     pending list.
+   *
+   * In the end, we expect both channels to be in the pending list after this
+   * scheduler run. */
+
+  /* Put the second channel in the pending list. */
+  scheduler_channel_wants_writes(chan2);
+  scheduler_channel_has_waiting_cells(chan2);
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 2);
+  tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+  /* This makes it that the first pass on socket_can_write() will be true but
+   * then when a single cell is flushed (514 + 29 bytes), the second call to
+   * socket_can_write() will be false. If it wasn't sending back false on the
+   * second run, we end up in an infinite loop of the scheduler. */
+  mock_update_socket_info_limit = 600;
+  /* We want to hit "Case 3:" of the scheduler so channel_more_to_flush() is
+   * true but socket_can_write() has to be false on the second check on the
+   * channel. */
+  mock_more_to_flush = 1;
+  mock_flush_some_cells_num = 1;
+  MOCK(channel_write_to_kernel, channel_write_to_kernel_mock_trigger_24700);
+  the_scheduler->run();
+  tt_int_op(smartlist_len(get_channels_pending()), OP_EQ, 2);
+  tt_int_op(chan1->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+  tt_int_op(chan2->scheduler_state, OP_EQ, SCHED_CHAN_PENDING);
+
+ done:
+  chan1->state = chan2->state = CHANNEL_STATE_CLOSED;
+  chan1->registered = chan2->registered = 0;
+  channel_free(chan1);
+  channel_free(chan2);
+  scheduler_free_all();
+
+  UNMOCK(get_options);
+  UNMOCK(channel_flush_some_cells);
+  UNMOCK(channel_more_to_flush);
+  UNMOCK(update_socket_info_impl);
+  UNMOCK(channel_write_to_kernel);
+  UNMOCK(channel_should_write_to_kernel);
+}
+
 struct testcase_t scheduler_tests[] = {
-  { "channel_states", test_scheduler_channel_states, TT_FORK, NULL, NULL },
   { "compare_channels", test_scheduler_compare_channels,
     TT_FORK, NULL, NULL },
+  { "channel_states", test_scheduler_channel_states, TT_FORK, NULL, NULL },
   { "initfree", test_scheduler_initfree, TT_FORK, NULL, NULL },
-  { "loop", test_scheduler_loop, TT_FORK, NULL, NULL },
-  { "queue_heuristic", test_scheduler_queue_heuristic,
-    TT_FORK, NULL, NULL },
+  { "loop_vanilla", test_scheduler_loop_vanilla, TT_FORK, NULL, NULL },
+  { "loop_kist", test_scheduler_loop_kist, TT_FORK, NULL, NULL },
+  { "ns_changed", test_scheduler_ns_changed, TT_FORK, NULL, NULL},
+  { "should_use_kist", test_scheduler_can_use_kist, TT_FORK, NULL, NULL },
+  { "kist_pending_list", test_scheduler_kist_pending_list, TT_FORK,
+    NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index 526fb09b67..630883e1fb 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -1,3 +1,6 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #define SHARED_RANDOM_PRIVATE
 #define SHARED_RANDOM_STATE_PRIVATE
 #define CONFIG_PRIVATE
@@ -6,15 +9,18 @@
 #include "or.h"
 #include "test.h"
 #include "config.h"
-#include "dirvote.h"
-#include "shared_random.h"
-#include "shared_random_state.h"
+#include "crypto_rand.h"
+#include "dirauth/dirvote.h"
+#include "dirauth/shared_random.h"
+#include "dirauth/shared_random_state.h"
+#include "log_test_helpers.h"
+#include "networkstatus.h"
+#include "router.h"
 #include "routerkeys.h"
 #include "routerlist.h"
-#include "router.h"
 #include "routerparse.h"
-#include "networkstatus.h"
-#include "log_test_helpers.h"
+#include "shared_random_client.h"
+#include "voting_schedule.h"
 
 static authority_cert_t *mock_cert;
 
@@ -48,7 +54,7 @@ init_authority_state(void)
   mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
-  tt_int_op(0, ==, load_ed_keys(options, time(NULL)));
+  tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
   sr_state_init(0, 0);
   /* It's possible a commit has been generated in our state depending on
    * the phase we are currently in which uses "now" as the starting
@@ -76,65 +82,73 @@ test_get_sr_protocol_phase(void *arg)
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 23:59:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 00:00:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_COMMIT);
+    tt_int_op(phase, OP_EQ, SR_PHASE_COMMIT);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 00:00:01 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_COMMIT);
+    tt_int_op(phase, OP_EQ, SR_PHASE_COMMIT);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 11:59:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_COMMIT);
+    tt_int_op(phase, OP_EQ, SR_PHASE_COMMIT);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 12:00:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 12:00:01 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
   {
     retval = parse_rfc1123_time("Wed, 20 Apr 2015 13:00:00 UTC", &the_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
 
     phase = get_sr_protocol_phase(the_time);
-    tt_int_op(phase, ==, SR_PHASE_REVEAL);
+    tt_int_op(phase, OP_EQ, SR_PHASE_REVEAL);
   }
 
  done:
   ;
 }
 
-static networkstatus_t *mock_consensus = NULL;
+static networkstatus_t mock_consensus;
+
+/* Mock function to immediately return our local 'mock_consensus'. */
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_consensus;
+}
 
 static void
 test_get_state_valid_until_time(void *arg)
@@ -146,11 +160,23 @@ test_get_state_valid_until_time(void *arg)
 
   (void) arg;
 
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 01:00:00 UTC",
+                              &mock_consensus.fresh_until);
+  tt_int_op(retval, OP_EQ, 0);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                              &mock_consensus.valid_after);
+  tt_int_op(retval, OP_EQ, 0);
+
   {
     /* Get the valid until time if called at 00:00:01 */
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     /* Compare it with the correct result */
@@ -161,7 +187,8 @@ test_get_state_valid_until_time(void *arg)
   {
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -171,7 +198,8 @@ test_get_state_valid_until_time(void *arg)
   {
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -181,7 +209,8 @@ test_get_state_valid_until_time(void *arg)
   {
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -189,82 +218,148 @@ test_get_state_valid_until_time(void *arg)
   }
 
  done:
-  ;
+  UNMOCK(networkstatus_get_live_consensus);
 }
 
-/* Mock function to immediately return our local 'mock_consensus'. */
-static networkstatus_t *
-mock_networkstatus_get_live_consensus(time_t now)
-{
-  (void) now;
-  return mock_consensus;
-}
-
-/** Test the get_next_valid_after_time() function. */
+/** Test the function that calculates the start time of the current SRV
+ *  protocol run. */
 static void
-test_get_next_valid_after_time(void *arg)
+test_get_start_time_of_current_run(void *arg)
 {
-  time_t current_time;
-  time_t valid_after_time;
-  char tbuf[ISO_TIME_LEN + 1];
   int retval;
+  char tbuf[ISO_TIME_LEN + 1];
+  time_t current_time, run_start_time;
 
   (void) arg;
 
-  {
-    /* Setup a fake consensus just to get the times out of it, since
-       get_next_valid_after_time() needs them. */
-    mock_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
 
-    retval = parse_rfc1123_time("Mon, 13 Jan 2016 16:00:00 UTC",
-                                &mock_consensus->fresh_until);
-    tt_int_op(retval, ==, 0);
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 01:00:00 UTC",
+                              &mock_consensus.fresh_until);
+  tt_int_op(retval, OP_EQ, 0);
 
-    retval = parse_rfc1123_time("Mon, 13 Jan 2016 15:00:00 UTC",
-                                &mock_consensus->valid_after);
-    tt_int_op(retval, ==, 0);
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                              &mock_consensus.valid_after);
+  tt_int_op(retval, OP_EQ, 0);
 
-    MOCK(networkstatus_get_live_consensus,
-         mock_networkstatus_get_live_consensus);
+  {
+    /* Get start time if called at 00:00:01 */
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
+                                &current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time =
+      sr_state_get_start_time_of_current_protocol_run(current_time);
+
+    /* Compare it with the correct result */
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:00:00", OP_EQ, tbuf);
   }
 
   {
-    /* Get the valid after time if called at 00:00:00 */
-    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:59 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
-    valid_after_time = get_next_valid_after_time(current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time =
+      sr_state_get_start_time_of_current_protocol_run(current_time);
 
     /* Compare it with the correct result */
-    format_iso_time(tbuf, valid_after_time);
-    tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf);
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:00:00", OP_EQ, tbuf);
   }
 
   {
-    /* Get the valid until time if called at 00:00:01 */
-    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
-    valid_after_time = get_next_valid_after_time(current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time =
+      sr_state_get_start_time_of_current_protocol_run(current_time);
 
     /* Compare it with the correct result */
-    format_iso_time(tbuf, valid_after_time);
-    tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf);
- }
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:00:00", OP_EQ, tbuf);
+  }
+
+  /* Next test is testing it without a consensus to use the testing voting
+   * interval . */
+  UNMOCK(networkstatus_get_live_consensus);
 
+  /* Now let's alter the voting schedule and check the correctness of the
+   * function. Voting interval of 10 seconds, means that an SRV protocol run
+   * takes 10 seconds * 24 rounds = 4 mins */
   {
-    retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:30:01 UTC",
+    or_options_t *options = get_options_mutable();
+    options->V3AuthVotingInterval = 10;
+    options->TestingV3AuthInitialVotingInterval = 10;
+    retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:15:32 UTC",
                                 &current_time);
-    tt_int_op(retval, ==, 0);
-    valid_after_time = get_next_valid_after_time(current_time);
+    tt_int_op(retval, OP_EQ, 0);
+    voting_schedule_recalculate_timing(get_options(), current_time);
+    run_start_time =
+      sr_state_get_start_time_of_current_protocol_run(current_time);
 
     /* Compare it with the correct result */
-    format_iso_time(tbuf, valid_after_time);
-    tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf);
- }
+    format_iso_time(tbuf, run_start_time);
+    tt_str_op("2015-04-20 00:12:00", OP_EQ, tbuf);
+  }
 
  done:
-  networkstatus_vote_free(mock_consensus);
+  ;
+}
+
+/** Do some rudimentary consistency checks between the functions that
+ *  understand the shared random protocol schedule */
+static void
+test_get_start_time_functions(void *arg)
+{
+  (void) arg;
+  int retval;
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 01:00:00 UTC",
+                              &mock_consensus.fresh_until);
+  tt_int_op(retval, OP_EQ, 0);
+
+  retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
+                              &mock_consensus.valid_after);
+  tt_int_op(retval, OP_EQ, 0);
+  time_t now = mock_consensus.valid_after;
+
+  voting_schedule_recalculate_timing(get_options(), now);
+  time_t start_time_of_protocol_run =
+    sr_state_get_start_time_of_current_protocol_run(now);
+  tt_assert(start_time_of_protocol_run);
+
+  /* Check that the round start time of the beginning of the run, is itself */
+  tt_int_op(get_start_time_of_current_round(), OP_EQ,
+            start_time_of_protocol_run);
+
+ done:
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+static void
+test_get_sr_protocol_duration(void *arg)
+{
+  (void) arg;
+
+  /* Check that by default an SR phase is 12 hours */
+  tt_int_op(sr_state_get_phase_duration(), OP_EQ, 12*60*60);
+  tt_int_op(sr_state_get_protocol_run_duration(), OP_EQ, 24*60*60);
+
+  /* Now alter the voting interval and check that the SR phase is 2 mins long
+   * if voting happens every 10 seconds (10*12 seconds = 2 mins) */
+  or_options_t *options = get_options_mutable();
+  options->V3AuthVotingInterval = 10;
+  tt_int_op(sr_state_get_phase_duration(), OP_EQ, 2*60);
+  tt_int_op(sr_state_get_protocol_run_duration(), OP_EQ, 4*60);
+
+ done: ;
 }
 
 /* In this test we are going to generate a sr_commit_t object and validate
@@ -289,7 +384,7 @@ test_sr_commit(void *arg)
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
-    tt_int_op(0, ==, load_ed_keys(options, now));
+    tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
   }
 
   /* Generate our commit object and validate it has the appropriate field
@@ -307,18 +402,18 @@ test_sr_commit(void *arg)
     tt_assert(!tor_mem_is_zero((char *) our_commit->random_number,
                                sizeof(our_commit->random_number)));
     /* Commit and reveal timestamp should be the same. */
-    tt_u64_op(our_commit->commit_ts, ==, our_commit->reveal_ts);
+    tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts);
     /* We should have a hashed reveal. */
     tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal,
                                sizeof(our_commit->hashed_reveal)));
     /* Do we have a valid encoded commit and reveal. Note the following only
      * tests if the generated values are correct. Their could be a bug in
-     * the decode function but we test them seperately. */
-    tt_int_op(0, ==, reveal_decode(our_commit->encoded_reveal,
+     * the decode function but we test them separately. */
+    tt_int_op(0, OP_EQ, reveal_decode(our_commit->encoded_reveal,
                                    &test_commit));
-    tt_int_op(0, ==, commit_decode(our_commit->encoded_commit,
+    tt_int_op(0, OP_EQ, commit_decode(our_commit->encoded_commit,
                                    &test_commit));
-    tt_int_op(0, ==, verify_commit_and_reveal(our_commit));
+    tt_int_op(0, OP_EQ, verify_commit_and_reveal(our_commit));
   }
 
   /* Let's make sure our verify commit and reveal function works. We'll
@@ -331,32 +426,32 @@ test_sr_commit(void *arg)
     /* Timestamp MUST match. */
     test_commit.commit_ts = test_commit.reveal_ts - 42;
     setup_full_capture_of_logs(LOG_WARN);
-    tt_int_op(-1, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(-1, OP_EQ, verify_commit_and_reveal(&test_commit));
     expect_log_msg_containing("doesn't match reveal timestamp");
     teardown_capture_of_logs();
     memcpy(&test_commit, our_commit, sizeof(test_commit));
-    tt_int_op(0, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(0, OP_EQ, verify_commit_and_reveal(&test_commit));
 
     /* Hashed reveal must match the H(encoded_reveal). */
     memset(test_commit.hashed_reveal, 'X',
            sizeof(test_commit.hashed_reveal));
     setup_full_capture_of_logs(LOG_WARN);
-    tt_int_op(-1, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(-1, OP_EQ, verify_commit_and_reveal(&test_commit));
     expect_single_log_msg_containing("doesn't match the commit value");
     teardown_capture_of_logs();
     memcpy(&test_commit, our_commit, sizeof(test_commit));
-    tt_int_op(0, ==, verify_commit_and_reveal(&test_commit));
+    tt_int_op(0, OP_EQ, verify_commit_and_reveal(&test_commit));
   }
 
   /* We'll build a list of values from our commit that our parsing function
    * takes from a vote line and see if we can parse it correctly. */
   {
-    smartlist_add(args, tor_strdup("1"));
-    smartlist_add(args,
-               tor_strdup(crypto_digest_algorithm_get_name(our_commit->alg)));
-    smartlist_add(args, tor_strdup(sr_commit_get_rsa_fpr(our_commit)));
-    smartlist_add(args, tor_strdup(our_commit->encoded_commit));
-    smartlist_add(args, tor_strdup(our_commit->encoded_reveal));
+    smartlist_add_strdup(args, "1");
+    smartlist_add_strdup(args,
+               crypto_digest_algorithm_get_name(our_commit->alg));
+    smartlist_add_strdup(args, sr_commit_get_rsa_fpr(our_commit));
+    smartlist_add_strdup(args, our_commit->encoded_commit);
+    smartlist_add_strdup(args, our_commit->encoded_reveal);
     parsed_commit = sr_parse_commit(args);
     tt_assert(parsed_commit);
     /* That parsed commit should be _EXACTLY_ like our original commit (we
@@ -399,26 +494,26 @@ test_encoding(void *arg)
   /* Hash random number because we don't expose bytes of the RNG. */
   ret = crypto_digest256(hashed_rand, raw_rand,
                          sizeof(raw_rand), SR_DIGEST_ALG);
-  tt_int_op(0, ==, ret);
+  tt_int_op(0, OP_EQ, ret);
   /* Hash reveal value. */
-  tt_int_op(SR_REVEAL_BASE64_LEN, ==, strlen(encoded_reveal));
+  tt_int_op(SR_REVEAL_BASE64_LEN, OP_EQ, strlen(encoded_reveal));
   ret = crypto_digest256(hashed_reveal, encoded_reveal,
                          strlen(encoded_reveal), SR_DIGEST_ALG);
-  tt_int_op(0, ==, ret);
-  tt_int_op(SR_COMMIT_BASE64_LEN, ==, strlen(encoded_commit));
+  tt_int_op(0, OP_EQ, ret);
+  tt_int_op(SR_COMMIT_BASE64_LEN, OP_EQ, strlen(encoded_commit));
 
   /* Test our commit/reveal decode functions. */
   {
     /* Test the reveal encoded value. */
-    tt_int_op(0, ==, reveal_decode(encoded_reveal, &parsed_commit));
-    tt_u64_op(ts, ==, parsed_commit.reveal_ts);
+    tt_int_op(0, OP_EQ, reveal_decode(encoded_reveal, &parsed_commit));
+    tt_u64_op(ts, OP_EQ, parsed_commit.reveal_ts);
     tt_mem_op(hashed_rand, OP_EQ, parsed_commit.random_number,
               sizeof(hashed_rand));
 
     /* Test the commit encoded value. */
     memset(&parsed_commit, 0, sizeof(parsed_commit));
-    tt_int_op(0, ==, commit_decode(encoded_commit, &parsed_commit));
-    tt_u64_op(ts, ==, parsed_commit.commit_ts);
+    tt_int_op(0, OP_EQ, commit_decode(encoded_commit, &parsed_commit));
+    tt_u64_op(ts, OP_EQ, parsed_commit.commit_ts);
     tt_mem_op(encoded_commit, OP_EQ, parsed_commit.encoded_commit,
               sizeof(parsed_commit.encoded_commit));
     tt_mem_op(hashed_reveal, OP_EQ, parsed_commit.hashed_reveal,
@@ -433,7 +528,7 @@ test_encoding(void *arg)
     memcpy(parsed_commit.random_number, hashed_rand,
            sizeof(parsed_commit.random_number));
     ret = reveal_encode(&parsed_commit, encoded, sizeof(encoded));
-    tt_int_op(SR_REVEAL_BASE64_LEN, ==, ret);
+    tt_int_op(SR_REVEAL_BASE64_LEN, OP_EQ, ret);
     tt_mem_op(encoded_reveal, OP_EQ, encoded, strlen(encoded_reveal));
   }
 
@@ -444,7 +539,7 @@ test_encoding(void *arg)
     memcpy(parsed_commit.hashed_reveal, hashed_reveal,
            sizeof(parsed_commit.hashed_reveal));
     ret = commit_encode(&parsed_commit, encoded, sizeof(encoded));
-    tt_int_op(SR_COMMIT_BASE64_LEN, ==, ret);
+    tt_int_op(SR_COMMIT_BASE64_LEN, OP_EQ, ret);
     tt_mem_op(encoded_commit, OP_EQ, encoded, strlen(encoded_commit));
   }
 
@@ -456,7 +551,7 @@ test_encoding(void *arg)
  *  If <b>also_current</b> is set, then set both current and previous SRVs.
  *  Otherwise, just set the previous SRV. (And clear the current SRV.)
  *
- * You must call sr_state_free() to free the state at the end of each test
+ * You must call sr_state_free_all() to free the state at the end of each test
  * function (on pass or fail). */
 static void
 test_sr_setup_srv(int also_current)
@@ -529,9 +624,9 @@ test_vote(void *arg)
     tt_assert(lines);
     /* Split the lines. We expect 2 here. */
     ret = smartlist_split_string(chunks, lines, "\n", SPLIT_IGNORE_BLANK, 0);
-    tt_int_op(ret, ==, 4);
+    tt_int_op(ret, OP_EQ, 4);
     tt_str_op(smartlist_get(chunks, 0), OP_EQ, "shared-rand-participate");
-    /* Get our commitment line and will validate it agains our commit. The
+    /* Get our commitment line and will validate it against our commit. The
      * format is as follow:
      * "shared-rand-commitment" SP version SP algname SP identity
      *                          SP COMMIT [SP REVEAL] NL
@@ -539,7 +634,7 @@ test_vote(void *arg)
     char *commit_line = smartlist_get(chunks, 1);
     tt_assert(commit_line);
     ret = smartlist_split_string(tokens, commit_line, " ", 0, 0);
-    tt_int_op(ret, ==, 6);
+    tt_int_op(ret, OP_EQ, 6);
     tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-commit");
     tt_str_op(smartlist_get(tokens, 1), OP_EQ, "1");
     tt_str_op(smartlist_get(tokens, 2), OP_EQ,
@@ -547,7 +642,7 @@ test_vote(void *arg)
     char digest[DIGEST_LEN];
     base16_decode(digest, sizeof(digest), smartlist_get(tokens, 3),
                   HEX_DIGEST_LEN);
-    tt_mem_op(digest, ==, our_commit->rsa_identity, sizeof(digest));
+    tt_mem_op(digest, OP_EQ, our_commit->rsa_identity, sizeof(digest));
     tt_str_op(smartlist_get(tokens, 4), OP_EQ, our_commit->encoded_commit);
     tt_str_op(smartlist_get(tokens, 5), OP_EQ, our_commit->encoded_reveal)
 ;
@@ -563,7 +658,7 @@ test_vote(void *arg)
     /* Set valid flag explicitly here to compare since it's not set by
      * simply parsing the commit. */
     parsed_commit->valid = 1;
-    tt_mem_op(parsed_commit, ==, our_commit, sizeof(*our_commit));
+    tt_mem_op(parsed_commit, OP_EQ, our_commit, sizeof(*our_commit));
 
     /* minor cleanup */
     SMARTLIST_FOREACH(tokens, char *, s, tor_free(s));
@@ -573,7 +668,7 @@ test_vote(void *arg)
     char *prev_srv_line = smartlist_get(chunks, 2);
     tt_assert(prev_srv_line);
     ret = smartlist_split_string(tokens, prev_srv_line, " ", 0, 0);
-    tt_int_op(ret, ==, 3);
+    tt_int_op(ret, OP_EQ, 3);
     tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-previous-value");
     tt_str_op(smartlist_get(tokens, 1), OP_EQ, "42");
     tt_str_op(smartlist_get(tokens, 2), OP_EQ,
@@ -587,7 +682,7 @@ test_vote(void *arg)
     char *current_srv_line = smartlist_get(chunks, 3);
     tt_assert(current_srv_line);
     ret = smartlist_split_string(tokens, current_srv_line, " ", 0, 0);
-    tt_int_op(ret, ==, 3);
+    tt_int_op(ret, OP_EQ, 3);
     tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-current-value");
     tt_str_op(smartlist_get(tokens, 1), OP_EQ, "128");
     tt_str_op(smartlist_get(tokens, 2), OP_EQ,
@@ -606,7 +701,7 @@ test_vote(void *arg)
 
  done:
   UNMOCK(trusteddirserver_get_by_v3_auth_digest);
-  sr_state_free();
+  sr_state_free_all();
 }
 
 static const char *sr_state_str = "Version 1\n"
@@ -640,7 +735,7 @@ test_state_load_from_disk(void *arg)
 
   /* First try with a nonexistent path. */
   ret = disk_state_load_from_disk_impl("NONEXISTENTNONEXISTENT");
-  tt_assert(ret == -ENOENT);
+  tt_int_op(ret, OP_EQ, -ENOENT);
 
   /* Now create a mock state directory and state file */
 #ifdef _WIN32
@@ -648,9 +743,9 @@ test_state_load_from_disk(void *arg)
 #else
   ret = mkdir(dir, 0700);
 #endif
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
   ret = write_str_to_file(sr_state_path, sr_state_str, 0);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Try to load the directory itself. Should fail. */
   ret = disk_state_load_from_disk_impl(dir);
@@ -658,11 +753,11 @@ test_state_load_from_disk(void *arg)
 
   /* State should be non-existent at this point. */
   the_sr_state = get_sr_state();
-  tt_assert(!the_sr_state);
+  tt_ptr_op(the_sr_state, OP_EQ, NULL);
 
   /* Now try to load the correct file! */
   ret = disk_state_load_from_disk_impl(sr_state_path);
-  tt_assert(ret == 0);
+  tt_int_op(ret, OP_EQ, 0);
 
   /* Check the content of the state */
   /* XXX check more deeply!!! */
@@ -700,7 +795,7 @@ test_sr_setup_commits(void)
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
-    tt_int_op(0, ==, load_ed_keys(options, now));
+    tt_int_op(0, OP_EQ, load_ed_keys(options, now));
   }
 
   /* Generate three dummy commits according to sr_srv_calc_ref.py .  Then
@@ -786,7 +881,7 @@ test_sr_setup_commits(void)
   save_commit_to_state(commit_b);
   save_commit_to_state(commit_c);
   save_commit_to_state(commit_d);
-  tt_int_op(digestmap_size(get_sr_state()->commits), ==, 4);
+  tt_int_op(digestmap_size(get_sr_state()->commits), OP_EQ, 4);
 
   /* Now during REVEAL phase save commit D by restoring its reveal. */
   set_sr_phase(SR_PHASE_REVEAL);
@@ -833,14 +928,14 @@ test_sr_compute_srv(void *arg)
   /* Check the result against the test vector */
   current_srv = sr_state_get_current_srv();
   tt_assert(current_srv);
-  tt_u64_op(current_srv->num_reveals, ==, 3);
+  tt_u64_op(current_srv->num_reveals, OP_EQ, 3);
   tt_str_op(hex_str((char*)current_srv->value, 32),
-            ==,
+            OP_EQ,
             SRV_TEST_VECTOR);
 
  done:
   UNMOCK(trusteddirserver_get_by_v3_auth_digest);
-  sr_state_free();
+  sr_state_free_all();
 }
 
 /** Return a minimal vote document with a current SRV value set to
@@ -892,7 +987,7 @@ test_sr_get_majority_srv_from_votes(void *arg)
   /* Since it's only one vote with an SRV, it should not achieve majority and
      hence no SRV will be returned. */
   chosen_srv = get_majority_srv_from_votes(votes, 1);
-  tt_assert(!chosen_srv);
+  tt_ptr_op(chosen_srv, OP_EQ, NULL);
 
   { /* Now put in 8 more votes. Let SRV_1 have majority. */
     int i;
@@ -911,21 +1006,21 @@ test_sr_get_majority_srv_from_votes(void *arg)
       smartlist_add(votes, vote);
     }
 
-    tt_int_op(smartlist_len(votes), ==, 9);
+    tt_int_op(smartlist_len(votes), OP_EQ, 9);
   }
 
   /* Now we achieve majority for SRV_1, but not the AuthDirNumSRVAgreements
      requirement. So still not picking an SRV. */
   set_num_srv_agreements(8);
   chosen_srv = get_majority_srv_from_votes(votes, 1);
-  tt_assert(!chosen_srv);
+  tt_ptr_op(chosen_srv, OP_EQ, NULL);
 
   /* We will now lower the AuthDirNumSRVAgreements requirement by tweaking the
    * consensus parameter and we will try again. This time it should work. */
   set_num_srv_agreements(7);
   chosen_srv = get_majority_srv_from_votes(votes, 1);
   tt_assert(chosen_srv);
-  tt_u64_op(chosen_srv->num_reveals, ==, 42);
+  tt_u64_op(chosen_srv->num_reveals, OP_EQ, 42);
   tt_mem_op(chosen_srv->value, OP_EQ, SRV_1, sizeof(chosen_srv->value));
 
  done:
@@ -949,7 +1044,7 @@ test_utils(void *arg)
     memcpy(srv->value, srv_value, sizeof(srv->value));
     dup_srv = srv_dup(srv);
     tt_assert(dup_srv);
-    tt_u64_op(dup_srv->num_reveals, ==, srv->num_reveals);
+    tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
     tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
     tor_free(srv);
     tor_free(dup_srv);
@@ -969,10 +1064,10 @@ test_utils(void *arg)
     sr_commit_t commit1, commit2;
     memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
     memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), ==, 1);
+    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
     /* Let's corrupt one of them. */
     memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), ==, 0);
+    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
   }
 
   /* Testing commit_is_authoritative(). */
@@ -983,32 +1078,32 @@ test_utils(void *arg)
 
     tt_assert(!crypto_pk_generate_key(k));
 
-    tt_int_op(0, ==, crypto_pk_get_digest(k, digest));
+    tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
     memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), ==, 1);
+    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
     /* Change the pubkey. */
     memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), ==, 0);
+    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
     crypto_pk_free(k);
   }
 
   /* Testing get_phase_str(). */
   {
-    tt_str_op(get_phase_str(SR_PHASE_REVEAL), ==, "reveal");
-    tt_str_op(get_phase_str(SR_PHASE_COMMIT), ==, "commit");
+    tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
+    tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
   }
 
   /* Testing phase transition */
   {
     init_authority_state();
     set_sr_phase(SR_PHASE_COMMIT);
-    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 1);
-    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 0);
+    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1);
+    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0);
     set_sr_phase(SR_PHASE_REVEAL);
-    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 0);
-    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 1);
+    tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 0);
+    tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 1);
     /* Junk. */
-    tt_int_op(is_phase_transition(42), ==, 1);
+    tt_int_op(is_phase_transition(42), OP_EQ, 1);
   }
 
  done:
@@ -1036,24 +1131,24 @@ test_state_transition(void *arg)
     sr_commit_t *commit = sr_generate_our_commit(now, mock_cert);
     tt_assert(commit);
     sr_state_add_commit(commit);
-    tt_int_op(digestmap_size(state->commits), ==, 1);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     /* Let's test our delete feature. */
     sr_state_delete_commits();
-    tt_int_op(digestmap_size(state->commits), ==, 0);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 0);
     /* Add it back so we can continue the rest of the test because after
      * deletiong our commit will be freed so generate a new one. */
     commit = sr_generate_our_commit(now, mock_cert);
     tt_assert(commit);
     sr_state_add_commit(commit);
-    tt_int_op(digestmap_size(state->commits), ==, 1);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     state->n_reveal_rounds = 42;
     state->n_commit_rounds = 43;
     state->n_protocol_runs = 44;
     reset_state_for_new_protocol_run(now);
-    tt_int_op(state->n_reveal_rounds, ==, 0);
-    tt_int_op(state->n_commit_rounds, ==, 0);
-    tt_u64_op(state->n_protocol_runs, ==, 45);
-    tt_int_op(digestmap_size(state->commits), ==, 0);
+    tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
+    tt_int_op(state->n_commit_rounds, OP_EQ, 0);
+    tt_u64_op(state->n_protocol_runs, OP_EQ, 45);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 0);
   }
 
   /* Test SRV rotation in our state. */
@@ -1066,7 +1161,7 @@ test_state_transition(void *arg)
     state_rotate_srv();
     prev = sr_state_get_previous_srv();
     tt_assert(prev == cur);
-    tt_assert(!sr_state_get_current_srv());
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
     sr_state_clean_srvs();
   }
 
@@ -1091,22 +1186,22 @@ test_state_transition(void *arg)
     /* Also, make sure we did change the current. */
     tt_assert(sr_state_get_current_srv() != cur);
     /* We should have our commitment alone. */
-    tt_int_op(digestmap_size(state->commits), ==, 1);
-    tt_int_op(state->n_reveal_rounds, ==, 0);
-    tt_int_op(state->n_commit_rounds, ==, 0);
+    tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
+    tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
+    tt_int_op(state->n_commit_rounds, OP_EQ, 0);
     /* 46 here since we were at 45 just before. */
-    tt_u64_op(state->n_protocol_runs, ==, 46);
+    tt_u64_op(state->n_protocol_runs, OP_EQ, 46);
   }
 
   /* Cleanup of SRVs. */
   {
     sr_state_clean_srvs();
-    tt_assert(!sr_state_get_current_srv());
-    tt_assert(!sr_state_get_previous_srv());
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
   }
 
  done:
-  sr_state_free();
+  sr_state_free_all();
 }
 
 static void
@@ -1131,6 +1226,8 @@ test_keep_commit(void *arg)
     state = get_sr_state();
   }
 
+  crypto_rand((char*)fp, sizeof(fp));
+
   /* Test this very important function that tells us if we should keep a
    * commit or not in our state. Most of it depends on the phase and what's
    * in the commit so we'll change the commit as we go. */
@@ -1139,21 +1236,21 @@ test_keep_commit(void *arg)
   /* Set us in COMMIT phase for starter. */
   set_sr_phase(SR_PHASE_COMMIT);
   /* We should never keep a commit from a non authoritative authority. */
-  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_COMMIT), ==, 0);
+  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_COMMIT), OP_EQ, 0);
   /* This should NOT be kept because it has a reveal value in it. */
   tt_assert(commit_has_reveal_value(commit));
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_COMMIT), ==, 0);
+                               SR_PHASE_COMMIT), OP_EQ, 0);
   /* Add it to the state which should return to not keep it. */
   sr_state_add_commit(commit);
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_COMMIT), ==, 0);
+                               SR_PHASE_COMMIT), OP_EQ, 0);
   /* Remove it from state so we can continue our testing. */
   digestmap_remove(state->commits, commit->rsa_identity);
   /* Let's remove our reveal value which should make it OK to keep it. */
   memset(commit->encoded_reveal, 0, sizeof(commit->encoded_reveal));
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_COMMIT), ==, 1);
+                               SR_PHASE_COMMIT), OP_EQ, 1);
 
   /* Let's reset our commit and go into REVEAL phase. */
   sr_commit_free(commit);
@@ -1165,17 +1262,17 @@ test_keep_commit(void *arg)
   memset(dup_commit->encoded_reveal, 0, sizeof(dup_commit->encoded_reveal));
   set_sr_phase(SR_PHASE_REVEAL);
   /* We should never keep a commit from a non authoritative authority. */
-  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_REVEAL), ==, 0);
+  tt_int_op(should_keep_commit(commit, fp, SR_PHASE_REVEAL), OP_EQ, 0);
   /* We shouldn't accept a commit that is not in our state. */
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 0);
+                               SR_PHASE_REVEAL), OP_EQ, 0);
   /* Important to add the commit _without_ the reveal here. */
   sr_state_add_commit(dup_commit);
-  tt_int_op(digestmap_size(state->commits), ==, 1);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
   /* Our commit should be valid that is authoritative, contains a reveal, be
    * in the state and commitment and reveal values match. */
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 1);
+                               SR_PHASE_REVEAL), OP_EQ, 1);
   /* The commit shouldn't be kept if it's not verified that is no matchin
    * hashed reveal. */
   {
@@ -1186,7 +1283,7 @@ test_keep_commit(void *arg)
     memset(commit->hashed_reveal, 0, sizeof(commit->hashed_reveal));
     setup_full_capture_of_logs(LOG_WARN);
     tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                                 SR_PHASE_REVEAL), ==, 0);
+                                 SR_PHASE_REVEAL), OP_EQ, 0);
     expect_log_msg_containing("doesn't match the commit value.");
     expect_log_msg_containing("has an invalid reveal value.");
     assert_log_predicate(mock_saved_log_n_entries() == 2,
@@ -1197,11 +1294,11 @@ test_keep_commit(void *arg)
   }
   /* We shouldn't keep a commit that has no reveal. */
   tt_int_op(should_keep_commit(dup_commit, dup_commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 0);
+                               SR_PHASE_REVEAL), OP_EQ, 0);
   /* We must not keep a commit that is not the same from the commit phase. */
   memset(commit->encoded_commit, 0, sizeof(commit->encoded_commit));
   tt_int_op(should_keep_commit(commit, commit->rsa_identity,
-                               SR_PHASE_REVEAL), ==, 0);
+                               SR_PHASE_REVEAL), OP_EQ, 0);
 
  done:
   teardown_capture_of_logs();
@@ -1239,39 +1336,39 @@ test_state_update(void *arg)
   /* We are in COMMIT phase here and we'll trigger a state update but no
    * transition. */
   sr_state_update(commit_phase_time);
-  tt_int_op(state->valid_after, ==, commit_phase_time);
-  tt_int_op(state->n_commit_rounds, ==, 1);
-  tt_int_op(state->phase, ==, SR_PHASE_COMMIT);
-  tt_int_op(digestmap_size(state->commits), ==, 1);
+  tt_int_op(state->valid_after, OP_EQ, commit_phase_time);
+  tt_int_op(state->n_commit_rounds, OP_EQ, 1);
+  tt_int_op(state->phase, OP_EQ, SR_PHASE_COMMIT);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
 
   /* We are still in the COMMIT phase here but we'll trigger a state
    * transition to the REVEAL phase. */
   sr_state_update(reveal_phase_time);
-  tt_int_op(state->phase, ==, SR_PHASE_REVEAL);
-  tt_int_op(state->valid_after, ==, reveal_phase_time);
+  tt_int_op(state->phase, OP_EQ, SR_PHASE_REVEAL);
+  tt_int_op(state->valid_after, OP_EQ, reveal_phase_time);
   /* Only our commit should be in there. */
-  tt_int_op(digestmap_size(state->commits), ==, 1);
-  tt_int_op(state->n_reveal_rounds, ==, 1);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
+  tt_int_op(state->n_reveal_rounds, OP_EQ, 1);
 
   /* We can't update a state with a valid after _lower_ than the creation
    * time so here it is. */
   sr_state_update(commit_phase_time);
-  tt_int_op(state->valid_after, ==, reveal_phase_time);
+  tt_int_op(state->valid_after, OP_EQ, reveal_phase_time);
 
   /* Finally, let's go back in COMMIT phase so we can test the state update
    * of a new protocol run. */
   state->valid_after = 0;
   sr_state_update(commit_phase_time);
-  tt_int_op(state->valid_after, ==, commit_phase_time);
-  tt_int_op(state->n_commit_rounds, ==, 1);
-  tt_int_op(state->n_reveal_rounds, ==, 0);
-  tt_u64_op(state->n_protocol_runs, ==, 1);
-  tt_int_op(state->phase, ==, SR_PHASE_COMMIT);
-  tt_int_op(digestmap_size(state->commits), ==, 1);
+  tt_int_op(state->valid_after, OP_EQ, commit_phase_time);
+  tt_int_op(state->n_commit_rounds, OP_EQ, 1);
+  tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
+  tt_u64_op(state->n_protocol_runs, OP_EQ, 1);
+  tt_int_op(state->phase, OP_EQ, SR_PHASE_COMMIT);
+  tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
   tt_assert(state->current_srv);
 
  done:
-  sr_state_free();
+  sr_state_free_all();
   UNMOCK(get_my_v3_authority_cert);
 }
 
@@ -1284,7 +1381,11 @@ struct testcase_t sr_tests[] = {
     NULL, NULL },
   { "encoding", test_encoding, TT_FORK,
     NULL, NULL },
-  { "get_next_valid_after_time", test_get_next_valid_after_time, TT_FORK,
+  { "get_start_time_of_current_run", test_get_start_time_of_current_run,
+    TT_FORK, NULL, NULL },
+  { "get_start_time_functions", test_get_start_time_functions,
+    TT_FORK, NULL, NULL },
+  { "get_sr_protocol_duration", test_get_sr_protocol_duration, TT_FORK,
     NULL, NULL },
   { "get_state_valid_until_time", test_get_state_valid_until_time, TT_FORK,
     NULL, NULL },
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index 7c9f0b1cc2..e640702499 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 62ff12fe15..8da7191e82 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -1,12 +1,14 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
 #include "buffers.h"
 #include "config.h"
+#include "proto_socks.h"
 #include "test.h"
+#include "log_test_helpers.h"
 
 typedef struct socks_test_data_t {
   socks_request_t *req;
@@ -43,7 +45,7 @@ static const struct testcase_setup_t socks_setup = {
   buf_t *buf = testdata->buf;                   \
   socks_request_t *socks = testdata->req;
 #define ADD_DATA(buf, s)                                        \
-  write_to_buf(s, sizeof(s)-1, buf)
+  buf_add(buf, s, sizeof(s)-1)
 
 static void
 socks_request_clear(socks_request_t *socks)
@@ -61,8 +63,9 @@ test_socks_4_unsupported_commands(void *ptr)
 
   /* SOCKS 4 Send BIND [02] to IP address 2.2.2.2:4369 */
   ADD_DATA(buf, "\x04\x02\x11\x11\x02\x02\x02\x02\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == -1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
 
@@ -80,8 +83,9 @@ test_socks_4_supported_commands(void *ptr)
 
   /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4370 */
   ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x03\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
   tt_int_op(SOCKS_COMMAND_CONNECT,OP_EQ, socks->command);
@@ -95,8 +99,8 @@ test_socks_4_supported_commands(void *ptr)
 
   /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4369 with userid*/
   ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x04me\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, 1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
   tt_int_op(SOCKS_COMMAND_CONNECT,OP_EQ, socks->command);
@@ -112,8 +116,9 @@ test_socks_4_supported_commands(void *ptr)
 
   /* SOCKS 4a Send RESOLVE [F0] request for torproject.org */
   ADD_DATA(buf, "\x04\xF0\x01\x01\x00\x00\x00\x02me\x00torproject.org\x00");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(4,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen); /* XXX: shouldn't tor reply? */
   tt_str_op("torproject.org",OP_EQ, socks->address);
@@ -124,6 +129,83 @@ test_socks_4_supported_commands(void *ptr)
   ;
 }
 
+static void
+test_socks_4_bad_arguments(void *ptr)
+{
+  SOCKS_TEST_INIT();
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Try with 0 IPv4 address */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x00\x00");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Port or DestIP is zero.");
+  mock_clean_saved_logs();
+
+  /* Try with 0 port */
+  ADD_DATA(buf, "\x04\x01\x00\x00\x01\x02\x03\x04\x00");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Port or DestIP is zero.");
+  mock_clean_saved_logs();
+
+  /* Try with 2000-byte username (!) */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x01\x02\x03\x04");
+  int i;
+  for (i = 0; i < 200; ++i) {
+    ADD_DATA(buf, "1234567890");
+  }
+  ADD_DATA(buf, "\x00");
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("user name too long; rejecting.");
+  mock_clean_saved_logs();
+
+  /* Try with 2000-byte hostname */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x01\x00");
+  for (i = 0; i < 200; ++i) {
+    ADD_DATA(buf, "1234567890");
+  }
+  ADD_DATA(buf, "\x00");
+  {
+    const char *p;
+    size_t s;
+    buf_pullup(buf, 9999, &p, &s);
+  }
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Destaddr too long. Rejecting.");
+  mock_clean_saved_logs();
+
+  /* Try with 2000-byte hostname, not terminated. */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x01\x00");
+  for (i = 0; i < 200; ++i) {
+    ADD_DATA(buf, "1234567890");
+  }
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0),
+            OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Destaddr too long.");
+  mock_clean_saved_logs();
+
+  /* Socks4, bogus hostname */
+  ADD_DATA(buf, "\x04\x01\x00\x50\x00\x00\x00\x01\x00" "---\x00" );
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0), OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Your application (using socks4 to port 80) "
+                            "gave Tor a malformed hostname: ");
+  mock_clean_saved_logs();
+
+ done:
+  teardown_capture_of_logs();
+}
+
 /**  Perform unsupported SOCKS 5 commands */
 static void
 test_socks_5_unsupported_commands(void *ptr)
@@ -199,10 +281,28 @@ test_socks_5_supported_commands(void *ptr)
   tt_int_op(0,OP_EQ, buf_datalen(buf));
   socks_request_clear(socks);
 
+  /* SOCKS 5 Send CONNECT [01] to one of the ipv6 addresses for
+     torproject.org:80 */
+  ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\x01\x00\x04"
+           "\x20\x02\x41\xb8\x02\x02\x0d\xeb\x02\x13\x21\xff\xfe\x20\x14\x26"
+           "\x00\x50");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                   get_options()->SafeSocks),OP_EQ, 1);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+  tt_str_op("[2002:41b8:202:deb:213:21ff:fe20:1426]",OP_EQ, socks->address);
+  tt_int_op(80,OP_EQ, socks->port);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+  socks_request_clear(socks);
+
   /* SOCKS 5 Send CONNECT [01] to FQDN torproject.org:4369 */
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\x01\x00\x03\x0Etorproject.org\x11\x11");
-  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1,
                                    get_options()->SafeSocks),OP_EQ, 1);
 
   tt_int_op(5,OP_EQ, socks->socks_version);
@@ -218,8 +318,9 @@ test_socks_5_supported_commands(void *ptr)
   /* SOCKS 5 Send RESOLVE [F0] request for torproject.org:4369 */
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF0\x00\x03\x0Etorproject.org\x01\x02");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(5,OP_EQ, socks->socks_version);
   tt_int_op(2,OP_EQ, socks->replylen);
   tt_int_op(5,OP_EQ, socks->reply[0]);
@@ -229,47 +330,64 @@ test_socks_5_supported_commands(void *ptr)
   tt_int_op(0,OP_EQ, buf_datalen(buf));
   socks_request_clear(socks);
 
-  /* SOCKS 5 Should reject RESOLVE [F0] request for IPv4 address
+  /* SOCKS 5 Should NOT reject RESOLVE [F0] request for IPv4 address
    * string if SafeSocks is enabled. */
 
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF0\x00\x03\x07");
   ADD_DATA(buf, "8.8.8.8");
-  ADD_DATA(buf, "\x01\x02");
-  tt_assert(fetch_from_buf_socks(buf,socks,get_options()->TestSocks,1)
-            == -1);
+  ADD_DATA(buf, "\x11\x11");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, 1),
+            OP_EQ, 1);
 
-  tt_int_op(5,OP_EQ,socks->socks_version);
-  tt_int_op(10,OP_EQ,socks->replylen);
-  tt_int_op(5,OP_EQ,socks->reply[0]);
-  tt_int_op(SOCKS5_NOT_ALLOWED,OP_EQ,socks->reply[1]);
-  tt_int_op(1,OP_EQ,socks->reply[3]);
+  tt_str_op("8.8.8.8", OP_EQ, socks->address);
+  tt_int_op(4369, OP_EQ, socks->port);
+
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
 
   socks_request_clear(socks);
 
-  /* SOCKS 5 should reject RESOLVE [F0] reject for IPv6 address
+  /* SOCKS 5 should NOT reject RESOLVE [F0] request for IPv6 address
    * string if SafeSocks is enabled. */
 
   ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\xF0\x00\x03\x29");
+  ADD_DATA(buf, "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]");
+  ADD_DATA(buf, "\x01\x02");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, 1),
+            OP_EQ, 1);
+
+  tt_str_op("[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", OP_EQ,
+    socks->address);
+  tt_int_op(258, OP_EQ, socks->port);
+
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
+
+  socks_request_clear(socks);
+
+  /* Also allow bracket-less form. */
+
+  ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF0\x00\x03\x27");
   ADD_DATA(buf, "2001:0db8:85a3:0000:0000:8a2e:0370:7334");
   ADD_DATA(buf, "\x01\x02");
-  tt_assert(fetch_from_buf_socks(buf,socks,get_options()->TestSocks,1)
-            == -1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, 1),
+            OP_EQ, 1);
 
-  tt_int_op(5,OP_EQ,socks->socks_version);
-  tt_int_op(10,OP_EQ,socks->replylen);
-  tt_int_op(5,OP_EQ,socks->reply[0]);
-  tt_int_op(SOCKS5_NOT_ALLOWED,OP_EQ,socks->reply[1]);
-  tt_int_op(1,OP_EQ,socks->reply[3]);
+  tt_str_op("2001:0db8:85a3:0000:0000:8a2e:0370:7334", OP_EQ,
+    socks->address);
+  tt_int_op(258, OP_EQ, socks->port);
+
+  tt_int_op(0, OP_EQ, buf_datalen(buf));
 
   socks_request_clear(socks);
 
   /* SOCKS 5 Send RESOLVE_PTR [F1] for IP address 2.2.2.5 */
   ADD_DATA(buf, "\x05\x01\x00");
   ADD_DATA(buf, "\x05\xF1\x00\x01\x02\x02\x02\x05\x01\x03");
-  tt_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(5,OP_EQ, socks->socks_version);
   tt_int_op(2,OP_EQ, socks->replylen);
   tt_int_op(5,OP_EQ, socks->reply[0]);
@@ -380,9 +498,9 @@ test_socks_5_authenticate_with_data(void *ptr)
   /* SOCKS 5 Send username/password */
   /* SOCKS 5 Send CONNECT [01] to IP address 2.2.2.2:4369 */
   ADD_DATA(buf, "\x01\x02me\x03you\x05\x01\x00\x01\x02\x02\x02\x02\x11\x11");
-  tt_assert(fetch_from_buf_socks(buf, socks,
-                                   get_options()->TestSocks,
-                                   get_options()->SafeSocks) == 1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
   tt_int_op(5,OP_EQ, socks->socks_version);
   tt_int_op(2,OP_EQ, socks->replylen);
   tt_int_op(1,OP_EQ, socks->reply[0]);
@@ -400,6 +518,48 @@ test_socks_5_authenticate_with_data(void *ptr)
   ;
 }
 
+/** Try to negotiate an unsupported authentication type */
+static void
+test_socks_5_auth_unsupported_type(void *ptr)
+{
+  SOCKS_TEST_INIT();
+
+  /* None of these authentication types are recognized. */
+  ADD_DATA(buf, "\x05\x03\x99\x21\x10");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+  tt_int_op(0,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0xff,OP_EQ, socks->reply[1]);
+
+ done:
+  ;
+}
+
+/** Try to negotiate an unsupported version of username/password auth. */
+static void
+test_socks_5_auth_unsupported_version(void *ptr)
+{
+  SOCKS_TEST_INIT();
+
+  /* Negotiate username/password */
+  ADD_DATA(buf, "\x05\x01\x02");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 0);
+  tt_int_op(0,OP_EQ, buf_datalen(buf)); /* buf should be drained */
+  /* Now, suggest an unrecognized username/password version */
+  ADD_DATA(buf, "\x02\x05" "hello" "\x05" "world");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
+
+ done:
+  ;
+}
+
 /** Perform SOCKS 5 authentication before method negotiated */
 static void
 test_socks_5_auth_before_negotiation(void *ptr)
@@ -408,9 +568,9 @@ test_socks_5_auth_before_negotiation(void *ptr)
 
   /* SOCKS 5 Send username/password */
   ADD_DATA(buf, "\x01\x02me\x02me");
-  tt_assert(fetch_from_buf_socks(buf, socks,
-                                   get_options()->TestSocks,
-                                   get_options()->SafeSocks) == -1);
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, -1);
   tt_int_op(0,OP_EQ, socks->socks_version);
   tt_int_op(0,OP_EQ, socks->replylen);
   tt_int_op(0,OP_EQ, socks->reply[0]);
@@ -492,20 +652,397 @@ test_socks_5_malformed_commands(void *ptr)
   ;
 }
 
+static void
+test_socks_5_bad_arguments(void *ptr)
+{
+  SOCKS_TEST_INIT();
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* Socks5, bogus hostname */
+  ADD_DATA(buf, "\x05\x01\x00" "\x05\x01\x00\x03\x03" "---" "\x00\x50" );
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0), OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Your application (using socks5 to port 80) "
+                            "gave Tor a malformed hostname: ");
+  mock_clean_saved_logs();
+  socks_request_clear(socks);
+
+ done:
+  teardown_capture_of_logs();
+}
+
+/** check for correct behavior when the socks command has not arrived. */
+static void
+test_socks_truncated(void *ptr)
+{
+  const struct {
+    enum { NONE, AUTH, ALL } setup;
+    const char *body;
+    size_t len;
+  } commands[] = {
+    /* SOCKS4 */
+    /* Connect, to an IP. */
+    { NONE, "\x04\x01\x05\x05\x01\x02\x03\x04\x00", 9},
+    /* Connect, to an IP, with authentication. */
+    { NONE, "\x04\x01\x05\x05\x01\x02\x03\x04hello\x00", 14},
+    /* SOCKS4A */
+    /* Connect, to a hostname */
+    { NONE, "\x04\x01\x09\x09\x00\x00\x00\x01\x00www.example.com\x00", 25},
+    /* Connect, to a hostname, with authentication */
+    { NONE, "\x04\x01\x09\x09\x00\x00\x00\x01hi\x00www.example.com\x00", 27},
+    /* SOCKS5 */
+    /* initial handshake */
+    { NONE, "\x05\x00", 2 },
+    /* no-auth handshake */
+    { NONE, "\x05\x03\x99\x21\x10", 5 },
+    /* SOCSK5, username-password, all empty. */
+    { AUTH, "\x01\x00\x00", 3 },
+    /* SOCSK5, username-password, 1 char each. */
+    { AUTH, "\x01\x01x\x01y", 5 },
+    /* SOCSK5, username-password, max length. */
+    { AUTH, "\x01\xff"
+      "Ogni tempo ha il suo fascismo: se ne notano i segni premonitori "
+      "dovunque la concentrazione di potere nega al cittadino la "
+      "possibilit\xc3\xa0 e la capacit\xc3\xa0 di esprimere ed attuare la "
+      "sua volont\xc3\xa0. A questo si arriva in molti modi, non "
+      "necessariamente col terror"
+      "\xff"
+      "e dell'intimidazione poliziesca, ma anche negando o distorcendo "
+      "l'informazione, inquinando la giustizia, paralizzando la scuola, "
+      "diffondendo in molti modi sottili la nostalgia per un mondo in cui "
+      "regnava sovrano l'ordine, ed in cui la sicurezza dei pochi "
+      /* privilegiati riposava sul lavoro forzato e sul silenzio forzato dei
+         molti. -- Primo Levi */ , 513 },
+    /* Socks5, IPv4 address */
+    { ALL, "\x05\x01\x00\x01\x01\x02\x03\x04\x20\x20", 10 },
+    /* Socks5, IPv6 address */
+    { ALL, "\x05\x01\x00\x04"
+      "\x49\x20\x48\x41\x5a\x20\x45\x41\x53\x54\x45\x52\x20\x45\x47\x47"
+      "\x20\x20", 22 },
+    /* Socks5, hostname, empty. */
+    { ALL, "\x05\x01\x00\x03" "\x00" "\x00\x50", 7 },
+    /* Socks5, hostname, moderate. */
+    { ALL, "\x05\x01\x00\x03" "\x11" "onion.example.com" "\x00\x50", 24 },
+    /* Socks5, hostname, maximum. */
+    { ALL, "\x05\x01\x00\x03" "\xff"
+      "whatsoever.I.shall.see.or.hear.in.the.course.of.my.profession.as.well."
+      "as.outside.my.profession.in.my.intercourse.with.men.if.it.be.what."
+      "should.not.be.published.abroad.I.will.never.divulge.holding.such."
+      "things.to.be.holy.secrets.x.hippocratic.oath.wikipedia"
+      "\x00\x50", 262 },
+  };
+  unsigned i, j;
+  SOCKS_TEST_INIT();
+  for (i = 0; i < ARRAY_LENGTH(commands); ++i) {
+    for (j = 0; j < commands[i].len; ++j) {
+      switch (commands[i].setup) {
+        default: /* Falls through */
+        case NONE:
+          /* This test calls for no setup on the socks state. */
+          break;
+        case AUTH:
+          /* This test calls for the socks state to be waiting for
+           * username/password authentication */
+          ADD_DATA(buf, "\x05\x01\x02");
+          tt_int_op(0, OP_EQ, fetch_from_buf_socks(buf, socks, 0, 0));
+          tt_int_op(0, OP_EQ, buf_datalen(buf));
+          break;
+        case ALL:
+          /* This test calls for the socks state to be waiting for
+           * the connection request */
+          ADD_DATA(buf, "\x05\x01\x00");
+          tt_int_op(0, OP_EQ, fetch_from_buf_socks(buf, socks, 0, 0));
+          tt_int_op(0, OP_EQ, buf_datalen(buf));
+      }
+
+      TT_BLATHER(("Checking command %u, length %u, omitting char %u", i, j,
+                  (unsigned)commands[i].body[j]));
+      buf_add(buf, commands[i].body, j);
+      /* This should return 0 meaning "not done yet" */
+      tt_int_op(0, OP_EQ, fetch_from_buf_socks(buf, socks, 0, 0));
+      tt_uint_op(j, OP_EQ, buf_datalen(buf)); /* Nothing was drained */
+      buf_clear(buf);
+      socks_request_free(testdata->req);
+      socks = testdata->req = socks_request_new();
+    }
+  }
+  done:
+  ;
+}
+
+static void
+test_socks_wrong_protocol(void *ptr)
+{
+  SOCKS_TEST_INIT();
+  setup_capture_of_logs(LOG_DEBUG);
+
+  /* HTTP request. */
+  ADD_DATA(buf, "GET /index.html HTTP/1.0" );
+  tt_int_op(fetch_from_buf_socks(buf, socks, 1, 0), OP_EQ, -1);
+  buf_clear(buf);
+  expect_log_msg_containing("Socks version 71 not recognized. "
+                            "(This port is not an HTTP proxy;");
+  mock_clean_saved_logs();
+  socks_request_clear(socks);
+
+ done:
+  teardown_capture_of_logs();
+}
+
+/* Check our client-side socks4 parsing (that is to say, our parsing of
+ * server responses).
+ */
+static void
+test_socks_client_v4(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+  /* Legit socks4 response, success */
+  ADD_DATA(buf, "\x04\x5a\x20\x25\x01\x02\x03\x04");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf, PROXY_SOCKS4_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks4 response, failure. */
+  ADD_DATA(buf, "\x04\x5b\x20\x25\x01\x02\x03\x04");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf, PROXY_SOCKS4_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "server rejected connection");
+
+ done:
+  buf_free(buf);
+  tor_free(reason);
+}
+
+/* Check our client-side socks5 authentication-negotiation parsing (that is to
+ * say, our parsing of server responses).
+ */
+static void
+test_socks_client_v5_auth(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+  /* Legit socks5 responses, got a method we like. */
+  ADD_DATA(buf, "\x05\x00");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_NONE,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Same, but we wanted something else. */
+  ADD_DATA(buf, "\x05\x00");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Same, and they offered a password. */
+  ADD_DATA(buf, "\x05\x02");
+  tt_int_op(2, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* They rejected our method, or selected something we don't know. */
+  ADD_DATA(buf, "\x05\xff");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_NONE,
+                                        &reason));
+  tt_str_op(reason, OP_EQ, "server doesn't support any of our available "
+            "authentication methods");
+  buf_clear(buf);
+  tor_free(reason);
+  ADD_DATA(buf, "\x05\xff");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929,
+                                        &reason));
+  tt_str_op(reason, OP_EQ, "server doesn't support any of our available "
+            "authentication methods");
+  tor_free(reason);
+  buf_clear(buf);
+
+  /* Now check for authentication responses: check success and failure. */
+  ADD_DATA(buf, "\x01\x00");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_RFC1929_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  ADD_DATA(buf, "\x01\xf0");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_AUTH_RFC1929_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "authentication failed");
+
+ done:
+  buf_free(buf);
+  tor_free(reason);
+}
+
+/* Check our client-side socks5 connect parsing (that is to say, our parsing
+ * of server responses).
+ */
+static void
+test_socks_client_v5_connect(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+  /* Legit socks5 responses, success, ipv4. */
+  ADD_DATA(buf, "\x05\x00\x00\x01\x01\x02\x03\x04\x00\x05");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks5 responses, success, ipv6. */
+  ADD_DATA(buf, "\x05\x00\x00\x04"
+           "abcdefghijklmnop"
+           "\x00\x05");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks5 responses, success, hostname. */
+  ADD_DATA(buf, "\x05\x00\x00\x03\x12"
+           "gopher.example.com"
+           "\x00\x05");
+  tt_int_op(1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_EQ, NULL);
+  tt_int_op(buf_datalen(buf), OP_EQ, 0);
+
+  /* Legit socks5 responses, failure, hostname. */
+  ADD_DATA(buf, "\x05\x03\x00\x03\x12"
+           "gopher.example.com"
+           "\x00\x05");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "Network unreachable");
+  tor_free(reason);
+  buf_clear(buf);
+
+  /* Bogus socks5 responses: what is address type 0x17? */
+  ADD_DATA(buf, "\x05\x03\x00\x17\x12 blah blah");
+  tt_int_op(-1, OP_EQ,
+            fetch_from_buf_socks_client(buf,
+                                        PROXY_SOCKS5_WANT_CONNECT_OK,
+                                        &reason));
+  tt_ptr_op(reason, OP_NE, NULL);
+  tt_str_op(reason, OP_EQ, "invalid response to connect request");
+  buf_clear(buf);
+
+ done:
+  buf_free(buf);
+  tor_free(reason);
+}
+
+static void
+test_socks_client_truncated(void *arg)
+{
+  (void)arg;
+  buf_t *buf = buf_new();
+  char *reason = NULL;
+
+#define S(str) str, (sizeof(str)-1)
+  const struct {
+    int state;
+    const char *body;
+    size_t len;
+  } replies[] = {
+    { PROXY_SOCKS4_WANT_CONNECT_OK, S("\x04\x5a\x20\x25\x01\x02\x03\x04") },
+    { PROXY_SOCKS4_WANT_CONNECT_OK, S("\x04\x5b\x20\x25\x01\x02\x03\x04") },
+    { PROXY_SOCKS5_WANT_AUTH_METHOD_NONE, S("\x05\x00") },
+    { PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929, S("\x05\x00") },
+    { PROXY_SOCKS5_WANT_AUTH_RFC1929_OK, S("\x01\x00") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x00\x00\x01\x01\x02\x03\x04\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x00\x00\x04" "abcdefghijklmnop" "\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x00\x00\x03\x12" "gopher.example.com" "\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x03\x00\x03\x12" "gopher.example.com""\x00\x05") },
+    { PROXY_SOCKS5_WANT_CONNECT_OK,
+      S("\x05\x03\x00\x17") },
+  };
+  unsigned i, j;
+  for (i = 0; i < ARRAY_LENGTH(replies); ++i) {
+    for (j = 0; j < replies[i].len; ++j) {
+      TT_BLATHER(("Checking command %u, length %u", i, j));
+      buf_add(buf, replies[i].body, j);
+      /* This should return 0 meaning "not done yet" */
+      tt_int_op(0, OP_EQ,
+                fetch_from_buf_socks_client(buf, replies[i].state, &reason));
+      tt_uint_op(j, OP_EQ, buf_datalen(buf)); /* Nothing was drained */
+      buf_clear(buf);
+      tt_ptr_op(reason, OP_EQ, NULL);
+    }
+  }
+
+ done:
+  tor_free(reason);
+  buf_free(buf);
+}
+
 #define SOCKSENT(name)                                  \
   { #name, test_socks_##name, TT_FORK, &socks_setup, NULL }
 
 struct testcase_t socks_tests[] = {
   SOCKSENT(4_unsupported_commands),
   SOCKSENT(4_supported_commands),
+  SOCKSENT(4_bad_arguments),
 
   SOCKSENT(5_unsupported_commands),
   SOCKSENT(5_supported_commands),
   SOCKSENT(5_no_authenticate),
+  SOCKSENT(5_auth_unsupported_type),
+  SOCKSENT(5_auth_unsupported_version),
   SOCKSENT(5_auth_before_negotiation),
   SOCKSENT(5_authenticate),
   SOCKSENT(5_authenticate_with_data),
   SOCKSENT(5_malformed_commands),
+  SOCKSENT(5_bad_arguments),
+
+  SOCKSENT(truncated),
+
+  SOCKSENT(wrong_protocol),
+
+  { "client/v4", test_socks_client_v4, TT_FORK, NULL, NULL },
+  { "client/v5_auth", test_socks_client_v5_auth, TT_FORK, NULL, NULL },
+  { "client/v5_connect", test_socks_client_v5_connect, TT_FORK, NULL, NULL },
+  { "client/truncated", test_socks_client_truncated, TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_status.c b/src/test/test_status.c
index a3b1a2af87..b4ca17891b 100644
--- a/src/test/test_status.c
+++ b/src/test/test_status.c
@@ -1,3 +1,6 @@
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
 #define STATUS_PRIVATE
 #define HIBERNATE_PRIVATE
 #define LOG_PRIVATE
@@ -337,7 +340,7 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 5);
+  tt_int_op(CALLED(logv), OP_EQ, 6);
 
   done:
     NS_UNMOCK(tls_get_write_overhead_ratio);
@@ -436,6 +439,16 @@ NS(logv)(int severity, log_domain_mask_t domain,
       tt_ptr_op(strstr(funcname, "rep_hist_log_link_protocol_counts"),
                 OP_NE, NULL);
       break;
+    case 5:
+      tt_int_op(severity, OP_EQ, LOG_NOTICE);
+      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s");
+      tt_str_op(va_arg(ap, char *), OP_EQ,
+                " 0 circuits killed with too many cells.");
+      tt_str_op(va_arg(ap, char *), OP_EQ, " [cc not enabled]");
+      tt_str_op(va_arg(ap, char *), OP_EQ, " [conn not enabled]");
+      tt_str_op(va_arg(ap, char *), OP_EQ, "");
+      break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
       break;
@@ -889,8 +902,8 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
       tt_str_op(format, OP_EQ,
           "Average packaged cell fullness: %2.3f%%. "
           "TLS write overhead: %.f%%");
-      tt_double_op(fabs(va_arg(ap, double) - 50.0), <=, DBL_EPSILON);
-      tt_double_op(fabs(va_arg(ap, double) - 0.0), <=, DBL_EPSILON);
+      tt_double_op(fabs(va_arg(ap, double) - 50.0), OP_LE, DBL_EPSILON);
+      tt_double_op(fabs(va_arg(ap, double) - 0.0), OP_LE, DBL_EPSILON);
       break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
@@ -1039,7 +1052,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
           "Average packaged cell fullness: %2.3f%%. "
           "TLS write overhead: %.f%%");
       tt_int_op(fabs(va_arg(ap, double) - 100.0) <= DBL_EPSILON, OP_EQ, 1);
-      tt_double_op(fabs(va_arg(ap, double) - 100.0), <=, DBL_EPSILON);
+      tt_double_op(fabs(va_arg(ap, double) - 100.0), OP_LE, DBL_EPSILON);
       break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c
new file mode 100644
index 0000000000..26606f9b6e
--- /dev/null
+++ b/src/test/test_storagedir.c
@@ -0,0 +1,376 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "crypto_rand.h"
+#include "storagedir.h"
+#include "test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_storagedir_empty(void *arg)
+{
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  (void)arg;
+
+  tt_int_op(FN_NOENT, OP_EQ, file_status(dirname));
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tt_int_op(FN_DIR, OP_EQ, file_status(dirname));
+
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+  storage_dir_free(d);
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tt_int_op(FN_DIR, OP_EQ, file_status(dirname));
+
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+}
+
+static void
+test_storagedir_basic(void *arg)
+{
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *junk = NULL, *bytes = NULL;
+  const size_t junklen = 1024;
+  char *fname1 = NULL, *fname2 = NULL;
+  const char hello_str[] = "then what are we but cold, alone ... ?";
+  tor_mmap_t *mapping = NULL;
+  (void)arg;
+
+  junk = tor_malloc(junklen);
+  crypto_rand((void*)junk, junklen);
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+  int r;
+  r = storage_dir_save_string_to_file(d, hello_str, 1, &fname1);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(fname1, OP_NE, NULL);
+  tt_u64_op(strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+
+  r = storage_dir_save_bytes_to_file(d, junk, junklen, 1, &fname2);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(fname2, OP_NE, NULL);
+
+  tt_str_op(fname1, OP_NE, fname2);
+
+  tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname1));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname2));
+
+  storage_dir_free(d);
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+  tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname1));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname2));
+
+  size_t n;
+  bytes = storage_dir_read(d, fname2, 1, &n);
+  tt_assert(bytes);
+  tt_u64_op(n, OP_EQ, junklen);
+  tt_mem_op(bytes, OP_EQ, junk, junklen);
+
+  mapping = storage_dir_map(d, fname1);
+  tt_assert(mapping);
+  tt_u64_op(mapping->size, OP_EQ, strlen(hello_str));
+  tt_mem_op(mapping->data, OP_EQ, hello_str, strlen(hello_str));
+
+ done:
+  tor_free(dirname);
+  tor_free(junk);
+  tor_free(bytes);
+  tor_munmap_file(mapping);
+  storage_dir_free(d);
+  tor_free(fname1);
+  tor_free(fname2);
+}
+
+static void
+test_storagedir_deletion(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  char *fn1 = NULL, *fn2 = NULL;
+  char *bytes = NULL;
+  int r;
+  const char str1[] = "There are nine and sixty ways to disguise communiques";
+  const char str2[] = "And rather more than one of them is right";
+
+  // Make sure the directory is there. */
+  d = storage_dir_new(dirname, 10);
+  storage_dir_free(d);
+  d = NULL;
+
+  tor_asprintf(&fn1, "%s/1007", dirname);
+  r = write_str_to_file(fn1, str1, 0);
+  tt_int_op(r, OP_EQ, 0);
+
+  tor_asprintf(&fn2, "%s/1003.tmp", dirname);
+  r = write_str_to_file(fn2, str2, 0);
+  tt_int_op(r, OP_EQ, 0);
+
+  // The tempfile should be deleted the next time we list the directory.
+  d = storage_dir_new(dirname, 10);
+  tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d));
+  tt_int_op(FN_FILE, OP_EQ, file_status(fn1));
+  tt_int_op(FN_NOENT, OP_EQ, file_status(fn2));
+
+  bytes = (char*) storage_dir_read(d, "1007", 1, NULL);
+  tt_str_op(bytes, OP_EQ, str1);
+
+  // Should have no effect; file already gone.
+  storage_dir_remove_file(d, "1003.tmp");
+  tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d));
+
+  // Actually remove a file.
+  storage_dir_remove_file(d, "1007");
+  tt_int_op(FN_NOENT, OP_EQ, file_status(fn1));
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  tor_free(fn1);
+  tor_free(fn2);
+  storage_dir_free(d);
+  tor_free(bytes);
+}
+
+static void
+test_storagedir_full(void *arg)
+{
+  (void)arg;
+
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  const char str[] = "enemies of the peephole";
+  int r;
+
+  d = storage_dir_new(dirname, 3);
+  tt_assert(d);
+
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  // These should fail!
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, -1);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, -1);
+
+  tt_u64_op(strlen(str) * 3, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  storage_dir_free(d);
+}
+
+static void
+test_storagedir_cleaning(void *arg)
+{
+  (void)arg;
+
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  const char str[] =
+    "On a mountain halfway between Reno and Rome / "
+    "We have a machine in a plexiglass dome / "
+    "Which listens and looks into everyone's home."
+    " -- Dr. Seuss";
+  char *fns[8];
+  int r, i;
+
+  memset(fns, 0, sizeof(fns));
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  for (i = 0; i < 8; ++i) {
+    r = storage_dir_save_string_to_file(d, str+i*2, 1, &fns[i]);
+    tt_int_op(r, OP_EQ, 0);
+  }
+
+  /* Now we're going to make sure all the files have distinct mtimes. */
+  time_t now = time(NULL);
+  struct utimbuf ub;
+  ub.actime = now;
+  ub.modtime = now - 1000;
+  for (i = 0; i < 8; ++i) {
+    char *f = NULL;
+    tor_asprintf(&f, "%s/%s", dirname, fns[i]);
+    r = utime(f, &ub);
+    tor_free(f);
+    tt_int_op(r, OP_EQ, 0);
+    ub.modtime += 5;
+  }
+
+  const uint64_t usage_orig = storage_dir_get_usage(d);
+  /* No changes needed if we are already under target. */
+  storage_dir_shrink(d, 1024*1024, 0);
+  tt_u64_op(usage_orig, OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of at least one byte.  This will delete fns[0]. */
+  storage_dir_shrink(d, usage_orig - 1, 0);
+  tt_u64_op(usage_orig, OP_GT, storage_dir_get_usage(d));
+  tt_u64_op(usage_orig - strlen(str), OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of at least two files.  This will delete fns[1] and fns[2]. */
+  storage_dir_shrink(d, 1024*1024, 2);
+  tt_u64_op(usage_orig - strlen(str)*3 + 6, OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of everything. */
+  storage_dir_remove_all(d);
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  storage_dir_free(d);
+  for (i = 0; i < 8; ++i) {
+    tor_free(fns[i]);
+  }
+}
+
+static void
+test_storagedir_save_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL;
+  char *fname = NULL;
+  uint8_t *saved = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  crypto_rand((char *)inp, 8192);
+
+  config_line_append(&labels, "Foo", "bar baz");
+  config_line_append(&labels, "quux", "quuzXxz");
+  const char expected[] =
+    "Foo bar baz\n"
+    "quux quuzXxz\n";
+
+  int r = storage_dir_save_labeled_to_file(d, labels, inp, 8192, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  size_t n;
+  saved = storage_dir_read(d, fname, 1, &n);
+  tt_assert(memchr(saved, '\0', n));
+  tt_str_op((char*)saved, OP_EQ, expected); /* NUL guarantees strcmp works */
+  tt_mem_op(saved+strlen(expected)+1, OP_EQ, inp, 8192);
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  tor_free(saved);
+}
+
+static void
+test_storagedir_read_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL, *labels2 = NULL;
+  char *fname = NULL;
+  tor_mmap_t *map = NULL;
+  uint8_t *as_read = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tor_snprintf((char*)inp, 8192,
+               "Hello world\n"
+               "This is a test\n"
+               "Yadda yadda.\n");
+  size_t bodylen = 8192 - strlen((char*)inp) - 1;
+  crypto_rand((char *)inp+strlen((char*)inp)+1, bodylen);
+
+  int r = storage_dir_save_bytes_to_file(d, inp, 8192, 1, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* Try mapping */
+  const uint8_t *datap = NULL;
+  size_t sz = 0;
+  map = storage_dir_map_labeled(d, fname, &labels, &datap, &sz);
+  tt_assert(map);
+  tt_assert(datap);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(datap, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(labels);
+  tt_str_op(labels->key, OP_EQ, "Hello");
+  tt_str_op(labels->value, OP_EQ, "world");
+  tt_assert(labels->next);
+  tt_str_op(labels->next->key, OP_EQ, "This");
+  tt_str_op(labels->next->value, OP_EQ, "is a test");
+  tt_assert(labels->next->next);
+  tt_str_op(labels->next->next->key, OP_EQ, "Yadda");
+  tt_str_op(labels->next->next->value, OP_EQ, "yadda.");
+  tt_ptr_op(labels->next->next->next, OP_EQ, NULL);
+
+  /* Try reading this time. */
+  sz = 0;
+  as_read = storage_dir_read_labeled(d, fname, &labels2, &sz);
+  tt_assert(as_read);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(as_read, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(config_lines_eq(labels, labels2));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  config_free_lines(labels2);
+  tor_munmap_file(map);
+  tor_free(as_read);
+}
+
+#define ENT(name)                                               \
+  { #name, test_storagedir_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t storagedir_tests[] = {
+  ENT(empty),
+  ENT(basic),
+  ENT(deletion),
+  ENT(full),
+  ENT(cleaning),
+  ENT(save_labeled),
+  ENT(read_labeled),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index e12205bb2e..fe36d8c6e6 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -71,7 +71,7 @@ check_can_bind_low_ports(void)
 
   return -1;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 int
 main(int argc, char **argv)
@@ -83,7 +83,7 @@ main(int argc, char **argv)
 
   fprintf(stderr, "This test is not supported on your OS.\n");
   return 77;
-#else
+#else /* !(defined(_WIN32)) */
   const char *username;
   const char *testname;
   if (argc != 3) {
@@ -174,7 +174,7 @@ main(int argc, char **argv)
         }
         cap_free(caps);
       }
-#endif
+#endif /* defined(HAVE_LINUX_CAPABILITIES) */
       break;
     default:
       fprintf(stderr, "Unsupported test '%s'\n", testname);
@@ -187,6 +187,6 @@ main(int argc, char **argv)
   }
 
   return (okay ? 0 : 1);
-#endif
+#endif /* defined(_WIN32) */
 }
 
diff --git a/src/test/test_threads.c b/src/test/test_threads.c
index 448ab2034b..4f37a40cc7 100644
--- a/src/test/test_threads.c
+++ b/src/test/test_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -139,8 +139,8 @@ test_threads_basic(void *arg)
               !strcmp(strmap_get(thread_test_strmap_, "thread 2"),
                       strmap_get(thread_test_strmap_, "last to run")));
 
-  tt_int_op(thread_fns_failed, ==, 0);
-  tt_int_op(thread_fn_tid1, !=, thread_fn_tid2);
+  tt_int_op(thread_fns_failed, OP_EQ, 0);
+  tt_int_op(thread_fn_tid1, OP_NE, thread_fn_tid2);
 
  done:
   tor_free(s1);
@@ -283,15 +283,15 @@ test_threads_conditionvar(void *arg)
   SPIN();
   tor_mutex_release(ti->mutex);
 
-  tt_int_op(ti->value, ==, 1337);
+  tt_int_op(ti->value, OP_EQ, 1337);
   if (!timeout) {
-    tt_int_op(ti->n_shutdown, ==, 4);
+    tt_int_op(ti->n_shutdown, OP_EQ, 4);
   } else {
     const int GIVE_UP_AFTER_SEC = 30;
     SPIN_UNTIL((ti->n_timeouts == 2 ||
                 time(NULL) >= started_at + GIVE_UP_AFTER_SEC), 10);
-    tt_int_op(ti->n_shutdown, ==, 2);
-    tt_int_op(ti->n_timeouts, ==, 2);
+    tt_int_op(ti->n_shutdown, OP_EQ, 2);
+    tt_int_op(ti->n_timeouts, OP_EQ, 2);
     tor_mutex_release(ti->mutex);
   }
 
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 4febd82ddc..5b5e69b002 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -1,7 +1,8 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
+#define TORTLS_OPENSSL_PRIVATE
 #define LOG_PRIVATE
 #include "orconfig.h"
 
@@ -62,7 +63,7 @@ fake_num_ciphers(void)
 {
   return 0;
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_errno_to_tls_error(void *data)
@@ -135,7 +136,7 @@ test_tortls_tor_tls_new(void *data)
   SSL_CTX_free(client_tls_context->ctx);
   client_tls_context->ctx = NULL;
   tls = tor_tls_new(-1, 0);
-  tt_assert(!tls);
+  tt_ptr_op(tls, OP_EQ, NULL);
 
 #ifndef OPENSSL_OPAQUE
   method = give_me_a_test_method();
@@ -143,8 +144,8 @@ test_tortls_tor_tls_new(void *data)
   method->num_ciphers = fake_num_ciphers;
   client_tls_context->ctx = ctx;
   tls = tor_tls_new(-1, 0);
-  tt_assert(!tls);
-#endif
+  tt_ptr_op(tls, OP_EQ, NULL);
+#endif /* !defined(OPENSSL_OPAQUE) */
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
@@ -202,6 +203,17 @@ test_tortls_tor_tls_get_error(void *data)
 }
 
 static void
+library_init(void)
+{
+#ifdef OPENSSL_1_1_API
+  OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL);
+#else
+  SSL_library_init();
+  SSL_load_error_strings();
+#endif
+}
+
+static void
 test_tortls_get_state_description(void *ignored)
 {
   (void)ignored;
@@ -209,9 +221,7 @@ test_tortls_get_state_description(void *ignored)
   char *buf;
   SSL_CTX *ctx;
 
-  SSL_library_init();
-  SSL_load_error_strings();
-
+  library_init();
   ctx = SSL_CTX_new(SSLv23_method());
 
   buf = tor_malloc_zero(1000);
@@ -273,8 +283,7 @@ test_tortls_get_by_ssl(void *ignored)
   SSL_CTX *ctx;
   SSL *ssl;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
   tor_tls_allocate_tor_tls_object_ex_data_index();
 
   ctx = SSL_CTX_new(SSLv23_method());
@@ -321,8 +330,7 @@ test_tortls_log_one_error(void *ignored)
   SSL_CTX *ctx;
   SSL *ssl = NULL;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
 
   ctx = SSL_CTX_new(SSLv23_method());
   tls = tor_malloc_zero(sizeof(tor_tls_t));
@@ -375,7 +383,7 @@ test_tortls_log_one_error(void *ignored)
   tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_RECORD_TOO_LARGE),
                         LOG_WARN, 0, NULL);
   expect_log_severity(LOG_INFO);
-#endif
+#endif /* !defined(OPENSSL_1_1_API) */
 
   mock_clean_saved_logs();
   tor_tls_log_one_error(tls, ERR_PACK(1, 2, SSL_R_UNKNOWN_PROTOCOL),
@@ -414,8 +422,7 @@ test_tortls_get_error(void *ignored)
   int ret;
   SSL_CTX *ctx;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
 
   ctx = SSL_CTX_new(SSLv23_method());
   setup_capture_of_logs(LOG_INFO);
@@ -489,7 +496,7 @@ test_tortls_get_error(void *ignored)
   tor_free(tls);
   SSL_CTX_free(ctx);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_always_accept_verify_cb(void *ignored)
@@ -515,11 +522,11 @@ test_tortls_x509_cert_free(void *ignored)
   tor_x509_cert_free(cert);
 
   cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  cert->cert = tor_malloc_zero(sizeof(X509));
+  cert->cert = X509_new();
   cert->encoded = tor_malloc_zero(1);
   tor_x509_cert_free(cert);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_x509_cert_get_id_digests(void *ignored)
@@ -546,6 +553,15 @@ test_tortls_x509_cert_get_id_digests(void *ignored)
 }
 
 #ifndef OPENSSL_OPAQUE
+/*
+ * Use only for the matching fake_x509_free() call
+ */
+static X509 *
+fake_x509_malloc(void)
+{
+  return tor_malloc_zero(sizeof(X509));
+}
+
 static void
 fake_x509_free(X509 *cert)
 {
@@ -642,7 +658,7 @@ test_tortls_cert_get_key(void *ignored)
   crypto_pk_t *res = NULL;
   cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
   X509 *key = NULL;
-  key = tor_malloc_zero(sizeof(X509));
+  key = fake_x509_malloc();
   key->references = 1;
 
   res = tor_tls_cert_get_key(cert);
@@ -662,7 +678,7 @@ test_tortls_cert_get_key(void *ignored)
   tor_free(cert);
   crypto_pk_free(res);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_get_my_client_auth_key(void *ignored)
@@ -744,7 +760,7 @@ get_cipher_by_name(const char *name)
 
   return NULL;
 }
-#endif
+#endif /* !defined(HAVE_SSL_GET_CLIENT_CIPHERS) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -792,8 +808,8 @@ test_tortls_classify_client_ciphers(void *ignored)
   STACK_OF(SSL_CIPHER) *ciphers;
   SSL_CIPHER *tmp_cipher;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
+
   tor_tls_allocate_tor_tls_object_ex_data_index();
 
   tls = tor_malloc_zero(sizeof(tor_tls_t));
@@ -841,8 +857,10 @@ test_tortls_classify_client_ciphers(void *ignored)
   sk_SSL_CIPHER_zero(ciphers);
 
   one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
+  tt_assert(one);
   one->id = 0x00ff;
   two = get_cipher_by_name("ECDHE-RSA-AES128-GCM-SHA256");
+  tt_assert(two);
   two->id = 0x0000;
   sk_SSL_CIPHER_push(ciphers, one);
   tls->client_cipher_list_type = 0;
@@ -879,7 +897,7 @@ test_tortls_classify_client_ciphers(void *ignored)
   tor_free(tls);
   SSL_CTX_free(ctx);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_client_is_using_v2_ciphers(void *ignored)
@@ -897,8 +915,7 @@ test_tortls_client_is_using_v2_ciphers(void *ignored)
   SSL_SESSION *sess;
   STACK_OF(SSL_CIPHER) *ciphers;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
 
   ctx = SSL_CTX_new(TLSv1_method());
   ssl = SSL_new(ctx);
@@ -913,6 +930,7 @@ test_tortls_client_is_using_v2_ciphers(void *ignored)
 
   ciphers = sk_SSL_CIPHER_new_null();
   SSL_CIPHER *one = get_cipher_by_name("ECDHE-RSA-AES256-GCM-SHA384");
+  tt_assert(one);
   one->id = 0x00ff;
   sk_SSL_CIPHER_push(ciphers, one);
   sess->ciphers = ciphers;
@@ -921,7 +939,7 @@ test_tortls_client_is_using_v2_ciphers(void *ignored)
  done:
   SSL_free(ssl);
   SSL_CTX_free(ctx);
-#endif
+#endif /* defined(HAVE_SSL_GET_CLIENT_CIPHERS) */
 }
 
 #ifndef OPENSSL_OPAQUE
@@ -937,7 +955,7 @@ fixed_try_to_extract_certs_from_tls(int severity, tor_tls_t *tls,
   *cert_out = fixed_try_to_extract_certs_from_tls_cert_out_result;
   *id_cert_out = fixed_try_to_extract_certs_from_tls_id_cert_out_result;
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static const char* notCompletelyValidCertString =
@@ -956,7 +974,7 @@ static const char* notCompletelyValidCertString =
   "jC9UeuErhaA/zzWi8ewMTFZW/WshOrm3fNvcMrMLKtH534JKvcdMg6qIdjTFINIr\n"
   "evnAhf0cwULaebn+lMs8Pdl7y37+sfluVok=\n"
   "-----END CERTIFICATE-----\n";
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static const char* validCertString = "-----BEGIN CERTIFICATE-----\n"
   "MIIDpTCCAY0CAg3+MA0GCSqGSIb3DQEBBQUAMF4xCzAJBgNVBAYTAlVTMREwDwYD\n"
@@ -1079,7 +1097,7 @@ test_tortls_verify(void *ignored)
   tor_free(tls);
   tor_free(k);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1092,13 +1110,13 @@ test_tortls_check_lifetime(void *ignored)
   time_t now = time(NULL);
 
   tls = tor_malloc_zero(sizeof(tor_tls_t));
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, 0);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   tls->ssl = tor_malloc_zero(sizeof(SSL));
   tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
   tls->ssl->session->peer = validCert;
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, 0);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   ASN1_STRING_free(validCert->cert_info->validity->notBefore);
@@ -1106,10 +1124,10 @@ test_tortls_check_lifetime(void *ignored)
   ASN1_STRING_free(validCert->cert_info->validity->notAfter);
   validCert->cert_info->validity->notAfter = ASN1_TIME_set(NULL, now+60);
 
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, -1000);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), 0, -1000);
   tt_int_op(ret, OP_EQ, -1);
 
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, -1000, 0);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), -1000, 0);
   tt_int_op(ret, OP_EQ, -1);
 
  done:
@@ -1118,7 +1136,7 @@ test_tortls_check_lifetime(void *ignored)
   tor_free(tls);
   X509_free(validCert);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static int fixed_ssl_pending_result = 0;
@@ -1153,7 +1171,7 @@ test_tortls_get_pending_bytes(void *ignored)
   tor_free(tls->ssl);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_get_forced_write_size(void *ignored)
@@ -1276,13 +1294,13 @@ test_tortls_SSL_SESSION_get_master_key(void *ignored)
   tt_int_op(out[0], OP_EQ, 43);
 
  done:
-#endif
+#endif /* !defined(HAVE_SSL_SESSION_GET_MASTER_KEY) */
   tor_free(tls->ssl->session);
   tor_free(tls->ssl);
   tor_free(tls);
   tor_free(out);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1290,7 +1308,7 @@ test_tortls_get_tlssecrets(void *ignored)
 {
   (void)ignored;
   int ret;
-  uint8_t *secret_out = tor_malloc_zero(DIGEST256_LEN);;
+  uint8_t *secret_out = tor_malloc_zero(DIGEST256_LEN);
   tor_tls_t *tls;
   tls = tor_malloc_zero(sizeof(tor_tls_t));
   tls->ssl = tor_malloc_zero(sizeof(SSL));
@@ -1308,7 +1326,7 @@ test_tortls_get_tlssecrets(void *ignored)
   tor_free(tls->ssl);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1350,7 +1368,7 @@ test_tortls_get_buffer_sizes(void *ignored)
   tt_int_op(rbuf_c, OP_EQ, 1);
   tt_int_op(wbuf_c, OP_EQ, 2);
 
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
 
  done:
   tor_free(tls->ssl->s3->rbuf.buf);
@@ -1359,7 +1377,7 @@ test_tortls_get_buffer_sizes(void *ignored)
   tor_free(tls->ssl);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_evaluate_ecgroup_for_tls(void *ignored)
@@ -1378,6 +1396,7 @@ test_tortls_evaluate_ecgroup_for_tls(void *ignored)
 
   ret = evaluate_ecgroup_for_tls("P224");
   //  tt_int_op(ret, OP_EQ, 1); This varies between machines
+  tt_assert(ret == 0 || ret == 1);
 
  done:
   (void)0;
@@ -1450,7 +1469,7 @@ test_tortls_try_to_extract_certs_from_tls(void *ignored)
   X509_free(c1);
   X509_free(c2);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1482,7 +1501,7 @@ test_tortls_get_peer_cert(void *ignored)
   tor_free(tls);
   X509_free(cert);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1512,7 +1531,7 @@ test_tortls_peer_has_cert(void *ignored)
   tor_free(tls);
   X509_free(cert);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_is_server(void *ignored)
@@ -1540,8 +1559,8 @@ test_tortls_session_secret_cb(void *ignored)
   STACK_OF(SSL_CIPHER) *ciphers = NULL;
   SSL_CIPHER *one;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
+
   tor_tls_allocate_tor_tls_object_ex_data_index();
 
   tls = tor_malloc_zero(sizeof(tor_tls_t));
@@ -1572,7 +1591,7 @@ test_tortls_session_secret_cb(void *ignored)
   SSL_CTX_free(ctx);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 /* TODO: It seems block_renegotiation and unblock_renegotiation and
@@ -1623,7 +1642,7 @@ test_tortls_unblock_renegotiation(void *ignored)
   tor_free(tls->ssl);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1642,7 +1661,7 @@ test_tortls_assert_renegotiation_unblocked(void *ignored)
   tor_free(tls->ssl);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static void
 test_tortls_set_logged_address(void *ignored)
@@ -1697,7 +1716,7 @@ test_tortls_set_renegotiate_callback(void *ignored)
   tor_free(tls->ssl);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static SSL_CIPHER *fixed_cipher1 = NULL;
@@ -1715,7 +1734,7 @@ fake_get_cipher(unsigned ncipher)
     return NULL;
   }
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1732,8 +1751,7 @@ test_tortls_find_cipher_by_id(void *ignored)
   fixed_cipher2 = tor_malloc_zero(sizeof(SSL_CIPHER));
   fixed_cipher2->id = 0xC00A;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
 
   ctx = SSL_CTX_new(m);
   ssl = SSL_new(ctx);
@@ -1785,7 +1803,7 @@ test_tortls_find_cipher_by_id(void *ignored)
   SSL_CTX_free(ctx);
   tor_free(fixed_cipher1);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1813,7 +1831,7 @@ test_tortls_debug_state_callback(void *ignored)
   tor_free(buf);
   tor_free(ssl);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -1824,8 +1842,7 @@ test_tortls_server_info_callback(void *ignored)
   SSL_CTX *ctx;
   SSL *ssl;
 
-  SSL_library_init();
-  SSL_load_error_strings();
+  library_init();
 
   ctx = SSL_CTX_new(TLSv1_method());
   ssl = SSL_new(ctx);
@@ -1877,7 +1894,7 @@ test_tortls_server_info_callback(void *ignored)
   SSL_CTX_free(ctx);
   tor_free(tls);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static int fixed_ssl_read_result_index;
@@ -1918,7 +1935,7 @@ setting_version_and_state_ssl_shutdown(SSL *s)
   s->version = SSL2_VERSION;
   return fixed_ssl_shutdown_result;
 }
-#endif
+#endif /* !defined(LIBRESSL_VERSION_NUMBER) */
 
 static int
 dummy_handshake_func(SSL *s)
@@ -2014,7 +2031,7 @@ test_tortls_shutdown(void *ignored)
   method->ssl_shutdown = setting_version_and_state_ssl_shutdown;
   ret = tor_tls_shutdown(tls);
   tt_int_op(ret, OP_EQ, TOR_TLS_ERROR_MISC);
-#endif
+#endif /* !defined(LIBRESSL_VERSION_NUMBER) */
 
  done:
   teardown_capture_of_logs();
@@ -2085,7 +2102,7 @@ test_tortls_read(void *ignored)
   ret = tor_tls_read(tls, buf, 10);
   tt_int_op(ret, OP_EQ, TOR_TLS_CLOSE);
   tt_int_op(tls->state, OP_EQ, TOR_TLS_ST_CLOSED);
-#endif
+#endif /* !defined(LIBRESSL_VERSION_NUMBER) */
   // TODO: fill up
 
  done:
@@ -2160,7 +2177,7 @@ test_tortls_write(void *ignored)
   tor_free(tls);
   tor_free(method);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static int fixed_ssl_accept_result;
@@ -2240,7 +2257,7 @@ test_tortls_handshake(void *ignored)
             "(null):SSLv3 write client hello B)\n");
   expect_log_msg("TLS error while handshaking: (null) (in system library:"
             "connect:SSLv3 write client hello B)\n");
-#endif
+#endif /* 0 */
   expect_log_severity(LOG_INFO);
 
   tls->isServer = 0;
@@ -2258,7 +2275,7 @@ test_tortls_handshake(void *ignored)
             "(null) (in bignum routines:(null):SSLv3 write client hello B)\n");
   expect_log_msg("TLS error while handshaking: "
             "(null) (in system library:connect:SSLv3 write client hello B)\n");
-#endif
+#endif /* 0 */
   expect_log_severity(LOG_WARN);
 
  done:
@@ -2268,7 +2285,7 @@ test_tortls_handshake(void *ignored)
   tor_free(tls);
   tor_free(method);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
@@ -2343,7 +2360,7 @@ test_tortls_finish_handshake(void *ignored)
   tor_free(method);
   teardown_capture_of_logs();
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static int fixed_crypto_pk_new_result_index;
 static crypto_pk_t *fixed_crypto_pk_new_result[5];
@@ -2468,8 +2485,8 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_generate_key_with_bits_result[1] = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
   fixed_tor_tls_create_certificate_result[0] = NULL;
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
   ret = tor_tls_context_new(NULL, 0, 0, 0);
   tt_assert(!ret);
 
@@ -2479,9 +2496,9 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_new_result[2] = NULL;
   fixed_crypto_pk_generate_key_with_bits_result_index = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
   fixed_tor_tls_create_certificate_result[1] = NULL;
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
   ret = tor_tls_context_new(NULL, 0, 0, 0);
   tt_assert(!ret);
 
@@ -2491,8 +2508,8 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_new_result[2] = NULL;
   fixed_crypto_pk_generate_key_with_bits_result_index = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
   fixed_tor_tls_create_certificate_result[2] = NULL;
   ret = tor_tls_context_new(NULL, 0, 0, 0);
   tt_assert(!ret);
@@ -2504,9 +2521,9 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_new_result[2] = NULL;
   fixed_crypto_pk_generate_key_with_bits_result_index = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
   fixed_tor_x509_cert_new_result_index = 0;
   fixed_tor_x509_cert_new_result[0] = NULL;
   fixed_tor_x509_cert_new_result[1] = NULL;
@@ -2520,9 +2537,9 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_new_result[2] = NULL;
   fixed_crypto_pk_generate_key_with_bits_result_index = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
   fixed_tor_x509_cert_new_result_index = 0;
   fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
   fixed_tor_x509_cert_new_result[1] = NULL;
@@ -2536,9 +2553,9 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_new_result[2] = NULL;
   fixed_crypto_pk_generate_key_with_bits_result_index = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
   fixed_tor_x509_cert_new_result_index = 0;
   fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
   fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t));
@@ -2552,9 +2569,9 @@ test_tortls_context_new(void *ignored)
   fixed_crypto_pk_new_result[2] = NULL;
   fixed_crypto_pk_generate_key_with_bits_result_index = 0;
   fixed_tor_tls_create_certificate_result_index = 0;
-  fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509));
-  fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509));
+  fixed_tor_tls_create_certificate_result[0] = X509_new();
+  fixed_tor_tls_create_certificate_result[1] = X509_new();
+  fixed_tor_tls_create_certificate_result[2] = X509_new();
   fixed_tor_x509_cert_new_result_index = 0;
   fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t));
   fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t));
@@ -2568,7 +2585,7 @@ test_tortls_context_new(void *ignored)
   UNMOCK(crypto_pk_generate_key_with_bits);
   UNMOCK(crypto_pk_new);
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 static int fixed_crypto_pk_get_evp_pkey_result_index = 0;
 static EVP_PKEY *fixed_crypto_pk_get_evp_pkey_result[5];
@@ -2637,7 +2654,7 @@ test_tortls_cert_new(void *ignored)
   X509_get_pubkey(cert)->type = EVP_PKEY_DSA;
   ret = tor_x509_cert_new(cert);
   tt_assert(ret);
-#endif
+#endif /* 0 */
 
 #ifndef OPENSSL_OPAQUE
   cert = read_cert_from(validCertString);
@@ -2645,7 +2662,7 @@ test_tortls_cert_new(void *ignored)
   cert->cert_info = NULL;
   ret = tor_x509_cert_new(cert);
   tt_assert(ret);
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
  done:
   tor_x509_cert_free(ret);
@@ -2659,18 +2676,18 @@ test_tortls_cert_is_valid(void *ignored)
   tor_x509_cert_t *cert = NULL, *scert = NULL;
 
   scert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
 
   cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(scert);
   tor_free(cert);
 
   cert = tor_x509_cert_new(read_cert_from(validCertString));
   scert = tor_x509_cert_new(read_cert_from(caCertString));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 1);
 
 #ifndef OPENSSL_OPAQUE
@@ -2681,7 +2698,7 @@ test_tortls_cert_is_valid(void *ignored)
   ASN1_TIME_free(cert->cert->cert_info->validity->notAfter);
   cert->cert->cert_info->validity->notAfter =
     ASN1_TIME_set(NULL, time(NULL)-1000000);
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
 
   tor_x509_cert_free(cert);
@@ -2690,9 +2707,9 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   X509_PUBKEY_free(cert->cert->cert_info->key);
   cert->cert->cert_info->key = NULL;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
   tt_int_op(ret, OP_EQ, 0);
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #if 0
   tor_x509_cert_free(cert);
@@ -2701,7 +2718,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   /* This doesn't actually change the key in the cert. XXXXXX */
   BN_one(EVP_PKEY_get1_RSA(X509_get_pubkey(cert->cert))->n);
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
   tt_int_op(ret, OP_EQ, 0);
 
   tor_x509_cert_free(cert);
@@ -2710,7 +2727,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   /* This doesn't actually change the key in the cert. XXXXXX */
   X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
   tt_int_op(ret, OP_EQ, 0);
 
   tor_x509_cert_free(cert);
@@ -2719,7 +2736,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   /* This doesn't actually change the key in the cert. XXXXXX */
   X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 1);
 
   tor_x509_cert_free(cert);
@@ -2729,9 +2746,9 @@ test_tortls_cert_is_valid(void *ignored)
   /* This doesn't actually change the key in the cert. XXXXXX */
   X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
   X509_get_pubkey(cert->cert)->ameth = NULL;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
-#endif
+#endif /* 0 */
 
  done:
   tor_x509_cert_free(cert);
@@ -2764,7 +2781,7 @@ test_tortls_context_init_one(void *ignored)
   { #name, NULL, TT_SKIP, NULL, NULL }
 #else
 #define INTRUSIVE_TEST_CASE(name, flags) LOCAL_TEST_CASE(name, flags)
-#endif
+#endif /* defined(OPENSSL_OPAQUE) */
 
 struct testcase_t tortls_tests[] = {
   LOCAL_TEST_CASE(errno_to_tls_error, 0),
diff --git a/src/test/test_util.c b/src/test/test_util.c
index dd122b250d..ec11bfd5f5 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -9,12 +9,15 @@
 #define CONTROL_PRIVATE
 #define UTIL_PRIVATE
 #include "or.h"
+#include "buffers.h"
 #include "config.h"
 #include "control.h"
+#include "crypto_rand.h"
 #include "test.h"
 #include "memarea.h"
 #include "util_process.h"
 #include "log_test_helpers.h"
+#include "compress_zstd.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -69,7 +72,7 @@ test_util_read_until_eof_impl(const char *fname, size_t file_len,
   fd = open(fifo_name, O_RDONLY|O_BINARY);
   tt_int_op(fd, OP_GE, 0);
   str = read_file_to_str_until_eof(fd, read_limit, &sz);
-  tt_assert(str != NULL);
+  tt_ptr_op(str, OP_NE, NULL);
 
   if (read_limit < file_len)
     tt_int_op(sz, OP_EQ, read_limit);
@@ -366,7 +369,7 @@ test_util_time(void *arg)
    * calculations internally, then catches the overflow. */
 #define TV_SEC_MAX TIME_MAX
 #define TV_SEC_MIN TIME_MIN
-#endif
+#endif /* defined(_WIN32) */
 
 /* Assume tv_usec is an unsigned integer until proven otherwise */
 #define TV_USEC_MAX UINT_MAX
@@ -634,13 +637,16 @@ test_util_time(void *arg)
    * time_t */
   a_time.tm_year = 2039-1900;
 #if SIZEOF_TIME_T == 4
+  setup_full_capture_of_logs(LOG_WARN);
   tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
+  expect_single_log_msg_containing("Result does not fit in tor_timegm");
+  teardown_capture_of_logs();
 #elif SIZEOF_TIME_T == 8
   t_res = 2178252895UL;
   tt_int_op(t_res, OP_EQ, tor_timegm(&a_time));
   tor_gmtime_r(&t_res, &b_time);
   TM_EQUAL(a_time, b_time);
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   /* Test tor_timegm out of range */
 
@@ -650,8 +656,7 @@ test_util_time(void *arg)
     setup_full_capture_of_logs(LOG_WARN);                       \
   } while (0)
 #define CHECK_TIMEGM_WARNING(msg) do { \
-    expect_log_msg_containing(msg);                                     \
-    tt_int_op(1, OP_EQ, smartlist_len(mock_saved_logs()));              \
+    expect_single_log_msg_containing(msg);                              \
     teardown_capture_of_logs();                                         \
   } while (0)
 
@@ -688,7 +693,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 4 || SIZEOF_INT == 8 */
 
 #if SIZEOF_INT == 8
     a_time.tm_year = -1*(1 << 48);
@@ -700,7 +705,7 @@ test_util_time(void *arg)
      * a "correct" retrospective gregorian negative year value,
      * which I'm pretty sure is:
      * -1*(2^63)/60/60/24*2000/730485 + 1970 = -292277022657
-     * 730485 is the number of days in two millenia, including leap days */
+     * 730485 is the number of days in two millennia, including leap days */
     a_time.tm_year = -292277022657-1900;
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
@@ -710,7 +715,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 8 */
 
   /* Wrong year >= INT32_MAX - 1900 */
 #if SIZEOF_INT == 4 || SIZEOF_INT == 8
@@ -723,7 +728,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 4 || SIZEOF_INT == 8 */
 
 #if SIZEOF_INT == 8
     /* one of the largest tm_year values my 64 bit system supports */
@@ -736,7 +741,7 @@ test_util_time(void *arg)
      * a "correct" proleptic gregorian year value,
      * which I'm pretty sure is:
      * (2^63-1)/60/60/24*2000/730485 + 1970 = 292277026596
-     * 730485 is the number of days in two millenia, including leap days */
+     * 730485 is the number of days in two millennia, including leap days */
     a_time.tm_year = 292277026596-1900;
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
@@ -751,7 +756,7 @@ test_util_time(void *arg)
     CAPTURE();
     tt_int_op((time_t) -1,OP_EQ, tor_timegm(&a_time));
     CHECK_TIMEGM_ARG_OUT_OF_RANGE();
-#endif
+#endif /* SIZEOF_INT == 8 */
 
   /* month */
   a_time.tm_year = 2007-1900;  /* restore valid year */
@@ -872,7 +877,7 @@ test_util_time(void *arg)
      * a "correct" retrospective gregorian negative year value,
      * which I'm pretty sure is:
      * -1*(2^63)/60/60/24*2000/730485 + 1970 = -292277022657
-     * 730485 is the number of days in two millenia, including leap days
+     * 730485 is the number of days in two millennia, including leap days
      * (int64_t)b_time.tm_year == (-292277022657LL-1900LL) without clamping */
     t_res = INT64_MIN;
     CAPTURE();
@@ -887,7 +892,7 @@ test_util_time(void *arg)
       teardown_capture_of_logs();
     }
   }
-#endif
+#endif /* SIZEOF_TIME_T == 8 */
 
   /* time_t >= INT_MAX yields a year clamped to 2037 or 9999,
    * depending on whether the implementation of the system gmtime(_r)
@@ -903,7 +908,7 @@ test_util_time(void *arg)
     tt_assert(b_time.tm_year == (2037-1900) ||
               b_time.tm_year == (2038-1900));
   }
-#endif
+#endif /* SIZEOF_TIME_T == 4 || SIZEOF_TIME_T == 8 */
 
 #if SIZEOF_TIME_T == 8
   {
@@ -918,7 +923,7 @@ test_util_time(void *arg)
      * a "correct" proleptic gregorian year value,
      * which I'm pretty sure is:
      * (2^63-1)/60/60/24*2000/730485 + 1970 = 292277026596
-     * 730485 is the number of days in two millenia, including leap days
+     * 730485 is the number of days in two millennia, including leap days
      * (int64_t)b_time.tm_year == (292277026596L-1900L) without clamping */
     t_res = INT64_MAX;
     CAPTURE();
@@ -928,7 +933,7 @@ test_util_time(void *arg)
     tt_assert(b_time.tm_year == (2037-1900) ||
               b_time.tm_year == (9999-1900));
   }
-#endif
+#endif /* SIZEOF_TIME_T == 8 */
 
   /* Test {format,parse}_rfc1123_time */
 
@@ -964,7 +969,9 @@ test_util_time(void *arg)
   strlcpy(timestr, "Wed, 17 Feb 2038 06:13:20 GMT", sizeof(timestr));
 
   t_res = 0;
+  CAPTURE();
   i = parse_rfc1123_time(timestr, &t_res);
+  CHECK_TIMEGM_WARNING("does not fit in tor_timegm");
   tt_int_op(-1,OP_EQ, i);
 #elif SIZEOF_TIME_T == 8
   tt_str_op("Wed, 17 Feb 2038 06:13:20 GMT",OP_EQ, timestr);
@@ -973,7 +980,7 @@ test_util_time(void *arg)
   i = parse_rfc1123_time(timestr, &t_res);
   tt_int_op(0,OP_EQ, i);
   tt_int_op(t_res,OP_EQ, (time_t)2150000000UL);
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   /* The timezone doesn't matter */
   t_res = 0;
@@ -1039,13 +1046,16 @@ test_util_time(void *arg)
   /* This value is out of range with 32 bit time_t, but in range for 64 bit
    * time_t */
   t_res = 0;
-  i = parse_iso_time("2038-02-17 06:13:20", &t_res);
 #if SIZEOF_TIME_T == 4
+  CAPTURE();
+  i = parse_iso_time("2038-02-17 06:13:20", &t_res);
   tt_int_op(-1,OP_EQ, i);
+  CHECK_TIMEGM_WARNING("does not fit in tor_timegm");
 #elif SIZEOF_TIME_T == 8
+  i = parse_iso_time("2038-02-17 06:13:20", &t_res);
   tt_int_op(0,OP_EQ, i);
   tt_int_op(t_res,OP_EQ, (time_t)2150000000UL);
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-zz 99-99x99", &t_res));
   tt_int_op(-1,OP_EQ, parse_iso_time("2011-03-32 00:00:00", &t_res));
@@ -1059,6 +1069,23 @@ test_util_time(void *arg)
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04 00:48:22.100", &t_res));
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04 00:48:22XYZ", &t_res));
 
+  /* but... that _is_ acceptable if we aren't being strict. */
+  t_res = 0;
+  i = parse_iso_time_("2004-08-04 00:48:22XYZ", &t_res, 0, 0);
+  tt_int_op(0,OP_EQ, i);
+  tt_int_op(t_res,OP_EQ, (time_t)1091580502UL);
+
+  /* try nospace variant. */
+  t_res = 0;
+  i = parse_iso_time_nospace("2004-08-04T00:48:22", &t_res);
+  tt_int_op(0,OP_EQ, i);
+  tt_int_op(t_res,OP_EQ, (time_t)1091580502UL);
+
+  tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04T00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time_nospace("2004-08-04 00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04x00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time_nospace("2004-08-04x00:48:22", &t_res));
+
   /* Test tor_gettimeofday */
 
   end.tv_sec = 4;
@@ -1111,7 +1138,7 @@ test_util_time(void *arg)
   /* This SHOULD work on windows too; see bug #18665 */
   tt_str_op("2038-02-17 06:13:20",OP_EQ, timestr);
 #endif
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
 #undef CAPTURE
 #undef CHECK_TIMEGM_ARG_OUT_OF_RANGE
@@ -1200,13 +1227,16 @@ test_util_parse_http_time(void *arg)
 #if SIZEOF_TIME_T == 4
   /* parse_http_time should indicate failure on overflow, but it doesn't yet.
    * Hopefully #18480 will improve the failure semantics in this case. */
+  setup_full_capture_of_logs(LOG_WARN);
   tt_int_op(0,OP_EQ,parse_http_time("Wed, 17 Feb 2038 06:13:20 GMT", &a_time));
   tt_int_op((time_t)-1,OP_EQ, tor_timegm(&a_time));
+  expect_single_log_msg_containing("does not fit in tor_timegm");
+  teardown_capture_of_logs();
 #elif SIZEOF_TIME_T == 8
   tt_int_op(0,OP_EQ,parse_http_time("Wed, 17 Feb 2038 06:13:20 GMT", &a_time));
   tt_int_op((time_t)2150000000UL,OP_EQ, tor_timegm(&a_time));
   T("2038-02-17 06:13:20");
-#endif
+#endif /* SIZEOF_TIME_T == 4 || ... */
 
   tt_int_op(-1,OP_EQ, parse_http_time("2004-08-zz 99-99x99 GMT", &a_time));
   tt_int_op(-1,OP_EQ, parse_http_time("2011-03-32 00:00:00 GMT", &a_time));
@@ -1219,7 +1249,7 @@ test_util_parse_http_time(void *arg)
 
 #undef T
  done:
-  ;
+  teardown_capture_of_logs();
 }
 
 static void
@@ -1453,7 +1483,7 @@ test_util_config_line_comment_character(void *arg)
   tor_free(k); tor_free(v);
 
   test_streq(str, "");
-#endif
+#endif /* 0 */
 
  done:
   tor_free(k);
@@ -1584,7 +1614,7 @@ test_util_config_line_escaped_content(void *arg)
   str = parse_config_line_from_str_verbose(str, &k, &v, NULL);
   tt_ptr_op(str, OP_EQ, NULL);
   tor_free(k); tor_free(v);
-#endif
+#endif /* 0 */
 
   str = buf6;
 
@@ -1656,14 +1686,14 @@ test_util_config_line_crlf(void *arg)
   tt_assert(str);
   tt_str_op(k,OP_EQ,"Hello");
   tt_str_op(v,OP_EQ,"world");
-  tt_assert(!err);
+  tt_ptr_op(err, OP_EQ, NULL);
   tor_free(k); tor_free(v);
 
   str = parse_config_line_from_str_verbose(str, &k, &v, &err);
   tt_assert(str);
   tt_str_op(k,OP_EQ,"Hello");
   tt_str_op(v,OP_EQ,"nice big world");
-  tt_assert(!err);
+  tt_ptr_op(err, OP_EQ, NULL);
   tor_free(k); tor_free(v);
   tt_str_op(str,OP_EQ, "");
 
@@ -1768,7 +1798,7 @@ test_util_expand_filename(void *arg)
  done:
   tor_free(str);
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 /** Test tor_escape_str_for_pt_args(). */
 static void
@@ -1923,7 +1953,7 @@ test_util_strmisc(void *arg)
   tt_assert(!tor_mem_is_zero(buf, 10));
 
   /* Test 'escaped' */
-  tt_assert(NULL == escaped(NULL));
+  tt_ptr_op(escaped(NULL), OP_EQ, NULL);
   tt_str_op("\"\"",OP_EQ, escaped(""));
   tt_str_op("\"abcd\"",OP_EQ, escaped("abcd"));
   tt_str_op("\"\\\\ \\n\\r\\t\\\"\\'\"",OP_EQ, escaped("\\ \n\r\t\"'"));
@@ -1981,23 +2011,23 @@ test_util_strmisc(void *arg)
   /* Test memmem and memstr */
   {
     const char *haystack = "abcde";
-    tt_assert(!tor_memmem(haystack, 5, "ef", 2));
+    tt_ptr_op(tor_memmem(haystack, 5, "ef", 2), OP_EQ, NULL);
     tt_ptr_op(tor_memmem(haystack, 5, "cd", 2),OP_EQ, haystack + 2);
     tt_ptr_op(tor_memmem(haystack, 5, "cde", 3),OP_EQ, haystack + 2);
-    tt_assert(!tor_memmem(haystack, 4, "cde", 3));
+    tt_ptr_op(tor_memmem(haystack, 4, "cde", 3), OP_EQ, NULL);
     haystack = "ababcad";
     tt_ptr_op(tor_memmem(haystack, 7, "abc", 3),OP_EQ, haystack + 2);
     tt_ptr_op(tor_memmem(haystack, 7, "ad", 2),OP_EQ, haystack + 5);
     tt_ptr_op(tor_memmem(haystack, 7, "cad", 3),OP_EQ, haystack + 4);
-    tt_assert(!tor_memmem(haystack, 7, "dadad", 5));
-    tt_assert(!tor_memmem(haystack, 7, "abcdefghij", 10));
+    tt_ptr_op(tor_memmem(haystack, 7, "dadad", 5), OP_EQ, NULL);
+    tt_ptr_op(tor_memmem(haystack, 7, "abcdefghij", 10), OP_EQ, NULL);
     /* memstr */
     tt_ptr_op(tor_memstr(haystack, 7, "abc"),OP_EQ, haystack + 2);
     tt_ptr_op(tor_memstr(haystack, 7, "cad"),OP_EQ, haystack + 4);
-    tt_assert(!tor_memstr(haystack, 6, "cad"));
-    tt_assert(!tor_memstr(haystack, 7, "cadd"));
-    tt_assert(!tor_memstr(haystack, 7, "fe"));
-    tt_assert(!tor_memstr(haystack, 7, "ababcade"));
+    tt_ptr_op(tor_memstr(haystack, 6, "cad"), OP_EQ, NULL);
+    tt_ptr_op(tor_memstr(haystack, 7, "cadd"), OP_EQ, NULL);
+    tt_ptr_op(tor_memstr(haystack, 7, "fe"), OP_EQ, NULL);
+    tt_ptr_op(tor_memstr(haystack, 7, "ababcade"), OP_EQ, NULL);
   }
 
   /* Test hex_str */
@@ -2107,10 +2137,13 @@ test_util_parse_integer(void *arg)
   /* Base different than 10 */
   tt_int_op(2L,OP_EQ,   tor_parse_long("10",2,0,100,NULL,NULL));
   tt_int_op(0L,OP_EQ,   tor_parse_long("2",2,0,100,NULL,NULL));
-  tt_int_op(0L,OP_EQ,   tor_parse_long("10",-2,0,100,NULL,NULL));
   tt_int_op(68284L,OP_EQ, tor_parse_long("10abc",16,0,70000,NULL,NULL));
   tt_int_op(68284L,OP_EQ, tor_parse_long("10ABC",16,0,70000,NULL,NULL));
+  tor_capture_bugs_(2);
+  tt_int_op(0L,OP_EQ,   tor_parse_long("10",-2,0,100,NULL,NULL));
   tt_int_op(0,OP_EQ, tor_parse_long("10ABC",-1,0,70000,&i,NULL));
+  tt_int_op(2, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
+  tor_end_capture_bugs_();
   tt_int_op(i,OP_EQ, 0);
 
   /* Test parse_ulong */
@@ -2123,7 +2156,10 @@ test_util_parse_integer(void *arg)
   tt_int_op(0UL,OP_EQ, tor_parse_ulong("8",8,0,100,NULL,NULL));
   tt_int_op(50UL,OP_EQ, tor_parse_ulong("50",10,50,100,NULL,NULL));
   tt_int_op(0UL,OP_EQ, tor_parse_ulong("-50",10,0,100,NULL,NULL));
+  tor_capture_bugs_(1);
   tt_int_op(0UL,OP_EQ, tor_parse_ulong("50",-1,50,100,&i,NULL));
+  tt_int_op(1, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
+  tor_end_capture_bugs_();
   tt_int_op(0,OP_EQ, i);
   tt_int_op(0UL,OP_EQ, tor_parse_ulong("-50",10,0,100,&i,NULL));
   tt_int_op(0,OP_EQ, i);
@@ -2139,8 +2175,11 @@ test_util_parse_integer(void *arg)
   tt_assert(U64_LITERAL(0) ==
               tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp));
   tt_int_op(0,OP_EQ, i);
+  tor_capture_bugs_(1);
   tt_assert(U64_LITERAL(0) ==
               tor_parse_uint64("123",-1,0,INT32_MAX, &i, &cp));
+  tt_int_op(1, OP_EQ, smartlist_len(tor_get_captured_bug_log_()));
+  tor_end_capture_bugs_();
   tt_int_op(0,OP_EQ, i);
 
   {
@@ -2152,10 +2191,13 @@ test_util_parse_integer(void *arg)
   tt_int_op(1,OP_EQ, i);
   tt_assert(DBL_TO_U64(d) == 0);
   d = tor_parse_double(" ", 0, (double)UINT64_MAX,&i,NULL);
+  tt_double_op(fabs(d), OP_LT, 1e-10);
   tt_int_op(0,OP_EQ, i);
   d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,NULL);
+  tt_double_op(fabs(d), OP_LT, 1e-10);
   tt_int_op(0,OP_EQ, i);
   d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,&cp);
+  tt_double_op(fabs(d), OP_LT, 1e-10);
   tt_int_op(1,OP_EQ, i);
   d = tor_parse_double("-.0", 0, (double)UINT64_MAX,&i,NULL);
   tt_int_op(1,OP_EQ, i);
@@ -2224,114 +2266,371 @@ test_util_pow2(void *arg)
   ;
 }
 
-/** Run unit tests for compression functions */
 static void
-test_util_gzip(void *arg)
+test_util_compress_impl(compress_method_t method)
 {
-  char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
-  const char *ccp2;
+  char *buf1=NULL, *buf2=NULL, *buf3=NULL;
   size_t len1, len2;
-  tor_zlib_state_t *state = NULL;
-
-  (void)arg;
-  buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
-  tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
 
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                               GZIP_METHOD));
-  tt_assert(buf2);
-  tt_assert(len1 < strlen(buf1));
-  tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD);
+  tt_assert(tor_compress_supports_method(method));
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1,
-                                 GZIP_METHOD, 1, LOG_INFO));
-  tt_assert(buf3);
-  tt_int_op(strlen(buf1) + 1,OP_EQ, len2);
-  tt_str_op(buf1,OP_EQ, buf3);
+  if (method != NO_METHOD) {
+    tt_ptr_op(tor_compress_version_str(method), OP_NE, NULL);
+    tt_ptr_op(tor_compress_header_version_str(method), OP_NE, NULL);
+  }
 
-  tor_free(buf2);
-  tor_free(buf3);
+  buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
+  tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
 
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                                 ZLIB_METHOD));
-  tt_assert(buf2);
-  tt_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD);
+  tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, method));
+  tt_ptr_op(buf2, OP_NE, NULL);
+  if (method == NO_METHOD) {
+    // The identity transform doesn't actually compress, and it isn't
+    // detectable as "the identity transform."
+    tt_int_op(len1, OP_EQ, strlen(buf1)+1);
+    tt_int_op(detect_compression_method(buf2, len1), OP_EQ, UNKNOWN_METHOD);
+  } else {
+    tt_int_op(len1, OP_LT, strlen(buf1));
+    tt_int_op(detect_compression_method(buf2, len1), OP_EQ, method);
+  }
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1,
-                                   ZLIB_METHOD, 1, LOG_INFO));
-  tt_assert(buf3);
-  tt_int_op(strlen(buf1) + 1,OP_EQ, len2);
-  tt_str_op(buf1,OP_EQ, buf3);
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, method, 1, LOG_INFO));
+  tt_ptr_op(buf3, OP_NE, NULL);
+  tt_int_op(strlen(buf1) + 1, OP_EQ, len2);
+  tt_str_op(buf1, OP_EQ, buf3);
+  tt_int_op(buf3[len2], OP_EQ, 0);
 
   /* Check whether we can uncompress concatenated, compressed strings. */
   tor_free(buf3);
   buf2 = tor_reallocarray(buf2, len1, 2);
   memcpy(buf2+len1, buf2, len1);
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2,
-                                   ZLIB_METHOD, 1, LOG_INFO));
-  tt_int_op((strlen(buf1)+1)*2,OP_EQ, len2);
-  tt_mem_op(buf3,OP_EQ,
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, method, 1, LOG_INFO));
+  tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2);
+  tt_mem_op(buf3, OP_EQ,
              "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0"
              "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0",
              (strlen(buf1)+1)*2);
+  tt_int_op(buf3[len2], OP_EQ, 0);
+
+  /* Check whether we can uncompress partial strings */
 
   tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
 
-  /* Check whether we can uncompress partial strings. */
-  buf1 =
-    tor_strdup("String with low redundancy that won't be compressed much.");
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                                 ZLIB_METHOD));
-  tt_assert(len1>16);
-  /* when we allow an incomplete string, we should succeed.*/
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16,
-                                  ZLIB_METHOD, 0, LOG_INFO));
-  tt_assert(len2 > 5);
-  buf3[len2]='\0';
-  tt_assert(!strcmpstart(buf1, buf3));
-
-  /* when we demand a complete string, this must fail. */
+  size_t b1len = 1<<10;
+  if (method == ZSTD_METHOD) {
+    // zstd needs a big input before it starts generating output that it
+    // can partially decompress.
+    b1len = 1<<18;
+  }
+  buf1 = tor_malloc(b1len);
+  crypto_rand(buf1, b1len);
+  tt_assert(!tor_compress(&buf2, &len1, buf1, b1len, method));
+  tt_int_op(len1, OP_GT, 16);
+  /* when we allow an incomplete output we should succeed.*/
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16,
+                            method, 0, LOG_INFO));
+  tt_int_op(len2, OP_GT, 5);
+  tt_int_op(len2, OP_LE, len1);
+  tt_assert(fast_memeq(buf1, buf3, len2));
+  tt_int_op(buf3[len2], OP_EQ, 0);
+
+  /* when we demand a complete output from a real compression method, this
+   * must fail. */
   tor_free(buf3);
-  tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16,
-                                 ZLIB_METHOD, 1, LOG_INFO));
-  tt_assert(!buf3);
+  if (method != NO_METHOD) {
+    tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16,
+                             method, 1, LOG_INFO));
+    tt_ptr_op(buf3, OP_EQ, NULL);
+  }
 
-  /* Now, try streaming compression. */
+ done:
   tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
-  state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
+}
+
+static void
+test_util_compress_stream_impl(compress_method_t method,
+                               compression_level_t level)
+{
+  char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
+  const char *ccp2;
+  size_t len1, len2;
+
+  tor_compress_state_t *state = NULL;
+  state = tor_compress_new(1, method, level);
   tt_assert(state);
   cp1 = buf1 = tor_malloc(1024);
   len1 = 1024;
   ccp2 = "ABCDEFGHIJABCDEFGHIJ";
   len2 = 21;
-  tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0)
-              == TOR_ZLIB_OK);
-  tt_int_op(0,OP_EQ, len2); /* Make sure we compressed it all. */
+  tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0),
+            OP_EQ, TOR_COMPRESS_OK);
+  tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */
   tt_assert(cp1 > buf1);
 
   len2 = 0;
   cp2 = cp1;
-  tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1)
-              == TOR_ZLIB_DONE);
-  tt_int_op(0,OP_EQ, len2);
-  tt_assert(cp1 > cp2); /* Make sure we really added something. */
+  tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1),
+            OP_EQ, TOR_COMPRESS_DONE);
+  tt_int_op(0, OP_EQ, len2);
+  if (method == NO_METHOD) {
+    tt_ptr_op(cp1, OP_EQ, cp2);
+  } else {
+    tt_assert(cp1 > cp2); /* Make sure we really added something. */
+  }
+
+  tt_int_op(tor_compress_state_size(state), OP_GT, 0);
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1,
-                                  ZLIB_METHOD, 1, LOG_WARN));
+  tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1,
+                            method, 1, LOG_WARN));
   /* Make sure it compressed right. */
   tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ");
-  tt_int_op(21,OP_EQ, len2);
+  tt_int_op(21, OP_EQ, len2);
 
  done:
   if (state)
-    tor_zlib_free(state);
+    tor_compress_free(state);
+  tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
-  tor_free(buf1);
+}
+
+/** Setup function for compression tests: handles x-zstd:nostatic
+ */
+static void *
+compression_test_setup(const struct testcase_t *testcase)
+{
+  tor_assert(testcase->setup_data);
+  tor_assert(testcase->setup_data != (void*)TT_SKIP);
+  const char *methodname = testcase->setup_data;
+
+  if (!strcmp(methodname, "x-zstd:nostatic")) {
+    methodname = "x-zstd";
+    tor_zstd_set_static_apis_disabled_for_testing(1);
+  }
+
+  return (void *)methodname;
+}
+
+/** Cleanup for compression tests: disables nostatic */
+static int
+compression_test_cleanup(const struct testcase_t *testcase, void *ptr)
+{
+  (void)testcase;
+  (void)ptr;
+  tor_zstd_set_static_apis_disabled_for_testing(0);
+  return 1;
+}
+
+static const struct testcase_setup_t compress_setup = {
+  compression_test_setup, compression_test_cleanup
+};
+
+/** Run unit tests for compression functions */
+static void
+test_util_compress(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  compression_level_t levels[] = {
+    BEST_COMPRESSION,
+    HIGH_COMPRESSION,
+    MEDIUM_COMPRESSION,
+    LOW_COMPRESSION
+  };
+
+  test_util_compress_impl(method);
+
+  for (unsigned l = 0; l < ARRAY_LENGTH(levels); ++l) {
+    compression_level_t level = levels[l];
+    test_util_compress_stream_impl(method, level);
+  }
+ done:
+  ;
+}
+
+static void
+test_util_decompress_concatenated_impl(compress_method_t method)
+{
+  char input[4096];
+  char *c1 = NULL, *c2 = NULL, *c3 = NULL;
+  char *result = NULL;
+  size_t sz1, sz2, sz3, szr;
+  int r;
+
+  crypto_rand(input, sizeof(input));
+
+  /* Compress the input in two chunks. */
+  r = tor_compress(&c1, &sz1, input, 2048, method);
+  tt_int_op(r, OP_EQ, 0);
+  r = tor_compress(&c2, &sz2, input+2048, 2048, method);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* concatenate the chunks. */
+  sz3 = sz1 + sz2;
+  c3 = tor_malloc(sz3);
+  memcpy(c3, c1, sz1);
+  memcpy(c3+sz1, c2, sz2);
+
+  /* decompress the concatenated result */
+  r = tor_uncompress(&result, &szr, c3, sz3, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(szr, OP_EQ, sizeof(input));
+  tt_mem_op(result, OP_EQ, input, sizeof(input));
+
+ done:
+  tor_free(c1);
+  tor_free(c2);
+  tor_free(c3);
+  tor_free(result);
+}
+
+static void
+test_util_decompress_concatenated(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_concatenated_impl(method);
+ done:
+  ;
+}
+
+static void
+test_util_decompress_junk_impl(compress_method_t method)
+{
+  char input[4096];
+  char *result = NULL, *result2 = NULL;
+  size_t szr, szr2, sz;
+  int r;
+
+  /* This shouldn't be a compressed string according to any method. */
+  strlcpy(input, "This shouldn't be a compressed string by any means.",
+          sizeof(input));
+  sz = strlen(input);
+  setup_capture_of_logs(LOG_WARN);
+  r = tor_uncompress(&result, &szr, input, sz, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, -1);
+  tt_ptr_op(result, OP_EQ, NULL);
+  expect_log_msg_containing("Error while uncompressing data: bad input?");
+  mock_clean_saved_logs();
+
+  /* Now try again, with a compressed object that starts out good and turns to
+     junk. */
+  crypto_rand(input, sizeof(input));
+  r = tor_compress(&result, &szr, input, sizeof(input), method);
+  tt_int_op(r, OP_EQ, 0);
+  crypto_rand(result+szr/2, szr-(szr/2)); // trash the 2nd half of the result
+  r = tor_uncompress(&result2, &szr2, result, szr, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("Error while uncompressing data: bad input?");
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(result);
+  tor_free(result2);
+}
+
+static void
+test_util_decompress_junk(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_junk_impl(method);
+ done:
+  ;
+}
+
+/* mock replacement for tor_compress_is_compression_bomb that doesn't
+ * believe in compression bombs. */
+static int
+mock_is_never_compression_bomb(size_t in, size_t out)
+{
+  (void)in;
+  (void) out;
+  return 0;
+}
+
+static void
+test_util_decompress_dos_impl(compress_method_t method)
+{
+  char *input;
+  char *result = NULL, *result2 = NULL;
+  size_t szr, szr2;
+  int r;
+
+  const size_t big = 1024*1024;
+  /* one megabyte of 0s. */
+  input = tor_malloc_zero(big);
+
+  /* Compress it into "result": it should fail. */
+  setup_full_capture_of_logs(LOG_WARN);
+  r = tor_compress(&result, &szr, input, big, method);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing(
+                 "other Tors would think this was a compression bomb");
+  teardown_capture_of_logs();
+
+  /* Try again, but this time suppress compression-bomb detection */
+  MOCK(tor_compress_is_compression_bomb, mock_is_never_compression_bomb);
+  r = tor_compress(&result, &szr, input, big, method);
+  UNMOCK(tor_compress_is_compression_bomb);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+
+  /* We should refuse to uncomrpess it again, since it looks like a
+   * compression bomb. */
+  setup_capture_of_logs(LOG_WARN);
+  r = tor_uncompress(&result2, &szr2, result, szr, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("bomb; abandoning stream");
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(input);
+  tor_free(result);
+  tor_free(result2);
+}
+
+static void
+test_util_decompress_dos(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_dos_impl(method);
+ done:
+  ;
 }
 
 static void
@@ -2346,44 +2645,44 @@ test_util_gzip_compression_bomb(void *arg)
   char *one_mb = tor_malloc_zero(one_million);
   char *result = NULL;
   size_t result_len = 0;
-  tor_zlib_state_t *state = NULL;
+  tor_compress_state_t *state = NULL;
 
   /* Make sure we can't produce a compression bomb */
   setup_full_capture_of_logs(LOG_WARN);
-  tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len,
-                                         one_mb, one_million,
-                                         ZLIB_METHOD));
+  tt_int_op(-1, OP_EQ, tor_compress(&result, &result_len,
+                                    one_mb, one_million,
+                                    ZLIB_METHOD));
   expect_single_log_msg_containing(
          "We compressed something and got an insanely high "
          "compression factor; other Tors would think this "
-         "was a zlib bomb.");
+         "was a compression bomb.");
   teardown_capture_of_logs();
 
   /* Here's a compression bomb that we made manually. */
   const char compression_bomb[1039] =
     { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2,
       0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ };
-  tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len,
-                                           compression_bomb, 1039,
-                                           ZLIB_METHOD, 0, LOG_WARN));
+  tt_int_op(-1, OP_EQ, tor_uncompress(&result, &result_len,
+                                      compression_bomb, 1039,
+                                      ZLIB_METHOD, 0, LOG_WARN));
 
   /* Now try streaming that. */
-  state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-  tor_zlib_output_t r;
+  state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
+  tor_compress_output_t r;
   const char *inp = compression_bomb;
   size_t inlen = 1039;
   do {
     char *outp = one_mb;
     size_t outleft = 4096; /* small on purpose */
-    r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0);
+    r = tor_compress_process(state, &outp, &outleft, &inp, &inlen, 0);
     tt_int_op(inlen, OP_NE, 0);
-  } while (r == TOR_ZLIB_BUF_FULL);
+  } while (r == TOR_COMPRESS_BUFFER_FULL);
 
-  tt_int_op(r, OP_EQ, TOR_ZLIB_ERR);
+  tt_int_op(r, OP_EQ, TOR_COMPRESS_ERROR);
 
  done:
   tor_free(one_mb);
-  tor_zlib_free(state);
+  tor_compress_free(state);
 }
 
 /** Run unit tests for mmap() wrapper functionality. */
@@ -2401,7 +2700,7 @@ test_util_mmap(void *arg)
   crypto_rand(buf, buflen);
 
   mapping = tor_mmap_file(fname1);
-  tt_assert(! mapping);
+  tt_ptr_op(mapping, OP_EQ, NULL);
 
   write_str_to_file(fname1, "Short file.", 1);
 
@@ -2419,7 +2718,7 @@ test_util_mmap(void *arg)
   tt_str_op(mapping->data,OP_EQ, "Short file.");
   tt_int_op(0, OP_EQ, tor_munmap_file(mapping));
   mapping = NULL;
-#endif
+#endif /* defined(_WIN32) */
 
   /* Now a zero-length file. */
   write_str_to_file(fname1, "", 1);
@@ -2430,7 +2729,7 @@ test_util_mmap(void *arg)
 
   /* Make sure that we fail to map a no-longer-existent file. */
   mapping = tor_mmap_file(fname1);
-  tt_assert(! mapping);
+  tt_ptr_op(mapping, OP_EQ, NULL);
 
   /* Now try a big file that stretches across a few pages and isn't aligned */
   write_bytes_to_file(fname2, buf, buflen, 1);
@@ -2748,7 +3047,7 @@ test_util_sscanf(void *arg)
   r = tor_sscanf("9223372036854775808. -9223372036854775809.",
                  "%d. %d.", &int1, &int2);
   tt_int_op(r,OP_EQ, 0);
-#endif
+#endif /* SIZEOF_INT == 4 || ... */
 
 #if SIZEOF_LONG == 4
   /* %lu */
@@ -2843,7 +3142,7 @@ test_util_sscanf(void *arg)
   r = tor_sscanf("9223372036854775808. -9223372036854775809.",
                  "%ld. %ld.", &lng1, &lng2);
   tt_int_op(r,OP_EQ, 0);
-#endif
+#endif /* SIZEOF_LONG == 4 || ... */
 
   r = tor_sscanf("123.456 .000007 -900123123.2000787 00003.2",
                  "%lf %lf %lf %lf", &d1,&d2,&d3,&d4);
@@ -2870,7 +3169,7 @@ strnlen(const char *s, size_t len)
     return len;
   return p - s;
 }
-#endif
+#endif /* !defined(HAVE_STRNLEN) */
 
 static void
 test_util_format_time_interval(void *arg)
@@ -3237,7 +3536,7 @@ test_util_format_time_interval(void *arg)
   tt_ci_char_op(label_m[0],OP_EQ, 'm');
   /* and 7 or 8 seconds - ignored */
 
-#endif
+#endif /* SIZEOF_LONG == 4 || SIZEOF_LONG == 8 */
 
 #if SIZEOF_LONG == 8
 
@@ -3275,7 +3574,7 @@ test_util_format_time_interval(void *arg)
   tt_ci_char_op(label_m[0],OP_EQ, 'm');
   /* and 7 or 8 seconds - ignored */
 
-#endif
+#endif /* SIZEOF_LONG == 8 */
 
  done:
   ;
@@ -3318,7 +3617,7 @@ test_util_path_is_relative(void *arg)
   tt_int_op(0,OP_EQ, path_is_relative("\\dir"));
   tt_int_op(0,OP_EQ, path_is_relative("a:\\dir"));
   tt_int_op(0,OP_EQ, path_is_relative("z:\\dir"));
-#endif
+#endif /* defined(_WIN32) */
 
  done:
   ;
@@ -3333,6 +3632,13 @@ test_util_memarea(void *arg)
   void *malloced_ptr = NULL;
   int i;
 
+#ifdef DISABLE_MEMORY_SENTINELS
+  /* If memory sentinels are disabled, this whole module is just an alias for
+     malloc(), which is free to lay out memory most any way it wants. */
+  if (1)
+    tt_skip();
+#endif /* defined(DISABLE_MEMORY_SENTINELS) */
+
   (void)arg;
   tt_assert(area);
 
@@ -3502,8 +3808,8 @@ test_util_strtok(void *arg)
     }
     tor_snprintf(buf, sizeof(buf), "%s", pad1);
     tor_snprintf(buf2, sizeof(buf2), "%s", pad2);
-    tt_assert(NULL == tor_strtok_r_impl(buf, " ", &cp1));
-    tt_assert(NULL == tor_strtok_r_impl(buf2, ".!..;!", &cp2));
+    tt_ptr_op(tor_strtok_r_impl(buf, " ", &cp1), OP_EQ, NULL);
+    tt_ptr_op(tor_strtok_r_impl(buf2, ".!..;!", &cp2), OP_EQ, NULL);
 
     tor_snprintf(buf, sizeof(buf),
                  "%sGraved on the dark  in gestures of descent%s", pad1, pad1);
@@ -3853,7 +4159,7 @@ test_util_load_win_lib(void *ptr)
   if (h)
     FreeLibrary(h);
 }
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef _WIN32
 static void
@@ -3905,7 +4211,7 @@ test_util_exit_status(void *ptr)
   tt_int_op(n,OP_EQ, strlen(hex_errno));
   tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
 
-#endif
+#endif /* SIZEOF_INT == 4 || ... */
 
   clear_hex_errno(hex_errno);
   n = format_helper_exit_status(0x7F, 0, hex_errno);
@@ -3923,20 +4229,16 @@ test_util_exit_status(void *ptr)
  done:
   ;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 #ifndef _WIN32
-/* Check that fgets with a non-blocking pipe returns partial lines and sets
- * EAGAIN, returns full lines and sets no error, and returns NULL on EOF and
- * sets no error */
 static void
-test_util_fgets_eagain(void *ptr)
+test_util_string_from_pipe(void *ptr)
 {
   int test_pipe[2] = {-1, -1};
-  int retval;
+  int retval = 0;
+  enum stream_status status = IO_STREAM_TERM;
   ssize_t retlen;
-  char *retptr;
-  FILE *test_stream = NULL;
   char buf[4] = { 0 };
 
   (void)ptr;
@@ -3947,91 +4249,115 @@ test_util_fgets_eagain(void *ptr)
   retval = pipe(test_pipe);
   tt_int_op(retval, OP_EQ, 0);
 
-  /* Set up the read-end to be non-blocking */
-  retval = fcntl(test_pipe[0], F_SETFL, O_NONBLOCK);
-  tt_int_op(retval, OP_EQ, 0);
+  /* Send in a string. */
+  retlen = write(test_pipe[1], "ABC", 3);
+  tt_int_op(retlen, OP_EQ, 3);
 
-  /* Open it as a stdio stream */
-  test_stream = fdopen(test_pipe[0], "r");
-  tt_ptr_op(test_stream, OP_NE, NULL);
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "ABC");
+  errno = 0;
 
-  /* Send in a partial line */
-  retlen = write(test_pipe[1], "A", 1);
+  /* Send in a string that contains a nul. */
+  retlen = write(test_pipe[1], "AB\0", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
+  errno = 0;
+
+  /* Send in a string that contains a nul only. */
+  retlen = write(test_pipe[1], "\0", 1);
   tt_int_op(retlen, OP_EQ, 1);
-  retptr = fgets(buf, sizeof(buf), test_stream);
-  tt_int_op(errno, OP_EQ, EAGAIN);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "A");
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "");
   errno = 0;
 
-  /* Send in the rest */
-  retlen = write(test_pipe[1], "B\n", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  /* Send in a string that contains a trailing newline. */
+  retlen = write(test_pipe[1], "AB\n", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
+  errno = 0;
+
+  /* Send in a string that contains a newline only. */
+  retlen = write(test_pipe[1], "\n", 1);
+  tt_int_op(retlen, OP_EQ, 1);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "B\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "");
   errno = 0;
 
-  /* Send in a full line */
-  retlen = write(test_pipe[1], "CD\n", 3);
+  /* Send in a string and check that we nul terminate return values. */
+  retlen = write(test_pipe[1], "AAA", 3);
   tt_int_op(retlen, OP_EQ, 3);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "CD\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AAA");
+  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
   errno = 0;
 
-  /* Send in a partial line */
-  retlen = write(test_pipe[1], "E", 1);
+  retlen = write(test_pipe[1], "B", 1);
   tt_int_op(retlen, OP_EQ, 1);
-  retptr = fgets(buf, sizeof(buf), test_stream);
-  tt_int_op(errno, OP_EQ, EAGAIN);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "E");
+
+  memset(buf, '\xff', sizeof(buf));
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "B");
+  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
   errno = 0;
 
-  /* Send in the rest */
-  retlen = write(test_pipe[1], "F\n", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  /* Send in multiple lines. */
+  retlen = write(test_pipe[1], "A\nB", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "F\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "A\nB");
   errno = 0;
 
-  /* Send in a full line and close */
-  retlen = write(test_pipe[1], "GH", 2);
+  /* Send in a line and close */
+  retlen = write(test_pipe[1], "AB", 2);
   tt_int_op(retlen, OP_EQ, 2);
   retval = close(test_pipe[1]);
   tt_int_op(retval, OP_EQ, 0);
   test_pipe[1] = -1;
-  retptr = fgets(buf, sizeof(buf), test_stream);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "GH");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
   errno = 0;
 
   /* Check for EOF */
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, NULL);
-  retval = feof(test_stream);
-  tt_int_op(retval, OP_NE, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
   errno = 0;
 
-  /* Check that buf is unchanged according to C99 and C11 */
-  tt_str_op(buf, OP_EQ, "GH");
-
  done:
-  if (test_stream != NULL)
-    fclose(test_stream);
   if (test_pipe[0] != -1)
     close(test_pipe[0]);
   if (test_pipe[1] != -1)
     close(test_pipe[1]);
 }
-#endif
+
+#endif /* !defined(_WIN32) */
 
 /**
  * Test for format_hex_number_sigsafe()
@@ -4227,7 +4553,7 @@ test_util_split_lines(void *ptr)
       /* Check we have not got too many lines */
       tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j);
       /* Check that there actually should be a line here */
-      tt_assert(tests[i].split_line[j] != NULL);
+      tt_ptr_op(tests[i].split_line[j], OP_NE, NULL);
       log_info(LD_GENERAL, "Line %d of test %d, should be <%s>",
                j, i, tests[i].split_line[j]);
       /* Check that the line is as expected */
@@ -4343,11 +4669,11 @@ test_util_di_map(void *arg)
 
   char dflt_entry[] = "'You have made a good beginning', but no more";
 
-  tt_int_op(32, ==, sizeof(key1));
-  tt_int_op(32, ==, sizeof(key2));
-  tt_int_op(32, ==, sizeof(key3));
+  tt_int_op(32, OP_EQ, sizeof(key1));
+  tt_int_op(32, OP_EQ, sizeof(key2));
+  tt_int_op(32, OP_EQ, sizeof(key3));
 
-  tt_ptr_op(dflt_entry, ==, dimap_search(dimap, key1, dflt_entry));
+  tt_ptr_op(dflt_entry, OP_EQ, dimap_search(dimap, key1, dflt_entry));
 
   char *str1 = tor_strdup("You are precisely as big as what you love"
                           " and precisely as small as what you allow"
@@ -4365,10 +4691,10 @@ test_util_di_map(void *arg)
   dimap_add_entry(&dimap, key2, str2);
   dimap_add_entry(&dimap, key3, str3);
 
-  tt_ptr_op(str1, ==, dimap_search(dimap, key1, dflt_entry));
-  tt_ptr_op(str3, ==, dimap_search(dimap, key3, dflt_entry));
-  tt_ptr_op(str2, ==, dimap_search(dimap, key2, dflt_entry));
-  tt_ptr_op(dflt_entry, ==, dimap_search(dimap, key4, dflt_entry));
+  tt_ptr_op(str1, OP_EQ, dimap_search(dimap, key1, dflt_entry));
+  tt_ptr_op(str3, OP_EQ, dimap_search(dimap, key3, dflt_entry));
+  tt_ptr_op(str2, OP_EQ, dimap_search(dimap, key2, dflt_entry));
+  tt_ptr_op(dflt_entry, OP_EQ, dimap_search(dimap, key4, dflt_entry));
 
  done:
   dimap_free(dimap, tor_free_);
@@ -4835,34 +5161,34 @@ test_util_round_to_next_multiple_of(void *arg)
 {
   (void)arg;
 
-  tt_u64_op(round_uint64_to_next_multiple_of(0,1), ==, 0);
-  tt_u64_op(round_uint64_to_next_multiple_of(0,7), ==, 0);
+  tt_u64_op(round_uint64_to_next_multiple_of(0,1), OP_EQ, 0);
+  tt_u64_op(round_uint64_to_next_multiple_of(0,7), OP_EQ, 0);
 
-  tt_u64_op(round_uint64_to_next_multiple_of(99,1), ==, 99);
-  tt_u64_op(round_uint64_to_next_multiple_of(99,7), ==, 105);
-  tt_u64_op(round_uint64_to_next_multiple_of(99,9), ==, 99);
+  tt_u64_op(round_uint64_to_next_multiple_of(99,1), OP_EQ, 99);
+  tt_u64_op(round_uint64_to_next_multiple_of(99,7), OP_EQ, 105);
+  tt_u64_op(round_uint64_to_next_multiple_of(99,9), OP_EQ, 99);
 
-  tt_u64_op(round_uint64_to_next_multiple_of(UINT64_MAX,2), ==,
+  tt_u64_op(round_uint64_to_next_multiple_of(UINT64_MAX,2), OP_EQ,
             UINT64_MAX);
 
-  tt_int_op(round_uint32_to_next_multiple_of(0,1), ==, 0);
-  tt_int_op(round_uint32_to_next_multiple_of(0,7), ==, 0);
+  tt_int_op(round_uint32_to_next_multiple_of(0,1), OP_EQ, 0);
+  tt_int_op(round_uint32_to_next_multiple_of(0,7), OP_EQ, 0);
 
-  tt_int_op(round_uint32_to_next_multiple_of(99,1), ==, 99);
-  tt_int_op(round_uint32_to_next_multiple_of(99,7), ==, 105);
-  tt_int_op(round_uint32_to_next_multiple_of(99,9), ==, 99);
+  tt_int_op(round_uint32_to_next_multiple_of(99,1), OP_EQ, 99);
+  tt_int_op(round_uint32_to_next_multiple_of(99,7), OP_EQ, 105);
+  tt_int_op(round_uint32_to_next_multiple_of(99,9), OP_EQ, 99);
 
-  tt_int_op(round_uint32_to_next_multiple_of(UINT32_MAX,2), ==,
+  tt_int_op(round_uint32_to_next_multiple_of(UINT32_MAX,2), OP_EQ,
             UINT32_MAX);
 
-  tt_uint_op(round_to_next_multiple_of(0,1), ==, 0);
-  tt_uint_op(round_to_next_multiple_of(0,7), ==, 0);
+  tt_uint_op(round_to_next_multiple_of(0,1), OP_EQ, 0);
+  tt_uint_op(round_to_next_multiple_of(0,7), OP_EQ, 0);
 
-  tt_uint_op(round_to_next_multiple_of(99,1), ==, 99);
-  tt_uint_op(round_to_next_multiple_of(99,7), ==, 105);
-  tt_uint_op(round_to_next_multiple_of(99,9), ==, 99);
+  tt_uint_op(round_to_next_multiple_of(99,1), OP_EQ, 99);
+  tt_uint_op(round_to_next_multiple_of(99,7), OP_EQ, 105);
+  tt_uint_op(round_to_next_multiple_of(99,9), OP_EQ, 99);
 
-  tt_uint_op(round_to_next_multiple_of(UINT_MAX,2), ==,
+  tt_uint_op(round_to_next_multiple_of(UINT_MAX,2), OP_EQ,
             UINT_MAX);
  done:
   ;
@@ -4883,26 +5209,26 @@ test_util_laplace(void *arg)
   const double delta_f = 15.0, epsilon = 0.3; /* b = 15.0 / 0.3 = 50.0 */
   (void)arg;
 
-  tt_i64_op(INT64_MIN, ==, sample_laplace_distribution(mu, b, 0.0));
-  tt_i64_op(-69, ==, sample_laplace_distribution(mu, b, 0.01));
-  tt_i64_op(24, ==, sample_laplace_distribution(mu, b, 0.5));
-  tt_i64_op(24, ==, sample_laplace_distribution(mu, b, 0.51));
-  tt_i64_op(117, ==, sample_laplace_distribution(mu, b, 0.99));
+  tt_i64_op(INT64_MIN, OP_EQ, sample_laplace_distribution(mu, b, 0.0));
+  tt_i64_op(-69, OP_EQ, sample_laplace_distribution(mu, b, 0.01));
+  tt_i64_op(24, OP_EQ, sample_laplace_distribution(mu, b, 0.5));
+  tt_i64_op(24, OP_EQ, sample_laplace_distribution(mu, b, 0.51));
+  tt_i64_op(117, OP_EQ, sample_laplace_distribution(mu, b, 0.99));
 
   /* >>> laplace.ppf([0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99],
    * ...             loc = 0, scale = 50)
    * array([         -inf,  -80.47189562,  -34.65735903,    0.        ,
    *          34.65735903,   80.47189562,  195.60115027])
    */
-  tt_i64_op(INT64_MIN + 20, ==,
+  tt_i64_op(INT64_MIN + 20, OP_EQ,
             add_laplace_noise(20, 0.0, delta_f, epsilon));
 
-  tt_i64_op(-60, ==, add_laplace_noise(20, 0.1, delta_f, epsilon));
-  tt_i64_op(-14, ==, add_laplace_noise(20, 0.25, delta_f, epsilon));
-  tt_i64_op(20, ==, add_laplace_noise(20, 0.5, delta_f, epsilon));
-  tt_i64_op(54, ==, add_laplace_noise(20, 0.75, delta_f, epsilon));
-  tt_i64_op(100, ==, add_laplace_noise(20, 0.9, delta_f, epsilon));
-  tt_i64_op(215, ==, add_laplace_noise(20, 0.99, delta_f, epsilon));
+  tt_i64_op(-60, OP_EQ, add_laplace_noise(20, 0.1, delta_f, epsilon));
+  tt_i64_op(-14, OP_EQ, add_laplace_noise(20, 0.25, delta_f, epsilon));
+  tt_i64_op(20, OP_EQ, add_laplace_noise(20, 0.5, delta_f, epsilon));
+  tt_i64_op(54, OP_EQ, add_laplace_noise(20, 0.75, delta_f, epsilon));
+  tt_i64_op(100, OP_EQ, add_laplace_noise(20, 0.9, delta_f, epsilon));
+  tt_i64_op(215, OP_EQ, add_laplace_noise(20, 0.99, delta_f, epsilon));
 
   /* Test extreme values of signal with maximally negative values of noise
    * 1.0000000000000002 is the smallest number > 1
@@ -4915,54 +5241,54 @@ test_util_laplace(void *arg)
    */
   const double noscale_df = 1.0, noscale_eps = 1.0;
 
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, 0.0, noscale_df, noscale_eps));
 
   /* is it clipped to INT64_MIN? */
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(-1, 0.0, noscale_df, noscale_eps));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, 0.0,
                               noscale_df, noscale_eps));
   /* ... even when scaled? */
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, 0.0, delta_f, epsilon));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, 0.0,
                               DBL_MAX, 1));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, 0.0,
                               DBL_MAX, 1));
 
   /* does it play nice with INT64_MAX? */
-  tt_i64_op((INT64_MIN + INT64_MAX), ==,
+  tt_i64_op((INT64_MIN + INT64_MAX), OP_EQ,
             add_laplace_noise(INT64_MAX, 0.0,
                               noscale_df, noscale_eps));
 
   /* do near-zero fractional values work? */
   const double min_dbl_error = 0.0000000000000002;
 
-  tt_i64_op(-35, ==,
+  tt_i64_op(-35, OP_EQ,
             add_laplace_noise(0, min_dbl_error,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, min_dbl_error,
                               noscale_df, noscale_eps));
-  tt_i64_op((-35 + INT64_MAX), ==,
+  tt_i64_op((-35 + INT64_MAX), OP_EQ,
             add_laplace_noise(INT64_MAX, min_dbl_error,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(0, min_dbl_error,
                               DBL_MAX, 1));
-  tt_i64_op((INT64_MAX + INT64_MIN), ==,
+  tt_i64_op((INT64_MAX + INT64_MIN), OP_EQ,
             add_laplace_noise(INT64_MAX, min_dbl_error,
                               DBL_MAX, 1));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             add_laplace_noise(INT64_MIN, min_dbl_error,
                               DBL_MAX, 1));
 
   /* does it play nice with INT64_MAX? */
-  tt_i64_op((INT64_MAX - 35), ==,
+  tt_i64_op((INT64_MAX - 35), OP_EQ,
             add_laplace_noise(INT64_MAX, min_dbl_error,
                               noscale_df, noscale_eps));
 
@@ -4977,31 +5303,31 @@ test_util_laplace(void *arg)
   const double max_dbl_lt_one = 0.9999999999999998;
 
   /* do near-one fractional values work? */
-  tt_i64_op(35, ==,
+  tt_i64_op(35, OP_EQ,
             add_laplace_noise(0, max_dbl_lt_one, noscale_df, noscale_eps));
 
   /* is it clipped to INT64_MAX? */
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX - 35, max_dbl_lt_one,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX - 34, max_dbl_lt_one,
                               noscale_df, noscale_eps));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX, max_dbl_lt_one,
                               noscale_df, noscale_eps));
   /* ... even when scaled? */
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX, max_dbl_lt_one,
                               delta_f, epsilon));
-  tt_i64_op((INT64_MIN + INT64_MAX), ==,
+  tt_i64_op((INT64_MIN + INT64_MAX), OP_EQ,
             add_laplace_noise(INT64_MIN, max_dbl_lt_one,
                               DBL_MAX, 1));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             add_laplace_noise(INT64_MAX, max_dbl_lt_one,
                               DBL_MAX, 1));
   /* does it play nice with INT64_MIN? */
-  tt_i64_op((INT64_MIN + 35), ==,
+  tt_i64_op((INT64_MIN + 35), OP_EQ,
             add_laplace_noise(INT64_MIN, max_dbl_lt_one,
                               noscale_df, noscale_eps));
 
@@ -5014,32 +5340,32 @@ test_util_clamp_double_to_int64(void *arg)
 {
   (void)arg;
 
-  tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-INFINITY_DBL));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ, clamp_double_to_int64(-INFINITY_DBL));
+  tt_i64_op(INT64_MIN, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 64.0) - 1.0));
-  tt_i64_op(INT64_MIN, ==,
+  tt_i64_op(INT64_MIN, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 63.0) - 1.0));
-  tt_i64_op(((uint64_t) -1) << 53, ==,
+  tt_i64_op(((uint64_t) -1) << 53, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 53.0)));
-  tt_i64_op((((uint64_t) -1) << 53) + 1, ==,
+  tt_i64_op((((uint64_t) -1) << 53) + 1, OP_EQ,
             clamp_double_to_int64(-1.0 * pow(2.0, 53.0) + 1.0));
-  tt_i64_op(-1, ==, clamp_double_to_int64(-1.0));
-  tt_i64_op(0, ==, clamp_double_to_int64(-0.9));
-  tt_i64_op(0, ==, clamp_double_to_int64(-0.1));
-  tt_i64_op(0, ==, clamp_double_to_int64(0.0));
-  tt_i64_op(0, ==, clamp_double_to_int64(NAN_DBL));
-  tt_i64_op(0, ==, clamp_double_to_int64(0.1));
-  tt_i64_op(0, ==, clamp_double_to_int64(0.9));
-  tt_i64_op(1, ==, clamp_double_to_int64(1.0));
-  tt_i64_op((((int64_t) 1) << 53) - 1, ==,
+  tt_i64_op(-1, OP_EQ, clamp_double_to_int64(-1.0));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(-0.9));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(-0.1));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(0.0));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(NAN_DBL));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(0.1));
+  tt_i64_op(0, OP_EQ, clamp_double_to_int64(0.9));
+  tt_i64_op(1, OP_EQ, clamp_double_to_int64(1.0));
+  tt_i64_op((((int64_t) 1) << 53) - 1, OP_EQ,
             clamp_double_to_int64(pow(2.0, 53.0) - 1.0));
-  tt_i64_op(((int64_t) 1) << 53, ==,
+  tt_i64_op(((int64_t) 1) << 53, OP_EQ,
             clamp_double_to_int64(pow(2.0, 53.0)));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             clamp_double_to_int64(pow(2.0, 63.0)));
-  tt_i64_op(INT64_MAX, ==,
+  tt_i64_op(INT64_MAX, OP_EQ,
             clamp_double_to_int64(pow(2.0, 64.0)));
-  tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(INFINITY_DBL));
+  tt_i64_op(INT64_MAX, OP_EQ, clamp_double_to_int64(INFINITY_DBL));
 
  done:
   ;
@@ -5053,7 +5379,7 @@ fd_is_cloexec(tor_socket_t fd)
   int flags = fcntl(fd, F_GETFD, 0);
   return (flags & FD_CLOEXEC) == FD_CLOEXEC;
 }
-#endif
+#endif /* defined(FD_CLOEXEC) */
 
 #ifndef _WIN32
 #define CAN_CHECK_NONBLOCK
@@ -5063,7 +5389,7 @@ fd_is_nonblocking(tor_socket_t fd)
   int flags = fcntl(fd, F_GETFL, 0);
   return (flags & O_NONBLOCK) == O_NONBLOCK;
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 #define ERRNO_IS_EPROTO(e)    (e == SOCK_ERRNO(EPROTONOSUPPORT))
 #define SOCK_ERR_IS_EPROTO(s) ERRNO_IS_EPROTO(tor_socket_errno(s))
@@ -5085,7 +5411,8 @@ test_util_socket(void *arg)
 
   fd1 = tor_open_socket_with_extensions(domain, SOCK_STREAM, 0, 0, 0);
   int err = tor_socket_errno(fd1);
-  if (fd1 < 0 && err == SOCK_ERRNO(EPROTONOSUPPORT)) {
+  if (fd1 < 0 && (err == SOCK_ERRNO(EPROTONOSUPPORT) ||
+                  err == SOCK_ERRNO(EAFNOSUPPORT))) {
     /* Assume we're on an IPv4-only or IPv6-only system, and give up now. */
     goto done;
   }
@@ -5106,13 +5433,13 @@ test_util_socket(void *arg)
   tt_int_op(fd_is_cloexec(fd2), OP_EQ, 0);
   tt_int_op(fd_is_cloexec(fd3), OP_EQ, 1);
   tt_int_op(fd_is_cloexec(fd4), OP_EQ, 1);
-#endif
+#endif /* defined(CAN_CHECK_CLOEXEC) */
 #ifdef CAN_CHECK_NONBLOCK
   tt_int_op(fd_is_nonblocking(fd1), OP_EQ, 0);
   tt_int_op(fd_is_nonblocking(fd2), OP_EQ, 1);
   tt_int_op(fd_is_nonblocking(fd3), OP_EQ, 0);
   tt_int_op(fd_is_nonblocking(fd4), OP_EQ, 1);
-#endif
+#endif /* defined(CAN_CHECK_NONBLOCK) */
 
   tor_assert(tor_close_socket == tor_close_socket__real);
 
@@ -5168,10 +5495,10 @@ is_there_a_localhost(int family)
 
   return result;
 }
-#endif
+#endif /* 0 */
 
 /* Test for socketpair and ersatz_socketpair().  We test them both, since
- * the latter is a tolerably good way to exersize tor_accept_socket(). */
+ * the latter is a tolerably good way to exercise tor_accept_socket(). */
 static void
 test_util_socketpair(void *arg)
 {
@@ -5195,7 +5522,7 @@ test_util_socketpair(void *arg)
      * Assume we're on a machine without 127.0.0.1 or ::1 and give up now. */
     tt_skip();
   }
-#endif
+#endif /* defined(__FreeBSD__) */
   tt_int_op(0, OP_EQ, socketpair_result);
 
   tt_assert(SOCKET_OK(fds[0]));
@@ -5248,47 +5575,74 @@ test_util_max_mem(void *arg)
 }
 
 static void
+test_util_dest_validation_edgecase(void *arg)
+{
+  (void)arg;
+
+  tt_assert(!string_is_valid_dest(NULL));
+  tt_assert(!string_is_valid_dest(""));
+
+  done:
+  return;
+}
+
+static void
 test_util_hostname_validation(void *arg)
 {
   (void)arg;
 
   // Lets try valid hostnames first.
-  tt_assert(string_is_valid_hostname("torproject.org"));
-  tt_assert(string_is_valid_hostname("ocw.mit.edu"));
-  tt_assert(string_is_valid_hostname("i.4cdn.org"));
-  tt_assert(string_is_valid_hostname("stanford.edu"));
-  tt_assert(string_is_valid_hostname("multiple-words-with-hypens.jp"));
+  tt_assert(string_is_valid_nonrfc_hostname("torproject.org"));
+  tt_assert(string_is_valid_nonrfc_hostname("ocw.mit.edu"));
+  tt_assert(string_is_valid_nonrfc_hostname("i.4cdn.org"));
+  tt_assert(string_is_valid_nonrfc_hostname("stanford.edu"));
+  tt_assert(string_is_valid_nonrfc_hostname("multiple-words-with-hypens.jp"));
 
   // Subdomain name cannot start with '-' or '_'.
-  tt_assert(!string_is_valid_hostname("-torproject.org"));
-  tt_assert(!string_is_valid_hostname("subdomain.-domain.org"));
-  tt_assert(!string_is_valid_hostname("-subdomain.domain.org"));
-  tt_assert(!string_is_valid_hostname("___abc.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("-torproject.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("subdomain.-domain.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("-subdomain.domain.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("___abc.org"));
 
   // Hostnames cannot contain non-alphanumeric characters.
-  tt_assert(!string_is_valid_hostname("%%domain.\\org."));
-  tt_assert(!string_is_valid_hostname("***x.net"));
-  tt_assert(!string_is_valid_hostname("\xff\xffxyz.org"));
-  tt_assert(!string_is_valid_hostname("word1 word2.net"));
+  tt_assert(!string_is_valid_nonrfc_hostname("%%domain.\\org."));
+  tt_assert(!string_is_valid_nonrfc_hostname("***x.net"));
+  tt_assert(!string_is_valid_nonrfc_hostname("\xff\xffxyz.org"));
+  tt_assert(!string_is_valid_nonrfc_hostname("word1 word2.net"));
 
   // Test workaround for nytimes.com stupidity, technically invalid,
   // but we allow it since they are big, even though they are failing to
   // comply with a ~30 year old standard.
-  tt_assert(string_is_valid_hostname("core3_euw1.fabrik.nytimes.com"));
+  tt_assert(string_is_valid_nonrfc_hostname("core3_euw1.fabrik.nytimes.com"));
 
   // Firefox passes FQDNs with trailing '.'s  directly to the SOCKS proxy,
   // which is redundant since the spec states DOMAINNAME addresses are fully
   // qualified.  While unusual, this should be tollerated.
-  tt_assert(string_is_valid_hostname("core9_euw1.fabrik.nytimes.com."));
-  tt_assert(!string_is_valid_hostname("..washingtonpost.is.better.com"));
-  tt_assert(!string_is_valid_hostname("so.is..ft.com"));
-  tt_assert(!string_is_valid_hostname("..."));
+  tt_assert(string_is_valid_nonrfc_hostname("core9_euw1.fabrik.nytimes.com."));
+  tt_assert(!string_is_valid_nonrfc_hostname(
+                                         "..washingtonpost.is.better.com"));
+  tt_assert(!string_is_valid_nonrfc_hostname("so.is..ft.com"));
+  tt_assert(!string_is_valid_nonrfc_hostname("..."));
 
   // XXX: do we allow single-label DNS names?
   // We shouldn't for SOCKS (spec says "contains a fully-qualified domain name"
   // but only test pathologically malformed traling '.' cases for now.
-  tt_assert(!string_is_valid_hostname("."));
-  tt_assert(!string_is_valid_hostname(".."));
+  tt_assert(!string_is_valid_nonrfc_hostname("."));
+  tt_assert(!string_is_valid_nonrfc_hostname(".."));
+
+  // IP address strings are not hostnames.
+  tt_assert(!string_is_valid_nonrfc_hostname("8.8.8.8"));
+  tt_assert(!string_is_valid_nonrfc_hostname("[2a00:1450:401b:800::200e]"));
+  tt_assert(!string_is_valid_nonrfc_hostname("2a00:1450:401b:800::200e"));
+
+  // We allow alphanumeric TLDs. For discussion, see ticket #25055.
+  tt_assert(string_is_valid_nonrfc_hostname("lucky.13"));
+  tt_assert(string_is_valid_nonrfc_hostname("luck.y13"));
+  tt_assert(string_is_valid_nonrfc_hostname("luck.y13."));
+
+  // We allow punycode TLDs. For examples, see
+  // http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+  tt_assert(string_is_valid_nonrfc_hostname("example.xn--l1acc"));
 
   done:
   return;
@@ -5356,7 +5710,7 @@ test_util_get_avail_disk_space(void *arg)
 #else
   tt_i64_op(val, OP_GT, 0); /* You have some space. */
   tt_i64_op(val, OP_LT, ((int64_t)1)<<56); /* You don't have a zebibyte */
-#endif
+#endif /* !defined(HAVE_STATVFS) && !defined(_WIN32) */
 
  done:
   ;
@@ -5406,25 +5760,25 @@ test_util_pwdb(void *arg)
   /* Uncached case. */
   /* Let's assume that we exist. */
   me = tor_getpwuid(getuid());
-  tt_assert(me != NULL);
+  tt_ptr_op(me, OP_NE, NULL);
   name = tor_strdup(me->pw_name);
 
   /* Uncached case */
   me2 = tor_getpwnam(name);
-  tt_assert(me2 != NULL);
+  tt_ptr_op(me2, OP_NE, NULL);
   tt_int_op(me2->pw_uid, OP_EQ, getuid());
 
   /* Cached case */
   me3 = tor_getpwuid(getuid());
-  tt_assert(me3 != NULL);
+  tt_ptr_op(me3, OP_NE, NULL);
   tt_str_op(me3->pw_name, OP_EQ, name);
 
   me3 = tor_getpwnam(name);
-  tt_assert(me3 != NULL);
+  tt_ptr_op(me3, OP_NE, NULL);
   tt_int_op(me3->pw_uid, OP_EQ, getuid());
 
   dir = get_user_homedir(name);
-  tt_assert(dir != NULL);
+  tt_ptr_op(dir, OP_NE, NULL);
 
   /* Try failing cases.  First find a user that doesn't exist by name */
   char randbytes[4];
@@ -5444,7 +5798,7 @@ test_util_pwdb(void *arg)
   /* We should do a LOG_ERR */
   setup_full_capture_of_logs(LOG_ERR);
   dir = get_user_homedir(badname);
-  tt_assert(dir == NULL);
+  tt_ptr_op(dir, OP_EQ, NULL);
   expect_log_msg_containing("not found");
   tt_int_op(smartlist_len(mock_saved_logs()), OP_EQ, 1);
   teardown_capture_of_logs();
@@ -5466,33 +5820,33 @@ test_util_pwdb(void *arg)
   tor_free(dir);
   teardown_capture_of_logs();
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 static void
 test_util_calloc_check(void *arg)
 {
   (void) arg;
   /* Easy cases that are good. */
-  tt_assert(size_mul_check__(0,0));
-  tt_assert(size_mul_check__(0,100));
-  tt_assert(size_mul_check__(100,0));
-  tt_assert(size_mul_check__(100,100));
+  tt_assert(size_mul_check(0,0));
+  tt_assert(size_mul_check(0,100));
+  tt_assert(size_mul_check(100,0));
+  tt_assert(size_mul_check(100,100));
 
   /* Harder cases that are still good. */
-  tt_assert(size_mul_check__(SIZE_MAX, 1));
-  tt_assert(size_mul_check__(1, SIZE_MAX));
-  tt_assert(size_mul_check__(SIZE_MAX / 10, 9));
-  tt_assert(size_mul_check__(11, SIZE_MAX / 12));
+  tt_assert(size_mul_check(SIZE_MAX, 1));
+  tt_assert(size_mul_check(1, SIZE_MAX));
+  tt_assert(size_mul_check(SIZE_MAX / 10, 9));
+  tt_assert(size_mul_check(11, SIZE_MAX / 12));
   const size_t sqrt_size_max_p1 = ((size_t)1) << (sizeof(size_t) * 4);
-  tt_assert(size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1 - 1));
+  tt_assert(size_mul_check(sqrt_size_max_p1, sqrt_size_max_p1 - 1));
 
   /* Cases that overflow */
-  tt_assert(! size_mul_check__(SIZE_MAX, 2));
-  tt_assert(! size_mul_check__(2, SIZE_MAX));
-  tt_assert(! size_mul_check__(SIZE_MAX / 10, 11));
-  tt_assert(! size_mul_check__(11, SIZE_MAX / 10));
-  tt_assert(! size_mul_check__(SIZE_MAX / 8, 9));
-  tt_assert(! size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1));
+  tt_assert(! size_mul_check(SIZE_MAX, 2));
+  tt_assert(! size_mul_check(2, SIZE_MAX));
+  tt_assert(! size_mul_check(SIZE_MAX / 10, 11));
+  tt_assert(! size_mul_check(11, SIZE_MAX / 10));
+  tt_assert(! size_mul_check(SIZE_MAX / 8, 9));
+  tt_assert(! size_mul_check(sqrt_size_max_p1, sqrt_size_max_p1));
 
  done:
   ;
@@ -5507,6 +5861,7 @@ test_util_monotonic_time(void *arg)
   monotime_coarse_t mtc1, mtc2;
   uint64_t nsec1, nsec2, usec1, msec1;
   uint64_t nsecc1, nsecc2, usecc1, msecc1;
+  uint32_t stamp1, stamp2;
 
   monotime_init();
 
@@ -5518,6 +5873,7 @@ test_util_monotonic_time(void *arg)
   nsecc1 = monotime_coarse_absolute_nsec();
   usecc1 = monotime_coarse_absolute_usec();
   msecc1 = monotime_coarse_absolute_msec();
+  stamp1 = monotime_coarse_to_stamp(&mtc1);
 
   tor_sleep_msec(200);
 
@@ -5525,6 +5881,7 @@ test_util_monotonic_time(void *arg)
   monotime_coarse_get(&mtc2);
   nsec2 = monotime_absolute_nsec();
   nsecc2 = monotime_coarse_absolute_nsec();
+  stamp2 = monotime_coarse_to_stamp(&mtc2);
 
   /* We need to be a little careful here since we don't know the system load.
    */
@@ -5546,6 +5903,18 @@ test_util_monotonic_time(void *arg)
   tt_u64_op(msecc1, OP_LE, nsecc1 / 1000000 + 10);
   tt_u64_op(usecc1, OP_LE, nsecc1 / 1000 + 10000);
 
+  uint64_t coarse_stamp_diff =
+    monotime_coarse_stamp_units_to_approx_msec(stamp2-stamp1);
+  tt_u64_op(coarse_stamp_diff, OP_GE, 120);
+  tt_u64_op(coarse_stamp_diff, OP_LE, 1200);
+
+  {
+    uint64_t units = monotime_msec_to_approx_coarse_stamp_units(5000);
+    uint64_t ms = monotime_coarse_stamp_units_to_approx_msec(units);
+    tt_u64_op(ms, OP_GE, 4950);
+    tt_u64_op(ms, OP_LT, 5050);
+  }
+
  done:
   ;
 }
@@ -5623,12 +5992,216 @@ test_util_monotonic_time_ratchet(void *arg)
   ;
 }
 
+static void
+test_util_monotonic_time_zero(void *arg)
+{
+  (void) arg;
+  monotime_t t1;
+  monotime_coarse_t ct1;
+  monotime_init();
+  /* Check 1: The current time is not zero. */
+  monotime_get(&t1);
+  monotime_coarse_get(&ct1);
+  tt_assert(!monotime_is_zero(&t1));
+  tt_assert(!monotime_coarse_is_zero(&ct1));
+
+  /* Check 2: The _zero() makes the time zero. */
+  monotime_zero(&t1);
+  monotime_coarse_zero(&ct1);
+  tt_assert(monotime_is_zero(&t1));
+  tt_assert(monotime_coarse_is_zero(&ct1));
+ done:
+  ;
+}
+
+static void
+test_util_monotonic_time_add_msec(void *arg)
+{
+  (void) arg;
+  monotime_t t1, t2;
+  monotime_coarse_t ct1, ct2;
+  monotime_init();
+
+  monotime_get(&t1);
+  monotime_coarse_get(&ct1);
+
+  /* adding zero does nothing */
+  monotime_add_msec(&t2, &t1, 0);
+  monotime_coarse_add_msec(&ct2, &ct1, 0);
+  tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 0);
+  tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 0);
+
+  /* Add 1337 msec; see if the diff function agree */
+  monotime_add_msec(&t2, &t1, 1337);
+  monotime_coarse_add_msec(&ct2, &ct1, 1337);
+  tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337);
+  tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337);
+  // The 32-bit variant must be within 1% of the regular one.
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 1323);
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 1350);
+
+  /* Add 1337 msec twice more; make sure that any second rollover issues
+   * worked. */
+  monotime_add_msec(&t2, &t2, 1337);
+  monotime_coarse_add_msec(&ct2, &ct2, 1337);
+  monotime_add_msec(&t2, &t2, 1337);
+  monotime_coarse_add_msec(&ct2, &ct2, 1337);
+  tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337*3);
+  tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337*3);
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 3970);
+  tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 4051);
+
+ done:
+  ;
+}
+
+static void
+test_util_nowrap_math(void *arg)
+{
+  (void)arg;
+
+  tt_u64_op(0, OP_EQ, tor_add_u32_nowrap(0, 0));
+  tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(0, 1));
+  tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(1, 0));
+  tt_u64_op(4, OP_EQ, tor_add_u32_nowrap(2, 2));
+  tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX-1, 2));
+  tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(2, UINT32_MAX-1));
+  tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX, UINT32_MAX));
+
+ done:
+  ;
+}
+
+static void
+test_util_htonll(void *arg)
+{
+  (void)arg;
+#ifdef WORDS_BIGENDIAN
+  const uint64_t res_be = 0x8877665544332211;
+#else
+  const uint64_t res_le = 0x1122334455667788;
+#endif
+
+  tt_u64_op(0, OP_EQ, tor_htonll(0));
+  tt_u64_op(0, OP_EQ, tor_ntohll(0));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_htonll(UINT64_MAX));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_ntohll(UINT64_MAX));
+
+#ifdef WORDS_BIGENDIAN
+  tt_u64_op(res_be, OP_EQ, tor_htonll(0x8877665544332211));
+  tt_u64_op(res_be, OP_EQ, tor_ntohll(0x8877665544332211));
+#else
+  tt_u64_op(res_le, OP_EQ, tor_htonll(0x8877665544332211));
+  tt_u64_op(res_le, OP_EQ, tor_ntohll(0x8877665544332211));
+#endif /* defined(WORDS_BIGENDIAN) */
+
+ done:
+  ;
+}
+
+static void
+test_util_get_unquoted_path(void *arg)
+{
+  (void)arg;
+
+  char *r = NULL;
+
+  r = get_unquoted_path("\""); // "
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"\"\""); // """
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\\\""); // \"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\\\"\\\""); // \"\"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\C\""); // A\B\C"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C"); // "A\B\C
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\"C\""); // "A\B"C"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\"C"); // A\B"C
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("");
+  tt_str_op(r, OP_EQ, "");
+  tor_free(r);
+
+  r = get_unquoted_path("\"\""); // ""
+  tt_str_op(r, OP_EQ, "");
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\C"); // A\B\C
+  tt_str_op(r, OP_EQ, "A\\B\\C"); // A\B\C
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C\""); // "A\B\C"
+  tt_str_op(r, OP_EQ, "A\\B\\C"); // A\B\C
+  tor_free(r);
+
+  r = get_unquoted_path("\"\\\""); // "\"
+  tt_str_op(r, OP_EQ, "\\"); // \ /* comment to prevent line continuation */
+  tor_free(r);
+
+  r = get_unquoted_path("\"\\\"\""); // "\""
+  tt_str_op(r, OP_EQ, "\""); // "
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C\\\"\""); // "A\B\C\""
+  tt_str_op(r, OP_EQ, "A\\B\\C\""); // A\B\C"
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\\"C"); // A\B\"C
+  tt_str_op(r, OP_EQ, "A\\B\"C"); // A\B"C
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\\"C\""); // "A\B\"C"
+  tt_str_op(r, OP_EQ, "A\\B\"C"); // A\B"C
+
+ done:
+  tor_free(r);
+}
+
 #define UTIL_LEGACY(name)                                               \
   { #name, test_util_ ## name , 0, NULL, NULL }
 
 #define UTIL_TEST(name, flags)                          \
   { #name, test_util_ ## name, flags, NULL, NULL }
 
+#define COMPRESS(name, identifier)              \
+  { "compress/" #name, test_util_compress, 0, &compress_setup,          \
+    (char*)(identifier) }
+
+#define COMPRESS_CONCAT(name, identifier)                               \
+  { "compress_concat/" #name, test_util_decompress_concatenated, 0,     \
+    &compress_setup,                                                    \
+    (char*)(identifier) }
+
+#define COMPRESS_JUNK(name, identifier)                                 \
+  { "compress_junk/" #name, test_util_decompress_junk, 0,               \
+    &compress_setup,                                                    \
+    (char*)(identifier) }
+
+#define COMPRESS_DOS(name, identifier)                                  \
+  { "compress_dos/" #name, test_util_decompress_dos, 0,                 \
+    &compress_setup,                                                    \
+    (char*)(identifier) }
+
 #ifdef _WIN32
 #define UTIL_TEST_NO_WIN(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_TEST_WIN_ONLY(n, f) UTIL_TEST(n, (f))
@@ -5637,7 +6210,7 @@ test_util_monotonic_time_ratchet(void *arg)
 #define UTIL_TEST_NO_WIN(n, f) UTIL_TEST(n, (f))
 #define UTIL_TEST_WIN_ONLY(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_LEGACY_NO_WIN(n) UTIL_LEGACY(n)
-#endif
+#endif /* defined(_WIN32) */
 
 struct testcase_t util_tests[] = {
   UTIL_LEGACY(time),
@@ -5653,7 +6226,26 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(strmisc),
   UTIL_TEST(parse_integer, 0),
   UTIL_LEGACY(pow2),
-  UTIL_LEGACY(gzip),
+  COMPRESS(zlib, "deflate"),
+  COMPRESS(gzip, "gzip"),
+  COMPRESS(lzma, "x-tor-lzma"),
+  COMPRESS(zstd, "x-zstd"),
+  COMPRESS(zstd_nostatic, "x-zstd:nostatic"),
+  COMPRESS(none, "identity"),
+  COMPRESS_CONCAT(zlib, "deflate"),
+  COMPRESS_CONCAT(gzip, "gzip"),
+  COMPRESS_CONCAT(lzma, "x-tor-lzma"),
+  COMPRESS_CONCAT(zstd, "x-zstd"),
+  COMPRESS_CONCAT(zstd_nostatic, "x-zstd:nostatic"),
+  COMPRESS_CONCAT(none, "identity"),
+  COMPRESS_JUNK(zlib, "deflate"),
+  COMPRESS_JUNK(gzip, "gzip"),
+  COMPRESS_JUNK(lzma, "x-tor-lzma"),
+  COMPRESS_DOS(zlib, "deflate"),
+  COMPRESS_DOS(gzip, "gzip"),
+  COMPRESS_DOS(lzma, "x-tor-lzma"),
+  COMPRESS_DOS(zstd, "x-zstd"),
+  COMPRESS_DOS(zstd_nostatic, "x-zstd:nostatic"),
   UTIL_TEST(gzip_compression_bomb, TT_FORK),
   UTIL_LEGACY(datadir),
   UTIL_LEGACY(memarea),
@@ -5674,10 +6266,11 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(listdir, 0),
   UTIL_TEST(parent_dir, 0),
   UTIL_TEST(ftruncate, 0),
+  UTIL_TEST(nowrap_math, 0),
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
   UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(fgets_eagain, 0),
+  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
   UTIL_TEST(join_win_cmdline, 0),
@@ -5708,6 +6301,7 @@ struct testcase_t util_tests[] = {
     &passthrough_setup, (void*)"1" },
   UTIL_TEST(max_mem, 0),
   UTIL_TEST(hostname_validation, 0),
+  UTIL_TEST(dest_validation_edgecase, 0),
   UTIL_TEST(ipv4_validation, 0),
   UTIL_TEST(writepid, 0),
   UTIL_TEST(get_avail_disk_space, 0),
@@ -5716,6 +6310,10 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(calloc_check, 0),
   UTIL_TEST(monotonic_time, 0),
   UTIL_TEST(monotonic_time_ratchet, TT_FORK),
+  UTIL_TEST(monotonic_time_zero, 0),
+  UTIL_TEST(monotonic_time_add_msec, 0),
+  UTIL_TEST(htonll, 0),
+  UTIL_TEST(get_unquoted_path, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index 63a668238c..10645fe117 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,30 +6,20 @@
 
 #include "test.h"
 
+#include "crypto_rand.h"
 #define UTIL_FORMAT_PRIVATE
 #include "util_format.h"
 
 #define NS_MODULE util_format
 
-#if !defined(HAVE_HTONLL) && !defined(htonll)
-#ifdef WORDS_BIGENDIAN
-#define htonll(x) (x)
-#else
-static uint64_t
-htonll(uint64_t a)
-{
-  return htonl((uint32_t)(a>>32)) | (((uint64_t)htonl((uint32_t)a))<<32);
-}
-#endif
-#endif
-
 static void
 test_util_format_unaligned_accessors(void *ignored)
 {
   (void)ignored;
   char buf[9] = "onionsoup"; // 6f6e696f6e736f7570
 
-  tt_u64_op(get_uint64(buf+1), OP_EQ, htonll(U64_LITERAL(0x6e696f6e736f7570)));
+  tt_u64_op(get_uint64(buf+1), OP_EQ,
+      tor_htonll(U64_LITERAL(0x6e696f6e736f7570)));
   tt_uint_op(get_uint32(buf+1), OP_EQ, htonl(0x6e696f6e));
   tt_uint_op(get_uint16(buf+1), OP_EQ, htons(0x6e69));
   tt_uint_op(get_uint8(buf+1), OP_EQ, 0x6e);
@@ -43,7 +33,7 @@ test_util_format_unaligned_accessors(void *ignored)
   set_uint32(buf+1, htonl(0x78696465));
   tt_mem_op(buf, OP_EQ, "oxidestop", 9);
 
-  set_uint64(buf+1, htonll(U64_LITERAL(0x6266757363617465)));
+  set_uint64(buf+1, tor_htonll(U64_LITERAL(0x6266757363617465)));
   tt_mem_op(buf, OP_EQ, "obfuscate", 9);
  done:
   ;
@@ -144,48 +134,54 @@ test_util_format_base64_encode(void *ignored)
 }
 
 static void
-test_util_format_base64_decode_nopad(void *ignored)
+test_util_format_base64_decode_oddsize(void *ignored)
 {
   (void)ignored;
   int res;
   int i;
   char *src;
-  uint8_t *dst, *real_dst;
-  uint8_t expected[] = {0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65};
+  char *dst, real_dst[7];
+  char expected[] = {0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65};
   char real_src[] = "ZXhhbXBsZQ";
+  char expected40[] = "testing40characteroddsizebase64encoding!";
+  char src40[] = "dGVzdGluZzQwY2hhcmFjdGVyb2Rkc2l6ZWJhc2U2NGVuY29kaW5nIQ";
+  char pad40[] = "dGVzdGluZzQwY2hhcmFjdGVyb2Rkc2l6ZWJhc2U2NGVuY29kaW5nIQ==";
 
   src = tor_malloc_zero(256);
   dst = tor_malloc_zero(1000);
-  real_dst = tor_malloc_zero(10);
 
   for (i=0;i<256;i++) {
     src[i] = (char)i;
   }
 
-  res = base64_decode_nopad(dst, 1, src, SIZE_T_CEILING);
-  tt_int_op(res, OP_EQ, -1);
-
-  res = base64_decode_nopad(dst, 1, src, 5);
+  res = base64_decode(dst, 1, src, 5);
   tt_int_op(res, OP_EQ, -1);
 
   const char *s = "SGVsbG8gd29ybGQ";
-  res = base64_decode_nopad(dst, 1000, s, strlen(s));
+  res = base64_decode(dst, 1000, s, strlen(s));
   tt_int_op(res, OP_EQ, 11);
   tt_mem_op(dst, OP_EQ, "Hello world", 11);
 
   s = "T3BhIG11bmRv";
-  res = base64_decode_nopad(dst, 9, s, strlen(s));
+  res = base64_decode(dst, 9, s, strlen(s));
   tt_int_op(res, OP_EQ, 9);
   tt_mem_op(dst, OP_EQ, "Opa mundo", 9);
 
-  res = base64_decode_nopad(real_dst, 10, real_src, 10);
+  res = base64_decode(real_dst, sizeof(real_dst), real_src, 10);
   tt_int_op(res, OP_EQ, 7);
   tt_mem_op(real_dst, OP_EQ, expected, 7);
 
+  res = base64_decode(dst, 40, src40, strlen(src40));
+  tt_int_op(res, OP_EQ, 40);
+  tt_mem_op(dst, OP_EQ, expected40, 40);
+
+  res = base64_decode(dst, 40, pad40, strlen(pad40));
+  tt_int_op(res, OP_EQ, 40);
+  tt_mem_op(dst, OP_EQ, expected40, 40);
+
  done:
   tor_free(src);
   tor_free(dst);
-  tor_free(real_dst);
 }
 
 static void
@@ -207,10 +203,10 @@ test_util_format_base64_decode(void *ignored)
     src[i] = (char)i;
   }
 
-  res = base64_decode(dst, 1, src, SIZE_T_CEILING);
+  res = base64_decode(dst, 1, src, 100);
   tt_int_op(res, OP_EQ, -1);
 
-  res = base64_decode(dst, SIZE_T_CEILING+1, src, 10);
+  res = base64_decode(dst, 1, real_src, 10);
   tt_int_op(res, OP_EQ, -1);
 
   const char *s = "T3BhIG11bmRv";
@@ -350,7 +346,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswqmjs";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, ==, 0);
+    tt_int_op(ret, OP_EQ, 0);
     tt_str_op(expected, OP_EQ, dst);
   }
 
@@ -361,7 +357,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswq";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, ==, 0);
+    tt_int_op(ret, OP_EQ, 0);
     tt_mem_op(expected, OP_EQ, dst, strlen(expected));
   }
 
@@ -369,20 +365,48 @@ test_util_format_base32_decode(void *arg)
   {
     /* Invalid character '#'. */
     ret = base32_decode(dst, real_dstlen, "#abcde", 6);
-    tt_int_op(ret, ==, -1);
+    tt_int_op(ret, OP_EQ, -1);
     /* Make sure the destination buffer has been zeroed even on error. */
-    tt_int_op(tor_mem_is_zero(dst, real_dstlen), ==, 1);
+    tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
   }
 
  done:
   tor_free(dst);
 }
 
+static void
+test_util_format_encoded_size(void *arg)
+{
+  (void)arg;
+  uint8_t inbuf[256];
+  char outbuf[1024];
+  unsigned i;
+
+  crypto_rand((char *)inbuf, sizeof(inbuf));
+  for (i = 0; i <= sizeof(inbuf); ++i) {
+    /* XXXX (Once the return values are consistent, check them too.) */
+
+    base32_encode(outbuf, sizeof(outbuf), (char *)inbuf, i);
+    /* The "+ 1" below is an API inconsistency. */
+    tt_int_op(strlen(outbuf) + 1, OP_EQ, base32_encoded_size(i));
+
+    base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
+    tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
+                  BASE64_ENCODE_MULTILINE);
+    tt_int_op(strlen(outbuf), OP_EQ,
+              base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+  }
+
+ done:
+  ;
+}
+
 struct testcase_t util_format_tests[] = {
   { "unaligned_accessors", test_util_format_unaligned_accessors, 0,
     NULL, NULL },
   { "base64_encode", test_util_format_base64_encode, 0, NULL, NULL },
-  { "base64_decode_nopad", test_util_format_base64_decode_nopad, 0,
+  { "base64_decode_oddsize", test_util_format_base64_decode_oddsize, 0,
     NULL, NULL },
   { "base64_decode", test_util_format_base64_decode, 0, NULL, NULL },
   { "base16_decode", test_util_format_base16_decode, 0, NULL, NULL },
@@ -390,6 +414,7 @@ struct testcase_t util_format_tests[] = {
     NULL, NULL },
   { "base32_decode", test_util_format_base32_decode, 0,
     NULL, NULL },
+  { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c
index 4e75b97f3d..68ce6cfd40 100644
--- a/src/test/test_util_process.c
+++ b/src/test/test_util_process.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define UTIL_PROCESS_PRIVATE
@@ -67,7 +67,7 @@ test_util_process_clear_waitpid_callback(void *ignored)
  done:
   teardown_capture_of_logs();
 }
-#endif /* _WIN32 */
+#endif /* !defined(_WIN32) */
 
 #ifndef _WIN32
 #define TEST(name) { #name, test_util_process_##name, 0, NULL, NULL }
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
index 1e7160598c..2cd68cf118 100644
--- a/src/test/test_util_slow.c
+++ b/src/test/test_util_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -22,14 +22,14 @@
 #else
 #define TEST_CHILD (BUILDDIR "/src/test/test-child")
 #define EOL "\n"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef _WIN32
 /* I've assumed Windows doesn't have the gap between fork and exec
  * that causes the race condition on unix-like platforms */
 #define MATCH_PROCESS_STATUS(s1,s2)         ((s1) == (s2))
 
-#else
+#else /* !(defined(_WIN32)) */
 /* work around a race condition of the timing of SIGCHLD handler updates
  * to the process_handle's fields, and checks of those fields
  *
@@ -46,7 +46,7 @@
      ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
         && IS_RUNNING_OR_NOTRUNNING(s1)))
 
-#endif // _WIN32
+#endif /* defined(_WIN32) */
 
 /** Helper function for testing tor_spawn_background */
 static void
@@ -78,7 +78,7 @@ run_util_spawn_background(const char *argv[], const char *expected_out,
     return;
   }
 
-  tt_assert(process_handle != NULL);
+  tt_ptr_op(process_handle, OP_NE, NULL);
 
   /* When a spawned process forks, fails, then exits very quickly,
    * (this typically occurs when exec fails)
@@ -102,7 +102,7 @@ run_util_spawn_background(const char *argv[], const char *expected_out,
    * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
   tt_assert(process_handle->waitpid_cb != NULL
               || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING);
-#endif
+#endif /* !defined(_WIN32) */
 
 #ifdef _WIN32
   tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE);
@@ -112,7 +112,7 @@ run_util_spawn_background(const char *argv[], const char *expected_out,
   tt_assert(process_handle->stdout_pipe >= 0);
   tt_assert(process_handle->stderr_pipe >= 0);
   tt_assert(process_handle->stdin_pipe >= 0);
-#endif
+#endif /* defined(_WIN32) */
 
   /* Check stdout */
   pos = tor_read_all_from_process_stdout(process_handle, stdout_buf,
@@ -178,7 +178,7 @@ test_util_spawn_background_fail(void *ptr)
   /* TODO: Once we can signal failure to exec, set this to be
    * PROCESS_STATUS_RUNNING_OR_ERROR */
   const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING;
-#endif
+#endif /* defined(_WIN32) */
 
   memset(expected_out, 0xf0, sizeof(expected_out));
   memset(code, 0xf0, sizeof(code));
@@ -242,9 +242,9 @@ test_util_spawn_background_partial_read_impl(int exit_early)
 #else
     /* Check that we didn't read the end of file last time */
     tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_handle, stdout_buf,
+    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
                               sizeof(stdout_buf) - 1, NULL, &eof);
-#endif
+#endif /* defined(_WIN32) */
     log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
 
     /* We would have blocked, keep on trying */
@@ -270,17 +270,17 @@ test_util_spawn_background_partial_read_impl(int exit_early)
                             sizeof(stdout_buf) - 1,
                             process_handle);
   tt_int_op(0,OP_EQ, pos);
-#else
+#else /* !(defined(_WIN32)) */
   if (!eof) {
     /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_handle, stdout_buf,
+    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
                               sizeof(stdout_buf) - 1,
                               process_handle, &eof);
     tt_int_op(0,OP_EQ, pos);
     tt_assert(eof);
   }
   /* Otherwise, we got the EOF on the last read */
-#endif
+#endif /* defined(_WIN32) */
 
   /* Check it terminated correctly */
   retval = tor_get_exit_code(process_handle, 1, &exit_code);
@@ -351,7 +351,7 @@ test_util_spawn_background_waitpid_notify(void *arg)
   }
   tt_int_op(ms_timer, OP_GT, 0);
   tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif
+#endif /* !defined(_WIN32) */
 
   ms_timer = 30*1000;
   while (((retval = tor_get_exit_code(process_handle, 0, &exit_code))
diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c
new file mode 100644
index 0000000000..df6058b74f
--- /dev/null
+++ b/src/test/test_voting_schedule.c
@@ -0,0 +1,64 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "or.h"
+#include "voting_schedule.h"
+
+#include "test.h"
+
+static void
+test_voting_schedule_interval_start(void *arg)
+{
+#define next_interval voting_schedule_get_start_of_next_interval
+  (void)arg;
+  char buf[ISO_TIME_LEN+1];
+
+  // Midnight UTC tonight (as I am writing this test)
+  const time_t midnight = 1525651200;
+  format_iso_time(buf, midnight);
+  tt_str_op(buf, OP_EQ, "2018-05-07 00:00:00");
+
+  /* Some simple tests with a 50-minute voting interval */
+
+  tt_i64_op(next_interval(midnight, 3000, 0), OP_EQ,
+            midnight+3000);
+
+  tt_i64_op(next_interval(midnight+100, 3000, 0), OP_EQ,
+            midnight+3000);
+
+  tt_i64_op(next_interval(midnight+3000, 3000, 0), OP_EQ,
+            midnight+6000);
+
+  tt_i64_op(next_interval(midnight+3001, 3000, 0), OP_EQ,
+            midnight+6000);
+
+  /* Make sure that we roll around properly at midnight */
+  tt_i64_op(next_interval(midnight+83000, 3000, 0), OP_EQ,
+            midnight+84000);
+
+  /* We start fresh at midnight UTC, even if there are leftover seconds. */
+  tt_i64_op(next_interval(midnight+84005, 3000, 0), OP_EQ,
+            midnight+86400);
+
+  /* Now try with offsets.  (These are only used for test networks.) */
+  tt_i64_op(next_interval(midnight, 3000, 99), OP_EQ,
+            midnight+99);
+
+  tt_i64_op(next_interval(midnight+100, 3000, 99), OP_EQ,
+            midnight+3099);
+
+ done:
+  ;
+#undef next_interval
+}
+
+#define VS(name,flags)                                          \
+  { #name, test_voting_schedule_##name, (flags), NULL, NULL }
+
+struct testcase_t voting_schedule_tests[] = {
+  VS(interval_start, 0),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index ccb8d0c8ca..cc7073850c 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -1,18 +1,17 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
 #include "compat_threads.h"
 #include "onion.h"
 #include "workqueue.h"
-#include "crypto.h"
 #include "crypto_curve25519.h"
+#include "crypto_rand.h"
 #include "compat_libevent.h"
 
 #include <stdio.h>
-#include <event2/event.h>
 
 #define MAX_INFLIGHT (1<<16)
 
@@ -61,9 +60,9 @@ mark_handled(int serial)
   tor_assert(! bitarray_is_set(handled, serial));
   bitarray_set(handled, serial);
   tor_mutex_release(&bitmap_mutex);
-#else
+#else /* !(defined(TRACK_RESPONSES)) */
   (void)serial;
-#endif
+#endif /* defined(TRACK_RESPONSES) */
 }
 
 static workqueue_reply_t
@@ -159,6 +158,7 @@ static tor_weak_rng_t weak_rng;
 static int n_sent = 0;
 static int rsa_sent = 0;
 static int ecdh_sent = 0;
+static int n_received_previously = 0;
 static int n_received = 0;
 static int no_shutdown = 0;
 
@@ -200,7 +200,9 @@ add_work(threadpool_t *tp)
     crypto_rand((char*)w->msg, 20);
     w->msglen = 20;
     ++rsa_sent;
-    return threadpool_queue_work(tp, workqueue_do_rsa, handle_reply, w);
+    return threadpool_queue_work_priority(tp,
+                                          WQ_PRI_MED,
+                                          workqueue_do_rsa, handle_reply, w);
   } else {
     ecdh_work_t *w = tor_malloc_zero(sizeof(*w));
     w->serial = n_sent++;
@@ -222,18 +224,24 @@ add_n_work_items(threadpool_t *tp, int n)
   workqueue_entry_t **to_cancel;
   workqueue_entry_t *ent;
 
-  to_cancel = tor_malloc(sizeof(workqueue_entry_t*) * opt_n_cancel);
+  // We'll choose randomly which entries to cancel.
+  to_cancel = tor_calloc(opt_n_cancel, sizeof(workqueue_entry_t*));
 
   while (n_queued++ < n) {
     ent = add_work(tp);
     if (! ent) {
       puts("Z");
-      tor_event_base_loopexit(tor_libevent_get_base(), NULL);
+      tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), NULL);
       return -1;
     }
-    if (n_try_cancel < opt_n_cancel &&
-        tor_weak_random_range(&weak_rng, n) < opt_n_cancel) {
+
+    if (n_try_cancel < opt_n_cancel) {
       to_cancel[n_try_cancel++] = ent;
+    } else {
+      int p = tor_weak_random_range(&weak_rng, n_queued);
+      if (p < n_try_cancel) {
+        to_cancel[p] = ent;
+      }
     }
   }
 
@@ -254,19 +262,13 @@ add_n_work_items(threadpool_t *tp, int n)
 static int shutting_down = 0;
 
 static void
-replysock_readable_cb(tor_socket_t sock, short what, void *arg)
+replysock_readable_cb(threadpool_t *tp)
 {
-  threadpool_t *tp = arg;
-  replyqueue_t *rq = threadpool_get_replyqueue(tp);
-
-  int old_r = n_received;
-  (void) sock;
-  (void) what;
-
-  replyqueue_process(rq);
-  if (old_r == n_received)
+  if (n_received_previously == n_received)
     return;
 
+  n_received_previously = n_received;
+
   if (opt_verbose) {
     printf("%d / %d", n_received, n_sent);
     if (opt_n_cancel)
@@ -286,7 +288,7 @@ replysock_readable_cb(tor_socket_t sock, short what, void *arg)
   }
   puts("");
   tor_mutex_release(&bitmap_mutex);
-#endif
+#endif /* defined(TRACK_RESPONSES) */
 
   if (n_sent - (n_received+n_successful_cancel) < opt_n_lowwater) {
     int n_to_send = n_received + opt_n_inflight - n_sent;
@@ -306,7 +308,7 @@ replysock_readable_cb(tor_socket_t sock, short what, void *arg)
                           handle_reply_shutdown, NULL);
     {
       struct timeval limit = { 2, 0 };
-      tor_event_base_loopexit(tor_libevent_get_base(), &limit);
+      tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit);
     }
   }
 }
@@ -335,7 +337,6 @@ main(int argc, char **argv)
   threadpool_t *tp;
   int i;
   tor_libevent_cfg evcfg;
-  struct event *ev;
   uint32_t as_flags = 0;
 
   for (i = 1; i < argc; ++i) {
@@ -409,18 +410,18 @@ main(int argc, char **argv)
   memset(&evcfg, 0, sizeof(evcfg));
   tor_libevent_initialize(&evcfg);
 
-  ev = tor_event_new(tor_libevent_get_base(),
-                     replyqueue_get_socket(rq), EV_READ|EV_PERSIST,
-                     replysock_readable_cb, tp);
-
-  event_add(ev, NULL);
+  {
+    int r = threadpool_register_reply_event(tp,
+                                            replysock_readable_cb);
+    tor_assert(r == 0);
+  }
 
 #ifdef TRACK_RESPONSES
   handled = bitarray_init_zero(opt_n_items);
   received = bitarray_init_zero(opt_n_items);
   tor_mutex_init(&bitmap_mutex);
   handled_len = opt_n_items;
-#endif
+#endif /* defined(TRACK_RESPONSES) */
 
   for (i = 0; i < opt_n_inflight; ++i) {
     if (! add_work(tp)) {
@@ -431,10 +432,10 @@ main(int argc, char **argv)
 
   {
     struct timeval limit = { 180, 0 };
-    tor_event_base_loopexit(tor_libevent_get_base(), &limit);
+    tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit);
   }
 
-  event_base_loop(tor_libevent_get_base(), 0);
+  tor_libevent_run_event_loop(tor_libevent_get_base(), 0);
 
   if (n_sent != opt_n_items || n_received+n_successful_cancel != n_sent) {
     printf("%d vs %d\n", n_sent, opt_n_items);
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index 9c6580f788..4c3fe15960 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -1,26 +1,24 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-extern const char tor_git_revision[];
-
-/* Ordinarily defined in tor_main.c; this bit is just here to provide one
- * since we're not linking to tor_main.c */
-const char tor_git_revision[] = "";
-
 /**
  * \file test_common.c
  * \brief Common pieces to implement unit tests.
  **/
 
+#define MAIN_PRIVATE
 #include "orconfig.h"
 #include "or.h"
 #include "control.h"
 #include "config.h"
+#include "crypto_rand.h"
 #include "rephist.h"
 #include "backtrace.h"
 #include "test.h"
+#include "channelpadding.h"
+#include "main.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -32,13 +30,10 @@ const char tor_git_revision[] = "";
 #include <direct.h>
 #else
 #include <dirent.h>
-#endif
-
-#include "or.h"
+#endif /* defined(_WIN32) */
 
 #ifdef USE_DMALLOC
 #include <dmalloc.h>
-#include <openssl/crypto.h>
 #include "main.h"
 #endif
 
@@ -84,7 +79,7 @@ setup_directory(void)
                  (int)getpid(), rnd32);
     r = mkdir(temp_dir);
   }
-#else
+#else /* !(defined(_WIN32)) */
   tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s",
                (int) getpid(), rnd32);
   r = mkdir(temp_dir, 0700);
@@ -92,7 +87,7 @@ setup_directory(void)
     /* undo sticky bit so tests don't get confused. */
     r = chown(temp_dir, getuid(), getgid());
   }
-#endif
+#endif /* defined(_WIN32) */
   if (r) {
     fprintf(stderr, "Can't create directory %s:", temp_dir);
     perror("");
@@ -178,65 +173,6 @@ remove_directory(void)
   rm_rf(temp_dir);
 }
 
-/** Define this if unit tests spend too much time generating public keys*/
-#define CACHE_GENERATED_KEYS
-
-#define N_PREGEN_KEYS 11
-static crypto_pk_t *pregen_keys[N_PREGEN_KEYS];
-static int next_key_idx;
-
-/** Generate and return a new keypair for use in unit tests.  If we're using
- * the key cache optimization, we might reuse keys. "idx" is ignored.
- * Our only guarantee is that we won't reuse a key till this function has been
- * called several times. The order in which keys are returned is slightly
- * randomized, so that tests that depend on a particular order will not be
- * reliable. */
-crypto_pk_t *
-pk_generate(int idx)
-{
-  (void) idx;
-#ifdef CACHE_GENERATED_KEYS
-  /* Either skip 1 or 2 keys. */
-  next_key_idx += crypto_rand_int_range(1,3);
-  next_key_idx %= N_PREGEN_KEYS;
-  return crypto_pk_dup_key(pregen_keys[next_key_idx]);
-#else
-  crypto_pk_t *result;
-  int res;
-  result = crypto_pk_new();
-  res = crypto_pk_generate_key__real(result);
-  tor_assert(!res);
-  return result;
-#endif
-}
-
-#ifdef CACHE_GENERATED_KEYS
-static int
-crypto_pk_generate_key_with_bits__get_cached(crypto_pk_t *env, int bits)
-{
-  if (bits != 1024)
-    return crypto_pk_generate_key_with_bits__real(env, bits);
-
-  crypto_pk_t *newkey = pk_generate(0);
-  crypto_pk_assign_(env, newkey);
-  crypto_pk_free(newkey);
-  return 0;
-}
-#endif
-
-/** Free all storage used for the cached key optimization. */
-static void
-free_pregenerated_keys(void)
-{
-  unsigned idx;
-  for (idx = 0; idx < N_PREGEN_KEYS; ++idx) {
-    if (pregen_keys[idx]) {
-      crypto_pk_free(pregen_keys[idx]);
-      pregen_keys[idx] = NULL;
-    }
-  }
-}
-
 static void *
 passthrough_test_setup(const struct testcase_t *testcase)
 {
@@ -297,14 +233,15 @@ main(int c, const char **v)
 
 #ifdef USE_DMALLOC
   {
-    int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
-    tor_assert(r);
+    int r = crypto_use_tor_alloc_functions();
+    tor_assert(r == 0);
   }
-#endif
+#endif /* defined(USE_DMALLOC) */
 
   update_approx_time(time(NULL));
   options = options_new();
   tor_threads_init();
+  tor_compress_init();
 
   network_init();
 
@@ -342,6 +279,7 @@ main(int c, const char **v)
     s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
+  init_protocol_warning_severity_level();
 
   options->command = CMD_RUN_UNITTESTS;
   if (crypto_global_init(accel_crypto, NULL, NULL)) {
@@ -355,25 +293,26 @@ main(int c, const char **v)
   }
   rep_hist_init();
   setup_directory();
+  initialize_mainloop_events();
   options_init(options);
   options->DataDirectory = tor_strdup(temp_dir);
+  tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys",
+               options->DataDirectory);
+  options->CacheDirectory = tor_strdup(temp_dir);
   options->EntryStatistics = 1;
   if (set_options(options, &errmsg) < 0) {
     printf("Failed to set initial options: %s\n", errmsg);
     tor_free(errmsg);
     return 1;
   }
+
   tor_set_failed_assertion_callback(an_assertion_failed);
 
-#ifdef CACHE_GENERATED_KEYS
-  for (i = 0; i < N_PREGEN_KEYS; ++i) {
-    pregen_keys[i] = crypto_pk_new();
-    int r = crypto_pk_generate_key(pregen_keys[i]);
-    tor_assert(r == 0);
-  }
-  MOCK(crypto_pk_generate_key_with_bits,
-       crypto_pk_generate_key_with_bits__get_cached);
-#endif
+  init_pregenerated_keys();
+
+  channelpadding_new_consensus_params(NULL);
+
+  predicted_ports_init();
 
   atexit(remove_directory);
 
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
new file mode 100644
index 0000000000..94d3db328a
--- /dev/null
+++ b/src/test/testing_rsakeys.c
@@ -0,0 +1,547 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "crypto_rand.h"
+#include "orconfig.h"
+#include "or.h"
+#include "test.h"
+
+/** Define this if unit tests spend too much time generating public keys.
+ * This module is meant to save time by using a bunch of pregenerated RSA
+keys among */
+#define USE_PREGENERATED_RSA_KEYS
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+
+static const char *PREGEN_KEYS_1024[] = {
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCZa39BCgq7KWBWFSjGYHhqmTCHvQ7WNEFAb9Mujb6Xn/Zy01fu\n"
+"WIpVvqmAKeLNEziItUm/gB8GwAN+/ZLwL9pufjIp2Ar+yqVXKySioZQxuCgTP2wm\n"
+"Ku0OfmAra1Xbtrkc2OCJllxkyNPrJ/kxfwjWR96UP0+VMbOlkBoEH1FtvwIDAQAB\n"
+"AoGAUXoygeMIYe+OdwkTt48CRHKIwH3aRE5KHSOGPyIOB05vvvmYqD8jcHgqYqNc\n"
+"DNdZXdkRin9LevU8phObFq4DTXp08XggUx4Kk4AdsFKubQtJ8gHm3xlSKbZXX2m/\n"
+"ZF0GRaZtVDQ3TRGh+OBLILt/2jT+BaFKGAyJ7al76F2nprECQQDJyLlteLDFBmrd\n"
+"0kAjNBE50S5YskBCQeQACROfyTKW8lG1J57UBeYjXvbrDFBR4alIS9DEexGai9Gz\n"
+"wxpgKg2nAkEAwqQmPstjHxvqGQRi41uXO026MLxY7dhEqs1aSw3tuT8v17pW3OEa\n"
+"Qxv7JINePZ3+sNN+Ic+3RXBR0QuD7lSSKQJAZjVSF21GvMXfY7SX4D0DbLHUNAE2\n"
+"I1mUz5/JXOpgwazETmpfPS4vwELd93kpRhBz2rbsbFmaNRoVgmSU+5jRiQJAZ1bV\n"
+"g2NilgKxEGU2x3U6Xt8Oqo9lO6omEvUCKnUTsNWuZf/l3FGbKuQxO5qPr3Ex5tny\n"
+"zqrEqBZRKgbOHfxCuQJAbJY5C3Nm5koemr031r00MY2YD1b6+hyKZyPdZ21HpyY8\n"
+"z1kWShL0POjYPX/BnKE1FkpklWcKBb7wkK7dvAKkEQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQCyqMM2TfFGV5tVBTVabxLVln8146nDavIdR6q78DCUMh8Zfzkk\n"
+"h9Lbl1NX4RU+AmrCZMPq21/EjIRxRQyRdgPYJVLdp96eGeYnEzmMkqvXiswXvDg/\n"
+"tXqsjyJeYsoHMQWDTpCLfjYo4K1ol1sg8VIs4wQeq5og6QSdmhBoz7MyqQIDAQAB\n"
+"AoGBAIJekey7nZeV8Bxva4ptSRIg+v0I/2VBUiG5nUX9NIW/uV/yrXERx/VDjKaw\n"
+"8b5JJzxpKWnk4RJc83xwRYaT1qMYHiQfybxEI0K9SjhtaThAjtXkQGtZgLJILl3t\n"
+"yh3LPTh1ocwafsKjU6eGYAe/DYn9/QwYHbtyaimcigu4etp9AkEA2DgC+HndoP1i\n"
+"np26Lx+4TG0vAfrVYGSLT9FXwf2iBV3oJvdKqu6wr8ipb1SbshRPcOQd31/mCh6+\n"
+"2BR+d4ddcwJBANOHrlBbGZdHnoEu6kKbPwwkc31IZYqyfSpkqm0Lb2oWZ9SInKfc\n"
+"cz0qpH91p610XUpYmycaJr4K+N8jgrz86HMCQQCoqGBg1Ca2OpCf66bctWB8dTqS\n"
+"z8d7rlIhC8npr1+f0hWRt5pN5Wx7YgoQpq3gZgllpPtMT7DQOhVh1fKkaDnTAkA4\n"
+"XuskPPLX7t0dvhvtviOSH9CrLXTp/mD+wC7uumJpmij3aaSd01DelxOZaAhUYDNQ\n"
+"UcafKAf1E0V5aaQ4qwljAkA9NVN6CtpzzcLrstTKxrx5P1Ylt/0UYQDo1lIaqwrT\n"
+"aOFbXmOungiC9+p/4U7RbX0MEzjFDHCWlaHASviGVgta\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQDDt2V63APj3JSqaRgofUzhtB+prm0wII4uHyxfOxnpYIELOW5z\n"
+"3UHmkr+B4D+Nif5jIp0i6W4OS4S+YHewKsDsXvXKRIW78KzOt6Le4JI9rSarNjy5\n"
+"aJKksWQRALLCmxP/BdolaBFqF3fIPD5+Zxu8ESgxhkEQI4p7awUp3E730QIDAQAB\n"
+"AoGAZktfAR4p8lkCYydW9yK2ommQ+xEuBK+fYL/uYz/yxSYpjIJSFsEYhrlA21Mo\n"
+"JIRxr8MRuoOjgFk8YnztUeimuHpslDlZDaCBzjRjBRFCMepZNG9xqSEL0u7C+SH6\n"
+"KU5f2x2P6PneBj6WaHZM+6Lf2xHlOoeuaVSUfq2Pk2VBF9kCQQDtawWWNwP0+xea\n"
+"oCAQpanaLzYPjlqZfHJQ1AAI5eSkdf1qmlypIHwOtjAEa6XuEO/Or8RNkNy4nQdw\n"
+"qhcQ7PXDAkEA0wjT6Z+Lrt67FnwPgoSvl4Nukcqw4OWHbBKhaQPsO9+oc3PAXLdD\n"
+"SclUUqDF6NX1yONTV1KrPdz4zElmEua+2wJABm4inZnp2oW+cuqpU6oY+pbSwQMb\n"
+"AxMyyWukgJkxYx7q+SsrHU2K7p8Sl9wOh28f/5oVGAC3aayfGfcRXtz8HwJAIqeO\n"
+"dQzYGU1GF7kjquEzHIRewd4xEZ1fkaW1j9MvFd3ygZL+gbsud41yJWd1WHjaNbTu\n"
+"2KYgrLX+vT1IX844hQJAbg0V7iHlttQqXL7yN09jIjQLprqVhDZCUHS9s9Dxe7fz\n"
+"Ac0ZZD0D6EVNmSmBB71q7kLUWX/W/10d447TLnnfew==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDhCAjPEockl4lqkvoIb5O3NJJG8NWD31c63e/cPWY6MX5nOM/q\n"
+"avof2eWJxFOk0HQ2BRVwIgNex6kLxtsdw7XE0A5uZorTp9DbRCGMqUqHNhHH9ci2\n"
+"mMPP9jptq3ieWg310bH4Tad8h3WE2npSCDBvxyV6EmuH2rlQW9ZlHNoiRQIDAQAB\n"
+"AoGBAI4PgWggPTqng7PJF5mNvsYQpSutzE0VCL977nmuNUQVjMPjRLarVD4ZU+QW\n"
+"EevhQQv9R5xjjJcgGqL5pchzjeKDm0/LA+AygnZoDMs2O68Neieqvr7cPqr5ALGs\n"
+"WuZvSn+bRJTenvV9sUh2ii0/u3GQbL1v7GWDkIdD7itDbmRhAkEA8iijuEY+W67w\n"
+"7JusjY2MQ2Cm6xxxR0YcnYPzT6UDm+Z7NNJwKscQ6AjayNmxmXGpbUdukzLzXf8y\n"
+"fccI9t6iHQJBAO3kx9nZay0Ktl51QP5o2gwoqRIbnogGfR06KJOlzIPGR0aPn8cg\n"
+"uKq2SiyjewEaSBM6S/4UlxYUmvc3VKnxCEkCQQDpTjg2YQ7RPGIIRA/iLV7Wx3bq\n"
+"C/QjjCwjoi44LK6mdE9928WPoUzrkSRg4EQYpwZqL6kcDrmkdSuLPMipOGQNAkA3\n"
+"KtzlujPOiDNuiEaAORSHyU4b8ue6p7aP9pK+Wq6oyGxzAo+NABuTCx78ZxT5Vnzs\n"
+"aJKC44d+CV0+g0hQ+KJxAkEAqFYzNWIzTHX8DVDdK9BpUaBg1DFxIeP5Kk+/X3FF\n"
+"5BafG08B6OiLf8qIGGsxLXNRjIE0GVp3Sy23FUKtUymP+A==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDMDk01VwPxQq/BAwOBmfGUP/x5BQn+uxI0Aat6bdWuz/2CsjbS\n"
+"CWD/YLCaPm+DpHp9RMwk4HONJaw4B2XOw3ELPx7y9DEgdC1wZ9wRkJmqr2IJZoZR\n"
+"C7x43nNv+/IXTiRkkljCcMpoL1Tld+L2VbmWR29PdZwvspWRILkEZu1mNwIDAQAB\n"
+"AoGANvFK3KfXSei4xfF3yjeXEmHAKx2uOUZJenNQpqBYPr+F9ODjXd5knZ59LqrM\n"
+"/9cTnBMgHHXK5yBTpKppQSjikLeQ2BF04Ktff9oGqVcS9x/rKo0CREuxsEfawZOW\n"
+"OzOWENp4YcDKGP1I/Ctr185QzStaWrXVQftxmYQ53T77ShECQQDnhabwtqW7rfe4\n"
+"+MfkWEJ9Y2s6iMs3JWnwPOX9G9R39PiAD4vAghHJyHHttS9Ipxmvp0hThu0x7a4g\n"
+"8BfUpqgjAkEA4aFAmzarWKigREAACVTYH2RHpXbuk05vF9WqfMPiEvQUd5a1q6vc\n"
+"xkGZsE3v/TExLjPRZP4FeUNV5sD7THzA3QJBAJxPoRlNx3GCEAlDdfnWGPX9JI09\n"
+"hC40RWUcSI7ttjJTI1+an1kWuBnLChhaRpU/tFjikTNLmmMmPHUihIRfDI8CQG7g\n"
+"3WzpKr8A7vFbOilbxnF2yDaqAYfmTXW7DHMPl/OUetJh/5kDdhT/e9VGF5+nIvH/\n"
+"iPFGW85Bpt8lCtmFnQkCQQDjpp9iy2qesE7KKX4Kv3++QfCJ2w3g7lwg4iyncoDd\n"
+"JrM53p29HROM21R6eekvqeWIe9tEX754b+E/N60ZjpGm\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDdDn3H+Eu0AW5GKohqDBntw6ubnd3VaJwZGzZyga4J2kLg8peP\n"
+"RAW6GDD6pcHzW+KZbFWHtRk70FSwvmyGcf+DY0r5tfyCHyDGmbJyPR0o6OVCgSFl\n"
+"ccf4eDvbyszzMdlx3uL05ABIpCShoKtEUqvyIQla3Jon+QBwuVkizMzyVwIDAQAB\n"
+"AoGACoKh4Fwh3VEkGRn0mnYw1Wk0Q5Xh8j+jDF6K3C7mQ3mpLGDca+dkDlEQIxq2\n"
+"egeoYnsQJf+qT3m8TRsAtfO9nj7+7IX4BfCtdIi4RNcorbs5YMWtFyaywnM6SQjS\n"
+"+1qf74aL4On9WRO2FtvnTMjFAAkiWNbQp7mWwTmB59i620ECQQDwde6/PwhUzvZh\n"
+"dyslKJdna5RjkDQyDIuh0zD/tFZ0Iko7Luec8q6n52ev/n0OiTLGetUh8goePsPP\n"
+"HVZHidNJAkEA61eMCmmu+GCAg2vJRtL5sDakAXsbP5M9Bf/QVHXtc4EVXHC6T2ld\n"
+"bldOJriNbBThBuPNmlQbssn9FApkyWT4nwJBAIuHIv3+CUuMvBJaH8L0BsaP+g67\n"
+"wk24Ud2Yujnl3rSMoR4uXV8IwqfS8quAs/gXTEs3QyzrUUuzh9NKZqIkK2ECQQCz\n"
+"vivBEDKIlPvSZBJYO25kfXcJgoKvLb9fw5/TwjXXD/HGpnpFiI3JZnjT7gRlVhT/\n"
+"9CDmC/MTvF3EXqPXhXy1AkEAo3a2me23Ljmub21jycSKaCk09dK85QTRRMe9c/hs\n"
+"i+pcGi9ZZW0Mm7cyQo47oXjNurkkv0fEvXIobVTEXAGU7w==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQCv8R1IbfYnE3R3kNeezJ7m02XnyCBDDy0YfrQldQ+urdg1CFye\n"
+"bO0iPniJb8fmV8NW7x6nUZTDznCg+igroKXtK/w0WYmJJiH4A7Oi5xNjAfRIPvJ/\n"
+"J5GI8szS8rH8tp8pW1h8k/kNg2pnBjwQ2U9omhp95RGaHDQSRYzzH/fEFQIDAQAB\n"
+"AoGAcy7+BcH/iZuB/xjzIIJDcUhqibCJ9n0D/+pLU85sYuZrCmUcBZe4M1gEn61v\n"
+"iExilRJc1hthskL/l1POYql8lk+aqeeDuh38fWJj60TCV/sENiuXOsTmoFVA5pNn\n"
+"lwlG8JlpBMsgr1fGqg1C/WLFfMmvXdKVGvpRqI06j7AYUa0CQQDfZ5rI+FhXBlxo\n"
+"PR5CM1LB90DuHUMW+Kqoj0c9d2esXEQM7UqQ/9BiBQbL6Py7Z3VwCxibOqyz7+V7\n"
+"2aGUMAKnAkEAyZy5Mu2tHs6YBBxPYam7huzMUYjddN7ixAZUyGwxQp9kTIF2NbSQ\n"
+"yVDjKrco3s2lO4qj4pSumwVe3GGlsi6G4wJAOOS3pIqqZK84BUvbUtyjLMZ9AKbv\n"
+"GQCG5ZpneB3ahyiQJAKiRL8BIJVLH87b3hYA8GHDCHUu2jwz4xCPd5+qbQJAV0TP\n"
+"pYvb9AnZI25drhiaY7z8dA6aTYxs/A0Bhf/PEteLwtIHKRgP1BR/QG4n8slxTGSm\n"
+"q91P9ypL9XkPECGzoQJBAIMvGEM7ZGevQHBjJ8HhU8IsgT4cYH/XEYb8jRy4F+Ui\n"
+"jKxHPxLuFK4urAZunNUNrqhT0PxbB7hRjtHZrmFkrcc=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDSpmV8ncLwc8gXzdFsZGPDtMO7C/IN9jKCIK13WIseMg1APlMt\n"
+"PB5lMQ9fa3m9ZRU0L8HzRo+u/Xdos3yIBI38X2Avy0laGKnQxiOKaDT/5ZHeiBBh\n"
+"nMZjP2WY5V1sgqNP9RD8enE6WaSvq1j0BM++mn9KEe//5+dWD8tboBKF4QIDAQAB\n"
+"AoGBALgVoerdE1Z+WAY1XyaSNHz6o3H6ZnW9CTaex/jb7/dbVikmThnhx842qXCB\n"
+"w8m3ZGhOs/edWkNaTde5wsI6+LhVGco/PWxN4v61jokxUU+5KvUvGacXhXIjzKwG\n"
+"DrNCYmle62QCI1z4+TLQW/Lq+jw2Wzk70NWEvoP58gt5SJoBAkEA9wubRKRs49LW\n"
+"5JNQZ9hjc+mAfP9YK/sMe4jkdloMMWXjSMlF3Z4mI9XQSpfbBqwWIBXsjU/15LIS\n"
+"ftmujZsMKQJBANpJEZI7UFoRdSP7AlM0YJuXWnVGyn/K+VIeEso5AlZdKXCTpxqp\n"
+"9blWq0UVC6jLesZ5UNPuBiAnrBaVwDA8YvkCQF+FQVfdK607TJO80g4VAP9EfcXX\n"
+"BUScIUtytsN8NdKzzpnKGRWDnMOmXI87ABkoWLW3RGuvSyhOIhCiInfmR2ECQASc\n"
+"FmroJcJBLCAeZOYs7P1cLOTdIdmhB7LcP7lVit8YCJAADj9Z536KfgNvdleSNH2M\n"
+"glB3blmvfMrdTrm2DMECQQDj6GJ/Tc2rCsq534xknasVjrgtJMQFxmQCTVgBx9pc\n"
+"gTflJAHAmNDvstacVqeObLCF2ZIvya8fSXGbDOJYeGDv\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDGgUJAm7vf/3focNGwzv4TkzYF2XwpAirnb61dyxvfug1zKv2k\n"
+"AUg3qACiurR7JrI+kAbmxEnNaKV7ts7uO763wP9KE8YAuFZsp7NFA295rEZhw38T\n"
+"rUlWHMCeaZ3mqW2q8gA14C/ZJCG4gS91SIHLjNGsbHwr2Jvri2ItwIP8FQIDAQAB\n"
+"AoGAONceb32oiHWQkkBr6uL6ogRPPdGO2fdC7c5uqCLWsnOGEmpHAsVTNoym0fIA\n"
+"aBsmgv+e2klukKDccdZg3prA+z7lHcc2a4bIFguF6ei80hLIis/dds66fFXofCzy\n"
+"DMlkncSbJwIvQHG9gblxp9qSKElZF7XjABZEImarfUlakGkCQQD//msGy5N0ZhMI\n"
+"yGMXkwXRJXfmRrIrOqHx6u1eUp4OuqDW+hBz4KCHnWfuRJkNGQIammSf18jPasP5\n"
+"YHyr/LifAkEAxoJ8R8Vusexo9ZjuU44qXCSvJQ26UBV7mn6TGEAn2DRK1RWKDaHv\n"
+"j2vnRjt3CO9WPDQL7SB/1HNAy+dIMPyqywJBAIB6tESIz8zPniX+TJ18UKMTZwXP\n"
+"3YQMvVKpUdDRLjq+OBMtFizSRD9MJOlUzGvibUfkzTPcHRDcyNbUMj4vbIkCQBx4\n"
+"6sqAjvgGKKfRX52sbnb47AYsieSisC/gp8h6qzxfg7w8cqix6WJw36M7ND+b1Iqe\n"
+"DHfeiXc3cLvOWJRuKTECQCEYkujtSjXWb26xaESFWGtUI/nEvCyqYPQAFBpaGzQ3\n"
+"tiTDeKHzypesWYoTxOiNQWCQMLrFGuUbDpYOuDOVNjw=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQCcwSAfytnspSSDX/sKmCPOMnpuCYeWA4wbz1wLyb63a8/KXhhG\n"
+"6o2W0kt3x1vnGZkeWwZOeBFUqwoc+xHhoNcZFsMOyqbqA3UMZW5cx27MsexRTQHs\n"
+"Go1newu/E+8NNCohY51G7z1Hdo0L6mi/Tldh7puuGsMwKqNG/Vvo/GQDgwIDAQAB\n"
+"AoGBAIUdpBAbjXDe1OET0vYuOMnUKA/l29RS8tpy/zGrg1/0GCM8QNWIPfEEaL4w\n"
+"+CSKonMazYI5iE4kaZQuygKXOdFqKxX8nrGK2hR0DIEUHhhiqyGMUKrf4ELkAJzK\n"
+"tHtcO64OFEU2EGa72wCmyk2MhqhLxWxA7E00x24uvW6pen6xAkEAzHhbzlRgLZ+K\n"
+"QuXmQHEqkGaS2Ccf6c9TA5Bf5S2/5zBl+OqVyJJQH0yrbPYR6Nn1NeSv3R4IDJYg\n"
+"fSZLaVzWHQJBAMRCU6QtTnZoQ97pLvXCSKRYKJF+CnE3zDFTyoJrpK0W1FSnb1EE\n"
+"DWjjdSdMLynf/InX+VOaLk3Gxwjme4NKjh8CQQCg2b4/HplayrsVzY3I/D2jw02Z\n"
+"xY2RfYusrhMCU284DBbsLn8OfiuRs9rXqOyF5ZDFiNXgeROT8zYzvcBtbp7xAkBU\n"
+"ZET9IvJLXjhZISItUXbVHIeNUIqC9sBaMbKx9EGioF97a2gliT2O7cgRtuPM+ODq\n"
+"ETHILlNc5G3vuNRBt4x3AkBV98Y1SZA3TQlUVTsjGraxkFTfU1IlomiOdOwTQ+xZ\n"
+"x+JxhhgZwZ+kgI3PidEufFCTZJ3WO6Wk9gk18Bx7CLjm\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDq/K7wNW3fcTbaRTjNZlM4W0G7tKeO+X0bca4+9uin3ML3ogNJ\n"
+"6qT/B0QAZB6Vyi9kKa3E8plQkjmPuX8Q27zj2QjEuDZ12RGFnikeOosUhOYiDh3Z\n"
+"T9CHnr6stozzgk79Xd6VI7bqRcgRwbY0uc9QVr6vwddyIfSploSpVcgspQIDAQAB\n"
+"AoGBAJfUpo/sZc6uzxtfCKGmkPTj+ef3hSBbUZuu60AhtxfnC06HrwpOg0eJAUYj\n"
+"aqOsHMziJTYQ7kDiCjE0UMaqxDNS5hueumznq2xM2mSN0nYoktU00kpANVkW4VPA\n"
+"33TB16DyqlKq2/21Rs1g8/8+IKkKDbRLTC//1WqNHASQVoGNAkEA/+z4hxTVXZkr\n"
+"9hz29tAHKURlqzxUEKLnS0eL+XGJRNfGJ+65eXL+gFiIbTnpVeidL1+lKWkZyYzl\n"
+"75cNRdUHhwJBAOsOJ9mUOqTbLW5tzh18ewZGOa1JcxhOvf2E1d56N8tDK6lvoqkF\n"
+"oUUb8kIweDxPLCVLCl8qFrbjn619fxDInXMCQAfEZGKNIlCd5nSoumIRPDZnagKB\n"
+"aTe8CfMB7+CZLoZVWiE6IIzsDYdNqI5QFKHT1nlqmLOiCfNRAGV+GxwEdB8CQQDE\n"
+"sHu4HclU2fMSTOAE3H01qt3om2WsGXfyBI3SNQMrG3IVvkymkwd4BQKbUGPMU5Pl\n"
+"QP3U1CtdruuXCUSijrzxAkEAoqYub6+0zM8fakSQZcZ01TG9Fuo2xVFDCQsvqR3m\n"
+"ZhRT/oinIvOxSh4fQs40bmt1RBmc2L1Is6YB2NTVQEBZDQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCrf0rPvHYaGYQrc1ciRwaONs8TUvSVmUU98HMYXoFEkBL4CAGH\n"
+"4oNHFk8kXHEOsBED0eccSYegWhqKHSz7PbjmJaXloExWrtx5ea3Twf8VTgcfDWQP\n"
+"0TzD3G1TYjAFPQ1/LAZCpQFmwpMmTGGxegUhOzkpEWXdLVEVc9Uw4C4L2QIDAQAB\n"
+"AoGAZXAJZA5pHM7y6nBynYe9TOkGWru6h7H8zsImkcd0VoWRcrvpi+JjG+0KKsuy\n"
+"46kop0XEmWq0mhgxknfnX0QG1MKTqGMIUGN4qCaezOabIpCOdA4d/pr/mWoNgOWw\n"
+"9Kc/tNCrKxPKsQMAlWP6ktHN30XRSlHgAjSeUVUiNHztvTECQQDUNin2nyIvj8ZA\n"
+"QAsFW9qW+TiTkeUK6yiZ9Gvgf20gwZRWOe5/xnMxVvtN6v7Av1ew/l4VhBoj/w5g\n"
+"ydIZk+2LAkEAzuJwdt+ccllG19qmEcbo9XFafgi2PvlEjPJmT1rHV2ns/7HIMu27\n"
+"PJY36GgExSfFco6VmicaoOt+RKg+5acgqwJBAKQxAEjcGWQ5VsgRhTVxO3DChX7Q\n"
+"TColhrWPwwPhM/s7K92HVzwvvKL5TNmdr9xMb7n3Ja56FouxZVuH6/J0XT8CQAat\n"
+"Mhnz/3WFQg8HRGLAe5YoMVZt64u+uaKe1ARtlo9QoNBjqWVTXL6IzocWjEjcjrey\n"
+"uEtARdC5qNqIX3dD3H8CP3pVCPvpHOTxkUaktmLYowSA1HSfO9wkE6bMCHhkLwXF\n"
+"yTIJ+N7c5u5YN1B6hhVqpKbdnSv+K0MQ0xbfwOWNMw==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDGQmrKfO3WovoXkOTSh/shO9qjbX4izhg4pccVU3Tp45v/dgAE\n"
+"uDUuaa/clToyH5AhOtuazO/asC3ZNajg1ia5VPzmQU3gtqiIZIEXFaOovPlOrXru\n"
+"wyQnxaGORndJwfDXicG6bUwI+PDpNq8c4VOTujReeF0r74qMSc7TQLVlUQIDAQAB\n"
+"AoGAakR/aTm9YibJVohbnl00xoOGlcLCsXU2lmaFZ3DsYdGWdD+TkvQJzW7ozJtQ\n"
+"Lj2sy6L4wujGR7nXWW3hr2IaLpoc1UoyJpieAZM5os6bMN+N4MCqdcZMlazMtSWV\n"
+"UDO7O7xQGFpcvvZmnfKCyluFaJ5K/tWxP+2TnS1/m0BDRIECQQD5DYvToA0eKBt+\n"
+"7K4eEI8pzDot9NlcL21D86kNgpmuY4pifALU7GvXr299JpFFiYa2A1JVRfpQaoI3\n"
+"hZzz0ze1AkEAy8opWJP+T2q4reD5Qq5UjjrHUXFID23KeJEjh5YF40/bHqyVpWVR\n"
+"UMntNgAzs+13vRij48Zn6I8GRhStaQ3ArQJASPyFS8GN1paeaDXoWPs1WWR2cF1f\n"
+"DbsAZHeVxVXOv+J//ZimI8wdVpodLCoPTLee+NxEVqUpVEPCYY8QjgwKOQJAATmj\n"
+"6f5pxvxzQ8hYd0gpBfngfOLbdgxI7VSiDAyg2G8AeDy9YZMsW/n6zRpPNUO2NpLR\n"
+"WWs18LX7aaxyJnGIuQJBAPPfy9pd4XEFsRBIIe3N23Gua1XkS/407RJtAGm73Vrt\n"
+"QhtWh3i6D5gfpEApMoaE8aaQQ7H0z+0Uh1t8SWesy10=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCc/M/X8etUqrxnmH3PyuAYLIPZhwNySch8qz9NB47izYjxzuBG\n"
+"GSls6H7WeKIrB8UJY1gW8TLkdOLcrI/0hTANNHEPaueOE0xdABFj7tAaiiGPIM25\n"
+"N0wc76me0ZAMYJrZTHk8JZK153y9wInYBwVZreXCVSVf11RuVwe+iFQa5QIDAQAB\n"
+"AoGAQC4XJtivdhDLL6snHFF7pkZkrQTGgu3pOhakrXA+mTigGQOTqvTUe8LdP/9X\n"
+"hTIK+tiTheWcAcxLhx5BSB0/VDKjYhS0ROpTc33Iq9KalOQaTJbBYGA4eagpQjwU\n"
+"jGwr9u2sUsM9WI/Jg0VvLSKhfnNwYIUzLpK3BbWb2qAdh+0CQQDQ2s/8DlibFSBK\n"
+"UsFK7lLpV8UgMk9CkaNM2BPzI8Hsjpp6s3pULVRd36m4YTSg15EEHv7bZ1N/+krX\n"
+"mXb9xUULAkEAwGy5wHsUSjTK+kntkNXjlCU/+9R+HFpzg9Bwm/PqXTBwEWeU24hV\n"
+"iRjPvqPtWFZrWi/nfcviuMaqtdliw1I1zwJAZ2mQxhtMYC2LuYFUWAe9YfClmJWQ\n"
+"jUOTef8bka5I3RqW/t5TWc7AEWMnpDXtWx6hnUrDolt9Cschu7MvKeQ9lQJAL18U\n"
+"46PpPNN+XNuyVoOxgRkihVasrUI/SeYYsuv7eHGiRUagyOLpW9T139LvbV3pE8zT\n"
+"So7VA/Q0towL2lX01QJAGcoBNNouSpum9+5NvGQK1XXsZweawE+pFR2BE5XcjG+n\n"
+"FnaLEUBX7nTxhTU2cSQET1PKRNp568a281NEna0nxw==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDFOqqGG/VtIScxayZYZ+BT+hcs5W1bD5qRxunbG9O36UVT18UE\n"
+"CWw9HUf0Q5sDMGvVmBxwZ4GjbR5FDPfhIXaRCzobnejJXq/0k+O5NAVkcSPtJvhK\n"
+"AaUqBrWA41vnjKOtJudTsZLfufKafzYwVonze7fXGyVsBRjVwHNS4iqq2QIDAQAB\n"
+"AoGAJCoStI6R3RXUKvKb0GATuTJFZ50WBTmCPTK9FMkwdCuY47vPy2Ky7y3cUMTI\n"
+"urf5PewrYs0H72CFyWGMXkKVi8aOYshsATEXMfGSqOcqXn+UDssRzvabZFlpnAUa\n"
+"WDVt/iN092AdakXNna7/DxrLisDpq8HHJfjtlWGPfkXRg4ECQQDpHeKimTvwJcPc\n"
+"iDa6Qb/n9gwLeRckfzhYtfX1luJYLIOHh+J9vjQN75thenBLQB/B6qlKtOn9ejxg\n"
+"5z+3zIOpAkEA2JbxXVTCOA802p9khvHxDtLHdKi3w/BjjJiC7Mgqo69ZI+s3PB9E\n"
+"F2HJA69kZqpGqvybWHDapjWsq7rcMlxrsQJBAME2yvR3y00VEAyGPc4M1vF8ZqlP\n"
+"uRW/+ETWtEDUyU/JvU6lGt2bu2tdkEyv/cjxIiFIzP4litdT7B1pLc+6S9kCQBwE\n"
+"usiWFGHoJbA6emiyl7qRLdg7kzo3uMkRWa6D3nA6WM+6t/SBHu/faH+fit91G5s2\n"
+"/mmcf8yMmP/GNoIVTqECQFl4Pt6yGiz/YVoYSp35ljY5n3JB6T8o2pOmIrRLuPmT\n"
+"6kgyygtJBAmx5nnQoeG8n08tl9QakWznKzkNJ0DIFKI=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDCaOqJ0lsSAEBcnNB6X7BvVcEcol+evi/nJsPe0uT1SbtW50Ch\n"
+"vYOHwK6aQR2C5x9VSs47cLynTL7tNt5d8oeryF3NpI8VTPLImDJCcvUZhS7p4bxn\n"
+"JO+Wm+D/e3TWfyjreuWtdL+Mfimw2gzwWuBEtmj51GzQ89eYm7fh11SB6QIDAQAB\n"
+"AoGAWaakMbZNxPlUtOCjyysBY/Y5vYira7rswD3CKak7aFn+CE9QIMYSN7IFUqEg\n"
+"iNMoQd7jR8nvVX8wtJeO5+gF48W13C3n8FZSrW7c5N3bmfMIgo0xa/TGfeXHP98o\n"
+"7vhH0I58j3ZZt0Q+3wTm7t7WPE/nJzgrCk30TqmoaEmstTkCQQDtV6YZ6juEK2Lp\n"
+"LGUiqohcS/WJxvFrF5+LNpk86Xdgomf6FphZlkq42KYkvl7qibKDcfDqLKTbHHle\n"
+"vQQeCgZ7AkEA0bFHi7F8o4iHtKleBvt4QCj1neA0q3CRDypCI5EqFSrNpxY4Krhh\n"
+"WYSVX+xT00QYaCpKKWfYQztCw7Anylv96wJACl86Mwe5ch0zRV1bThiFvQLUyCCZ\n"
+"jESMBFlueOr6/I4cXSF/puqaeVl+aTyoiTdbRcNE8/bffXPRGgLIm0d04QJBAJSY\n"
+"lmTN789Lby99Xh6AkaSV4ghw26Ip8QHYJmph8npxjK69Niw/4Oy44cnKBVUPSmR2\n"
+"o3tYFY7/Lb7S1D+4lOUCQQDbMQUGVsZT+ZjuOG1bAjIuXoAOfOd3mgH5VgQHjSgJ\n"
+"ourZtlJ4OUpNrq9IfWqPkM+zSE8+0Dk8/9MS5ngBA/SJ\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDNbHjwg+7tVNr9erMLowXRnIcttp4pUJbr3B7Jo/u+kD/Yo3F3\n"
+"4rIKhHpJl1uEHP1QmvAD+4ApFFI2hNG54xYI8dGflxL5HOs5xxyOPpkrwzQ8Qvnv\n"
+"LPg7Gf6PAW9zF4McG4wK0TkrV28G6NhqcPs5VFY6UyvfZ0fEdWAeoWTIfQIDAQAB\n"
+"AoGBAKOmkMp7MLLd8QAS6eSRYSdWHdLrMyES1MjduaFGBF4SKOr7en/Zl6ENXSaX\n"
+"cA7V0XCPnjpt9/HCAKTyNupx4LCeFWiqdu8VGXhlzX8bdb896OSR2brKbxgRY5tF\n"
+"36uL8akrZdrYgocykQCxmRARMB7/rHwDusiamjL6RUZ3+c45AkEA6UPTVmKZQRMr\n"
+"A7Qgg5nXrXo9117Lpqf3FdZ1wdni9V59Ptf5xrx9oGZNZzctJPXSAH4M4cumSJrV\n"
+"sZ1V8qE7AwJBAOFx+5luLrVKrdlG7MyOhTAdhKYUvKIvL4wvVSY6y+L2nNEx/cTx\n"
+"KYbxGC+H1RJbkCS09rYir3VfDRWQ3W1c1n8CQH+X4hn2hO3blkPIW6CgniD+JKWR\n"
+"7MOUTMtdK7yFemfM76VYbgAPSohabSxwOfllnSE30cQQqTw9tXYaIdE98BECQG+M\n"
+"QWxSS0QillB6unIgVqBPCrJOcmNhK4qWZPBMiVNcqI0Nyj2nAeAl7MyfzfqOWY0A\n"
+"CU5nbR+LD2NLUXRqSisCQQCN3IGv1WOWInmA5xhU6vCFDX5u48Dcji7VLJO/Nv/i\n"
+"b/zHKAgjHk5Js7bi5ZWEGaUgA4Jt6cKmGdERheqTMKxx\n"
+"-----END RSA PRIVATE KEY-----\n"
+};
+
+static const char *PREGEN_KEYS_2048[] = {
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEpAIBAAKCAQEAoksI1qIuIaFCqT4QbgDvOQCmr9Z9F0E7ku+U5Ep/5dWNANqB\n"
+"bSzAOq0+cxiisfF+H4desoqiWDUwlOwXH74qD3ZsbChhvFUD78cQBWQkF+whLVHb\n"
+"296QmF0LZqosqz9HMS9CdoMUc1brZb78Hb25QIOOjrg25KYHLZHaqcet1wfhHow6\n"
+"Uehc6QTuWgOWFhJnfiXzYgen2o8lnLixxZozhk7Lm7Aix9ur2ckXdQ2Wgny4xw70\n"
+"JW84Hapnd8oFUD98XXrExk4VFuIcA8qo7r7y18II6wx4Cw1suKru6bhW65cM/y51\n"
+"KC4lB7VkvuoJCelRFdM1PfKZLv2tJP63oAqJrQIDAQABAoIBAQCWc38PEqw3avqU\n"
+"UMAEaoNa0bq1Gd8/Nq8WqVnbRSFKHO2pk+cWIb1W6BITuwvgcGKesezdEV4s7apK\n"
+"9I7/U1hEm2Ep50mrwRh0KZM1nD9Fmharn851Bt//D4qpMytT2caS1yADI8NKpZJ1\n"
+"8VZh7+cT4qG+txHUaAIRgbw3VrBWvTIMu6SOSOZm+e3eOr5UU3du1KvjdJHJ2c2k\n"
+"TceHvUdKxV7OYt+BBSN1oBOhs3ajUSRge1v3twRDg3cmbwG0DeXvwHNhGUTcF8IH\n"
+"JO1RF5njbkFvyqdAi3ltjU41zYd4OMuPtrwzFOtxUjKT62Soz109HUXXE2CGKFPZ\n"
+"PVi5/BIhAoGBANN1xqS5BgHszIB0nXbw5ImYpTRmyhO0KsTblBT9+8Q/B7BCK7bM\n"
+"zl+dOPeyvEadSwE7RSMMt6CAlTakWIf3Quw/VZajvXy9C9/LHf52pEKXjxMFMPKE\n"
+"aGLHpQnwMtDi8/H8AEAXxI3hpxB2KVR7sAYHWihSGjRJ6oPGvEmKEkb5AoGBAMR6\n"
+"G2PKz0xk1vFrjfjSY+y13gH/t7xHaXUggjggUSGKaknQh2BDUllXjadeI0fi1eLW\n"
+"r98ZImZZgntAgjaIZ4bAlooTDk4gRHaz9jI+z8lsRwOKnWdiigM7txiXZTMVwMqj\n"
+"o5mMNGMA+A+ACkTViRHmkDI7S/9FqAvnbOqVwgFVAoGBALUcY6WDvwx5B3Jh7tgH\n"
+"XIYpEh3+h8c2gYcX1g3gtvkPTwN8uToY0gz8eOVV1YHZiHsmi4GIi+HRH3usaRMT\n"
+"COOVHzYlSc8Dj57+tdLTRL6wVl9hC9o647ju64DGlI9qQquYPZKniLZIdbFYsu9j\n"
+"/JA9Tc/I+h6czFpPJccKlbrpAoGAAPWXrKUQ3g6f/g3IY66jTkSVEO1uuDyhBzFh\n"
+"cWS3ALLsUe/yuUWa4VTMHEUZZwB0iucBdNVqlZVaTb/C4wFHgCDwmzv8leUScIHw\n"
+"cc5ctV8R+bJzkk2o3tsrybLzi4xPpK2n3tgQaWtXyruVUUC5qpy1l4kylcyBRY2b\n"
+"uomAqQECgYAiCNWtuWIDlRBcvtIB+kHguzcoFT3vTCCNhalTEn0zi/tbi+voQgVJ\n"
+"SDJNptZv+6vRwQ/HfcQtljKIPO6hUZPYaFWRNhgbh7Ay85lRXYXQOottE8ayReBk\n"
+"zZb0fl853Qah4DPsaOugAvhjjKeBmKg6bFWO1z6hj18I3UpDf2YnVQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEpQIBAAKCAQEAssO0r37mSJNAkc/ISwXBsu9JjyLeWlsHPAhylQGkSAdp2rjz\n"
+"E6AT0Eh3wrocNO31I4pvHReAuh1QedGY6T1cQwO/WAAhQtRCBQDK12qWRgfbC11y\n"
+"Xu7zNYPd1Z7YIRy+FxhbL5f+lv3rEUv0HUG5c3CWhLtbANKg+jOieIDzA4Yp1s55\n"
+"ynodQBUkTZrwQiT0P8yDSjiasf+clgJRfA1k2XK12KSAMRgyDuPTE4OtBxBvUM3L\n"
+"Zvxs81PsmcOuAG4DLaFTg2a/QkCjt2VC1SYYuh/LVxpL41FFh3eMoK5g5deHkgRe\n"
+"tlywKjAHIDJu/qgNzNgNW7ymwn2CfBvry9h0/wIDAQABAoIBAEMZ4wDdCWPEokAZ\n"
+"Vn2Ss5qO53WrCPuxn42RPjFgZGIFJl7LfbKoK8fK6+lUIrJbf+DPXdX1tIQn7MVN\n"
+"P7CNL8yX44MMyW9kbUOjgIBLqgyvdjFV6lBoMTKtRN+iuE31lATnR5Md4pqaxVnA\n"
+"wOkaepoycM1x5j7w0SwZparF/HIdkYv0y/MysqT9ByupPA4Fqp/iRSrosHXahNtI\n"
+"KZYj1TyERYtuDXq91P4dr/pWq3FmDNI8O3upblkL0YouvG/ZlFLdiNy77XbAyWcX\n"
+"ps3YDddM+vECnXO3+sa3ZxgBYvXJdWrrIzM5A+jCkDRZQGsFAzK5I5/S7C2ljt6i\n"
+"SmzqvMECgYEA16bGy2XTi6KBPb8aev/OBgK9XuGLwUqK1m15mS9Y2qPHmuc22qaZ\n"
+"hw6zginPFrxAEtQWKanhZy4aVqlLkDPLwRnyeuMo1EZAc5B1gZ5ViSAKxBq99hA9\n"
+"eqyakdb+IUQsEnRDxSc2gqUQ0EagksUyw5wGG5Q/CVEALmS/r1SU3KUCgYEA1DYf\n"
+"6JYdzuRtule3vYeWXKf8sOJpdplgWV7tvLrKkQhdE564uwMCYB23HvYfwWqEdDYG\n"
+"fsYg/ur/stk9MDZ3wZKffTEM8V3sX1t1JXnC3ogSAgMGhLZ3ILOLqkoO4BEZJnsS\n"
+"dMdiNijlAtQkqs/BO/UVUAKysCtKP3v/+1775dMCgYEAvLjGFjApfnSbV/cK7IM6\n"
+"wEXbhdIqZOCgOeEaXjVyM/zKbMRVW+oaR3hVHd8KzSG3jQKv1oxFpu9Qu3ByoWLC\n"
+"uF3Ft0debs6ADuJoAyQWROeWpGGmxlUWCGpO5rxYL7KiQxAeUsXrTU+5NBvq4CbV\n"
+"MxwyuCX3OGb7mp4upfiGQcUCgYEAuhVsDYv1P4LXJVvd5viKRV2ZG5KuYC1Ga5fu\n"
+"aFxzXJI07At2eaa94oKsHR494mEBHNZzA5/BN0fiSHZuTWS1xqxH5oOokc6Gg2ez\n"
+"ZdVLp88x20nD4YQPGkHW6tBeEuVrZG7vVC+yU0Ow7bYRISdkjqrusWZsQkbzqI+X\n"
+"fFliEbkCgYEAu8x+47M1ordbI7NmbBGyiyP0r7nMRCZ+KEvGeCNYracWmsnCNnfV\n"
+"zR2UzmwtSainw3Ho8Jv/rWDC8RIDauyBRYEi2VqOnUzT2ca0iymQyLeBCudAQuio\n"
+"drOu4JU8RzZ3Ad6V3DNFnaqmX/7GA9Pa2GI8NJMyb8p1GAGv7Gi8nxc=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEowIBAAKCAQEAt01S8JuEwWy/Hzb90yO2O7oGWq3GfvfDpFOF4OQnwG3kQ/BP\n"
+"4MoPDCYHdqb3iI9aD3vykZA6Q8zpdfGwjm4+bHrgRdiSmZWv8NvRwuQ5Ji9xbiGn\n"
+"hA1XwqH9hvgFTiy6tRvirWSJ7kzH3Q/bEGpCbHUQkwMog4v6yCNKNrjlwjN++eCi\n"
+"gFK/0RMOJMLOs8BD3zY+lKjd/pd8LBRujkMyUF5SryeRueAFjD2sq4OXq8DPABGt\n"
+"zdR6vbTcsi4JwP1Q6y4x0/LIWEprzzewNU63I5E2zj0WnoRGAIM4aF+VuqcHjWUx\n"
+"VWnyLZldSen6lScZ4xj4seitiDbSFvtFkDF6VwIDAQABAoIBAGTP9im2ntDyyjqU\n"
+"uA0DuxomOZBtupniEouyFBOX5/UBe2WSKZxsBNKdp8UuFz3X+aRCeyprtF/NtyjT\n"
+"AFOVdmebPPWtIxOtK9LAUyFo+7VwqmXzxHnwDLBS/2jXx7MzDozFBWpvvRx+xf1i\n"
+"1wy0JEwaJj90oTeYKRkhr5NhJZwkX8zCNYaemBd3kHB3aGWGJasI1Y81UezeRKCn\n"
+"hSbn2CrWalI7pyJ4lsavM11nIq1Eu2ZthJiNCMghbYrHoBHd+iVWiCYchP2rNEWV\n"
+"sdHtaVHtQ9zdZ43bao3OzPu7lAjd6UAbxsuhUe+a2YdDz/+Up+6+BvQf1FCfYIjW\n"
+"KFUdCoECgYEA4t5O+u0V9gkMUhKsevYb0zgc7O/mo8ivN+V++EpAtL0mhiwxeO8p\n"
+"oef0szLyhdULQeLN9pJQDCeAbkGdwIe3L+AKU8o8BFGEWLFysZjMg9In/UTrp5MN\n"
+"mMDy2SRKKu5BqsvdYH302xpZfHq1T2cMNDWE8lrZffduH06Cgq/XEtECgYEAztbj\n"
+"bhFneADnrvk609VnOQvoQEjySeCQKFQFRRI6k/FguqMisL2IRXnMaWammosdeCAg\n"
+"m7eZchnszHIst9cwZUKXUFqmAqeDuWSNdTI7uKZH6nT/A6IDlgdjaHsqhvpK0Ac9\n"
+"ngycdHONitOZh0ZG74pdWjf828Dwzf+CuYjl9KcCgYEAmIvI6ZqvkJ8m5Kzfw1Jn\n"
+"BVCOypbJK8oOX3R2Orea6KzjEYb3wQx3nwFcHX6danYFOskpmqlpH7MT/Y8rZsEa\n"
+"4RsxdoPedTzm08iFiXtn0R9nejp0hlov402iPXXUVSedih3IflBTa1w9XaEY9wog\n"
+"P57ZBSknYzcTmgNtaDiaUnECgYA5sWauhNw/dMEq5QmrnJK2LsQRakdqo+CR3x25\n"
+"LmR4b5Nze51pfvRLrLV/kMpXwQXvQ8bUqFl8og6S2CXxAWzWUcSy/RXhF6h+RbXP\n"
+"Qru1vWvB0fBvqvklF9p6giBSle3YKKzfMNVTBggs+OiR+uA+YHG5gHRfN2nzi5mC\n"
+"9tRtcQKBgBnDSi4lRCjRe9pPnyAYaa4iyBUGhjPysScSLY9orel89+qmTBQ/Py6J\n"
+"0+sefL4ZJaOsuaR2mSSPP/lbSkF9DMFs4tHbBqY+WkVNYLshAkauHwqv26HTVCSd\n"
+"QKzeb7uZw9lNaRIzDvy/3wfCLvXfdDozPFrOUgkyaBN5pJSA/4sv\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEogIBAAKCAQEA9qtiDoJWqU/eSlpj381eG6UcDzfMguFh/q4e4s7QVdRYj5J0\n"
+"Msv0PCkti8JHuvQUyncRpOPccBkhNbVjNbjIgw1pHaIZNdVotUDhP0kseRyJ6z3M\n"
+"qbZ5qKn+0mHjVjPNItVDDe6tebYMT1BZpVyRrCOqY2v5z1ecLC+ReygmHgDpzg+L\n"
+"0rWfIxGT10IPZ8pAlcdEn6xt5aEhi7mPCX/xwqfQChPIJz6zVLEC8UaPtvDBohPR\n"
+"6NQTBTeZZAAtzrQ7+oNxfz1v6Fz6RwMei7Q+qOBnMiwpQmbcDBKABM2RnXSpD0LA\n"
+"1GR7/+CiV1HQoShWVvEwrSIlM6jVAJo6iqF6WQIDAQABAoIBAHqwcdxPnfUm4aTP\n"
+"4r9NcZKEhDlZgqJSoiA/0OL1BRC7xrTanmspoLhPrvTF1FG715+Aq8j9AQbMqQUC\n"
+"zG7LEwiEIhV4K9vn4uXMeHy206UFud/E5EhBl695pmJUB/Q3XcAGnQyP+77++o50\n"
+"o7IpIdeiAbzj1uP3aplbq5u7M4JV7fUZWA/368G4HolqFTxcAfBJ05GXlp97BBwY\n"
+"AnY3/pNrKMz0NiPf3nsJHYWK18up0JCLPL3tomc94wuNZ66spIazHIL9aaKY0q3V\n"
+"LkBrelndfYM1m4xRTnSOy6STu0qKTPOpX0C8XBLYs6uiXjRsChqSYwndCCeASaH3\n"
+"LGNIcbUCgYEA/m4qvt8tdT4wEvnE+QUxEELmBtT4UFa3NnQISrzNlhNeI0Zd2xlp\n"
+"SG0/pcw83mG2uX+V5xSaWL5LYfLBkvy83Y0yIWgYbbIkyyCOUZnTpwaDGU/FjWip\n"
+"3TfXf5qpAgiez94sV+MsFpKfG05yxJh5u+3sIyGTVUAxp0HPx4LVgbMCgYEA+DD1\n"
+"fu6ttpuV1UMrsFdjuk6gBvSbyJ9OilY2jT+yE7hSRc/yP3O9ikuR74tNlVrWTnO2\n"
+"0kcYbyLJXE2cGUC2q5e4r8TDGiozNfQ7/OC2M3XaJ+xJk4zMf/8PuDDpWr+18ZXA\n"
+"Pf+ibXWTFvZ6ZeUmpbrrfCrXdvmIZnwVuOI0FcMCgYAZn26emksxq3mb75tumJ9A\n"
+"S/xuY7Q+Iv2Adl7/Z9QscPbiBowdLIn1yUrHn7Hhk2WbeMXX57NDjKZ6zr+/1cQP\n"
+"a9DInHsZUP9zlWu/vAYcpAM/4VC71PaGWMFTEHhExCl6NZ2xnCcsfseXMGdOdSyN\n"
+"SICnaRI1W6mkdnQ+W2a1EQKBgGEKA3KVr6XuPy8bDEHuaTe29irCCQbwAq1j+ABS\n"
+"HzZGoyRYocbdYgZoda7LMJJs6c3SwHCHC66oU0KbtaTKAKImuDdBH2djiJJX4/yD\n"
+"f7mvIpTpdfsS2gJRn7vMo/CvdFv4ySl0gfV6OwCHbmPYrLuv0dLCjWwfNI2dhoC7\n"
+"MNIxAoGAIPSIG4BrShzbeX4c2L18iwIg+NlOcUbtl0Ccr1t6uLGI+ge/6I6T/5XH\n"
+"DPKqYIf0IRYV8suxpfQNKiz/C0NPffA1d1M2hvuAg2v09o2cSwvdcQwdmakKZ5bl\n"
+"sdCuYKdCIwomEUOz/4XgQrJl4XDUqxftJT6/egAjWvcIYvfNCsY=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEowIBAAKCAQEA1yHZMsgRLckL+v6rgpGq9qmxVBNDxeuul1V/QlFyOlcAk5n/\n"
+"uduTalSqGQhc4NEePMxq6nFui4ucpkZOozmcEnhV0N9jld9IB9rLGt4erdg7RKl9\n"
+"+gQ+zTn69j69U36E2I47H4dM69uxeSOyWP2Odxpw+biisa3o8mMz1zCmuj4GMDtG\n"
+"DlnSpthFzgQR6N1pbvxLXrWg5F16GqFiJOD7kXDfy4/l6kB/mDs1T/3r8kav6DqR\n"
+"c/t3aQZxgWGIpI7hc9Qgvp7coZRMey5dNOZEna3tqS8dn2tZlhkpYV5uyFUjmxjG\n"
+"TERSULQ7hvUqW+eshGGsnxFtL7ANnTSc4xECowIDAQABAoIBAFhJJMhpQFuIySjd\n"
+"AGeZ/g4x/3rgWQzNNp4WUR5XLEhy0eLA7ShJywp06kVRoEQGraEHxsyldldAGS5H\n"
+"ZhgoGTufNKB+PHER646FpJpHE1IGjfQUloVW3qr8I1iQ0MOGBWCVpf+/V7rnMsLi\n"
+"+lr421FXgYuJ0QKXuyRVv72M0q9U6i+ml3aVAhgW/19oFg+dW7YccX+9iVyD05Q5\n"
+"KR64tX8xd4wrAqfAgYA3erbbE6GTyHYD5K54kIgfRr/+pIU4qc1L7XOCblnqc/rI\n"
+"BilFysEC634r2MNe66uQvNui4oQTfBcFFlXg0zAmp7d5QE0ApOL6HpCsmbImm2uJ\n"
+"sdFNYyECgYEA716kfEv7HfnF0P3pAP2AOuEsW6t8q0UtWvnHrwRQXQw8Yv90g7kD\n"
+"pUV3/BjD9VQgsQZosbdSn5wbT4j7dypRdrzYk+8m/hBk4Q8M/tWoRGVOn46NudvK\n"
+"/KX0A4ODLuulj8yAZVc7CM5Cdy4GCGJBVO+oVvBUAnHxfZziOyqBw9MCgYEA5hQg\n"
+"HEORzdxvbbfAx1ggvH1Eg1lqRhmpI43PpRkaoqb8jLwXb2CyBeuv3RBft/X2Tr6F\n"
+"mHpe0U1kN/5YEjii/Q/jUX8azIHaUNNSAjrriEeMQZOqFxmhCdiyeXuqg2fbFbhe\n"
+"K3Q6/fsB1xj9OOSwyPMqm/M5U0LsoGjmg8TFE/ECgYAlImKUIdlwOgp1NJ7MF4eo\n"
+"Gryd8AmkLFQv8+YFgb7R4I8RsJ2rva0SG6fUhScJTSbRL7RYNZ9swXP/L7oLL5Z5\n"
+"vCxBLu22pmZv/7y9X/n9ulWrLRtRhQaFkV08mk9knQwPNeOJVTIEWLM49/vZmxyV\n"
+"h6Ru8FOoGXMkUI1MLnj5HwKBgGJLkNhiacVYeuaWDa9c0EeXARFYvxWJ2wAMkvzG\n"
+"9+ErlFQP+7ciyYvMAItidnJii8NilDLrfNzQwpNFf5zxQ3j4M7bapblfdMT5M10u\n"
+"jPfhEWPm0VEjKvDI+p76HYQcd7YU2W6ZLqbZeRTLYUvQMFL5yGduBzyyJ+P0TR9Y\n"
+"jpYRAoGBAM7vYGTprw4w2tTZPFICXVk1bQ0LO06oNRtwkiQTUT6UqPjWMFyvHnmN\n"
+"11SVVBmRZ0RAk6e5eZLFX8WelJ4J4nSOGRcJheCtoEFlO7D1ewAUSbqWJ0pBqp2T\n"
+"gV4oCS8LYe8zReVoYZJjuLwoHvxZzs/hUjc3SI2HRW2W/HQRPC25\n"
+"-----END RSA PRIVATE KEY-----\n"
+};
+
+#define N_PREGEN_KEYS_1024 ARRAY_LENGTH(PREGEN_KEYS_1024)
+static crypto_pk_t *pregen_keys_1024[N_PREGEN_KEYS_1024];
+static int next_key_idx_1024;
+#define N_PREGEN_KEYS_2048 ARRAY_LENGTH(PREGEN_KEYS_2048)
+static crypto_pk_t *pregen_keys_2048[N_PREGEN_KEYS_2048];
+static int next_key_idx_2048;
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+
+/** Generate and return a new keypair for use in unit tests.  If we're using
+ * the key cache optimization, we might reuse keys. "idx" is ignored.
+ * Our only guarantee is that we won't reuse a key till this function has been
+ * called several times. The order in which keys are returned is slightly
+ * randomized, so that tests that depend on a particular order will not be
+ * reliable. */
+static crypto_pk_t *
+pk_generate_internal(int bits)
+{
+  tor_assert(bits == 2048 || bits == 1024);
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+  int *idxp;
+  int n_pregen;
+  crypto_pk_t **pregen_array;
+  if (bits == 2048) {
+    idxp = &next_key_idx_2048;
+    n_pregen = N_PREGEN_KEYS_2048;
+    pregen_array = pregen_keys_2048;
+  } else {
+    idxp = &next_key_idx_1024;
+    n_pregen = N_PREGEN_KEYS_1024;
+    pregen_array = pregen_keys_1024;
+  }
+  /* Either skip 1 or 2 keys. */
+  *idxp += crypto_rand_int_range(1,3);
+  *idxp %= n_pregen;
+  return crypto_pk_dup_key(pregen_array[*idxp]);
+#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */
+  crypto_pk_t *result;
+  int res;
+  result = crypto_pk_new();
+  res = crypto_pk_generate_key_with_bits__real(result, bits);
+  tor_assert(!res);
+  return result;
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+}
+
+crypto_pk_t *
+pk_generate(int idx)
+{
+  (void) idx;
+  return pk_generate_internal(1024);
+}
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+static int
+crypto_pk_generate_key_with_bits__get_cached(crypto_pk_t *env, int bits)
+{
+  if (bits == 1024 || bits == 2048)  {
+    crypto_pk_t *newkey = pk_generate_internal(bits);
+    crypto_pk_assign_(env, newkey);
+    crypto_pk_free(newkey);
+  } else {
+    return crypto_pk_generate_key_with_bits__real(env, bits);
+  }
+  return 0;
+}
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+
+/** Free all storage used for the cached key optimization. */
+void
+free_pregenerated_keys(void)
+{
+#ifdef USE_PREGENERATED_RSA_KEYS
+  unsigned idx;
+  for (idx = 0; idx < N_PREGEN_KEYS_1024; ++idx) {
+    if (pregen_keys_1024[idx]) {
+      crypto_pk_free(pregen_keys_1024[idx]);
+      pregen_keys_1024[idx] = NULL;
+    }
+  }
+  for (idx = 0; idx < N_PREGEN_KEYS_2048; ++idx) {
+    if (pregen_keys_2048[idx]) {
+      crypto_pk_free(pregen_keys_2048[idx]);
+      pregen_keys_2048[idx] = NULL;
+    }
+  }
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+}
+
+void
+init_pregenerated_keys(void)
+{
+#ifdef USE_PREGENERATED_RSA_KEYS
+  const char *s;
+  crypto_pk_t *pk;
+  unsigned i;
+  for (i = 0; i < N_PREGEN_KEYS_1024; ++i) {
+    pk = pregen_keys_1024[i] = crypto_pk_new();
+    s = PREGEN_KEYS_1024[i];
+    int r = crypto_pk_read_private_key_from_string(pk, s, strlen(s));
+    tor_assert(r == 0);
+  }
+  for (i = 0; i < N_PREGEN_KEYS_2048; ++i) {
+    pk = pregen_keys_2048[i] = crypto_pk_new();
+    s = PREGEN_KEYS_2048[i];
+    int r = crypto_pk_read_private_key_from_string(pk, s, strlen(s));
+    tor_assert(r == 0);
+  }
+
+  MOCK(crypto_pk_generate_key_with_bits,
+       crypto_pk_generate_key_with_bits__get_cached);
+#endif /* defined(USE_PREGENERATED_RSA_KEYS) */
+}
+
diff --git a/src/tools/include.am b/src/tools/include.am
index d0185b5887..016cf3b124 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -1,5 +1,4 @@
 bin_PROGRAMS+= src/tools/tor-resolve src/tools/tor-gencert
-noinst_PROGRAMS+=  src/tools/tor-checkkey
 
 if COVERAGE_ENABLED
 noinst_PROGRAMS+= src/tools/tor-cov-resolve src/tools/tor-cov-gencert
@@ -9,7 +8,8 @@ src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_resolve_LDFLAGS =
 src_tools_tor_resolve_LDADD = src/common/libor.a \
 	src/common/libor-ctime.a \
-	@TOR_LIB_MATH@ @TOR_LIB_WS32@
+	$(rust_ldadd) \
+	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_USERENV@ 
 
 if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
@@ -23,11 +23,12 @@ endif
 src_tools_tor_gencert_SOURCES = src/tools/tor-gencert.c
 src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_tools_tor_gencert_LDADD = src/common/libor.a src/common/libor-crypto.a \
-    src/common/libor-ctime.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	src/common/libor-ctime.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	$(rust_ldadd) \
+	@TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@
 
 if COVERAGE_ENABLED
 src_tools_tor_cov_gencert_SOURCES = src/tools/tor-gencert.c
@@ -43,14 +44,7 @@ src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \
     @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
 endif
 
-src_tools_tor_checkkey_SOURCES = src/tools/tor-checkkey.c
-src_tools_tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
-src_tools_tor_checkkey_LDADD = src/common/libor.a \
-    src/common/libor-ctime.a \
-    src/common/libor-crypto.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
-
-EXTRA_DIST += src/tools/tor-fw-helper/README
+if BUILD_LIBTORRUNNER
+noinst_LIBRARIES += src/tools/libtorrunner.a
+src_tools_libtorrunner_a_SOURCES = src/tools/tor_runner.c src/or/tor_api.c
+endif
diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c
deleted file mode 100644
index 3e16fd0336..0000000000
--- a/src/tools/tor-checkkey.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* Copyright (c) 2008-2015, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include "crypto.h"
-#include "torlog.h"
-#include "util.h"
-#include "compat.h"
-#include "compat_openssl.h"
-#include <openssl/bn.h>
-#include <openssl/rsa.h>
-
-int
-main(int c, char **v)
-{
-  crypto_pk_t *env;
-  char *str;
-  RSA *rsa;
-  int wantdigest=0;
-  int fname_idx;
-  char *fname=NULL;
-  init_logging(1);
-
-  if (c < 2) {
-    fprintf(stderr, "Hi. I'm tor-checkkey.  Tell me a filename that "
-            "has a PEM-encoded RSA public key (like in a cert) and I'll "
-            "dump the modulus.  Use the --digest option too and I'll "
-            "dump the digest.\n");
-    return 1;
-  }
-
-  if (crypto_global_init(0, NULL, NULL)) {
-    fprintf(stderr, "Couldn't initialize crypto library.\n");
-    return 1;
-  }
-
-  if (!strcmp(v[1], "--digest")) {
-    wantdigest = 1;
-    fname_idx = 2;
-    if (c<3) {
-      fprintf(stderr, "too few arguments");
-      return 1;
-    }
-  } else {
-    wantdigest = 0;
-    fname_idx = 1;
-  }
-
-  fname = expand_filename(v[fname_idx]);
-  str = read_file_to_str(fname, 0, NULL);
-  tor_free(fname);
-  if (!str) {
-    fprintf(stderr, "Couldn't read %s\n", v[fname_idx]);
-    return 1;
-  }
-
-  env = crypto_pk_new();
-  if (crypto_pk_read_public_key_from_string(env, str, strlen(str))<0) {
-    fprintf(stderr, "Couldn't parse key.\n");
-    return 1;
-  }
-  tor_free(str);
-
-  if (wantdigest) {
-    char digest[HEX_DIGEST_LEN+1];
-    if (crypto_pk_get_fingerprint(env, digest, 0)<0)
-      return 1;
-    printf("%s\n",digest);
-  } else {
-    rsa = crypto_pk_get_rsa_(env);
-
-    const BIGNUM *rsa_n;
-#ifdef OPENSSL_1_1_API
-    const BIGNUM *rsa_e, *rsa_d;
-    RSA_get0_key(rsa, &rsa_n, &rsa_e, &rsa_d);
-#else
-    rsa_n = rsa->n;
-#endif
-    str = BN_bn2hex(rsa_n);
-
-    printf("%s\n", str);
-  }
-
-  return 0;
-}
-
diff --git a/src/tools/tor-fw-helper/README b/src/tools/tor-fw-helper/README
deleted file mode 100644
index 6a1ecaa1e4..0000000000
--- a/src/tools/tor-fw-helper/README
+++ /dev/null
@@ -1,10 +0,0 @@
-
-We no longer recommend the use of this tool.  Instead, please use the
-pure-Go version of tor-fw-helper available at
-     https://gitweb.torproject.org/tor-fw-helper.git
-
-Why?
-
-The C code here was fine, but frankly: we don't trust the underlying
-libraries. They don't seem to have been written with network security
-in mind, and we have very little faith in their safety.
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index db308485e6..aafefdad74 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -36,10 +36,12 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include <assert.h>
 #endif
 
-#include "compat.h"
 #include "util.h"
 #include "torlog.h"
 #include "crypto.h"
+#include "crypto_digest.h"
+#include "crypto_rand.h"
+#include "crypto_util.h"
 #include "address.h"
 #include "util_format.h"
 
@@ -430,7 +432,7 @@ key_to_string(EVP_PKEY *key)
 static int
 get_fingerprint(EVP_PKEY *pkey, char *out)
 {
-  int r = 1;
+  int r = -1;
   crypto_pk_t *pk = crypto_new_pk_from_rsa_(EVP_PKEY_get1_RSA(pkey));
   if (pk) {
     r = crypto_pk_get_fingerprint(pk, out, 0);
@@ -443,7 +445,7 @@ get_fingerprint(EVP_PKEY *pkey, char *out)
 static int
 get_digest(EVP_PKEY *pkey, char *out)
 {
-  int r = 1;
+  int r = -1;
   crypto_pk_t *pk = crypto_new_pk_from_rsa_(EVP_PKEY_get1_RSA(pkey));
   if (pk) {
     r = crypto_pk_get_digest(pk, out);
@@ -464,16 +466,20 @@ generate_certificate(void)
   char expires[ISO_TIME_LEN+1];
   char id_digest[DIGEST_LEN];
   char fingerprint[FINGERPRINT_LEN+1];
-  char *ident = key_to_string(identity_key);
-  char *signing = key_to_string(signing_key);
   FILE *f;
   size_t signed_len;
   char digest[DIGEST_LEN];
   char signature[1024]; /* handles up to 8192-bit keys. */
   int r;
 
-  get_fingerprint(identity_key, fingerprint);
-  get_digest(identity_key, id_digest);
+  if (get_fingerprint(identity_key, fingerprint) < 0) {
+    return -1;
+  }
+  if (get_digest(identity_key, id_digest)) {
+    return -1;
+  }
+  char *ident = key_to_string(identity_key);
+  char *signing = key_to_string(signing_key);
 
   tor_localtime_r(&now, &tm);
   tm.tm_mon += months_lifetime;
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6ac866d3c0..966b88b3e8 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
- * Copyright (c) 2007-2015, The Tor Project, Inc.
+ * Copyright (c) 2007-2017, The Tor Project, Inc.
  */
 /* See LICENSE for licensing information */
 
@@ -347,7 +347,6 @@ main(int argc, char **argv)
   int n_args;
   tor_addr_t result;
   char *result_hostname = NULL;
-  log_severity_list_t *s = tor_malloc_zero(sizeof(log_severity_list_t));
 
   init_logging(1);
   sandbox_disable_getaddrinfo_cache();
@@ -398,11 +397,14 @@ main(int argc, char **argv)
     usage();
   }
 
+  log_severity_list_t *severities =
+    tor_malloc_zero(sizeof(log_severity_list_t));
   if (isVerbose)
-    set_log_severity_config(LOG_DEBUG, LOG_ERR, s);
+    set_log_severity_config(LOG_DEBUG, LOG_ERR, severities);
   else
-    set_log_severity_config(LOG_WARN, LOG_ERR, s);
-  add_stream_log(s, "<stderr>", fileno(stderr));
+    set_log_severity_config(LOG_WARN, LOG_ERR, severities);
+  add_stream_log(severities, "<stderr>", fileno(stderr));
+  tor_free(severities);
 
   if (n_args == 1) {
     log_debug(LD_CONFIG, "defaulting to localhost");
diff --git a/src/tools/tor_runner.c b/src/tools/tor_runner.c
new file mode 100644
index 0000000000..cad57a7665
--- /dev/null
+++ b/src/tools/tor_runner.c
@@ -0,0 +1,102 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file tor_runner.c
+ * @brief Experimental module to emulate tor_run_main() API with fork+exec
+ *
+ * The functions here are meant to allow the application developer to
+ * use the tor_run_main() API without having to care whether Tor is
+ * running in-process or out-of-process.  For in-process usage, the
+ * developer can link Tor as a library and call tor_run_main(); for
+ * out-of-process usage, the developer can link this library instead.
+ *
+ * This interface is EXPERIMENTAL; please let us know if you would like
+ * to depend on it.  We don't know yet whether it will be reliable in
+ * practice.
+ */
+
+/* NOTE: This module is supposed to work without the standard Tor utility
+ * functions.  Don't add more dependencies!
+ */
+
+#include "tor_api.h"
+#include "tor_api_internal.h"
+
+#include "orconfig.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef __GNUC__
+#define __attribute__(x)
+#endif
+
+static void child(const tor_main_configuration_t *cfg)
+  __attribute__((noreturn));
+
+int
+tor_run_main(const tor_main_configuration_t *cfg)
+{
+  pid_t pid = fork();
+  if (pid == 0) {
+    child(cfg);
+    exit(0); /* Unreachable */
+  }
+
+  pid_t stopped_pid;
+  int status = 0;
+  do {
+    stopped_pid = waitpid(pid, &status, 0);
+  } while (stopped_pid == -1);
+
+  /* Note: these return values are not documented.  No return value is
+   * documented! */
+
+  if (stopped_pid != pid) {
+    return -99999;
+  }
+  if (WIFSTOPPED(status)) {
+    return WEXITSTATUS(status);
+  }
+  if (WIFSIGNALED(status)) {
+    return -WTERMSIG(status);
+  }
+
+  return -999988;
+}
+
+/* circumlocution to avoid getting warned about calling calloc instead of
+ * tor_calloc. */
+#define real_calloc calloc
+#define real_free free
+
+static void
+child(const tor_main_configuration_t *cfg)
+{
+  /* XXXX Close unused file descriptors. */
+
+  char **args = real_calloc(cfg->argc+1, sizeof(char *));
+  memcpy(args, cfg->argv, cfg->argc * sizeof(char *));
+  args[cfg->argc] = NULL;
+
+  int rv = execv(BINDIR "/tor", args);
+
+  if (rv < 0) {
+    real_free(args);
+    exit(254);
+  } else {
+    abort(); /* Unreachable */
+  }
+}
diff --git a/src/trace/debug.h b/src/trace/debug.h
new file mode 100644
index 0000000000..3a1652543a
--- /dev/null
+++ b/src/trace/debug.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TRACE_LOG_DEBUG_H
+#define TOR_TRACE_LOG_DEBUG_H
+
+#include "torlog.h"
+
+/* Stringify pre-processor trick. */
+#define XSTR(d) STR(d)
+#define STR(s) #s
+
+/* Send every event to a debug log level. This is useful to debug new trace
+ * events without implementing them for a specific event tracing framework.
+ * Note that the arguments are ignored since at this step we do not know the
+ * types and amount there is. */
+
+/* Example on how to map a tracepoint to log_debug(). */
+#undef tor_trace
+#define tor_trace(subsystem, name, args...) \
+  log_debug(LD_GENERAL, "Trace event \"" XSTR(name) "\" from " \
+                        "\"" XSTR(subsystem) "\" hit. " \
+                        "(line "XSTR(__LINE__) ")")
+
+#endif /* TOR_TRACE_LOG_DEBUG_H */
diff --git a/src/trace/events.h b/src/trace/events.h
new file mode 100644
index 0000000000..1be1fd596e
--- /dev/null
+++ b/src/trace/events.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file events.h
+ * \brief Header file for Tor event tracing.
+ **/
+
+#ifndef TOR_TRACE_EVENTS_H
+#define TOR_TRACE_EVENTS_H
+
+/*
+ * The following defines a generic event tracing function name that has to be
+ * used to trace events in the code base.
+ *
+ * That generic function is then defined by a event tracing framework. For
+ * instance, the "log debug" framework sends all trace events to log_debug()
+ * which is defined in src/trace/debug.h which can only be enabled at compile
+ * time (--enable-event-tracing-debug).
+ *
+ * By default, every trace events in the code base are replaced by a NOP. See
+ * doc/HACKING/Tracing.md for more information on how to use event tracing or
+ * add events.
+ */
+
+#ifdef TOR_EVENT_TRACING_ENABLED
+/* Map every trace event to a per subsystem macro. */
+#define tor_trace(subsystem, name, ...) \
+  tor_trace_##subsystem(name, __VA_ARGS__)
+
+/* Enable event tracing for the debug framework where all trace events are
+ * mapped to a log_debug(). */
+#ifdef USE_EVENT_TRACING_DEBUG
+#include "trace/debug.h"
+#endif
+
+#else /* TOR_EVENT_TRACING_ENABLED */
+
+/* Reaching this point, we NOP every event declaration because event tracing
+ * is not been enabled at compile time. */
+#define tor_trace(subsystem, name, args...)
+
+#endif /* TOR_EVENT_TRACING_ENABLED */
+
+#endif /* TOR_TRACE_EVENTS_H */
diff --git a/src/trace/include.am b/src/trace/include.am
new file mode 100644
index 0000000000..3285b04de6
--- /dev/null
+++ b/src/trace/include.am
@@ -0,0 +1,22 @@
+# Include the src/ so we can use the trace/events.h statement when including
+# any file in that directory.
+AM_CPPFLAGS += -I$(srcdir)/src
+
+noinst_LIBRARIES += \
+	src/trace/libor-trace.a
+LIBOR_TRACE_A_SOURCES = \
+	src/trace/trace.c
+
+TRACEHEADERS = \
+	src/trace/trace.h \
+	src/trace/events.h
+
+if USE_EVENT_TRACING_DEBUG
+TRACEHEADERS += \
+	src/trace/debug.h
+endif
+
+# Library source files.
+src_trace_libor_trace_a_SOURCES = $(LIBOR_TRACE_A_SOURCES)
+
+noinst_HEADERS+= $(TRACEHEADERS)
diff --git a/src/trace/trace.c b/src/trace/trace.c
new file mode 100644
index 0000000000..fcdb80091f
--- /dev/null
+++ b/src/trace/trace.c
@@ -0,0 +1,11 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "trace.h"
+
+/** Initialize the tracing library. */
+void
+tor_trace_init(void)
+{
+}
+
diff --git a/src/trace/trace.h b/src/trace/trace.h
new file mode 100644
index 0000000000..28fcd8eea8
--- /dev/null
+++ b/src/trace/trace.h
@@ -0,0 +1,10 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TRACE_TRACE_H
+#define TOR_TRACE_TRACE_H
+
+void tor_trace_init(void);
+
+#endif // TOR_TRACE_TRACE_H
+
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
new file mode 100644
index 0000000000..59e6b38384
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -0,0 +1,281 @@
+/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "channelpadding_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int channelpaddingnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || channelpaddingnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+channelpadding_negotiate_t *
+channelpadding_negotiate_new(void)
+{
+  channelpadding_negotiate_t *val = trunnel_calloc(1, sizeof(channelpadding_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CHANNELPADDING_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+channelpadding_negotiate_clear(channelpadding_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+channelpadding_negotiate_free(channelpadding_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  channelpadding_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(channelpadding_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+channelpadding_negotiate_get_version(const channelpadding_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+channelpadding_negotiate_set_version(channelpadding_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+channelpadding_negotiate_get_command(const channelpadding_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+channelpadding_negotiate_set_command(channelpadding_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CHANNELPADDING_COMMAND_START || val == CHANNELPADDING_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint16_t
+channelpadding_negotiate_get_ito_low_ms(const channelpadding_negotiate_t *inp)
+{
+  return inp->ito_low_ms;
+}
+int
+channelpadding_negotiate_set_ito_low_ms(channelpadding_negotiate_t *inp, uint16_t val)
+{
+  inp->ito_low_ms = val;
+  return 0;
+}
+uint16_t
+channelpadding_negotiate_get_ito_high_ms(const channelpadding_negotiate_t *inp)
+{
+  return inp->ito_high_ms;
+}
+int
+channelpadding_negotiate_set_ito_high_ms(channelpadding_negotiate_t *inp, uint16_t val)
+{
+  inp->ito_high_ms = val;
+  return 0;
+}
+const char *
+channelpadding_negotiate_check(const channelpadding_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CHANNELPADDING_COMMAND_START || obj->command == CHANNELPADDING_COMMAND_STOP))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+channelpadding_negotiate_encoded_len(const channelpadding_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != channelpadding_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u16 ito_low_ms */
+  result += 2;
+
+  /* Length of u16 ito_high_ms */
+  result += 2;
+  return result;
+}
+int
+channelpadding_negotiate_clear_errors(channelpadding_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+channelpadding_negotiate_encode(uint8_t *output, const size_t avail, const channelpadding_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = channelpadding_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = channelpadding_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u16 ito_low_ms */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->ito_low_ms));
+  written += 2; ptr += 2;
+
+  /* Encode u16 ito_high_ms */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->ito_high_ms));
+  written += 2; ptr += 2;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As channelpadding_negotiate_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+channelpadding_negotiate_parse_into(channelpadding_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CHANNELPADDING_COMMAND_START || obj->command == CHANNELPADDING_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u16 ito_low_ms */
+  CHECK_REMAINING(2, truncated);
+  obj->ito_low_ms = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u16 ito_high_ms */
+  CHECK_REMAINING(2, truncated);
+  obj->ito_high_ms = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+channelpadding_negotiate_parse(channelpadding_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = channelpadding_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = channelpadding_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    channelpadding_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
new file mode 100644
index 0000000000..fcfc232fea
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -0,0 +1,98 @@
+/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CHANNELPADDING_NEGOTIATION_H
+#define TRUNNEL_CHANNELPADDING_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CHANNELPADDING_COMMAND_STOP 1
+#define CHANNELPADDING_COMMAND_START 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CHANNELPADDING_NEGOTIATE)
+struct channelpadding_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  uint16_t ito_low_ms;
+  uint16_t ito_high_ms;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct channelpadding_negotiate_st channelpadding_negotiate_t;
+/** Return a newly allocated channelpadding_negotiate with all
+ * elements set to zero.
+ */
+channelpadding_negotiate_t *channelpadding_negotiate_new(void);
+/** Release all storage held by the channelpadding_negotiate in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void channelpadding_negotiate_free(channelpadding_negotiate_t *victim);
+/** Try to parse a channelpadding_negotiate from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated channelpadding_negotiate_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t channelpadding_negotiate_parse(channelpadding_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * channelpadding_negotiate in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t channelpadding_negotiate_encoded_len(const channelpadding_negotiate_t *obj);
+/** Try to encode the channelpadding_negotiate from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t channelpadding_negotiate_encode(uint8_t *output, size_t avail, const channelpadding_negotiate_t *input);
+/** Check whether the internal state of the channelpadding_negotiate
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *channelpadding_negotiate_check(const channelpadding_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int channelpadding_negotiate_clear_errors(channelpadding_negotiate_t *obj);
+/** Return the value of the version field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint8_t channelpadding_negotiate_get_version(const channelpadding_negotiate_t *inp);
+/** Set the value of the version field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_version(channelpadding_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint8_t channelpadding_negotiate_get_command(const channelpadding_negotiate_t *inp);
+/** Set the value of the command field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_command(channelpadding_negotiate_t *inp, uint8_t val);
+/** Return the value of the ito_low_ms field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint16_t channelpadding_negotiate_get_ito_low_ms(const channelpadding_negotiate_t *inp);
+/** Set the value of the ito_low_ms field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_ito_low_ms(channelpadding_negotiate_t *inp, uint16_t val);
+/** Return the value of the ito_high_ms field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint16_t channelpadding_negotiate_get_ito_high_ms(const channelpadding_negotiate_t *inp);
+/** Set the value of the ito_high_ms field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_ito_high_ms(channelpadding_negotiate_t *inp, uint16_t val);
+
+
+#endif
diff --git a/src/trunnel/channelpadding_negotiation.trunnel b/src/trunnel/channelpadding_negotiation.trunnel
new file mode 100644
index 0000000000..7f2d4795b0
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.trunnel
@@ -0,0 +1,17 @@
+const CHANNELPADDING_COMMAND_STOP = 1;
+const CHANNELPADDING_COMMAND_START = 2;
+
+/* This command tells the relay to alter its min and max netflow
+   timeout range values, and send padding at that rate (resuming
+   if stopped). */
+struct channelpadding_negotiate {
+  u8 version IN [0];
+  u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP];
+
+  /* Min must not be lower than the current consensus parameter
+     nf_ito_low. */
+  u16 ito_low_ms;
+
+  /* Max must not be lower than ito_low_ms */
+  u16 ito_high_ms;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 24988d510b..1276c7a505 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.4.6.
+/* ed25519_cert.c -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -13,7 +13,7 @@
   } while (0)
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
-/* If we're runnning a static analysis tool, we don't want it to complain
+/* If we're running a static analysis tool, we don't want it to complain
  * that some of our remaining-bytes checks are dead-code. */
 int edcert_deadcode_dummy__ = 0;
 #define OR_DEADCODE_DUMMY || edcert_deadcode_dummy__
@@ -28,6 +28,281 @@ int edcert_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
+create2_cell_body_t *
+create2_cell_body_new(void)
+{
+  create2_cell_body_t *val = trunnel_calloc(1, sizeof(create2_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+create2_cell_body_clear(create2_cell_body_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->handshake_data);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->handshake_data);
+}
+
+void
+create2_cell_body_free(create2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  create2_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(create2_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint16_t
+create2_cell_body_get_handshake_type(const create2_cell_body_t *inp)
+{
+  return inp->handshake_type;
+}
+int
+create2_cell_body_set_handshake_type(create2_cell_body_t *inp, uint16_t val)
+{
+  inp->handshake_type = val;
+  return 0;
+}
+uint16_t
+create2_cell_body_get_handshake_len(const create2_cell_body_t *inp)
+{
+  return inp->handshake_len;
+}
+int
+create2_cell_body_set_handshake_len(create2_cell_body_t *inp, uint16_t val)
+{
+  inp->handshake_len = val;
+  return 0;
+}
+size_t
+create2_cell_body_getlen_handshake_data(const create2_cell_body_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->handshake_data);
+}
+
+uint8_t
+create2_cell_body_get_handshake_data(create2_cell_body_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->handshake_data, idx);
+}
+
+uint8_t
+create2_cell_body_getconst_handshake_data(const create2_cell_body_t *inp, size_t idx)
+{
+  return create2_cell_body_get_handshake_data((create2_cell_body_t*)inp, idx);
+}
+int
+create2_cell_body_set_handshake_data(create2_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->handshake_data, idx, elt);
+  return 0;
+}
+int
+create2_cell_body_add_handshake_data(create2_cell_body_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->handshake_data.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->handshake_data, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+create2_cell_body_getarray_handshake_data(create2_cell_body_t *inp)
+{
+  return inp->handshake_data.elts_;
+}
+const uint8_t  *
+create2_cell_body_getconstarray_handshake_data(const create2_cell_body_t *inp)
+{
+  return (const uint8_t  *)create2_cell_body_getarray_handshake_data((create2_cell_body_t*)inp);
+}
+int
+create2_cell_body_setlen_handshake_data(create2_cell_body_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->handshake_data.allocated_,
+                 &inp->handshake_data.n_, inp->handshake_data.elts_, newlen,
+                 sizeof(inp->handshake_data.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->handshake_data.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+create2_cell_body_check(const create2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->handshake_data) != obj->handshake_len)
+    return "Length mismatch for handshake_data";
+  return NULL;
+}
+
+ssize_t
+create2_cell_body_encoded_len(const create2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != create2_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u16 handshake_type */
+  result += 2;
+
+  /* Length of u16 handshake_len */
+  result += 2;
+
+  /* Length of u8 handshake_data[handshake_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->handshake_data);
+  return result;
+}
+int
+create2_cell_body_clear_errors(create2_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+create2_cell_body_encode(uint8_t *output, const size_t avail, const create2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = create2_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = create2_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u16 handshake_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->handshake_type));
+  written += 2; ptr += 2;
+
+  /* Encode u16 handshake_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->handshake_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 handshake_data[handshake_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->handshake_data);
+    trunnel_assert(obj->handshake_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->handshake_data.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As create2_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+create2_cell_body_parse_into(create2_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u16 handshake_type */
+  CHECK_REMAINING(2, truncated);
+  obj->handshake_type = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u16 handshake_len */
+  CHECK_REMAINING(2, truncated);
+  obj->handshake_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 handshake_data[handshake_len] */
+  CHECK_REMAINING(obj->handshake_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->handshake_data, obj->handshake_len, {});
+  obj->handshake_data.n_ = obj->handshake_len;
+  if (obj->handshake_len)
+    memcpy(obj->handshake_data.elts_, ptr, obj->handshake_len);
+  ptr += obj->handshake_len; remaining -= obj->handshake_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+create2_cell_body_parse(create2_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = create2_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = create2_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    create2_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 ed25519_cert_extension_t *
 ed25519_cert_extension_new(void)
 {
@@ -58,7 +333,7 @@ ed25519_cert_extension_free(ed25519_cert_extension_t *obj)
 }
 
 uint16_t
-ed25519_cert_extension_get_ext_length(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_length(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_length;
 }
@@ -69,7 +344,7 @@ ed25519_cert_extension_set_ext_length(ed25519_cert_extension_t *inp, uint16_t va
   return 0;
 }
 uint8_t
-ed25519_cert_extension_get_ext_type(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_type(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_type;
 }
@@ -80,7 +355,7 @@ ed25519_cert_extension_set_ext_type(ed25519_cert_extension_t *inp, uint8_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_extension_get_ext_flags(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_flags(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_flags;
 }
@@ -97,12 +372,17 @@ ed25519_cert_extension_getlen_un_signing_key(const ed25519_cert_extension_t *inp
 }
 
 uint8_t
-ed25519_cert_extension_get_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx)
+ed25519_cert_extension_get_un_signing_key(ed25519_cert_extension_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->un_signing_key[idx];
 }
 
+uint8_t
+ed25519_cert_extension_getconst_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx)
+{
+  return ed25519_cert_extension_get_un_signing_key((ed25519_cert_extension_t*)inp, idx);
+}
 int
 ed25519_cert_extension_set_un_signing_key(ed25519_cert_extension_t *inp, size_t idx, uint8_t elt)
 {
@@ -116,6 +396,11 @@ ed25519_cert_extension_getarray_un_signing_key(ed25519_cert_extension_t *inp)
 {
   return inp->un_signing_key;
 }
+const uint8_t  *
+ed25519_cert_extension_getconstarray_un_signing_key(const ed25519_cert_extension_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_extension_getarray_un_signing_key((ed25519_cert_extension_t*)inp);
+}
 size_t
 ed25519_cert_extension_getlen_un_unparsed(const ed25519_cert_extension_t *inp)
 {
@@ -128,6 +413,11 @@ ed25519_cert_extension_get_un_unparsed(ed25519_cert_extension_t *inp, size_t idx
   return TRUNNEL_DYNARRAY_GET(&inp->un_unparsed, idx);
 }
 
+uint8_t
+ed25519_cert_extension_getconst_un_unparsed(const ed25519_cert_extension_t *inp, size_t idx)
+{
+  return ed25519_cert_extension_get_un_unparsed((ed25519_cert_extension_t*)inp, idx);
+}
 int
 ed25519_cert_extension_set_un_unparsed(ed25519_cert_extension_t *inp, size_t idx, uint8_t elt)
 {
@@ -149,6 +439,11 @@ ed25519_cert_extension_getarray_un_unparsed(ed25519_cert_extension_t *inp)
 {
   return inp->un_unparsed.elts_;
 }
+const uint8_t  *
+ed25519_cert_extension_getconstarray_un_unparsed(const ed25519_cert_extension_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_extension_getarray_un_unparsed((ed25519_cert_extension_t*)inp);
+}
 int
 ed25519_cert_extension_setlen_un_unparsed(ed25519_cert_extension_t *inp, size_t newlen)
 {
@@ -410,6 +705,878 @@ ed25519_cert_extension_parse(ed25519_cert_extension_t **output, const uint8_t *i
   }
   return result;
 }
+extend1_cell_body_t *
+extend1_cell_body_new(void)
+{
+  extend1_cell_body_t *val = trunnel_calloc(1, sizeof(extend1_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+extend1_cell_body_clear(extend1_cell_body_t *obj)
+{
+  (void) obj;
+}
+
+void
+extend1_cell_body_free(extend1_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  extend1_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(extend1_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+extend1_cell_body_get_ipv4addr(const extend1_cell_body_t *inp)
+{
+  return inp->ipv4addr;
+}
+int
+extend1_cell_body_set_ipv4addr(extend1_cell_body_t *inp, uint32_t val)
+{
+  inp->ipv4addr = val;
+  return 0;
+}
+uint16_t
+extend1_cell_body_get_port(const extend1_cell_body_t *inp)
+{
+  return inp->port;
+}
+int
+extend1_cell_body_set_port(extend1_cell_body_t *inp, uint16_t val)
+{
+  inp->port = val;
+  return 0;
+}
+size_t
+extend1_cell_body_getlen_onionskin(const extend1_cell_body_t *inp)
+{
+  (void)inp;  return 186;
+}
+
+uint8_t
+extend1_cell_body_get_onionskin(extend1_cell_body_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 186);
+  return inp->onionskin[idx];
+}
+
+uint8_t
+extend1_cell_body_getconst_onionskin(const extend1_cell_body_t *inp, size_t idx)
+{
+  return extend1_cell_body_get_onionskin((extend1_cell_body_t*)inp, idx);
+}
+int
+extend1_cell_body_set_onionskin(extend1_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 186);
+  inp->onionskin[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+extend1_cell_body_getarray_onionskin(extend1_cell_body_t *inp)
+{
+  return inp->onionskin;
+}
+const uint8_t  *
+extend1_cell_body_getconstarray_onionskin(const extend1_cell_body_t *inp)
+{
+  return (const uint8_t  *)extend1_cell_body_getarray_onionskin((extend1_cell_body_t*)inp);
+}
+size_t
+extend1_cell_body_getlen_identity(const extend1_cell_body_t *inp)
+{
+  (void)inp;  return 20;
+}
+
+uint8_t
+extend1_cell_body_get_identity(extend1_cell_body_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 20);
+  return inp->identity[idx];
+}
+
+uint8_t
+extend1_cell_body_getconst_identity(const extend1_cell_body_t *inp, size_t idx)
+{
+  return extend1_cell_body_get_identity((extend1_cell_body_t*)inp, idx);
+}
+int
+extend1_cell_body_set_identity(extend1_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 20);
+  inp->identity[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+extend1_cell_body_getarray_identity(extend1_cell_body_t *inp)
+{
+  return inp->identity;
+}
+const uint8_t  *
+extend1_cell_body_getconstarray_identity(const extend1_cell_body_t *inp)
+{
+  return (const uint8_t  *)extend1_cell_body_getarray_identity((extend1_cell_body_t*)inp);
+}
+const char *
+extend1_cell_body_check(const extend1_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+extend1_cell_body_encoded_len(const extend1_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != extend1_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u32 ipv4addr */
+  result += 4;
+
+  /* Length of u16 port */
+  result += 2;
+
+  /* Length of u8 onionskin[186] */
+  result += 186;
+
+  /* Length of u8 identity[20] */
+  result += 20;
+  return result;
+}
+int
+extend1_cell_body_clear_errors(extend1_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+extend1_cell_body_encode(uint8_t *output, const size_t avail, const extend1_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = extend1_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = extend1_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 ipv4addr */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->ipv4addr));
+  written += 4; ptr += 4;
+
+  /* Encode u16 port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->port));
+  written += 2; ptr += 2;
+
+  /* Encode u8 onionskin[186] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 186)
+    goto truncated;
+  memcpy(ptr, obj->onionskin, 186);
+  written += 186; ptr += 186;
+
+  /* Encode u8 identity[20] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 20)
+    goto truncated;
+  memcpy(ptr, obj->identity, 20);
+  written += 20; ptr += 20;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As extend1_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+extend1_cell_body_parse_into(extend1_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 ipv4addr */
+  CHECK_REMAINING(4, truncated);
+  obj->ipv4addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse u16 port */
+  CHECK_REMAINING(2, truncated);
+  obj->port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 onionskin[186] */
+  CHECK_REMAINING(186, truncated);
+  memcpy(obj->onionskin, ptr, 186);
+  remaining -= 186; ptr += 186;
+
+  /* Parse u8 identity[20] */
+  CHECK_REMAINING(20, truncated);
+  memcpy(obj->identity, ptr, 20);
+  remaining -= 20; ptr += 20;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+extend1_cell_body_parse(extend1_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = extend1_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = extend1_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    extend1_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+link_specifier_t *
+link_specifier_new(void)
+{
+  link_specifier_t *val = trunnel_calloc(1, sizeof(link_specifier_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+link_specifier_clear(link_specifier_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->un_unrecognized);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->un_unrecognized);
+}
+
+void
+link_specifier_free(link_specifier_t *obj)
+{
+  if (obj == NULL)
+    return;
+  link_specifier_clear(obj);
+  trunnel_memwipe(obj, sizeof(link_specifier_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+link_specifier_get_ls_type(const link_specifier_t *inp)
+{
+  return inp->ls_type;
+}
+int
+link_specifier_set_ls_type(link_specifier_t *inp, uint8_t val)
+{
+  inp->ls_type = val;
+  return 0;
+}
+uint8_t
+link_specifier_get_ls_len(const link_specifier_t *inp)
+{
+  return inp->ls_len;
+}
+int
+link_specifier_set_ls_len(link_specifier_t *inp, uint8_t val)
+{
+  inp->ls_len = val;
+  return 0;
+}
+uint32_t
+link_specifier_get_un_ipv4_addr(const link_specifier_t *inp)
+{
+  return inp->un_ipv4_addr;
+}
+int
+link_specifier_set_un_ipv4_addr(link_specifier_t *inp, uint32_t val)
+{
+  inp->un_ipv4_addr = val;
+  return 0;
+}
+uint16_t
+link_specifier_get_un_ipv4_port(const link_specifier_t *inp)
+{
+  return inp->un_ipv4_port;
+}
+int
+link_specifier_set_un_ipv4_port(link_specifier_t *inp, uint16_t val)
+{
+  inp->un_ipv4_port = val;
+  return 0;
+}
+size_t
+link_specifier_getlen_un_ipv6_addr(const link_specifier_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+link_specifier_get_un_ipv6_addr(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->un_ipv6_addr[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_ipv6_addr(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_ipv6_addr((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_ipv6_addr(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->un_ipv6_addr[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_ipv6_addr(link_specifier_t *inp)
+{
+  return inp->un_ipv6_addr;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_ipv6_addr(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_ipv6_addr((link_specifier_t*)inp);
+}
+uint16_t
+link_specifier_get_un_ipv6_port(const link_specifier_t *inp)
+{
+  return inp->un_ipv6_port;
+}
+int
+link_specifier_set_un_ipv6_port(link_specifier_t *inp, uint16_t val)
+{
+  inp->un_ipv6_port = val;
+  return 0;
+}
+size_t
+link_specifier_getlen_un_legacy_id(const link_specifier_t *inp)
+{
+  (void)inp;  return 20;
+}
+
+uint8_t
+link_specifier_get_un_legacy_id(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 20);
+  return inp->un_legacy_id[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_legacy_id(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_legacy_id((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_legacy_id(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 20);
+  inp->un_legacy_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_legacy_id(link_specifier_t *inp)
+{
+  return inp->un_legacy_id;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_legacy_id(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_legacy_id((link_specifier_t*)inp);
+}
+size_t
+link_specifier_getlen_un_ed25519_id(const link_specifier_t *inp)
+{
+  (void)inp;  return 32;
+}
+
+uint8_t
+link_specifier_get_un_ed25519_id(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 32);
+  return inp->un_ed25519_id[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_ed25519_id(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_ed25519_id((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_ed25519_id(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 32);
+  inp->un_ed25519_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_ed25519_id(link_specifier_t *inp)
+{
+  return inp->un_ed25519_id;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_ed25519_id(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_ed25519_id((link_specifier_t*)inp);
+}
+size_t
+link_specifier_getlen_un_unrecognized(const link_specifier_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->un_unrecognized);
+}
+
+uint8_t
+link_specifier_get_un_unrecognized(link_specifier_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->un_unrecognized, idx);
+}
+
+uint8_t
+link_specifier_getconst_un_unrecognized(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_unrecognized((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_unrecognized(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->un_unrecognized, idx, elt);
+  return 0;
+}
+int
+link_specifier_add_un_unrecognized(link_specifier_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->un_unrecognized, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+link_specifier_getarray_un_unrecognized(link_specifier_t *inp)
+{
+  return inp->un_unrecognized.elts_;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_unrecognized(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_unrecognized((link_specifier_t*)inp);
+}
+int
+link_specifier_setlen_un_unrecognized(link_specifier_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->un_unrecognized.allocated_,
+                 &inp->un_unrecognized.n_, inp->un_unrecognized.elts_, newlen,
+                 sizeof(inp->un_unrecognized.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->un_unrecognized.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+link_specifier_check(const link_specifier_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->ls_type) {
+
+    case LS_IPV4:
+      break;
+
+    case LS_IPV6:
+      break;
+
+    case LS_LEGACY_ID:
+      break;
+
+    case LS_ED25519_ID:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+link_specifier_encoded_len(const link_specifier_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != link_specifier_check(obj))
+     return -1;
+
+
+  /* Length of u8 ls_type */
+  result += 1;
+
+  /* Length of u8 ls_len */
+  result += 1;
+  switch (obj->ls_type) {
+
+    case LS_IPV4:
+
+      /* Length of u32 un_ipv4_addr */
+      result += 4;
+
+      /* Length of u16 un_ipv4_port */
+      result += 2;
+      break;
+
+    case LS_IPV6:
+
+      /* Length of u8 un_ipv6_addr[16] */
+      result += 16;
+
+      /* Length of u16 un_ipv6_port */
+      result += 2;
+      break;
+
+    case LS_LEGACY_ID:
+
+      /* Length of u8 un_legacy_id[20] */
+      result += 20;
+      break;
+
+    case LS_ED25519_ID:
+
+      /* Length of u8 un_ed25519_id[32] */
+      result += 32;
+      break;
+
+    default:
+
+      /* Length of u8 un_unrecognized[] */
+      result += TRUNNEL_DYNARRAY_LEN(&obj->un_unrecognized);
+      break;
+  }
+  return result;
+}
+int
+link_specifier_clear_errors(link_specifier_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+link_specifier_encode(uint8_t *output, const size_t avail, const link_specifier_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = link_specifier_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_ls_len = NULL;
+
+  if (NULL != (msg = link_specifier_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 ls_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->ls_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 ls_len */
+  backptr_ls_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->ls_len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union un[ls_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->ls_type) {
+
+      case LS_IPV4:
+
+        /* Encode u32 un_ipv4_addr */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->un_ipv4_addr));
+        written += 4; ptr += 4;
+
+        /* Encode u16 un_ipv4_port */
+        trunnel_assert(written <= avail);
+        if (avail - written < 2)
+          goto truncated;
+        trunnel_set_uint16(ptr, trunnel_htons(obj->un_ipv4_port));
+        written += 2; ptr += 2;
+        break;
+
+      case LS_IPV6:
+
+        /* Encode u8 un_ipv6_addr[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->un_ipv6_addr, 16);
+        written += 16; ptr += 16;
+
+        /* Encode u16 un_ipv6_port */
+        trunnel_assert(written <= avail);
+        if (avail - written < 2)
+          goto truncated;
+        trunnel_set_uint16(ptr, trunnel_htons(obj->un_ipv6_port));
+        written += 2; ptr += 2;
+        break;
+
+      case LS_LEGACY_ID:
+
+        /* Encode u8 un_legacy_id[20] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 20)
+          goto truncated;
+        memcpy(ptr, obj->un_legacy_id, 20);
+        written += 20; ptr += 20;
+        break;
+
+      case LS_ED25519_ID:
+
+        /* Encode u8 un_ed25519_id[32] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 32)
+          goto truncated;
+        memcpy(ptr, obj->un_ed25519_id, 32);
+        written += 32; ptr += 32;
+        break;
+
+      default:
+
+        /* Encode u8 un_unrecognized[] */
+        {
+          size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->un_unrecognized);
+          trunnel_assert(written <= avail);
+          if (avail - written < elt_len)
+            goto truncated;
+          if (elt_len)
+            memcpy(ptr, obj->un_unrecognized.elts_, elt_len);
+          written += elt_len; ptr += elt_len;
+        }
+        break;
+    }
+    /* Write the length field back to ls_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_ls_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As link_specifier_parse(), but do not allocate the output object.
+ */
+static ssize_t
+link_specifier_parse_into(link_specifier_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 ls_type */
+  CHECK_REMAINING(1, truncated);
+  obj->ls_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 ls_len */
+  CHECK_REMAINING(1, truncated);
+  obj->ls_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->ls_len, truncated);
+    remaining_after = remaining - obj->ls_len;
+    remaining = obj->ls_len;
+
+    /* Parse union un[ls_type] */
+    switch (obj->ls_type) {
+
+      case LS_IPV4:
+
+        /* Parse u32 un_ipv4_addr */
+        CHECK_REMAINING(4, fail);
+        obj->un_ipv4_addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+
+        /* Parse u16 un_ipv4_port */
+        CHECK_REMAINING(2, fail);
+        obj->un_ipv4_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+        remaining -= 2; ptr += 2;
+        break;
+
+      case LS_IPV6:
+
+        /* Parse u8 un_ipv6_addr[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->un_ipv6_addr, ptr, 16);
+        remaining -= 16; ptr += 16;
+
+        /* Parse u16 un_ipv6_port */
+        CHECK_REMAINING(2, fail);
+        obj->un_ipv6_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+        remaining -= 2; ptr += 2;
+        break;
+
+      case LS_LEGACY_ID:
+
+        /* Parse u8 un_legacy_id[20] */
+        CHECK_REMAINING(20, fail);
+        memcpy(obj->un_legacy_id, ptr, 20);
+        remaining -= 20; ptr += 20;
+        break;
+
+      case LS_ED25519_ID:
+
+        /* Parse u8 un_ed25519_id[32] */
+        CHECK_REMAINING(32, fail);
+        memcpy(obj->un_ed25519_id, ptr, 32);
+        remaining -= 32; ptr += 32;
+        break;
+
+      default:
+
+        /* Parse u8 un_unrecognized[] */
+        TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->un_unrecognized, remaining, {});
+        obj->un_unrecognized.n_ = remaining;
+        if (remaining)
+          memcpy(obj->un_unrecognized.elts_, ptr, remaining);
+        ptr += remaining; remaining -= remaining;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = link_specifier_new();
+  if (NULL == *output)
+    return -1;
+  result = link_specifier_parse_into(*output, input, len_in);
+  if (result < 0) {
+    link_specifier_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 ed25519_cert_t *
 ed25519_cert_new(void)
 {
@@ -448,7 +1615,7 @@ ed25519_cert_free(ed25519_cert_t *obj)
 }
 
 uint8_t
-ed25519_cert_get_version(ed25519_cert_t *inp)
+ed25519_cert_get_version(const ed25519_cert_t *inp)
 {
   return inp->version;
 }
@@ -463,7 +1630,7 @@ ed25519_cert_set_version(ed25519_cert_t *inp, uint8_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_get_cert_type(ed25519_cert_t *inp)
+ed25519_cert_get_cert_type(const ed25519_cert_t *inp)
 {
   return inp->cert_type;
 }
@@ -474,7 +1641,7 @@ ed25519_cert_set_cert_type(ed25519_cert_t *inp, uint8_t val)
   return 0;
 }
 uint32_t
-ed25519_cert_get_exp_field(ed25519_cert_t *inp)
+ed25519_cert_get_exp_field(const ed25519_cert_t *inp)
 {
   return inp->exp_field;
 }
@@ -485,7 +1652,7 @@ ed25519_cert_set_exp_field(ed25519_cert_t *inp, uint32_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_get_cert_key_type(ed25519_cert_t *inp)
+ed25519_cert_get_cert_key_type(const ed25519_cert_t *inp)
 {
   return inp->cert_key_type;
 }
@@ -502,12 +1669,17 @@ ed25519_cert_getlen_certified_key(const ed25519_cert_t *inp)
 }
 
 uint8_t
-ed25519_cert_get_certified_key(const ed25519_cert_t *inp, size_t idx)
+ed25519_cert_get_certified_key(ed25519_cert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->certified_key[idx];
 }
 
+uint8_t
+ed25519_cert_getconst_certified_key(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_certified_key((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_certified_key(ed25519_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -521,8 +1693,13 @@ ed25519_cert_getarray_certified_key(ed25519_cert_t *inp)
 {
   return inp->certified_key;
 }
+const uint8_t  *
+ed25519_cert_getconstarray_certified_key(const ed25519_cert_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_getarray_certified_key((ed25519_cert_t*)inp);
+}
 uint8_t
-ed25519_cert_get_n_extensions(ed25519_cert_t *inp)
+ed25519_cert_get_n_extensions(const ed25519_cert_t *inp)
 {
   return inp->n_extensions;
 }
@@ -544,6 +1721,11 @@ ed25519_cert_get_ext(ed25519_cert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->ext, idx);
 }
 
+ const struct ed25519_cert_extension_st *
+ed25519_cert_getconst_ext(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_ext((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_ext(ed25519_cert_t *inp, size_t idx, struct ed25519_cert_extension_st * elt)
 {
@@ -577,6 +1759,11 @@ ed25519_cert_getarray_ext(ed25519_cert_t *inp)
 {
   return inp->ext.elts_;
 }
+const struct ed25519_cert_extension_st *  const  *
+ed25519_cert_getconstarray_ext(const ed25519_cert_t *inp)
+{
+  return (const struct ed25519_cert_extension_st *  const  *)ed25519_cert_getarray_ext((ed25519_cert_t*)inp);
+}
 int
 ed25519_cert_setlen_ext(ed25519_cert_t *inp, size_t newlen)
 {
@@ -604,12 +1791,17 @@ ed25519_cert_getlen_signature(const ed25519_cert_t *inp)
 }
 
 uint8_t
-ed25519_cert_get_signature(const ed25519_cert_t *inp, size_t idx)
+ed25519_cert_get_signature(ed25519_cert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 64);
   return inp->signature[idx];
 }
 
+uint8_t
+ed25519_cert_getconst_signature(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_signature((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_signature(ed25519_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -623,6 +1815,11 @@ ed25519_cert_getarray_signature(ed25519_cert_t *inp)
 {
   return inp->signature;
 }
+const uint8_t  *
+ed25519_cert_getconstarray_signature(const ed25519_cert_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_getarray_signature((ed25519_cert_t*)inp);
+}
 const char *
 ed25519_cert_check(const ed25519_cert_t *obj)
 {
@@ -887,3 +2084,630 @@ ed25519_cert_parse(ed25519_cert_t **output, const uint8_t *input, const size_t l
   }
   return result;
 }
+extend2_cell_body_t *
+extend2_cell_body_new(void)
+{
+  extend2_cell_body_t *val = trunnel_calloc(1, sizeof(extend2_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+extend2_cell_body_clear(extend2_cell_body_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->ls);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->ls);
+  create2_cell_body_free(obj->create2);
+  obj->create2 = NULL;
+}
+
+void
+extend2_cell_body_free(extend2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  extend2_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(extend2_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+extend2_cell_body_get_n_spec(const extend2_cell_body_t *inp)
+{
+  return inp->n_spec;
+}
+int
+extend2_cell_body_set_n_spec(extend2_cell_body_t *inp, uint8_t val)
+{
+  inp->n_spec = val;
+  return 0;
+}
+size_t
+extend2_cell_body_getlen_ls(const extend2_cell_body_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->ls);
+}
+
+struct link_specifier_st *
+extend2_cell_body_get_ls(extend2_cell_body_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->ls, idx);
+}
+
+ const struct link_specifier_st *
+extend2_cell_body_getconst_ls(const extend2_cell_body_t *inp, size_t idx)
+{
+  return extend2_cell_body_get_ls((extend2_cell_body_t*)inp, idx);
+}
+int
+extend2_cell_body_set_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->ls, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return extend2_cell_body_set0_ls(inp, idx, elt);
+}
+int
+extend2_cell_body_set0_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->ls, idx, elt);
+  return 0;
+}
+int
+extend2_cell_body_add_ls(extend2_cell_body_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->ls.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->ls, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+extend2_cell_body_getarray_ls(extend2_cell_body_t *inp)
+{
+  return inp->ls.elts_;
+}
+const struct link_specifier_st *  const  *
+extend2_cell_body_getconstarray_ls(const extend2_cell_body_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)extend2_cell_body_getarray_ls((extend2_cell_body_t*)inp);
+}
+int
+extend2_cell_body_setlen_ls(extend2_cell_body_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->ls.allocated_,
+                 &inp->ls.n_, inp->ls.elts_, newlen,
+                 sizeof(inp->ls.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->ls.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct create2_cell_body_st *
+extend2_cell_body_get_create2(extend2_cell_body_t *inp)
+{
+  return inp->create2;
+}
+const struct create2_cell_body_st *
+extend2_cell_body_getconst_create2(const extend2_cell_body_t *inp)
+{
+  return extend2_cell_body_get_create2((extend2_cell_body_t*) inp);
+}
+int
+extend2_cell_body_set_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val)
+{
+  if (inp->create2 && inp->create2 != val)
+    create2_cell_body_free(inp->create2);
+  return extend2_cell_body_set0_create2(inp, val);
+}
+int
+extend2_cell_body_set0_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val)
+{
+  inp->create2 = val;
+  return 0;
+}
+const char *
+extend2_cell_body_check(const extend2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->ls, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->ls) != obj->n_spec)
+    return "Length mismatch for ls";
+  {
+    const char *msg;
+    if (NULL != (msg = create2_cell_body_check(obj->create2)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+extend2_cell_body_encoded_len(const extend2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != extend2_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_spec */
+  result += 1;
+
+  /* Length of struct link_specifier ls[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+    }
+  }
+
+  /* Length of struct create2_cell_body create2 */
+  result += create2_cell_body_encoded_len(obj->create2);
+  return result;
+}
+int
+extend2_cell_body_clear_errors(extend2_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+extend2_cell_body_encode(uint8_t *output, const size_t avail, const extend2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = extend2_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = extend2_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_spec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_spec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier ls[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+  /* Encode struct create2_cell_body create2 */
+  trunnel_assert(written <= avail);
+  result = create2_cell_body_encode(ptr, avail - written, obj->create2);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As extend2_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+extend2_cell_body_parse_into(extend2_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_spec */
+  CHECK_REMAINING(1, truncated);
+  obj->n_spec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier ls[n_spec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->ls, obj->n_spec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_spec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->ls, elt, {link_specifier_free(elt);});
+    }
+  }
+
+  /* Parse struct create2_cell_body create2 */
+  result = create2_cell_body_parse(&obj->create2, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+extend2_cell_body_parse(extend2_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = extend2_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = extend2_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    extend2_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+link_specifier_list_t *
+link_specifier_list_new(void)
+{
+  link_specifier_list_t *val = trunnel_calloc(1, sizeof(link_specifier_list_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+link_specifier_list_clear(link_specifier_list_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->spec);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->spec);
+}
+
+void
+link_specifier_list_free(link_specifier_list_t *obj)
+{
+  if (obj == NULL)
+    return;
+  link_specifier_list_clear(obj);
+  trunnel_memwipe(obj, sizeof(link_specifier_list_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+link_specifier_list_get_n_spec(const link_specifier_list_t *inp)
+{
+  return inp->n_spec;
+}
+int
+link_specifier_list_set_n_spec(link_specifier_list_t *inp, uint8_t val)
+{
+  inp->n_spec = val;
+  return 0;
+}
+size_t
+link_specifier_list_getlen_spec(const link_specifier_list_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->spec);
+}
+
+struct link_specifier_st *
+link_specifier_list_get_spec(link_specifier_list_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->spec, idx);
+}
+
+ const struct link_specifier_st *
+link_specifier_list_getconst_spec(const link_specifier_list_t *inp, size_t idx)
+{
+  return link_specifier_list_get_spec((link_specifier_list_t*)inp, idx);
+}
+int
+link_specifier_list_set_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->spec, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return link_specifier_list_set0_spec(inp, idx, elt);
+}
+int
+link_specifier_list_set0_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->spec, idx, elt);
+  return 0;
+}
+int
+link_specifier_list_add_spec(link_specifier_list_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->spec.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->spec, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+link_specifier_list_getarray_spec(link_specifier_list_t *inp)
+{
+  return inp->spec.elts_;
+}
+const struct link_specifier_st *  const  *
+link_specifier_list_getconstarray_spec(const link_specifier_list_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)link_specifier_list_getarray_spec((link_specifier_list_t*)inp);
+}
+int
+link_specifier_list_setlen_spec(link_specifier_list_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->spec.allocated_,
+                 &inp->spec.n_, inp->spec.elts_, newlen,
+                 sizeof(inp->spec.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->spec.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+link_specifier_list_check(const link_specifier_list_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->spec, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->spec) != obj->n_spec)
+    return "Length mismatch for spec";
+  return NULL;
+}
+
+ssize_t
+link_specifier_list_encoded_len(const link_specifier_list_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != link_specifier_list_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_spec */
+  result += 1;
+
+  /* Length of struct link_specifier spec[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+    }
+  }
+  return result;
+}
+int
+link_specifier_list_clear_errors(link_specifier_list_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+link_specifier_list_encode(uint8_t *output, const size_t avail, const link_specifier_list_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = link_specifier_list_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = link_specifier_list_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_spec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_spec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier spec[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As link_specifier_list_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+link_specifier_list_parse_into(link_specifier_list_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_spec */
+  CHECK_REMAINING(1, truncated);
+  obj->n_spec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier spec[n_spec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->spec, obj->n_spec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_spec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->spec, elt, {link_specifier_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+link_specifier_list_parse(link_specifier_list_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = link_specifier_list_new();
+  if (NULL == *output)
+    return -1;
+  result = link_specifier_list_parse_into(*output, input, len_in);
+  if (result < 0) {
+    link_specifier_list_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index 28f6feef31..e086c6fced 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by by Trunnel v1.4.6.
+/* ed25519_cert.h -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,19 @@
 
 #define CERTEXT_SIGNED_WITH_KEY 4
 #define CERTEXT_FLAG_AFFECTS_VALIDATION 1
+#define LS_IPV4 0
+#define LS_IPV6 1
+#define LS_LEGACY_ID 2
+#define LS_ED25519_ID 3
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CREATE2_CELL_BODY)
+struct create2_cell_body_st {
+  uint16_t handshake_type;
+  uint16_t handshake_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) handshake_data;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct create2_cell_body_st create2_cell_body_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_ED25519_CERT_EXTENSION)
 struct ed25519_cert_extension_st {
   uint16_t ext_length;
@@ -21,6 +34,31 @@ struct ed25519_cert_extension_st {
 };
 #endif
 typedef struct ed25519_cert_extension_st ed25519_cert_extension_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_EXTEND1_CELL_BODY)
+struct extend1_cell_body_st {
+  uint32_t ipv4addr;
+  uint16_t port;
+  uint8_t onionskin[186];
+  uint8_t identity[20];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct extend1_cell_body_st extend1_cell_body_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_LINK_SPECIFIER)
+struct link_specifier_st {
+  uint8_t ls_type;
+  uint8_t ls_len;
+  uint32_t un_ipv4_addr;
+  uint16_t un_ipv4_port;
+  uint8_t un_ipv6_addr[16];
+  uint16_t un_ipv6_port;
+  uint8_t un_legacy_id[20];
+  uint8_t un_ed25519_id[32];
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) un_unrecognized;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct link_specifier_st link_specifier_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_ED25519_CERT)
 struct ed25519_cert_st {
   uint8_t version;
@@ -35,6 +73,112 @@ struct ed25519_cert_st {
 };
 #endif
 typedef struct ed25519_cert_st ed25519_cert_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_EXTEND2_CELL_BODY)
+struct extend2_cell_body_st {
+  uint8_t n_spec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) ls;
+  struct create2_cell_body_st *create2;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct extend2_cell_body_st extend2_cell_body_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_LINK_SPECIFIER_LIST)
+struct link_specifier_list_st {
+  uint8_t n_spec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) spec;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct link_specifier_list_st link_specifier_list_t;
+/** Return a newly allocated create2_cell_body with all elements set
+ * to zero.
+ */
+create2_cell_body_t *create2_cell_body_new(void);
+/** Release all storage held by the create2_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void create2_cell_body_free(create2_cell_body_t *victim);
+/** Try to parse a create2_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * create2_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t create2_cell_body_parse(create2_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * create2_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t create2_cell_body_encoded_len(const create2_cell_body_t *obj);
+/** Try to encode the create2_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t create2_cell_body_encode(uint8_t *output, size_t avail, const create2_cell_body_t *input);
+/** Check whether the internal state of the create2_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *create2_cell_body_check(const create2_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int create2_cell_body_clear_errors(create2_cell_body_t *obj);
+/** Return the value of the handshake_type field of the
+ * create2_cell_body_t in 'inp'
+ */
+uint16_t create2_cell_body_get_handshake_type(const create2_cell_body_t *inp);
+/** Set the value of the handshake_type field of the
+ * create2_cell_body_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int create2_cell_body_set_handshake_type(create2_cell_body_t *inp, uint16_t val);
+/** Return the value of the handshake_len field of the
+ * create2_cell_body_t in 'inp'
+ */
+uint16_t create2_cell_body_get_handshake_len(const create2_cell_body_t *inp);
+/** Set the value of the handshake_len field of the
+ * create2_cell_body_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int create2_cell_body_set_handshake_len(create2_cell_body_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the handshake_data
+ * field of the create2_cell_body_t in 'inp'.
+ */
+size_t create2_cell_body_getlen_handshake_data(const create2_cell_body_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp'.
+ */
+uint8_t create2_cell_body_get_handshake_data(create2_cell_body_t *inp, size_t idx);
+/** As create2_cell_body_get_handshake_data, but take and return a
+ * const pointer
+ */
+uint8_t create2_cell_body_getconst_handshake_data(const create2_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int create2_cell_body_set_handshake_data(create2_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp'.
+ */
+int create2_cell_body_add_handshake_data(create2_cell_body_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field handshake_data
+ * of 'inp'.
+ */
+uint8_t * create2_cell_body_getarray_handshake_data(create2_cell_body_t *inp);
+/** As create2_cell_body_get_handshake_data, but take and return a
+ * const pointer
+ */
+const uint8_t  * create2_cell_body_getconstarray_handshake_data(const create2_cell_body_t *inp);
+/** Change the length of the variable-length array field
+ * handshake_data of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int create2_cell_body_setlen_handshake_data(create2_cell_body_t *inp, size_t newlen);
 /** Return a newly allocated ed25519_cert_extension with all elements
  * set to zero.
  */
@@ -74,7 +218,7 @@ int ed25519_cert_extension_clear_errors(ed25519_cert_extension_t *obj);
 /** Return the value of the ext_length field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint16_t ed25519_cert_extension_get_ext_length(ed25519_cert_extension_t *inp);
+uint16_t ed25519_cert_extension_get_ext_length(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_length field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -83,7 +227,7 @@ int ed25519_cert_extension_set_ext_length(ed25519_cert_extension_t *inp, uint16_
 /** Return the value of the ext_type field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint8_t ed25519_cert_extension_get_ext_type(ed25519_cert_extension_t *inp);
+uint8_t ed25519_cert_extension_get_ext_type(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_type field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -92,7 +236,7 @@ int ed25519_cert_extension_set_ext_type(ed25519_cert_extension_t *inp, uint8_t v
 /** Return the value of the ext_flags field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint8_t ed25519_cert_extension_get_ext_flags(ed25519_cert_extension_t *inp);
+uint8_t ed25519_cert_extension_get_ext_flags(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_flags field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -105,7 +249,11 @@ size_t ed25519_cert_extension_getlen_un_signing_key(const ed25519_cert_extension
 /** Return the element at position 'idx' of the fixed array field
  * un_signing_key of the ed25519_cert_extension_t in 'inp'.
  */
-uint8_t ed25519_cert_extension_get_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx);
+uint8_t ed25519_cert_extension_get_un_signing_key(ed25519_cert_extension_t *inp, size_t idx);
+/** As ed25519_cert_extension_get_un_signing_key, but take and return
+ * a const pointer
+ */
+uint8_t ed25519_cert_extension_getconst_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * un_signing_key of the ed25519_cert_extension_t in 'inp', so that it
  * will hold the value 'elt'.
@@ -115,6 +263,10 @@ int ed25519_cert_extension_set_un_signing_key(ed25519_cert_extension_t *inp, siz
  * 'inp'.
  */
 uint8_t * ed25519_cert_extension_getarray_un_signing_key(ed25519_cert_extension_t *inp);
+/** As ed25519_cert_extension_get_un_signing_key, but take and return
+ * a const pointer
+ */
+const uint8_t  * ed25519_cert_extension_getconstarray_un_signing_key(const ed25519_cert_extension_t *inp);
 /** Return the length of the dynamic array holding the un_unparsed
  * field of the ed25519_cert_extension_t in 'inp'.
  */
@@ -123,6 +275,10 @@ size_t ed25519_cert_extension_getlen_un_unparsed(const ed25519_cert_extension_t
  * un_unparsed of the ed25519_cert_extension_t in 'inp'.
  */
 uint8_t ed25519_cert_extension_get_un_unparsed(ed25519_cert_extension_t *inp, size_t idx);
+/** As ed25519_cert_extension_get_un_unparsed, but take and return a
+ * const pointer
+ */
+uint8_t ed25519_cert_extension_getconst_un_unparsed(const ed25519_cert_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * un_unparsed of the ed25519_cert_extension_t in 'inp', so that it
  * will hold the value 'elt'.
@@ -136,11 +292,308 @@ int ed25519_cert_extension_add_un_unparsed(ed25519_cert_extension_t *inp, uint8_
  * 'inp'.
  */
 uint8_t * ed25519_cert_extension_getarray_un_unparsed(ed25519_cert_extension_t *inp);
+/** As ed25519_cert_extension_get_un_unparsed, but take and return a
+ * const pointer
+ */
+const uint8_t  * ed25519_cert_extension_getconstarray_un_unparsed(const ed25519_cert_extension_t *inp);
 /** Change the length of the variable-length array field un_unparsed
  * of 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on
  * success; return -1 and set the error code on 'inp' on failure.
  */
 int ed25519_cert_extension_setlen_un_unparsed(ed25519_cert_extension_t *inp, size_t newlen);
+/** Return a newly allocated extend1_cell_body with all elements set
+ * to zero.
+ */
+extend1_cell_body_t *extend1_cell_body_new(void);
+/** Release all storage held by the extend1_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void extend1_cell_body_free(extend1_cell_body_t *victim);
+/** Try to parse a extend1_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * extend1_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t extend1_cell_body_parse(extend1_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * extend1_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t extend1_cell_body_encoded_len(const extend1_cell_body_t *obj);
+/** Try to encode the extend1_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t extend1_cell_body_encode(uint8_t *output, size_t avail, const extend1_cell_body_t *input);
+/** Check whether the internal state of the extend1_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *extend1_cell_body_check(const extend1_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int extend1_cell_body_clear_errors(extend1_cell_body_t *obj);
+/** Return the value of the ipv4addr field of the extend1_cell_body_t
+ * in 'inp'
+ */
+uint32_t extend1_cell_body_get_ipv4addr(const extend1_cell_body_t *inp);
+/** Set the value of the ipv4addr field of the extend1_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend1_cell_body_set_ipv4addr(extend1_cell_body_t *inp, uint32_t val);
+/** Return the value of the port field of the extend1_cell_body_t in
+ * 'inp'
+ */
+uint16_t extend1_cell_body_get_port(const extend1_cell_body_t *inp);
+/** Set the value of the port field of the extend1_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend1_cell_body_set_port(extend1_cell_body_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the onionskin
+ * field of the extend1_cell_body_t in 'inp'.
+ */
+size_t extend1_cell_body_getlen_onionskin(const extend1_cell_body_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * onionskin of the extend1_cell_body_t in 'inp'.
+ */
+uint8_t extend1_cell_body_get_onionskin(extend1_cell_body_t *inp, size_t idx);
+/** As extend1_cell_body_get_onionskin, but take and return a const
+ * pointer
+ */
+uint8_t extend1_cell_body_getconst_onionskin(const extend1_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * onionskin of the extend1_cell_body_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int extend1_cell_body_set_onionskin(extend1_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 186-element array field onionskin of
+ * 'inp'.
+ */
+uint8_t * extend1_cell_body_getarray_onionskin(extend1_cell_body_t *inp);
+/** As extend1_cell_body_get_onionskin, but take and return a const
+ * pointer
+ */
+const uint8_t  * extend1_cell_body_getconstarray_onionskin(const extend1_cell_body_t *inp);
+/** Return the (constant) length of the array holding the identity
+ * field of the extend1_cell_body_t in 'inp'.
+ */
+size_t extend1_cell_body_getlen_identity(const extend1_cell_body_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * identity of the extend1_cell_body_t in 'inp'.
+ */
+uint8_t extend1_cell_body_get_identity(extend1_cell_body_t *inp, size_t idx);
+/** As extend1_cell_body_get_identity, but take and return a const
+ * pointer
+ */
+uint8_t extend1_cell_body_getconst_identity(const extend1_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * identity of the extend1_cell_body_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int extend1_cell_body_set_identity(extend1_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 20-element array field identity of 'inp'.
+ */
+uint8_t * extend1_cell_body_getarray_identity(extend1_cell_body_t *inp);
+/** As extend1_cell_body_get_identity, but take and return a const
+ * pointer
+ */
+const uint8_t  * extend1_cell_body_getconstarray_identity(const extend1_cell_body_t *inp);
+/** Return a newly allocated link_specifier with all elements set to
+ * zero.
+ */
+link_specifier_t *link_specifier_new(void);
+/** Release all storage held by the link_specifier in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void link_specifier_free(link_specifier_t *victim);
+/** Try to parse a link_specifier from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * link_specifier_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * link_specifier in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t link_specifier_encoded_len(const link_specifier_t *obj);
+/** Try to encode the link_specifier from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t link_specifier_encode(uint8_t *output, size_t avail, const link_specifier_t *input);
+/** Check whether the internal state of the link_specifier in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *link_specifier_check(const link_specifier_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int link_specifier_clear_errors(link_specifier_t *obj);
+/** Return the value of the ls_type field of the link_specifier_t in
+ * 'inp'
+ */
+uint8_t link_specifier_get_ls_type(const link_specifier_t *inp);
+/** Set the value of the ls_type field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_ls_type(link_specifier_t *inp, uint8_t val);
+/** Return the value of the ls_len field of the link_specifier_t in
+ * 'inp'
+ */
+uint8_t link_specifier_get_ls_len(const link_specifier_t *inp);
+/** Set the value of the ls_len field of the link_specifier_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int link_specifier_set_ls_len(link_specifier_t *inp, uint8_t val);
+/** Return the value of the un_ipv4_addr field of the link_specifier_t
+ * in 'inp'
+ */
+uint32_t link_specifier_get_un_ipv4_addr(const link_specifier_t *inp);
+/** Set the value of the un_ipv4_addr field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv4_addr(link_specifier_t *inp, uint32_t val);
+/** Return the value of the un_ipv4_port field of the link_specifier_t
+ * in 'inp'
+ */
+uint16_t link_specifier_get_un_ipv4_port(const link_specifier_t *inp);
+/** Set the value of the un_ipv4_port field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv4_port(link_specifier_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the un_ipv6_addr
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_ipv6_addr(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_ipv6_addr of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_ipv6_addr(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_ipv6_addr, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_ipv6_addr(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_ipv6_addr of the link_specifier_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int link_specifier_set_un_ipv6_addr(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field un_ipv6_addr of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_ipv6_addr(link_specifier_t *inp);
+/** As link_specifier_get_un_ipv6_addr, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_ipv6_addr(const link_specifier_t *inp);
+/** Return the value of the un_ipv6_port field of the link_specifier_t
+ * in 'inp'
+ */
+uint16_t link_specifier_get_un_ipv6_port(const link_specifier_t *inp);
+/** Set the value of the un_ipv6_port field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv6_port(link_specifier_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the un_legacy_id
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_legacy_id(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_legacy_id of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_legacy_id(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_legacy_id, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_legacy_id(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_legacy_id of the link_specifier_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int link_specifier_set_un_legacy_id(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 20-element array field un_legacy_id of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_legacy_id(link_specifier_t *inp);
+/** As link_specifier_get_un_legacy_id, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_legacy_id(const link_specifier_t *inp);
+/** Return the (constant) length of the array holding the
+ * un_ed25519_id field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_ed25519_id(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_ed25519_id of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_ed25519_id(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_ed25519_id, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_ed25519_id(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_ed25519_id of the link_specifier_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int link_specifier_set_un_ed25519_id(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 32-element array field un_ed25519_id of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_ed25519_id(link_specifier_t *inp);
+/** As link_specifier_get_un_ed25519_id, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_ed25519_id(const link_specifier_t *inp);
+/** Return the length of the dynamic array holding the un_unrecognized
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_unrecognized(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_unrecognized(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_unrecognized, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_unrecognized(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int link_specifier_set_un_unrecognized(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp'.
+ */
+int link_specifier_add_un_unrecognized(link_specifier_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field
+ * un_unrecognized of 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_unrecognized(link_specifier_t *inp);
+/** As link_specifier_get_un_unrecognized, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_unrecognized(const link_specifier_t *inp);
+/** Change the length of the variable-length array field
+ * un_unrecognized of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int link_specifier_setlen_un_unrecognized(link_specifier_t *inp, size_t newlen);
 /** Return a newly allocated ed25519_cert with all elements set to
  * zero.
  */
@@ -179,7 +632,7 @@ int ed25519_cert_clear_errors(ed25519_cert_t *obj);
 /** Return the value of the version field of the ed25519_cert_t in
  * 'inp'
  */
-uint8_t ed25519_cert_get_version(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_version(const ed25519_cert_t *inp);
 /** Set the value of the version field of the ed25519_cert_t in 'inp'
  * to 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -188,7 +641,7 @@ int ed25519_cert_set_version(ed25519_cert_t *inp, uint8_t val);
 /** Return the value of the cert_type field of the ed25519_cert_t in
  * 'inp'
  */
-uint8_t ed25519_cert_get_cert_type(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_cert_type(const ed25519_cert_t *inp);
 /** Set the value of the cert_type field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -197,7 +650,7 @@ int ed25519_cert_set_cert_type(ed25519_cert_t *inp, uint8_t val);
 /** Return the value of the exp_field field of the ed25519_cert_t in
  * 'inp'
  */
-uint32_t ed25519_cert_get_exp_field(ed25519_cert_t *inp);
+uint32_t ed25519_cert_get_exp_field(const ed25519_cert_t *inp);
 /** Set the value of the exp_field field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -206,7 +659,7 @@ int ed25519_cert_set_exp_field(ed25519_cert_t *inp, uint32_t val);
 /** Return the value of the cert_key_type field of the ed25519_cert_t
  * in 'inp'
  */
-uint8_t ed25519_cert_get_cert_key_type(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_cert_key_type(const ed25519_cert_t *inp);
 /** Set the value of the cert_key_type field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -219,7 +672,11 @@ size_t ed25519_cert_getlen_certified_key(const ed25519_cert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * certified_key of the ed25519_cert_t in 'inp'.
  */
-uint8_t ed25519_cert_get_certified_key(const ed25519_cert_t *inp, size_t idx);
+uint8_t ed25519_cert_get_certified_key(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_certified_key, but take and return a const
+ * pointer
+ */
+uint8_t ed25519_cert_getconst_certified_key(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * certified_key of the ed25519_cert_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -229,10 +686,14 @@ int ed25519_cert_set_certified_key(ed25519_cert_t *inp, size_t idx, uint8_t elt)
  * 'inp'.
  */
 uint8_t * ed25519_cert_getarray_certified_key(ed25519_cert_t *inp);
+/** As ed25519_cert_get_certified_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * ed25519_cert_getconstarray_certified_key(const ed25519_cert_t *inp);
 /** Return the value of the n_extensions field of the ed25519_cert_t
  * in 'inp'
  */
-uint8_t ed25519_cert_get_n_extensions(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_n_extensions(const ed25519_cert_t *inp);
 /** Set the value of the n_extensions field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -246,6 +707,9 @@ size_t ed25519_cert_getlen_ext(const ed25519_cert_t *inp);
  * ext of the ed25519_cert_t in 'inp'.
  */
 struct ed25519_cert_extension_st * ed25519_cert_get_ext(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_ext, but take and return a const pointer
+ */
+ const struct ed25519_cert_extension_st * ed25519_cert_getconst_ext(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * ext of the ed25519_cert_t in 'inp', so that it will hold the value
  * 'elt'. Free the previous value, if any.
@@ -261,6 +725,9 @@ int ed25519_cert_add_ext(ed25519_cert_t *inp, struct ed25519_cert_extension_st *
 /** Return a pointer to the variable-length array field ext of 'inp'.
  */
 struct ed25519_cert_extension_st * * ed25519_cert_getarray_ext(ed25519_cert_t *inp);
+/** As ed25519_cert_get_ext, but take and return a const pointer
+ */
+const struct ed25519_cert_extension_st *  const  * ed25519_cert_getconstarray_ext(const ed25519_cert_t *inp);
 /** Change the length of the variable-length array field ext of 'inp'
  * to 'newlen'.Fill extra elements with NULL; free removed elements.
  * Return 0 on success; return -1 and set the error code on 'inp' on
@@ -274,7 +741,10 @@ size_t ed25519_cert_getlen_signature(const ed25519_cert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * signature of the ed25519_cert_t in 'inp'.
  */
-uint8_t ed25519_cert_get_signature(const ed25519_cert_t *inp, size_t idx);
+uint8_t ed25519_cert_get_signature(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_signature, but take and return a const pointer
+ */
+uint8_t ed25519_cert_getconst_signature(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * signature of the ed25519_cert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -283,6 +753,190 @@ int ed25519_cert_set_signature(ed25519_cert_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 64-element array field signature of 'inp'.
  */
 uint8_t * ed25519_cert_getarray_signature(ed25519_cert_t *inp);
+/** As ed25519_cert_get_signature, but take and return a const pointer
+ */
+const uint8_t  * ed25519_cert_getconstarray_signature(const ed25519_cert_t *inp);
+/** Return a newly allocated extend2_cell_body with all elements set
+ * to zero.
+ */
+extend2_cell_body_t *extend2_cell_body_new(void);
+/** Release all storage held by the extend2_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void extend2_cell_body_free(extend2_cell_body_t *victim);
+/** Try to parse a extend2_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * extend2_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t extend2_cell_body_parse(extend2_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * extend2_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t extend2_cell_body_encoded_len(const extend2_cell_body_t *obj);
+/** Try to encode the extend2_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t extend2_cell_body_encode(uint8_t *output, size_t avail, const extend2_cell_body_t *input);
+/** Check whether the internal state of the extend2_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *extend2_cell_body_check(const extend2_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int extend2_cell_body_clear_errors(extend2_cell_body_t *obj);
+/** Return the value of the n_spec field of the extend2_cell_body_t in
+ * 'inp'
+ */
+uint8_t extend2_cell_body_get_n_spec(const extend2_cell_body_t *inp);
+/** Set the value of the n_spec field of the extend2_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend2_cell_body_set_n_spec(extend2_cell_body_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the ls field of the
+ * extend2_cell_body_t in 'inp'.
+ */
+size_t extend2_cell_body_getlen_ls(const extend2_cell_body_t *inp);
+/** Return the element at position 'idx' of the dynamic array field ls
+ * of the extend2_cell_body_t in 'inp'.
+ */
+struct link_specifier_st * extend2_cell_body_get_ls(extend2_cell_body_t *inp, size_t idx);
+/** As extend2_cell_body_get_ls, but take and return a const pointer
+ */
+ const struct link_specifier_st * extend2_cell_body_getconst_ls(const extend2_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field ls
+ * of the extend2_cell_body_t in 'inp', so that it will hold the value
+ * 'elt'. Free the previous value, if any.
+ */
+int extend2_cell_body_set_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As extend2_cell_body_set_ls, but does not free the previous value.
+ */
+int extend2_cell_body_set0_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field ls of the
+ * extend2_cell_body_t in 'inp'.
+ */
+int extend2_cell_body_add_ls(extend2_cell_body_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field ls of 'inp'.
+ */
+struct link_specifier_st * * extend2_cell_body_getarray_ls(extend2_cell_body_t *inp);
+/** As extend2_cell_body_get_ls, but take and return a const pointer
+ */
+const struct link_specifier_st *  const  * extend2_cell_body_getconstarray_ls(const extend2_cell_body_t *inp);
+/** Change the length of the variable-length array field ls of 'inp'
+ * to 'newlen'.Fill extra elements with NULL; free removed elements.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int extend2_cell_body_setlen_ls(extend2_cell_body_t *inp, size_t newlen);
+/** Return the value of the create2 field of the extend2_cell_body_t
+ * in 'inp'
+ */
+struct create2_cell_body_st * extend2_cell_body_get_create2(extend2_cell_body_t *inp);
+/** As extend2_cell_body_get_create2, but take and return a const
+ * pointer
+ */
+const struct create2_cell_body_st * extend2_cell_body_getconst_create2(const extend2_cell_body_t *inp);
+/** Set the value of the create2 field of the extend2_cell_body_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int extend2_cell_body_set_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val);
+/** As extend2_cell_body_set_create2, but does not free the previous
+ * value.
+ */
+int extend2_cell_body_set0_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val);
+/** Return a newly allocated link_specifier_list with all elements set
+ * to zero.
+ */
+link_specifier_list_t *link_specifier_list_new(void);
+/** Release all storage held by the link_specifier_list in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void link_specifier_list_free(link_specifier_list_t *victim);
+/** Try to parse a link_specifier_list from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated link_specifier_list_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t link_specifier_list_parse(link_specifier_list_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * link_specifier_list in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t link_specifier_list_encoded_len(const link_specifier_list_t *obj);
+/** Try to encode the link_specifier_list from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t link_specifier_list_encode(uint8_t *output, size_t avail, const link_specifier_list_t *input);
+/** Check whether the internal state of the link_specifier_list in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *link_specifier_list_check(const link_specifier_list_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int link_specifier_list_clear_errors(link_specifier_list_t *obj);
+/** Return the value of the n_spec field of the link_specifier_list_t
+ * in 'inp'
+ */
+uint8_t link_specifier_list_get_n_spec(const link_specifier_list_t *inp);
+/** Set the value of the n_spec field of the link_specifier_list_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_list_set_n_spec(link_specifier_list_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the spec field of
+ * the link_specifier_list_t in 'inp'.
+ */
+size_t link_specifier_list_getlen_spec(const link_specifier_list_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * spec of the link_specifier_list_t in 'inp'.
+ */
+struct link_specifier_st * link_specifier_list_get_spec(link_specifier_list_t *inp, size_t idx);
+/** As link_specifier_list_get_spec, but take and return a const
+ * pointer
+ */
+ const struct link_specifier_st * link_specifier_list_getconst_spec(const link_specifier_list_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * spec of the link_specifier_list_t in 'inp', so that it will hold
+ * the value 'elt'. Free the previous value, if any.
+ */
+int link_specifier_list_set_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As link_specifier_list_set_spec, but does not free the previous
+ * value.
+ */
+int link_specifier_list_set0_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field spec of the
+ * link_specifier_list_t in 'inp'.
+ */
+int link_specifier_list_add_spec(link_specifier_list_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field spec of 'inp'.
+ */
+struct link_specifier_st * * link_specifier_list_getarray_spec(link_specifier_list_t *inp);
+/** As link_specifier_list_get_spec, but take and return a const
+ * pointer
+ */
+const struct link_specifier_st *  const  * link_specifier_list_getconstarray_spec(const link_specifier_list_t *inp);
+/** Change the length of the variable-length array field spec of 'inp'
+ * to 'newlen'.Fill extra elements with NULL; free removed elements.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int link_specifier_list_setlen_spec(link_specifier_list_t *inp, size_t newlen);
 
 
 #endif
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index c46f1b6c6b..8d6483d558 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -23,44 +23,17 @@ struct ed25519_cert_extension {
   };
 }
 
-/*
-struct cert_revocation {
-  u8 prefix[8];
-  u8 version IN [1];
-  u8 keytype;
-  u8 identity_key[32];
-  u8 revoked_key[32];
-  u64 published;
-  u8 n_extensions;
-  struct cert_extension ext[n_extensions];
-  u8 signature[64];
-}
-
-struct crosscert_ed_rsa {
-  u8 ed_key[32];
-  u32 expiration_date;
-  u8 signature[128];
-}
-
-struct auth02_cell {
-  u8 type[8];
-  u8 cid[32];
-  u8 sid[32];
-  u8 cid_ed[32];
-  u8 sid_ed[32];
-  u8 slog[32];
-  u8 clog[32];
-  u8 scert[32];
-  u8 tlssecrets[32];
-  u8 rand[24];
-  u8 sig[64];
-}
-
 const LS_IPV4 = 0x00;
 const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
 const LS_ED25519_ID = 0x03;
 
+// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by
+//  taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or
+//  if we ever make link_specifier contain other types, we will
+//  need to refactor that function to do the copy by encoding and decoding the
+//  object.
+
 // amended from tor.trunnel
 struct link_specifier {
   u8 ls_type;
@@ -73,4 +46,27 @@ struct link_specifier {
     default: u8 unrecognized[];
   };
 }
-*/
\ No newline at end of file
+
+struct link_specifier_list {
+  u8 n_spec;
+  struct link_specifier spec[n_spec];
+}
+
+struct extend1_cell_body {
+  u32 ipv4addr;
+  u16 port;
+  u8 onionskin[186];
+  u8 identity[20];
+}
+
+struct create2_cell_body {
+  u16 handshake_type;
+  u16 handshake_len;
+  u8 handshake_data[handshake_len];
+}
+
+struct extend2_cell_body {
+  u8 n_spec;
+  struct link_specifier ls[n_spec];
+  struct create2_cell_body create2;
+}
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
new file mode 100644
index 0000000000..af223560c1
--- /dev/null
+++ b/src/trunnel/hs/cell_common.c
@@ -0,0 +1,595 @@
+/* cell_common.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_common.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellcommon_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellcommon_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+trn_cell_extension_fields_t *
+trn_cell_extension_fields_new(void)
+{
+  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->field);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->field);
+}
+
+void
+trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_fields_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+{
+  return inp->field_type;
+}
+int
+trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+{
+  inp->field_type = val;
+  return 0;
+}
+uint8_t
+trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+{
+  return inp->field_len;
+}
+int
+trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+{
+  inp->field_len = val;
+  return 0;
+}
+size_t
+trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->field);
+}
+
+uint8_t
+trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
+}
+
+uint8_t
+trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+{
+  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+}
+int
+trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->field.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->field, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+{
+  return inp->field.elts_;
+}
+const uint8_t  *
+trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+{
+  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+}
+int
+trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->field.allocated_,
+                 &inp->field.n_, inp->field.elts_, newlen,
+                 sizeof(inp->field.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->field.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->field) != obj->field_len)
+    return "Length mismatch for field";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_fields_check(obj))
+     return -1;
+
+
+  /* Length of u8 field_type */
+  result += 1;
+
+  /* Length of u8 field_len */
+  result += 1;
+
+  /* Length of u8 field[field_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->field);
+  return result;
+}
+int
+trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 field_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->field_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 field_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->field_len));
+  written += 1; ptr += 1;
+
+  /* Encode u8 field[field_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->field);
+    trunnel_assert(obj->field_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->field.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_fields_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 field_type */
+  CHECK_REMAINING(1, truncated);
+  obj->field_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 field_len */
+  CHECK_REMAINING(1, truncated);
+  obj->field_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 field[field_len] */
+  CHECK_REMAINING(obj->field_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->field, obj->field_len, {});
+  obj->field.n_ = obj->field_len;
+  if (obj->field_len)
+    memcpy(obj->field.elts_, ptr, obj->field_len);
+  ptr += obj->field_len; remaining -= obj->field_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_fields_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_fields_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_extension_t *
+trn_cell_extension_new(void)
+{
+  trn_cell_extension_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_clear(trn_cell_extension_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->fields);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->fields);
+}
+
+void
+trn_cell_extension_free(trn_cell_extension_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_get_num(const trn_cell_extension_t *inp)
+{
+  return inp->num;
+}
+int
+trn_cell_extension_set_num(trn_cell_extension_t *inp, uint8_t val)
+{
+  inp->num = val;
+  return 0;
+}
+size_t
+trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->fields);
+}
+
+struct trn_cell_extension_fields_st *
+trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+}
+
+ const struct trn_cell_extension_fields_st *
+trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
+{
+  return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
+}
+int
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+{
+  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_fields_free(oldval);
+  return trn_cell_extension_set0_fields(inp, idx, elt);
+}
+int
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->fields.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_fields_st * *
+trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
+{
+  return inp->fields.elts_;
+}
+const struct trn_cell_extension_fields_st *  const  *
+trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
+{
+  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+}
+int
+trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_fields_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
+                 &inp->fields.n_, inp->fields.elts_, newlen,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->fields.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_check(const trn_cell_extension_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->fields) != obj->num)
+    return "Length mismatch for fields";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_check(obj))
+     return -1;
+
+
+  /* Length of u8 num */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_fields fields[num] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_clear_errors(trn_cell_extension_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_extension_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 num */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->num));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_fields fields[num] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 num */
+  CHECK_REMAINING(1, truncated);
+  obj->num = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_fields fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  {
+    trn_cell_extension_fields_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->num; ++idx) {
+      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
new file mode 100644
index 0000000000..e08eedfdb3
--- /dev/null
+++ b/src/trunnel/hs/cell_common.h
@@ -0,0 +1,203 @@
+/* cell_common.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_COMMON_H
+#define TRUNNEL_CELL_COMMON_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
+struct trn_cell_extension_fields_st {
+  uint8_t field_type;
+  uint8_t field_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
+struct trn_cell_extension_st {
+  uint8_t num;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+/** Return a newly allocated trn_cell_extension_fields with all
+ * elements set to zero.
+ */
+trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
+/** Release all storage held by the trn_cell_extension_fields in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
+/** Try to parse a trn_cell_extension_fields from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
+/** Try to encode the trn_cell_extension_fields from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
+/** Check whether the internal state of the trn_cell_extension_fields
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+/** Return the value of the field_type field of the
+ * trn_cell_extension_fields_t in 'inp'
+ */
+uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+/** Set the value of the field_type field of the
+ * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+/** Return the value of the field_len field of the
+ * trn_cell_extension_fields_t in 'inp'
+ */
+uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+/** Set the value of the field_len field of the
+ * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the field field of
+ * the trn_cell_extension_fields_t in 'inp'.
+ */
+size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * field of the trn_cell_extension_fields_t in 'inp'.
+ */
+uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
+/** As trn_cell_extension_fields_get_field, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field field of the
+ * trn_cell_extension_fields_t in 'inp'.
+ */
+int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field field of
+ * 'inp'.
+ */
+uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
+/** As trn_cell_extension_fields_get_field, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+/** Change the length of the variable-length array field field of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension with all elements set
+ * to zero.
+ */
+trn_cell_extension_t *trn_cell_extension_new(void);
+/** Release all storage held by the trn_cell_extension in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+/** Try to parse a trn_cell_extension from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+/** Try to encode the trn_cell_extension from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+/** Check whether the internal state of the trn_cell_extension in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+/** Return the value of the num field of the trn_cell_extension_t in
+ * 'inp'
+ */
+uint8_t trn_cell_extension_get_num(const trn_cell_extension_t *inp);
+/** Set the value of the num field of the trn_cell_extension_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_extension_set_num(trn_cell_extension_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the fields field of
+ * the trn_cell_extension_t in 'inp'.
+ */
+size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * fields of the trn_cell_extension_t in 'inp'.
+ */
+struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+/** As trn_cell_extension_get_fields, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * fields of the trn_cell_extension_t in 'inp', so that it will hold
+ * the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+/** As trn_cell_extension_set_fields, but does not free the previous
+ * value.
+ */
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+/** Append a new element 'elt' to the dynamic array field fields of
+ * the trn_cell_extension_t in 'inp'.
+ */
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+/** Return a pointer to the variable-length array field fields of
+ * 'inp'.
+ */
+struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+/** As trn_cell_extension_get_fields, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+/** Change the length of the variable-length array field fields of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
new file mode 100644
index 0000000000..1aa6999de7
--- /dev/null
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -0,0 +1,12 @@
+/* This file contains common data structure that cells use. */
+
+struct trn_cell_extension_fields {
+  u8 field_type;
+  u8 field_len;
+  u8 field[field_len];
+};
+
+struct trn_cell_extension {
+  u8 num;
+  struct trn_cell_extension_fields fields[num];
+};
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
new file mode 100644
index 0000000000..ae3b7b1bc8
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -0,0 +1,735 @@
+/* cell_establish_intro.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_establish_intro.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellestablishintro_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellestablishintro_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+trn_cell_extension_t *trn_cell_extension_new(void);
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_establish_intro_t *
+trn_cell_establish_intro_new(void)
+{
+  trn_cell_establish_intro_t *val = trunnel_calloc(1, sizeof(trn_cell_establish_intro_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_establish_intro_clear(trn_cell_establish_intro_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->auth_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->auth_key);
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->sig);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->sig);
+}
+
+void
+trn_cell_establish_intro_free(trn_cell_establish_intro_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_establish_intro_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_establish_intro_t));
+  trunnel_free_(obj);
+}
+
+const uint8_t *
+trn_cell_establish_intro_get_start_cell(const trn_cell_establish_intro_t *inp)
+{
+  return inp->start_cell;
+}
+uint8_t
+trn_cell_establish_intro_get_auth_key_type(const trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key_type;
+}
+int
+trn_cell_establish_intro_set_auth_key_type(trn_cell_establish_intro_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->auth_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_establish_intro_get_auth_key_len(const trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key_len;
+}
+int
+trn_cell_establish_intro_set_auth_key_len(trn_cell_establish_intro_t *inp, uint16_t val)
+{
+  inp->auth_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_establish_intro_getlen_auth_key(const trn_cell_establish_intro_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->auth_key);
+}
+
+uint8_t
+trn_cell_establish_intro_get_auth_key(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->auth_key, idx);
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_auth_key(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_auth_key((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_auth_key(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->auth_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_establish_intro_add_auth_key(trn_cell_establish_intro_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->auth_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->auth_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_auth_key(trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key.elts_;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_auth_key(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_auth_key((trn_cell_establish_intro_t*)inp);
+}
+int
+trn_cell_establish_intro_setlen_auth_key(trn_cell_establish_intro_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->auth_key.allocated_,
+                 &inp->auth_key.n_, inp->auth_key.elts_, newlen,
+                 sizeof(inp->auth_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->auth_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct trn_cell_extension_st *
+trn_cell_establish_intro_get_extensions(trn_cell_establish_intro_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_establish_intro_getconst_extensions(const trn_cell_establish_intro_t *inp)
+{
+  return trn_cell_establish_intro_get_extensions((trn_cell_establish_intro_t*) inp);
+}
+int
+trn_cell_establish_intro_set_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_establish_intro_set0_extensions(inp, val);
+}
+int
+trn_cell_establish_intro_set0_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const uint8_t *
+trn_cell_establish_intro_get_end_mac_fields(const trn_cell_establish_intro_t *inp)
+{
+  return inp->end_mac_fields;
+}
+size_t
+trn_cell_establish_intro_getlen_handshake_mac(const trn_cell_establish_intro_t *inp)
+{
+  (void)inp;  return TRUNNEL_SHA3_256_LEN;
+}
+
+uint8_t
+trn_cell_establish_intro_get_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SHA3_256_LEN);
+  return inp->handshake_mac[idx];
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_handshake_mac(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_handshake_mac((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SHA3_256_LEN);
+  inp->handshake_mac[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_handshake_mac(trn_cell_establish_intro_t *inp)
+{
+  return inp->handshake_mac;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_handshake_mac(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_handshake_mac((trn_cell_establish_intro_t*)inp);
+}
+const uint8_t *
+trn_cell_establish_intro_get_end_sig_fields(const trn_cell_establish_intro_t *inp)
+{
+  return inp->end_sig_fields;
+}
+uint16_t
+trn_cell_establish_intro_get_sig_len(const trn_cell_establish_intro_t *inp)
+{
+  return inp->sig_len;
+}
+int
+trn_cell_establish_intro_set_sig_len(trn_cell_establish_intro_t *inp, uint16_t val)
+{
+  inp->sig_len = val;
+  return 0;
+}
+size_t
+trn_cell_establish_intro_getlen_sig(const trn_cell_establish_intro_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->sig);
+}
+
+uint8_t
+trn_cell_establish_intro_get_sig(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_sig(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_sig((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_sig(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->sig, idx, elt);
+  return 0;
+}
+int
+trn_cell_establish_intro_add_sig(trn_cell_establish_intro_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->sig.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->sig, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_sig(trn_cell_establish_intro_t *inp)
+{
+  return inp->sig.elts_;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_sig(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_sig((trn_cell_establish_intro_t*)inp);
+}
+int
+trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->sig.allocated_,
+                 &inp->sig.n_, inp->sig.elts_, newlen,
+                 sizeof(inp->sig.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->sig.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_establish_intro_check(const trn_cell_establish_intro_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->auth_key) != obj->auth_key_len)
+    return "Length mismatch for auth_key";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->sig) != obj->sig_len)
+    return "Length mismatch for sig";
+  return NULL;
+}
+
+ssize_t
+trn_cell_establish_intro_encoded_len(const trn_cell_establish_intro_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_establish_intro_check(obj))
+     return -1;
+
+
+  /* Length of u8 auth_key_type IN [0, 1, 2] */
+  result += 1;
+
+  /* Length of u16 auth_key_len */
+  result += 2;
+
+  /* Length of u8 auth_key[auth_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  result += TRUNNEL_SHA3_256_LEN;
+
+  /* Length of u16 sig_len */
+  result += 2;
+
+  /* Length of u8 sig[sig_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->sig);
+  return result;
+}
+int
+trn_cell_establish_intro_clear_errors(trn_cell_establish_intro_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_establish_intro_encode(uint8_t *output, const size_t avail, const trn_cell_establish_intro_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_establish_intro_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_establish_intro_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 auth_key_type IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->auth_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 auth_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->auth_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 auth_key[auth_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+    trunnel_assert(obj->auth_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->auth_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_SHA3_256_LEN)
+    goto truncated;
+  memcpy(ptr, obj->handshake_mac, TRUNNEL_SHA3_256_LEN);
+  written += TRUNNEL_SHA3_256_LEN; ptr += TRUNNEL_SHA3_256_LEN;
+
+  /* Encode u16 sig_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->sig_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 sig[sig_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->sig);
+    trunnel_assert(obj->sig_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->sig.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_establish_intro_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_establish_intro_parse_into(trn_cell_establish_intro_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+  obj->start_cell = ptr;
+
+  /* Parse u8 auth_key_type IN [0, 1, 2] */
+  CHECK_REMAINING(1, truncated);
+  obj->auth_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    goto fail;
+
+  /* Parse u16 auth_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->auth_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 auth_key[auth_key_len] */
+  CHECK_REMAINING(obj->auth_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->auth_key, obj->auth_key_len, {});
+  obj->auth_key.n_ = obj->auth_key_len;
+  if (obj->auth_key_len)
+    memcpy(obj->auth_key.elts_, ptr, obj->auth_key_len);
+  ptr += obj->auth_key_len; remaining -= obj->auth_key_len;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  obj->end_mac_fields = ptr;
+
+  /* Parse u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  CHECK_REMAINING(TRUNNEL_SHA3_256_LEN, truncated);
+  memcpy(obj->handshake_mac, ptr, TRUNNEL_SHA3_256_LEN);
+  remaining -= TRUNNEL_SHA3_256_LEN; ptr += TRUNNEL_SHA3_256_LEN;
+  obj->end_sig_fields = ptr;
+
+  /* Parse u16 sig_len */
+  CHECK_REMAINING(2, truncated);
+  obj->sig_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 sig[sig_len] */
+  CHECK_REMAINING(obj->sig_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->sig, obj->sig_len, {});
+  obj->sig.n_ = obj->sig_len;
+  if (obj->sig_len)
+    memcpy(obj->sig.elts_, ptr, obj->sig_len);
+  ptr += obj->sig_len; remaining -= obj->sig_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_establish_intro_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_establish_intro_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_establish_intro_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_intro_established_t *
+trn_cell_intro_established_new(void)
+{
+  trn_cell_intro_established_t *val = trunnel_calloc(1, sizeof(trn_cell_intro_established_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_intro_established_clear(trn_cell_intro_established_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+}
+
+void
+trn_cell_intro_established_free(trn_cell_intro_established_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_intro_established_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_intro_established_t));
+  trunnel_free_(obj);
+}
+
+struct trn_cell_extension_st *
+trn_cell_intro_established_get_extensions(trn_cell_intro_established_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_intro_established_getconst_extensions(const trn_cell_intro_established_t *inp)
+{
+  return trn_cell_intro_established_get_extensions((trn_cell_intro_established_t*) inp);
+}
+int
+trn_cell_intro_established_set_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_intro_established_set0_extensions(inp, val);
+}
+int
+trn_cell_intro_established_set0_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const char *
+trn_cell_intro_established_check(const trn_cell_intro_established_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_intro_established_encoded_len(const trn_cell_intro_established_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_intro_established_check(obj))
+     return -1;
+
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+  return result;
+}
+int
+trn_cell_intro_established_clear_errors(trn_cell_intro_established_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_intro_established_encode(uint8_t *output, const size_t avail, const trn_cell_intro_established_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_intro_established_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_intro_established_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_intro_established_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_intro_established_parse_into(trn_cell_intro_established_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+ssize_t
+trn_cell_intro_established_parse(trn_cell_intro_established_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_intro_established_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_intro_established_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_intro_established_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
new file mode 100644
index 0000000000..ccaef5488c
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -0,0 +1,276 @@
+/* cell_establish_intro.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_ESTABLISH_INTRO_H
+#define TRUNNEL_CELL_ESTABLISH_INTRO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+struct trn_cell_extension_st;
+#define TRUNNEL_SHA3_256_LEN 32
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
+struct trn_cell_establish_intro_st {
+  const uint8_t *start_cell;
+  uint8_t auth_key_type;
+  uint16_t auth_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) auth_key;
+  struct trn_cell_extension_st *extensions;
+  const uint8_t *end_mac_fields;
+  uint8_t handshake_mac[TRUNNEL_SHA3_256_LEN];
+  const uint8_t *end_sig_fields;
+  uint16_t sig_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) sig;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
+struct trn_cell_intro_established_st {
+  struct trn_cell_extension_st *extensions;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_establish_intro with all
+ * elements set to zero.
+ */
+trn_cell_establish_intro_t *trn_cell_establish_intro_new(void);
+/** Release all storage held by the trn_cell_establish_intro in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_establish_intro_free(trn_cell_establish_intro_t *victim);
+/** Try to parse a trn_cell_establish_intro from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_establish_intro_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_establish_intro in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_establish_intro_encoded_len(const trn_cell_establish_intro_t *obj);
+/** Try to encode the trn_cell_establish_intro from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_establish_intro_encode(uint8_t *output, size_t avail, const trn_cell_establish_intro_t *input);
+/** Check whether the internal state of the trn_cell_establish_intro
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_establish_intro_check(const trn_cell_establish_intro_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_establish_intro_clear_errors(trn_cell_establish_intro_t *obj);
+/** Return the position for start_cell when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_start_cell(const trn_cell_establish_intro_t *inp);
+/** Return the value of the auth_key_type field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint8_t trn_cell_establish_intro_get_auth_key_type(const trn_cell_establish_intro_t *inp);
+/** Set the value of the auth_key_type field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_auth_key_type(trn_cell_establish_intro_t *inp, uint8_t val);
+/** Return the value of the auth_key_len field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint16_t trn_cell_establish_intro_get_auth_key_len(const trn_cell_establish_intro_t *inp);
+/** Set the value of the auth_key_len field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_auth_key_len(trn_cell_establish_intro_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the auth_key field
+ * of the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_auth_key(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_auth_key(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_auth_key, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_auth_key(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_establish_intro_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_auth_key(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field auth_key of
+ * the trn_cell_establish_intro_t in 'inp'.
+ */
+int trn_cell_establish_intro_add_auth_key(trn_cell_establish_intro_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field auth_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_auth_key(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_auth_key, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_auth_key(const trn_cell_establish_intro_t *inp);
+/** Change the length of the variable-length array field auth_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_setlen_auth_key(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return the value of the extensions field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_establish_intro_get_extensions(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_extensions, but take and return a
+ * const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_establish_intro_getconst_extensions(const trn_cell_establish_intro_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_establish_intro_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_establish_intro_set0_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val);
+/** Return the position for end_mac_fields when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_end_mac_fields(const trn_cell_establish_intro_t *inp);
+/** Return the (constant) length of the array holding the
+ * handshake_mac field of the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_handshake_mac(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * handshake_mac of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_handshake_mac, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_handshake_mac(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * handshake_mac of the trn_cell_establish_intro_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SHA3_256_LEN-element array field
+ * handshake_mac of 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_handshake_mac(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_handshake_mac, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_handshake_mac(const trn_cell_establish_intro_t *inp);
+/** Return the position for end_sig_fields when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_end_sig_fields(const trn_cell_establish_intro_t *inp);
+/** Return the value of the sig_len field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint16_t trn_cell_establish_intro_get_sig_len(const trn_cell_establish_intro_t *inp);
+/** Set the value of the sig_len field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_sig_len(trn_cell_establish_intro_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the sig field of
+ * the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_sig(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * sig of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_sig(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_sig, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_sig(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * sig of the trn_cell_establish_intro_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_sig(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field sig of the
+ * trn_cell_establish_intro_t in 'inp'.
+ */
+int trn_cell_establish_intro_add_sig(trn_cell_establish_intro_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field sig of 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_sig(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_sig, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_establish_intro_t *inp);
+/** Change the length of the variable-length array field sig of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_intro_established with all
+ * elements set to zero.
+ */
+trn_cell_intro_established_t *trn_cell_intro_established_new(void);
+/** Release all storage held by the trn_cell_intro_established in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_intro_established_free(trn_cell_intro_established_t *victim);
+/** Try to parse a trn_cell_intro_established from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_intro_established_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_intro_established_parse(trn_cell_intro_established_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_intro_established in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_intro_established_encoded_len(const trn_cell_intro_established_t *obj);
+/** Try to encode the trn_cell_intro_established from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_intro_established_encode(uint8_t *output, size_t avail, const trn_cell_intro_established_t *input);
+/** Check whether the internal state of the trn_cell_intro_established
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_intro_established_check(const trn_cell_intro_established_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_intro_established_clear_errors(trn_cell_intro_established_t *obj);
+/** Return the value of the extensions field of the
+ * trn_cell_intro_established_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_intro_established_get_extensions(trn_cell_intro_established_t *inp);
+/** As trn_cell_intro_established_get_extensions, but take and return
+ * a const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_intro_established_getconst_extensions(const trn_cell_intro_established_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_intro_established_t in 'inp' to 'val'. Free the old value
+ * if any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_intro_established_set_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_intro_established_set_extensions, but does not free
+ * the previous value.
+ */
+int trn_cell_intro_established_set0_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
new file mode 100644
index 0000000000..011ee62a15
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -0,0 +1,41 @@
+/*
+ * This contains the definition of the ESTABLISH_INTRO and INTRO_ESTABLISHED
+ * cell for onion service version 3 and onward. The following format is
+ * specified in proposal 224 section 3.1.
+ */
+
+extern struct trn_cell_extension;
+
+const TRUNNEL_SHA3_256_LEN = 32;
+
+/* ESTABLISH_INTRO payload. See details in section 3.1.1 */
+struct trn_cell_establish_intro {
+  /* Indicate the start of the handshake authentication data. */
+  @ptr start_cell;
+
+  /* Authentication key material. */
+  u8 auth_key_type IN [0x00, 0x01, 0x02];
+  u16 auth_key_len;
+  u8 auth_key[auth_key_len];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+  @ptr end_mac_fields;
+
+  /* Handshake MAC. */
+  u8 handshake_mac[TRUNNEL_SHA3_256_LEN];
+
+  /* Signature */
+  /* Indicate the end of the handshake authentication data. */
+  @ptr end_sig_fields;
+  u16 sig_len;
+  u8 sig[sig_len];
+};
+
+/* INTRO_ESTABLISHED payload which is an acknowledge of the ESTABLISH_INTRO
+ * cell. For legacy node, this payload is empty so the following only applies
+ * to version >= 3. */
+struct trn_cell_intro_established {
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
new file mode 100644
index 0000000000..358b355cda
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -0,0 +1,1357 @@
+/* cell_introduce1.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_introduce1.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellintroduce_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellintroduce_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+trn_cell_extension_t *trn_cell_extension_new(void);
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+typedef struct link_specifier_st link_specifier_t;
+link_specifier_t *link_specifier_new(void);
+void link_specifier_free(link_specifier_t *victim);
+ssize_t link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in);
+ssize_t link_specifier_encoded_len(const link_specifier_t *obj);
+ssize_t link_specifier_encode(uint8_t *output, size_t avail, const link_specifier_t *input);
+const char *link_specifier_check(const link_specifier_t *obj);
+int link_specifier_clear_errors(link_specifier_t *obj);
+trn_cell_introduce1_t *
+trn_cell_introduce1_new(void)
+{
+  trn_cell_introduce1_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce1_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce1_clear(trn_cell_introduce1_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->auth_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->auth_key);
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->encrypted);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->encrypted);
+}
+
+void
+trn_cell_introduce1_free(trn_cell_introduce1_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce1_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce1_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_introduce1_getlen_legacy_key_id(const trn_cell_introduce1_t *inp)
+{
+  (void)inp;  return TRUNNEL_SHA1_LEN;
+}
+
+uint8_t
+trn_cell_introduce1_get_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SHA1_LEN);
+  return inp->legacy_key_id[idx];
+}
+
+uint8_t
+trn_cell_introduce1_getconst_legacy_key_id(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_legacy_key_id((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SHA1_LEN);
+  inp->legacy_key_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_legacy_key_id(trn_cell_introduce1_t *inp)
+{
+  return inp->legacy_key_id;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_legacy_key_id(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_legacy_key_id((trn_cell_introduce1_t*)inp);
+}
+uint8_t
+trn_cell_introduce1_get_auth_key_type(const trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key_type;
+}
+int
+trn_cell_introduce1_set_auth_key_type(trn_cell_introduce1_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->auth_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_introduce1_get_auth_key_len(const trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key_len;
+}
+int
+trn_cell_introduce1_set_auth_key_len(trn_cell_introduce1_t *inp, uint16_t val)
+{
+  inp->auth_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_introduce1_getlen_auth_key(const trn_cell_introduce1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->auth_key);
+}
+
+uint8_t
+trn_cell_introduce1_get_auth_key(trn_cell_introduce1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->auth_key, idx);
+}
+
+uint8_t
+trn_cell_introduce1_getconst_auth_key(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_auth_key((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_auth_key(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->auth_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce1_add_auth_key(trn_cell_introduce1_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->auth_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->auth_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_auth_key(trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key.elts_;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_auth_key(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_auth_key((trn_cell_introduce1_t*)inp);
+}
+int
+trn_cell_introduce1_setlen_auth_key(trn_cell_introduce1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->auth_key.allocated_,
+                 &inp->auth_key.n_, inp->auth_key.elts_, newlen,
+                 sizeof(inp->auth_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->auth_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct trn_cell_extension_st *
+trn_cell_introduce1_get_extensions(trn_cell_introduce1_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce1_getconst_extensions(const trn_cell_introduce1_t *inp)
+{
+  return trn_cell_introduce1_get_extensions((trn_cell_introduce1_t*) inp);
+}
+int
+trn_cell_introduce1_set_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce1_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce1_set0_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+size_t
+trn_cell_introduce1_getlen_encrypted(const trn_cell_introduce1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->encrypted);
+}
+
+uint8_t
+trn_cell_introduce1_get_encrypted(trn_cell_introduce1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->encrypted, idx);
+}
+
+uint8_t
+trn_cell_introduce1_getconst_encrypted(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_encrypted((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_encrypted(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->encrypted, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce1_add_encrypted(trn_cell_introduce1_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->encrypted, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_encrypted(trn_cell_introduce1_t *inp)
+{
+  return inp->encrypted.elts_;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_encrypted(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_encrypted((trn_cell_introduce1_t*)inp);
+}
+int
+trn_cell_introduce1_setlen_encrypted(trn_cell_introduce1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->encrypted.allocated_,
+                 &inp->encrypted.n_, inp->encrypted.elts_, newlen,
+                 sizeof(inp->encrypted.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->encrypted.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_introduce1_check(const trn_cell_introduce1_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->auth_key) != obj->auth_key_len)
+    return "Length mismatch for auth_key";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce1_encoded_len(const trn_cell_introduce1_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce1_check(obj))
+     return -1;
+
+
+  /* Length of u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  result += TRUNNEL_SHA1_LEN;
+
+  /* Length of u8 auth_key_type IN [0, 1, 2] */
+  result += 1;
+
+  /* Length of u16 auth_key_len */
+  result += 2;
+
+  /* Length of u8 auth_key[auth_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 encrypted[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->encrypted);
+  return result;
+}
+int
+trn_cell_introduce1_clear_errors(trn_cell_introduce1_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce1_encode(uint8_t *output, const size_t avail, const trn_cell_introduce1_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce1_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce1_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_SHA1_LEN)
+    goto truncated;
+  memcpy(ptr, obj->legacy_key_id, TRUNNEL_SHA1_LEN);
+  written += TRUNNEL_SHA1_LEN; ptr += TRUNNEL_SHA1_LEN;
+
+  /* Encode u8 auth_key_type IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->auth_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 auth_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->auth_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 auth_key[auth_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+    trunnel_assert(obj->auth_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->auth_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 encrypted[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->encrypted);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->encrypted.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce1_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_introduce1_parse_into(trn_cell_introduce1_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  CHECK_REMAINING(TRUNNEL_SHA1_LEN, truncated);
+  memcpy(obj->legacy_key_id, ptr, TRUNNEL_SHA1_LEN);
+  remaining -= TRUNNEL_SHA1_LEN; ptr += TRUNNEL_SHA1_LEN;
+
+  /* Parse u8 auth_key_type IN [0, 1, 2] */
+  CHECK_REMAINING(1, truncated);
+  obj->auth_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    goto fail;
+
+  /* Parse u16 auth_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->auth_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 auth_key[auth_key_len] */
+  CHECK_REMAINING(obj->auth_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->auth_key, obj->auth_key_len, {});
+  obj->auth_key.n_ = obj->auth_key_len;
+  if (obj->auth_key_len)
+    memcpy(obj->auth_key.elts_, ptr, obj->auth_key_len);
+  ptr += obj->auth_key_len; remaining -= obj->auth_key_len;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 encrypted[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->encrypted, remaining, {});
+  obj->encrypted.n_ = remaining;
+  if (remaining)
+    memcpy(obj->encrypted.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce1_parse(trn_cell_introduce1_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce1_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce1_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce1_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_introduce_ack_t *
+trn_cell_introduce_ack_new(void)
+{
+  trn_cell_introduce_ack_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce_ack_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce_ack_clear(trn_cell_introduce_ack_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+}
+
+void
+trn_cell_introduce_ack_free(trn_cell_introduce_ack_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce_ack_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce_ack_t));
+  trunnel_free_(obj);
+}
+
+uint16_t
+trn_cell_introduce_ack_get_status(const trn_cell_introduce_ack_t *inp)
+{
+  return inp->status;
+}
+int
+trn_cell_introduce_ack_set_status(trn_cell_introduce_ack_t *inp, uint16_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->status = val;
+  return 0;
+}
+struct trn_cell_extension_st *
+trn_cell_introduce_ack_get_extensions(trn_cell_introduce_ack_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce_ack_getconst_extensions(const trn_cell_introduce_ack_t *inp)
+{
+  return trn_cell_introduce_ack_get_extensions((trn_cell_introduce_ack_t*) inp);
+}
+int
+trn_cell_introduce_ack_set_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce_ack_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce_ack_set0_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const char *
+trn_cell_introduce_ack_check(const trn_cell_introduce_ack_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->status == 0 || obj->status == 1 || obj->status == 2))
+    return "Integer out of bounds";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce_ack_encoded_len(const trn_cell_introduce_ack_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce_ack_check(obj))
+     return -1;
+
+
+  /* Length of u16 status IN [0, 1, 2] */
+  result += 2;
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+  return result;
+}
+int
+trn_cell_introduce_ack_clear_errors(trn_cell_introduce_ack_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce_ack_encode(uint8_t *output, const size_t avail, const trn_cell_introduce_ack_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce_ack_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce_ack_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u16 status IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->status));
+  written += 2; ptr += 2;
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce_ack_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_introduce_ack_parse_into(trn_cell_introduce_ack_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u16 status IN [0, 1, 2] */
+  CHECK_REMAINING(2, truncated);
+  obj->status = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  if (! (obj->status == 0 || obj->status == 1 || obj->status == 2))
+    goto fail;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce_ack_parse(trn_cell_introduce_ack_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce_ack_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce_ack_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce_ack_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_introduce_encrypted_t *
+trn_cell_introduce_encrypted_new(void)
+{
+  trn_cell_introduce_encrypted_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce_encrypted_t));
+  if (NULL == val)
+    return NULL;
+  val->onion_key_type = 1;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce_encrypted_clear(trn_cell_introduce_encrypted_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->onion_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->onion_key);
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->nspecs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->nspecs);
+  TRUNNEL_DYNARRAY_WIPE(&obj->pad);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->pad);
+}
+
+void
+trn_cell_introduce_encrypted_free(trn_cell_introduce_encrypted_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce_encrypted_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce_encrypted_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_introduce_encrypted_getlen_rend_cookie(const trn_cell_introduce_encrypted_t *inp)
+{
+  (void)inp;  return TRUNNEL_REND_COOKIE_LEN;
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  return inp->rend_cookie[idx];
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_rend_cookie(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_rend_cookie((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  inp->rend_cookie[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_rend_cookie(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->rend_cookie;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_rend_cookie(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_rend_cookie((trn_cell_introduce_encrypted_t*)inp);
+}
+struct trn_cell_extension_st *
+trn_cell_introduce_encrypted_get_extensions(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce_encrypted_getconst_extensions(const trn_cell_introduce_encrypted_t *inp)
+{
+  return trn_cell_introduce_encrypted_get_extensions((trn_cell_introduce_encrypted_t*) inp);
+}
+int
+trn_cell_introduce_encrypted_set_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce_encrypted_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce_encrypted_set0_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+uint8_t
+trn_cell_introduce_encrypted_get_onion_key_type(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key_type;
+}
+int
+trn_cell_introduce_encrypted_set_onion_key_type(trn_cell_introduce_encrypted_t *inp, uint8_t val)
+{
+  if (! ((val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->onion_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_introduce_encrypted_get_onion_key_len(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key_len;
+}
+int
+trn_cell_introduce_encrypted_set_onion_key_len(trn_cell_introduce_encrypted_t *inp, uint16_t val)
+{
+  inp->onion_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_onion_key(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->onion_key);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->onion_key, idx);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_onion_key(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_onion_key((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->onion_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_onion_key(trn_cell_introduce_encrypted_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->onion_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->onion_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_onion_key(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key.elts_;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_onion_key(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_onion_key((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_onion_key(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->onion_key.allocated_,
+                 &inp->onion_key.n_, inp->onion_key.elts_, newlen,
+                 sizeof(inp->onion_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->onion_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+uint8_t
+trn_cell_introduce_encrypted_get_nspec(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->nspec;
+}
+int
+trn_cell_introduce_encrypted_set_nspec(trn_cell_introduce_encrypted_t *inp, uint8_t val)
+{
+  inp->nspec = val;
+  return 0;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_nspecs(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->nspecs);
+}
+
+struct link_specifier_st *
+trn_cell_introduce_encrypted_get_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->nspecs, idx);
+}
+
+ const struct link_specifier_st *
+trn_cell_introduce_encrypted_getconst_nspecs(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_nspecs((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->nspecs, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return trn_cell_introduce_encrypted_set0_nspecs(inp, idx, elt);
+}
+int
+trn_cell_introduce_encrypted_set0_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->nspecs, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_nspecs(trn_cell_introduce_encrypted_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->nspecs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->nspecs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+trn_cell_introduce_encrypted_getarray_nspecs(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->nspecs.elts_;
+}
+const struct link_specifier_st *  const  *
+trn_cell_introduce_encrypted_getconstarray_nspecs(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)trn_cell_introduce_encrypted_getarray_nspecs((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_nspecs(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->nspecs.allocated_,
+                 &inp->nspecs.n_, inp->nspecs.elts_, newlen,
+                 sizeof(inp->nspecs.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->nspecs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_pad(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->pad);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_pad(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->pad, idx);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_pad(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_pad((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_pad(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->pad, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_pad(trn_cell_introduce_encrypted_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->pad, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_pad(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->pad.elts_;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_pad(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_pad((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_pad(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->pad.allocated_,
+                 &inp->pad.n_, inp->pad.elts_, newlen,
+                 sizeof(inp->pad.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->pad.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_introduce_encrypted_check(const trn_cell_introduce_encrypted_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  if (! (obj->onion_key_type == 1))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->onion_key) != obj->onion_key_len)
+    return "Length mismatch for onion_key";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->nspecs) != obj->nspec)
+    return "Length mismatch for nspecs";
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce_encrypted_encoded_len(const trn_cell_introduce_encrypted_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce_encrypted_check(obj))
+     return -1;
+
+
+  /* Length of u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  result += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 onion_key_type IN [1] */
+  result += 1;
+
+  /* Length of u16 onion_key_len */
+  result += 2;
+
+  /* Length of u8 onion_key[onion_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->onion_key);
+
+  /* Length of u8 nspec */
+  result += 1;
+
+  /* Length of struct link_specifier nspecs[nspec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+    }
+  }
+
+  /* Length of u8 pad[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->pad);
+  return result;
+}
+int
+trn_cell_introduce_encrypted_clear_errors(trn_cell_introduce_encrypted_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce_encrypted_encode(uint8_t *output, const size_t avail, const trn_cell_introduce_encrypted_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce_encrypted_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce_encrypted_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_REND_COOKIE_LEN)
+    goto truncated;
+  memcpy(ptr, obj->rend_cookie, TRUNNEL_REND_COOKIE_LEN);
+  written += TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 onion_key_type IN [1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->onion_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 onion_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->onion_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 onion_key[onion_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->onion_key);
+    trunnel_assert(obj->onion_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->onion_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode u8 nspec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->nspec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier nspecs[nspec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+  /* Encode u8 pad[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->pad);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->pad.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce_encrypted_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_introduce_encrypted_parse_into(trn_cell_introduce_encrypted_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  CHECK_REMAINING(TRUNNEL_REND_COOKIE_LEN, truncated);
+  memcpy(obj->rend_cookie, ptr, TRUNNEL_REND_COOKIE_LEN);
+  remaining -= TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 onion_key_type IN [1] */
+  CHECK_REMAINING(1, truncated);
+  obj->onion_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->onion_key_type == 1))
+    goto fail;
+
+  /* Parse u16 onion_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->onion_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 onion_key[onion_key_len] */
+  CHECK_REMAINING(obj->onion_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->onion_key, obj->onion_key_len, {});
+  obj->onion_key.n_ = obj->onion_key_len;
+  if (obj->onion_key_len)
+    memcpy(obj->onion_key.elts_, ptr, obj->onion_key_len);
+  ptr += obj->onion_key_len; remaining -= obj->onion_key_len;
+
+  /* Parse u8 nspec */
+  CHECK_REMAINING(1, truncated);
+  obj->nspec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier nspecs[nspec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->nspecs, obj->nspec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->nspec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->nspecs, elt, {link_specifier_free(elt);});
+    }
+  }
+
+  /* Parse u8 pad[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->pad, remaining, {});
+  obj->pad.n_ = remaining;
+  if (remaining)
+    memcpy(obj->pad.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce_encrypted_parse(trn_cell_introduce_encrypted_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce_encrypted_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce_encrypted_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce_encrypted_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
new file mode 100644
index 0000000000..fa218adc6d
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -0,0 +1,493 @@
+/* cell_introduce1.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_INTRODUCE1_H
+#define TRUNNEL_CELL_INTRODUCE1_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+struct trn_cell_extension_st;
+struct link_specifier_st;
+#define TRUNNEL_SHA1_LEN 20
+#define TRUNNEL_REND_COOKIE_LEN 20
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE1)
+struct trn_cell_introduce1_st {
+  uint8_t legacy_key_id[TRUNNEL_SHA1_LEN];
+  uint8_t auth_key_type;
+  uint16_t auth_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) auth_key;
+  struct trn_cell_extension_st *extensions;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) encrypted;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce1_st trn_cell_introduce1_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE_ACK)
+struct trn_cell_introduce_ack_st {
+  uint16_t status;
+  struct trn_cell_extension_st *extensions;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce_ack_st trn_cell_introduce_ack_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE_ENCRYPTED)
+struct trn_cell_introduce_encrypted_st {
+  uint8_t rend_cookie[TRUNNEL_REND_COOKIE_LEN];
+  struct trn_cell_extension_st *extensions;
+  uint8_t onion_key_type;
+  uint16_t onion_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) onion_key;
+  uint8_t nspec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) nspecs;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) pad;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce_encrypted_st trn_cell_introduce_encrypted_t;
+/** Return a newly allocated trn_cell_introduce1 with all elements set
+ * to zero.
+ */
+trn_cell_introduce1_t *trn_cell_introduce1_new(void);
+/** Release all storage held by the trn_cell_introduce1 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce1_free(trn_cell_introduce1_t *victim);
+/** Try to parse a trn_cell_introduce1 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_introduce1_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_introduce1_parse(trn_cell_introduce1_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce1 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce1_encoded_len(const trn_cell_introduce1_t *obj);
+/** Try to encode the trn_cell_introduce1 from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_introduce1_encode(uint8_t *output, size_t avail, const trn_cell_introduce1_t *input);
+/** Check whether the internal state of the trn_cell_introduce1 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_introduce1_check(const trn_cell_introduce1_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce1_clear_errors(trn_cell_introduce1_t *obj);
+/** Return the (constant) length of the array holding the
+ * legacy_key_id field of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_legacy_key_id(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * legacy_key_id of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_legacy_key_id, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_introduce1_getconst_legacy_key_id(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * legacy_key_id of the trn_cell_introduce1_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SHA1_LEN-element array field
+ * legacy_key_id of 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_legacy_key_id(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_legacy_key_id, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_legacy_key_id(const trn_cell_introduce1_t *inp);
+/** Return the value of the auth_key_type field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+uint8_t trn_cell_introduce1_get_auth_key_type(const trn_cell_introduce1_t *inp);
+/** Set the value of the auth_key_type field of the
+ * trn_cell_introduce1_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_auth_key_type(trn_cell_introduce1_t *inp, uint8_t val);
+/** Return the value of the auth_key_len field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+uint16_t trn_cell_introduce1_get_auth_key_len(const trn_cell_introduce1_t *inp);
+/** Set the value of the auth_key_len field of the
+ * trn_cell_introduce1_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_auth_key_len(trn_cell_introduce1_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the auth_key field
+ * of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_auth_key(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_auth_key(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_auth_key, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_introduce1_getconst_auth_key(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_introduce1_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_auth_key(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field auth_key of
+ * the trn_cell_introduce1_t in 'inp'.
+ */
+int trn_cell_introduce1_add_auth_key(trn_cell_introduce1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field auth_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_auth_key(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_auth_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_auth_key(const trn_cell_introduce1_t *inp);
+/** Change the length of the variable-length array field auth_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_setlen_auth_key(trn_cell_introduce1_t *inp, size_t newlen);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce1_get_extensions(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_extensions, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce1_getconst_extensions(const trn_cell_introduce1_t *inp);
+/** Set the value of the extensions field of the trn_cell_introduce1_t
+ * in 'inp' to 'val'. Free the old value if any. Steals the
+ * referenceto 'val'.Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce1_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce1_set0_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val);
+/** Return the length of the dynamic array holding the encrypted field
+ * of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_encrypted(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * encrypted of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_encrypted(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_encrypted, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_introduce1_getconst_encrypted(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * encrypted of the trn_cell_introduce1_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_encrypted(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field encrypted of
+ * the trn_cell_introduce1_t in 'inp'.
+ */
+int trn_cell_introduce1_add_encrypted(trn_cell_introduce1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field encrypted of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_encrypted(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_encrypted, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_encrypted(const trn_cell_introduce1_t *inp);
+/** Change the length of the variable-length array field encrypted of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_setlen_encrypted(trn_cell_introduce1_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_introduce_ack with all elements
+ * set to zero.
+ */
+trn_cell_introduce_ack_t *trn_cell_introduce_ack_new(void);
+/** Release all storage held by the trn_cell_introduce_ack in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce_ack_free(trn_cell_introduce_ack_t *victim);
+/** Try to parse a trn_cell_introduce_ack from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_introduce_ack_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_introduce_ack_parse(trn_cell_introduce_ack_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce_ack in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce_ack_encoded_len(const trn_cell_introduce_ack_t *obj);
+/** Try to encode the trn_cell_introduce_ack from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_introduce_ack_encode(uint8_t *output, size_t avail, const trn_cell_introduce_ack_t *input);
+/** Check whether the internal state of the trn_cell_introduce_ack in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_introduce_ack_check(const trn_cell_introduce_ack_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce_ack_clear_errors(trn_cell_introduce_ack_t *obj);
+/** Return the value of the status field of the
+ * trn_cell_introduce_ack_t in 'inp'
+ */
+uint16_t trn_cell_introduce_ack_get_status(const trn_cell_introduce_ack_t *inp);
+/** Set the value of the status field of the trn_cell_introduce_ack_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_introduce_ack_set_status(trn_cell_introduce_ack_t *inp, uint16_t val);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce_ack_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce_ack_get_extensions(trn_cell_introduce_ack_t *inp);
+/** As trn_cell_introduce_ack_get_extensions, but take and return a
+ * const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce_ack_getconst_extensions(const trn_cell_introduce_ack_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_introduce_ack_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_ack_set_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce_ack_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce_ack_set0_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val);
+/** Return a newly allocated trn_cell_introduce_encrypted with all
+ * elements set to zero.
+ */
+trn_cell_introduce_encrypted_t *trn_cell_introduce_encrypted_new(void);
+/** Release all storage held by the trn_cell_introduce_encrypted in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce_encrypted_free(trn_cell_introduce_encrypted_t *victim);
+/** Try to parse a trn_cell_introduce_encrypted from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_introduce_encrypted_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_introduce_encrypted_parse(trn_cell_introduce_encrypted_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce_encrypted in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce_encrypted_encoded_len(const trn_cell_introduce_encrypted_t *obj);
+/** Try to encode the trn_cell_introduce_encrypted from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_introduce_encrypted_encode(uint8_t *output, size_t avail, const trn_cell_introduce_encrypted_t *input);
+/** Check whether the internal state of the
+ * trn_cell_introduce_encrypted in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_introduce_encrypted_check(const trn_cell_introduce_encrypted_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce_encrypted_clear_errors(trn_cell_introduce_encrypted_t *obj);
+/** Return the (constant) length of the array holding the rend_cookie
+ * field of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_rend_cookie(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * rend_cookie of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_rend_cookie, but take and
+ * return a const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_rend_cookie(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * rend_cookie of the trn_cell_introduce_encrypted_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_REND_COOKIE_LEN-element array
+ * field rend_cookie of 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_rend_cookie(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_rend_cookie, but take and
+ * return a const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_rend_cookie(const trn_cell_introduce_encrypted_t *inp);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce_encrypted_get_extensions(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_extensions, but take and
+ * return a const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce_encrypted_getconst_extensions(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Free the old
+ * value if any. Steals the referenceto 'val'.Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce_encrypted_set_extensions, but does not free
+ * the previous value.
+ */
+int trn_cell_introduce_encrypted_set0_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val);
+/** Return the value of the onion_key_type field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint8_t trn_cell_introduce_encrypted_get_onion_key_type(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the onion_key_type field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_onion_key_type(trn_cell_introduce_encrypted_t *inp, uint8_t val);
+/** Return the value of the onion_key_len field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint16_t trn_cell_introduce_encrypted_get_onion_key_len(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the onion_key_len field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_onion_key_len(trn_cell_introduce_encrypted_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the onion_key field
+ * of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_onion_key(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * onion_key of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_onion_key, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_onion_key(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * onion_key of the trn_cell_introduce_encrypted_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field onion_key of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_onion_key(trn_cell_introduce_encrypted_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field onion_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_onion_key(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_onion_key, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_onion_key(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field onion_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_onion_key(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+/** Return the value of the nspec field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint8_t trn_cell_introduce_encrypted_get_nspec(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the nspec field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_nspec(trn_cell_introduce_encrypted_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the nspecs field of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_nspecs(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * nspecs of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+struct link_specifier_st * trn_cell_introduce_encrypted_get_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_nspecs, but take and return a
+ * const pointer
+ */
+ const struct link_specifier_st * trn_cell_introduce_encrypted_getconst_nspecs(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * nspecs of the trn_cell_introduce_encrypted_t in 'inp', so that it
+ * will hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_introduce_encrypted_set_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As trn_cell_introduce_encrypted_set_nspecs, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce_encrypted_set0_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field nspecs of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_nspecs(trn_cell_introduce_encrypted_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field nspecs of
+ * 'inp'.
+ */
+struct link_specifier_st * * trn_cell_introduce_encrypted_getarray_nspecs(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_nspecs, but take and return a
+ * const pointer
+ */
+const struct link_specifier_st *  const  * trn_cell_introduce_encrypted_getconstarray_nspecs(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field nspecs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_nspecs(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+/** Return the length of the dynamic array holding the pad field of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_pad(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * pad of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_pad(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_pad, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_pad(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * pad of the trn_cell_introduce_encrypted_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_pad(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field pad of the
+ * trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_pad(trn_cell_introduce_encrypted_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field pad of 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_pad(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_pad, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_pad(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field pad of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_pad(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_introduce1.trunnel b/src/trunnel/hs/cell_introduce1.trunnel
new file mode 100644
index 0000000000..574382b163
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.trunnel
@@ -0,0 +1,60 @@
+/*
+ * This contains the definition of the INTRODUCE1 and INTRODUCE_ACK cell for
+ * onion service version 3 and onward. The following format is specified in
+ * proposal 224 section 3.2.
+ */
+
+/* From cell_common.trunnel. */
+extern struct trn_cell_extension;
+/* From ed25519_cert.trunnel. */
+extern struct link_specifier;
+
+const TRUNNEL_SHA1_LEN = 20;
+const TRUNNEL_REND_COOKIE_LEN = 20;
+
+/* INTRODUCE1 payload. See details in section 3.2.1. */
+struct trn_cell_introduce1 {
+  /* Always zeroed. MUST be checked explicitly by the caller. */
+  u8 legacy_key_id[TRUNNEL_SHA1_LEN];
+
+  /* Authentication key material. */
+  u8 auth_key_type IN [0x00, 0x01, 0x02];
+  u16 auth_key_len;
+  u8 auth_key[auth_key_len];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+
+  /* Variable length, up to the end of cell. */
+  u8 encrypted[];
+};
+
+/* INTRODUCE_ACK payload. See details in section 3.2.2. */
+struct trn_cell_introduce_ack {
+  /* Status of introduction. */
+  u16 status IN [0x0000, 0x0001, 0x0002];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+};
+
+/* Encrypted section of the INTRODUCE1/INTRODUCE2 cell. */
+struct trn_cell_introduce_encrypted {
+  /* Rendezvous cookie. */
+  u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+
+  /* Onion key material. */
+  u8 onion_key_type IN [0x01];
+  u16 onion_key_len;
+  u8 onion_key[onion_key_len];
+
+  /* Link specifiers(s) */
+  u8 nspec;
+  struct link_specifier nspecs[nspec];
+
+  /* Optional padding. This might be empty or not. */
+  u8 pad[];
+};
diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c
new file mode 100644
index 0000000000..53cb609138
--- /dev/null
+++ b/src/trunnel/hs/cell_rendezvous.c
@@ -0,0 +1,470 @@
+/* cell_rendezvous.c -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_rendezvous.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellrendezvous_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellrendezvous_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+trn_cell_rendezvous1_t *
+trn_cell_rendezvous1_new(void)
+{
+  trn_cell_rendezvous1_t *val = trunnel_calloc(1, sizeof(trn_cell_rendezvous1_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_rendezvous1_clear(trn_cell_rendezvous1_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->handshake_info);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->handshake_info);
+}
+
+void
+trn_cell_rendezvous1_free(trn_cell_rendezvous1_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_rendezvous1_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_rendezvous1_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_rendezvous1_getlen_rendezvous_cookie(const trn_cell_rendezvous1_t *inp)
+{
+  (void)inp;  return TRUNNEL_REND_COOKIE_LEN;
+}
+
+uint8_t
+trn_cell_rendezvous1_get_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  return inp->rendezvous_cookie[idx];
+}
+
+uint8_t
+trn_cell_rendezvous1_getconst_rendezvous_cookie(const trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  return trn_cell_rendezvous1_get_rendezvous_cookie((trn_cell_rendezvous1_t*)inp, idx);
+}
+int
+trn_cell_rendezvous1_set_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  inp->rendezvous_cookie[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_rendezvous1_getarray_rendezvous_cookie(trn_cell_rendezvous1_t *inp)
+{
+  return inp->rendezvous_cookie;
+}
+const uint8_t  *
+trn_cell_rendezvous1_getconstarray_rendezvous_cookie(const trn_cell_rendezvous1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_rendezvous1_getarray_rendezvous_cookie((trn_cell_rendezvous1_t*)inp);
+}
+size_t
+trn_cell_rendezvous1_getlen_handshake_info(const trn_cell_rendezvous1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->handshake_info);
+}
+
+uint8_t
+trn_cell_rendezvous1_get_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->handshake_info, idx);
+}
+
+uint8_t
+trn_cell_rendezvous1_getconst_handshake_info(const trn_cell_rendezvous1_t *inp, size_t idx)
+{
+  return trn_cell_rendezvous1_get_handshake_info((trn_cell_rendezvous1_t*)inp, idx);
+}
+int
+trn_cell_rendezvous1_set_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->handshake_info, idx, elt);
+  return 0;
+}
+int
+trn_cell_rendezvous1_add_handshake_info(trn_cell_rendezvous1_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->handshake_info, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_rendezvous1_getarray_handshake_info(trn_cell_rendezvous1_t *inp)
+{
+  return inp->handshake_info.elts_;
+}
+const uint8_t  *
+trn_cell_rendezvous1_getconstarray_handshake_info(const trn_cell_rendezvous1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_rendezvous1_getarray_handshake_info((trn_cell_rendezvous1_t*)inp);
+}
+int
+trn_cell_rendezvous1_setlen_handshake_info(trn_cell_rendezvous1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->handshake_info.allocated_,
+                 &inp->handshake_info.n_, inp->handshake_info.elts_, newlen,
+                 sizeof(inp->handshake_info.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->handshake_info.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_rendezvous1_check(const trn_cell_rendezvous1_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_rendezvous1_encoded_len(const trn_cell_rendezvous1_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_rendezvous1_check(obj))
+     return -1;
+
+
+  /* Length of u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  result += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Length of u8 handshake_info[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->handshake_info);
+  return result;
+}
+int
+trn_cell_rendezvous1_clear_errors(trn_cell_rendezvous1_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_rendezvous1_encode(uint8_t *output, const size_t avail, const trn_cell_rendezvous1_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_rendezvous1_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_rendezvous1_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_REND_COOKIE_LEN)
+    goto truncated;
+  memcpy(ptr, obj->rendezvous_cookie, TRUNNEL_REND_COOKIE_LEN);
+  written += TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Encode u8 handshake_info[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->handshake_info);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->handshake_info.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_rendezvous1_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_rendezvous1_parse_into(trn_cell_rendezvous1_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  CHECK_REMAINING(TRUNNEL_REND_COOKIE_LEN, truncated);
+  memcpy(obj->rendezvous_cookie, ptr, TRUNNEL_REND_COOKIE_LEN);
+  remaining -= TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Parse u8 handshake_info[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->handshake_info, remaining, {});
+  obj->handshake_info.n_ = remaining;
+  if (remaining)
+    memcpy(obj->handshake_info.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_rendezvous1_parse(trn_cell_rendezvous1_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_rendezvous1_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_rendezvous1_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_rendezvous1_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_rendezvous2_t *
+trn_cell_rendezvous2_new(void)
+{
+  trn_cell_rendezvous2_t *val = trunnel_calloc(1, sizeof(trn_cell_rendezvous2_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_rendezvous2_clear(trn_cell_rendezvous2_t *obj)
+{
+  (void) obj;
+}
+
+void
+trn_cell_rendezvous2_free(trn_cell_rendezvous2_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_rendezvous2_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_rendezvous2_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_rendezvous2_getlen_handshake_info(const trn_cell_rendezvous2_t *inp)
+{
+  (void)inp;  return TRUNNEL_HANDSHAKE_INFO_LEN;
+}
+
+uint8_t
+trn_cell_rendezvous2_get_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_HANDSHAKE_INFO_LEN);
+  return inp->handshake_info[idx];
+}
+
+uint8_t
+trn_cell_rendezvous2_getconst_handshake_info(const trn_cell_rendezvous2_t *inp, size_t idx)
+{
+  return trn_cell_rendezvous2_get_handshake_info((trn_cell_rendezvous2_t*)inp, idx);
+}
+int
+trn_cell_rendezvous2_set_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_HANDSHAKE_INFO_LEN);
+  inp->handshake_info[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_rendezvous2_getarray_handshake_info(trn_cell_rendezvous2_t *inp)
+{
+  return inp->handshake_info;
+}
+const uint8_t  *
+trn_cell_rendezvous2_getconstarray_handshake_info(const trn_cell_rendezvous2_t *inp)
+{
+  return (const uint8_t  *)trn_cell_rendezvous2_getarray_handshake_info((trn_cell_rendezvous2_t*)inp);
+}
+const char *
+trn_cell_rendezvous2_check(const trn_cell_rendezvous2_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_rendezvous2_encoded_len(const trn_cell_rendezvous2_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_rendezvous2_check(obj))
+     return -1;
+
+
+  /* Length of u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN] */
+  result += TRUNNEL_HANDSHAKE_INFO_LEN;
+  return result;
+}
+int
+trn_cell_rendezvous2_clear_errors(trn_cell_rendezvous2_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_rendezvous2_encode(uint8_t *output, const size_t avail, const trn_cell_rendezvous2_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_rendezvous2_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_rendezvous2_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_HANDSHAKE_INFO_LEN)
+    goto truncated;
+  memcpy(ptr, obj->handshake_info, TRUNNEL_HANDSHAKE_INFO_LEN);
+  written += TRUNNEL_HANDSHAKE_INFO_LEN; ptr += TRUNNEL_HANDSHAKE_INFO_LEN;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_rendezvous2_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_rendezvous2_parse_into(trn_cell_rendezvous2_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN] */
+  CHECK_REMAINING(TRUNNEL_HANDSHAKE_INFO_LEN, truncated);
+  memcpy(obj->handshake_info, ptr, TRUNNEL_HANDSHAKE_INFO_LEN);
+  remaining -= TRUNNEL_HANDSHAKE_INFO_LEN; ptr += TRUNNEL_HANDSHAKE_INFO_LEN;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+trn_cell_rendezvous2_parse(trn_cell_rendezvous2_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_rendezvous2_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_rendezvous2_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_rendezvous2_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h
new file mode 100644
index 0000000000..39e14da25b
--- /dev/null
+++ b/src/trunnel/hs/cell_rendezvous.h
@@ -0,0 +1,187 @@
+/* cell_rendezvous.h -- generated by Trunnel v1.5.2.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_RENDEZVOUS_H
+#define TRUNNEL_CELL_RENDEZVOUS_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define TRUNNEL_REND_COOKIE_LEN 20
+#define TRUNNEL_HANDSHAKE_INFO_LEN 64
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_RENDEZVOUS1)
+struct trn_cell_rendezvous1_st {
+  uint8_t rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN];
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) handshake_info;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_rendezvous1_st trn_cell_rendezvous1_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_RENDEZVOUS2)
+struct trn_cell_rendezvous2_st {
+  uint8_t handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_rendezvous2_st trn_cell_rendezvous2_t;
+/** Return a newly allocated trn_cell_rendezvous1 with all elements
+ * set to zero.
+ */
+trn_cell_rendezvous1_t *trn_cell_rendezvous1_new(void);
+/** Release all storage held by the trn_cell_rendezvous1 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_rendezvous1_free(trn_cell_rendezvous1_t *victim);
+/** Try to parse a trn_cell_rendezvous1 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_rendezvous1_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_rendezvous1_parse(trn_cell_rendezvous1_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_rendezvous1 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_rendezvous1_encoded_len(const trn_cell_rendezvous1_t *obj);
+/** Try to encode the trn_cell_rendezvous1 from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_rendezvous1_encode(uint8_t *output, size_t avail, const trn_cell_rendezvous1_t *input);
+/** Check whether the internal state of the trn_cell_rendezvous1 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_rendezvous1_check(const trn_cell_rendezvous1_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_rendezvous1_clear_errors(trn_cell_rendezvous1_t *obj);
+/** Return the (constant) length of the array holding the
+ * rendezvous_cookie field of the trn_cell_rendezvous1_t in 'inp'.
+ */
+size_t trn_cell_rendezvous1_getlen_rendezvous_cookie(const trn_cell_rendezvous1_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * rendezvous_cookie of the trn_cell_rendezvous1_t in 'inp'.
+ */
+uint8_t trn_cell_rendezvous1_get_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx);
+/** As trn_cell_rendezvous1_get_rendezvous_cookie, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_rendezvous1_getconst_rendezvous_cookie(const trn_cell_rendezvous1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * rendezvous_cookie of the trn_cell_rendezvous1_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_rendezvous1_set_rendezvous_cookie(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_REND_COOKIE_LEN-element array
+ * field rendezvous_cookie of 'inp'.
+ */
+uint8_t * trn_cell_rendezvous1_getarray_rendezvous_cookie(trn_cell_rendezvous1_t *inp);
+/** As trn_cell_rendezvous1_get_rendezvous_cookie, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_rendezvous1_getconstarray_rendezvous_cookie(const trn_cell_rendezvous1_t *inp);
+/** Return the length of the dynamic array holding the handshake_info
+ * field of the trn_cell_rendezvous1_t in 'inp'.
+ */
+size_t trn_cell_rendezvous1_getlen_handshake_info(const trn_cell_rendezvous1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * handshake_info of the trn_cell_rendezvous1_t in 'inp'.
+ */
+uint8_t trn_cell_rendezvous1_get_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx);
+/** As trn_cell_rendezvous1_get_handshake_info, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_rendezvous1_getconst_handshake_info(const trn_cell_rendezvous1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * handshake_info of the trn_cell_rendezvous1_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_rendezvous1_set_handshake_info(trn_cell_rendezvous1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * handshake_info of the trn_cell_rendezvous1_t in 'inp'.
+ */
+int trn_cell_rendezvous1_add_handshake_info(trn_cell_rendezvous1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field handshake_info
+ * of 'inp'.
+ */
+uint8_t * trn_cell_rendezvous1_getarray_handshake_info(trn_cell_rendezvous1_t *inp);
+/** As trn_cell_rendezvous1_get_handshake_info, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_rendezvous1_getconstarray_handshake_info(const trn_cell_rendezvous1_t *inp);
+/** Change the length of the variable-length array field
+ * handshake_info of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int trn_cell_rendezvous1_setlen_handshake_info(trn_cell_rendezvous1_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_rendezvous2 with all elements
+ * set to zero.
+ */
+trn_cell_rendezvous2_t *trn_cell_rendezvous2_new(void);
+/** Release all storage held by the trn_cell_rendezvous2 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_rendezvous2_free(trn_cell_rendezvous2_t *victim);
+/** Try to parse a trn_cell_rendezvous2 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_rendezvous2_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_rendezvous2_parse(trn_cell_rendezvous2_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_rendezvous2 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_rendezvous2_encoded_len(const trn_cell_rendezvous2_t *obj);
+/** Try to encode the trn_cell_rendezvous2 from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_rendezvous2_encode(uint8_t *output, size_t avail, const trn_cell_rendezvous2_t *input);
+/** Check whether the internal state of the trn_cell_rendezvous2 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_rendezvous2_check(const trn_cell_rendezvous2_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_rendezvous2_clear_errors(trn_cell_rendezvous2_t *obj);
+/** Return the (constant) length of the array holding the
+ * handshake_info field of the trn_cell_rendezvous2_t in 'inp'.
+ */
+size_t trn_cell_rendezvous2_getlen_handshake_info(const trn_cell_rendezvous2_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * handshake_info of the trn_cell_rendezvous2_t in 'inp'.
+ */
+uint8_t trn_cell_rendezvous2_get_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx);
+/** As trn_cell_rendezvous2_get_handshake_info, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_rendezvous2_getconst_handshake_info(const trn_cell_rendezvous2_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * handshake_info of the trn_cell_rendezvous2_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_rendezvous2_set_handshake_info(trn_cell_rendezvous2_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_HANDSHAKE_INFO_LEN-element array
+ * field handshake_info of 'inp'.
+ */
+uint8_t * trn_cell_rendezvous2_getarray_handshake_info(trn_cell_rendezvous2_t *inp);
+/** As trn_cell_rendezvous2_get_handshake_info, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_rendezvous2_getconstarray_handshake_info(const trn_cell_rendezvous2_t *inp);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_rendezvous.trunnel b/src/trunnel/hs/cell_rendezvous.trunnel
new file mode 100644
index 0000000000..2128b4d126
--- /dev/null
+++ b/src/trunnel/hs/cell_rendezvous.trunnel
@@ -0,0 +1,29 @@
+/*
+ * This contains the definition of the RENDEZVOUS1/2 cell for onion service
+ * version 3 and onward. The following format is specified in proposal 224
+ * section 4.2.
+ */
+
+/* Rendezvous cookie length. */
+const TRUNNEL_REND_COOKIE_LEN = 20;
+/* The HANDSHAKE_INFO field layout is as follow:
+ *    SERVER_PK   [PK_PUBKEY_LEN bytes]
+ *    AUTH        [MAC_LEN bytes]
+ * This means, the size is 32 bytes + 32 bytes. */
+const TRUNNEL_HANDSHAKE_INFO_LEN = 64;
+
+/* RENDEZVOUS1 payload. See details in section 4.2. */
+struct trn_cell_rendezvous1 {
+  /* The RENDEZVOUS_COOKIE field. */
+  u8 rendezvous_cookie[TRUNNEL_REND_COOKIE_LEN];
+  
+  /* The HANDSHAKE_INFO field which has a variable length depending on the
+   * handshake type used. */
+  u8 handshake_info[];
+};
+
+/* RENDEZVOUS2 payload. See details in section 4.2. */
+struct trn_cell_rendezvous2 {
+  /* The HANDSHAKE_INFO field. */
+  u8 handshake_info[TRUNNEL_HANDSHAKE_INFO_LEN];
+};
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index b1448b7cb2..b249fb302c 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -1,4 +1,3 @@
-
 noinst_LIBRARIES += \
 	src/trunnel/libor-trunnel.a
 
@@ -12,26 +11,42 @@ AM_CPPFLAGS += -I$(srcdir)/src/ext/trunnel -I$(srcdir)/src/trunnel
 TRUNNELINPUTS = \
 	src/trunnel/ed25519_cert.trunnel \
 	src/trunnel/link_handshake.trunnel \
-	src/trunnel/pwbox.trunnel
+	src/trunnel/pwbox.trunnel \
+	src/trunnel/channelpadding_negotiation.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
 	src/trunnel/ed25519_cert.c \
 	src/trunnel/link_handshake.c \
-	src/trunnel/pwbox.c
+	src/trunnel/pwbox.c			\
+	src/trunnel/hs/cell_common.c            \
+	src/trunnel/hs/cell_establish_intro.c	\
+	src/trunnel/hs/cell_introduce1.c \
+	src/trunnel/hs/cell_rendezvous.c \
+	src/trunnel/channelpadding_negotiation.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
 	src/ext/trunnel/trunnel-impl.h		\
-	src/trunnel/trunnel-local.h 		\
+	src/trunnel/trunnel-local.h		\
 	src/trunnel/ed25519_cert.h		\
-	src/trunnel/link_handshake.h	 	\
-	src/trunnel/pwbox.h
+	src/trunnel/link_handshake.h		\
+	src/trunnel/pwbox.h			\
+	src/trunnel/hs/cell_common.h            \
+	src/trunnel/hs/cell_establish_intro.h	\
+	src/trunnel/hs/cell_introduce1.h \
+	src/trunnel/hs/cell_rendezvous.h \
+	src/trunnel/channelpadding_negotiation.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
 src_trunnel_libor_trunnel_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS)
 
+if UNITTESTS_ENABLED
 src_trunnel_libor_trunnel_testing_a_SOURCES = $(TRUNNELSOURCES)
+else
+src_trunnel_libor_trunnel_testing_a_SOURCES =
+endif
+
 src_trunnel_libor_trunnel_testing_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_trunnel_libor_trunnel_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index c2717f36bf..03ead31c62 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.4.6.
+/* link_handshake.c -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -13,7 +13,7 @@
   } while (0)
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
-/* If we're runnning a static analysis tool, we don't want it to complain
+/* If we're running a static analysis tool, we don't want it to complain
  * that some of our remaining-bytes checks are dead-code. */
 int linkhandshake_deadcode_dummy__ = 0;
 #define OR_DEADCODE_DUMMY || linkhandshake_deadcode_dummy__
@@ -64,12 +64,17 @@ auth_challenge_cell_getlen_challenge(const auth_challenge_cell_t *inp)
 }
 
 uint8_t
-auth_challenge_cell_get_challenge(const auth_challenge_cell_t *inp, size_t idx)
+auth_challenge_cell_get_challenge(auth_challenge_cell_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->challenge[idx];
 }
 
+uint8_t
+auth_challenge_cell_getconst_challenge(const auth_challenge_cell_t *inp, size_t idx)
+{
+  return auth_challenge_cell_get_challenge((auth_challenge_cell_t*)inp, idx);
+}
 int
 auth_challenge_cell_set_challenge(auth_challenge_cell_t *inp, size_t idx, uint8_t elt)
 {
@@ -83,8 +88,13 @@ auth_challenge_cell_getarray_challenge(auth_challenge_cell_t *inp)
 {
   return inp->challenge;
 }
+const uint8_t  *
+auth_challenge_cell_getconstarray_challenge(const auth_challenge_cell_t *inp)
+{
+  return (const uint8_t  *)auth_challenge_cell_getarray_challenge((auth_challenge_cell_t*)inp);
+}
 uint16_t
-auth_challenge_cell_get_n_methods(auth_challenge_cell_t *inp)
+auth_challenge_cell_get_n_methods(const auth_challenge_cell_t *inp)
 {
   return inp->n_methods;
 }
@@ -106,6 +116,11 @@ auth_challenge_cell_get_methods(auth_challenge_cell_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->methods, idx);
 }
 
+uint16_t
+auth_challenge_cell_getconst_methods(const auth_challenge_cell_t *inp, size_t idx)
+{
+  return auth_challenge_cell_get_methods((auth_challenge_cell_t*)inp, idx);
+}
 int
 auth_challenge_cell_set_methods(auth_challenge_cell_t *inp, size_t idx, uint16_t elt)
 {
@@ -131,6 +146,11 @@ auth_challenge_cell_getarray_methods(auth_challenge_cell_t *inp)
 {
   return inp->methods.elts_;
 }
+const uint16_t  *
+auth_challenge_cell_getconstarray_methods(const auth_challenge_cell_t *inp)
+{
+  return (const uint16_t  *)auth_challenge_cell_getarray_methods((auth_challenge_cell_t*)inp);
+}
 int
 auth_challenge_cell_setlen_methods(auth_challenge_cell_t *inp, size_t newlen)
 {
@@ -342,7 +362,7 @@ auth_ctx_free(auth_ctx_t *obj)
 }
 
 uint8_t
-auth_ctx_get_is_ed(auth_ctx_t *inp)
+auth_ctx_get_is_ed(const auth_ctx_t *inp)
 {
   return inp->is_ed;
 }
@@ -382,7 +402,7 @@ certs_cell_cert_free(certs_cell_cert_t *obj)
 }
 
 uint8_t
-certs_cell_cert_get_cert_type(certs_cell_cert_t *inp)
+certs_cell_cert_get_cert_type(const certs_cell_cert_t *inp)
 {
   return inp->cert_type;
 }
@@ -393,7 +413,7 @@ certs_cell_cert_set_cert_type(certs_cell_cert_t *inp, uint8_t val)
   return 0;
 }
 uint16_t
-certs_cell_cert_get_cert_len(certs_cell_cert_t *inp)
+certs_cell_cert_get_cert_len(const certs_cell_cert_t *inp)
 {
   return inp->cert_len;
 }
@@ -415,6 +435,11 @@ certs_cell_cert_get_body(certs_cell_cert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->body, idx);
 }
 
+uint8_t
+certs_cell_cert_getconst_body(const certs_cell_cert_t *inp, size_t idx)
+{
+  return certs_cell_cert_get_body((certs_cell_cert_t*)inp, idx);
+}
 int
 certs_cell_cert_set_body(certs_cell_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -440,6 +465,11 @@ certs_cell_cert_getarray_body(certs_cell_cert_t *inp)
 {
   return inp->body.elts_;
 }
+const uint8_t  *
+certs_cell_cert_getconstarray_body(const certs_cell_cert_t *inp)
+{
+  return (const uint8_t  *)certs_cell_cert_getarray_body((certs_cell_cert_t*)inp);
+}
 int
 certs_cell_cert_setlen_body(certs_cell_cert_t *inp, size_t newlen)
 {
@@ -652,12 +682,17 @@ rsa_ed_crosscert_getlen_ed_key(const rsa_ed_crosscert_t *inp)
 }
 
 uint8_t
-rsa_ed_crosscert_get_ed_key(const rsa_ed_crosscert_t *inp, size_t idx)
+rsa_ed_crosscert_get_ed_key(rsa_ed_crosscert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->ed_key[idx];
 }
 
+uint8_t
+rsa_ed_crosscert_getconst_ed_key(const rsa_ed_crosscert_t *inp, size_t idx)
+{
+  return rsa_ed_crosscert_get_ed_key((rsa_ed_crosscert_t*)inp, idx);
+}
 int
 rsa_ed_crosscert_set_ed_key(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt)
 {
@@ -671,8 +706,13 @@ rsa_ed_crosscert_getarray_ed_key(rsa_ed_crosscert_t *inp)
 {
   return inp->ed_key;
 }
+const uint8_t  *
+rsa_ed_crosscert_getconstarray_ed_key(const rsa_ed_crosscert_t *inp)
+{
+  return (const uint8_t  *)rsa_ed_crosscert_getarray_ed_key((rsa_ed_crosscert_t*)inp);
+}
 uint32_t
-rsa_ed_crosscert_get_expiration(rsa_ed_crosscert_t *inp)
+rsa_ed_crosscert_get_expiration(const rsa_ed_crosscert_t *inp)
 {
   return inp->expiration;
 }
@@ -688,7 +728,7 @@ rsa_ed_crosscert_get_end_of_signed(const rsa_ed_crosscert_t *inp)
   return inp->end_of_signed;
 }
 uint8_t
-rsa_ed_crosscert_get_sig_len(rsa_ed_crosscert_t *inp)
+rsa_ed_crosscert_get_sig_len(const rsa_ed_crosscert_t *inp)
 {
   return inp->sig_len;
 }
@@ -710,6 +750,11 @@ rsa_ed_crosscert_get_sig(rsa_ed_crosscert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
 }
 
+uint8_t
+rsa_ed_crosscert_getconst_sig(const rsa_ed_crosscert_t *inp, size_t idx)
+{
+  return rsa_ed_crosscert_get_sig((rsa_ed_crosscert_t*)inp, idx);
+}
 int
 rsa_ed_crosscert_set_sig(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt)
 {
@@ -735,6 +780,11 @@ rsa_ed_crosscert_getarray_sig(rsa_ed_crosscert_t *inp)
 {
   return inp->sig.elts_;
 }
+const uint8_t  *
+rsa_ed_crosscert_getconstarray_sig(const rsa_ed_crosscert_t *inp)
+{
+  return (const uint8_t  *)rsa_ed_crosscert_getarray_sig((rsa_ed_crosscert_t*)inp);
+}
 int
 rsa_ed_crosscert_setlen_sig(rsa_ed_crosscert_t *inp, size_t newlen)
 {
@@ -964,12 +1014,17 @@ auth1_getlen_type(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_type(const auth1_t *inp, size_t idx)
+auth1_get_type(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 8);
   return inp->type[idx];
 }
 
+uint8_t
+auth1_getconst_type(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_type((auth1_t*)inp, idx);
+}
 int
 auth1_set_type(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -983,6 +1038,11 @@ auth1_getarray_type(auth1_t *inp)
 {
   return inp->type;
 }
+const uint8_t  *
+auth1_getconstarray_type(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_type((auth1_t*)inp);
+}
 size_t
 auth1_getlen_cid(const auth1_t *inp)
 {
@@ -990,12 +1050,17 @@ auth1_getlen_cid(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_cid(const auth1_t *inp, size_t idx)
+auth1_get_cid(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->cid[idx];
 }
 
+uint8_t
+auth1_getconst_cid(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_cid((auth1_t*)inp, idx);
+}
 int
 auth1_set_cid(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1009,6 +1074,11 @@ auth1_getarray_cid(auth1_t *inp)
 {
   return inp->cid;
 }
+const uint8_t  *
+auth1_getconstarray_cid(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_cid((auth1_t*)inp);
+}
 size_t
 auth1_getlen_sid(const auth1_t *inp)
 {
@@ -1016,12 +1086,17 @@ auth1_getlen_sid(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_sid(const auth1_t *inp, size_t idx)
+auth1_get_sid(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->sid[idx];
 }
 
+uint8_t
+auth1_getconst_sid(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_sid((auth1_t*)inp, idx);
+}
 int
 auth1_set_sid(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1035,6 +1110,11 @@ auth1_getarray_sid(auth1_t *inp)
 {
   return inp->sid;
 }
+const uint8_t  *
+auth1_getconstarray_sid(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_sid((auth1_t*)inp);
+}
 size_t
 auth1_getlen_u1_cid_ed(const auth1_t *inp)
 {
@@ -1042,12 +1122,17 @@ auth1_getlen_u1_cid_ed(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_u1_cid_ed(const auth1_t *inp, size_t idx)
+auth1_get_u1_cid_ed(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->u1_cid_ed[idx];
 }
 
+uint8_t
+auth1_getconst_u1_cid_ed(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_u1_cid_ed((auth1_t*)inp, idx);
+}
 int
 auth1_set_u1_cid_ed(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1061,6 +1146,11 @@ auth1_getarray_u1_cid_ed(auth1_t *inp)
 {
   return inp->u1_cid_ed;
 }
+const uint8_t  *
+auth1_getconstarray_u1_cid_ed(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_u1_cid_ed((auth1_t*)inp);
+}
 size_t
 auth1_getlen_u1_sid_ed(const auth1_t *inp)
 {
@@ -1068,12 +1158,17 @@ auth1_getlen_u1_sid_ed(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_u1_sid_ed(const auth1_t *inp, size_t idx)
+auth1_get_u1_sid_ed(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->u1_sid_ed[idx];
 }
 
+uint8_t
+auth1_getconst_u1_sid_ed(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_u1_sid_ed((auth1_t*)inp, idx);
+}
 int
 auth1_set_u1_sid_ed(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1087,6 +1182,11 @@ auth1_getarray_u1_sid_ed(auth1_t *inp)
 {
   return inp->u1_sid_ed;
 }
+const uint8_t  *
+auth1_getconstarray_u1_sid_ed(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_u1_sid_ed((auth1_t*)inp);
+}
 size_t
 auth1_getlen_slog(const auth1_t *inp)
 {
@@ -1094,12 +1194,17 @@ auth1_getlen_slog(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_slog(const auth1_t *inp, size_t idx)
+auth1_get_slog(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->slog[idx];
 }
 
+uint8_t
+auth1_getconst_slog(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_slog((auth1_t*)inp, idx);
+}
 int
 auth1_set_slog(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1113,6 +1218,11 @@ auth1_getarray_slog(auth1_t *inp)
 {
   return inp->slog;
 }
+const uint8_t  *
+auth1_getconstarray_slog(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_slog((auth1_t*)inp);
+}
 size_t
 auth1_getlen_clog(const auth1_t *inp)
 {
@@ -1120,12 +1230,17 @@ auth1_getlen_clog(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_clog(const auth1_t *inp, size_t idx)
+auth1_get_clog(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->clog[idx];
 }
 
+uint8_t
+auth1_getconst_clog(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_clog((auth1_t*)inp, idx);
+}
 int
 auth1_set_clog(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1139,6 +1254,11 @@ auth1_getarray_clog(auth1_t *inp)
 {
   return inp->clog;
 }
+const uint8_t  *
+auth1_getconstarray_clog(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_clog((auth1_t*)inp);
+}
 size_t
 auth1_getlen_scert(const auth1_t *inp)
 {
@@ -1146,12 +1266,17 @@ auth1_getlen_scert(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_scert(const auth1_t *inp, size_t idx)
+auth1_get_scert(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->scert[idx];
 }
 
+uint8_t
+auth1_getconst_scert(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_scert((auth1_t*)inp, idx);
+}
 int
 auth1_set_scert(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1165,6 +1290,11 @@ auth1_getarray_scert(auth1_t *inp)
 {
   return inp->scert;
 }
+const uint8_t  *
+auth1_getconstarray_scert(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_scert((auth1_t*)inp);
+}
 size_t
 auth1_getlen_tlssecrets(const auth1_t *inp)
 {
@@ -1172,12 +1302,17 @@ auth1_getlen_tlssecrets(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_tlssecrets(const auth1_t *inp, size_t idx)
+auth1_get_tlssecrets(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->tlssecrets[idx];
 }
 
+uint8_t
+auth1_getconst_tlssecrets(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_tlssecrets((auth1_t*)inp, idx);
+}
 int
 auth1_set_tlssecrets(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1191,6 +1326,11 @@ auth1_getarray_tlssecrets(auth1_t *inp)
 {
   return inp->tlssecrets;
 }
+const uint8_t  *
+auth1_getconstarray_tlssecrets(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_tlssecrets((auth1_t*)inp);
+}
 const uint8_t *
 auth1_get_end_of_fixed_part(const auth1_t *inp)
 {
@@ -1203,12 +1343,17 @@ auth1_getlen_rand(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_rand(const auth1_t *inp, size_t idx)
+auth1_get_rand(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 24);
   return inp->rand[idx];
 }
 
+uint8_t
+auth1_getconst_rand(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_rand((auth1_t*)inp, idx);
+}
 int
 auth1_set_rand(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1222,6 +1367,11 @@ auth1_getarray_rand(auth1_t *inp)
 {
   return inp->rand;
 }
+const uint8_t  *
+auth1_getconstarray_rand(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_rand((auth1_t*)inp);
+}
 const uint8_t *
 auth1_get_end_of_signed(const auth1_t *inp)
 {
@@ -1239,6 +1389,11 @@ auth1_get_sig(auth1_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
 }
 
+uint8_t
+auth1_getconst_sig(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_sig((auth1_t*)inp, idx);
+}
 int
 auth1_set_sig(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1260,6 +1415,11 @@ auth1_getarray_sig(auth1_t *inp)
 {
   return inp->sig.elts_;
 }
+const uint8_t  *
+auth1_getconstarray_sig(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_sig((auth1_t*)inp);
+}
 int
 auth1_setlen_sig(auth1_t *inp, size_t newlen)
 {
@@ -1647,7 +1807,7 @@ certs_cell_free(certs_cell_t *obj)
 }
 
 uint8_t
-certs_cell_get_n_certs(certs_cell_t *inp)
+certs_cell_get_n_certs(const certs_cell_t *inp)
 {
   return inp->n_certs;
 }
@@ -1669,6 +1829,11 @@ certs_cell_get_certs(certs_cell_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->certs, idx);
 }
 
+ const struct certs_cell_cert_st *
+certs_cell_getconst_certs(const certs_cell_t *inp, size_t idx)
+{
+  return certs_cell_get_certs((certs_cell_t*)inp, idx);
+}
 int
 certs_cell_set_certs(certs_cell_t *inp, size_t idx, struct certs_cell_cert_st * elt)
 {
@@ -1702,6 +1867,11 @@ certs_cell_getarray_certs(certs_cell_t *inp)
 {
   return inp->certs.elts_;
 }
+const struct certs_cell_cert_st *  const  *
+certs_cell_getconstarray_certs(const certs_cell_t *inp)
+{
+  return (const struct certs_cell_cert_st *  const  *)certs_cell_getarray_certs((certs_cell_t*)inp);
+}
 int
 certs_cell_setlen_certs(certs_cell_t *inp, size_t newlen)
 {
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 54611b96e8..6a23483adc 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by by Trunnel v1.4.6.
+/* link_handshake.h -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -121,7 +121,11 @@ size_t auth_challenge_cell_getlen_challenge(const auth_challenge_cell_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * challenge of the auth_challenge_cell_t in 'inp'.
  */
-uint8_t auth_challenge_cell_get_challenge(const auth_challenge_cell_t *inp, size_t idx);
+uint8_t auth_challenge_cell_get_challenge(auth_challenge_cell_t *inp, size_t idx);
+/** As auth_challenge_cell_get_challenge, but take and return a const
+ * pointer
+ */
+uint8_t auth_challenge_cell_getconst_challenge(const auth_challenge_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * challenge of the auth_challenge_cell_t in 'inp', so that it will
  * hold the value 'elt'.
@@ -130,10 +134,14 @@ int auth_challenge_cell_set_challenge(auth_challenge_cell_t *inp, size_t idx, ui
 /** Return a pointer to the 32-element array field challenge of 'inp'.
  */
 uint8_t * auth_challenge_cell_getarray_challenge(auth_challenge_cell_t *inp);
+/** As auth_challenge_cell_get_challenge, but take and return a const
+ * pointer
+ */
+const uint8_t  * auth_challenge_cell_getconstarray_challenge(const auth_challenge_cell_t *inp);
 /** Return the value of the n_methods field of the
  * auth_challenge_cell_t in 'inp'
  */
-uint16_t auth_challenge_cell_get_n_methods(auth_challenge_cell_t *inp);
+uint16_t auth_challenge_cell_get_n_methods(const auth_challenge_cell_t *inp);
 /** Set the value of the n_methods field of the auth_challenge_cell_t
  * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -147,6 +155,10 @@ size_t auth_challenge_cell_getlen_methods(const auth_challenge_cell_t *inp);
  * methods of the auth_challenge_cell_t in 'inp'.
  */
 uint16_t auth_challenge_cell_get_methods(auth_challenge_cell_t *inp, size_t idx);
+/** As auth_challenge_cell_get_methods, but take and return a const
+ * pointer
+ */
+uint16_t auth_challenge_cell_getconst_methods(const auth_challenge_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * methods of the auth_challenge_cell_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -160,6 +172,10 @@ int auth_challenge_cell_add_methods(auth_challenge_cell_t *inp, uint16_t elt);
  * 'inp'.
  */
 uint16_t * auth_challenge_cell_getarray_methods(auth_challenge_cell_t *inp);
+/** As auth_challenge_cell_get_methods, but take and return a const
+ * pointer
+ */
+const uint16_t  * auth_challenge_cell_getconstarray_methods(const auth_challenge_cell_t *inp);
 /** Change the length of the variable-length array field methods of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -174,7 +190,7 @@ auth_ctx_t *auth_ctx_new(void);
 void auth_ctx_free(auth_ctx_t *victim);
 /** Return the value of the is_ed field of the auth_ctx_t in 'inp'
  */
-uint8_t auth_ctx_get_is_ed(auth_ctx_t *inp);
+uint8_t auth_ctx_get_is_ed(const auth_ctx_t *inp);
 /** Set the value of the is_ed field of the auth_ctx_t in 'inp' to
  * 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -219,7 +235,7 @@ int certs_cell_cert_clear_errors(certs_cell_cert_t *obj);
 /** Return the value of the cert_type field of the certs_cell_cert_t
  * in 'inp'
  */
-uint8_t certs_cell_cert_get_cert_type(certs_cell_cert_t *inp);
+uint8_t certs_cell_cert_get_cert_type(const certs_cell_cert_t *inp);
 /** Set the value of the cert_type field of the certs_cell_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -228,7 +244,7 @@ int certs_cell_cert_set_cert_type(certs_cell_cert_t *inp, uint8_t val);
 /** Return the value of the cert_len field of the certs_cell_cert_t in
  * 'inp'
  */
-uint16_t certs_cell_cert_get_cert_len(certs_cell_cert_t *inp);
+uint16_t certs_cell_cert_get_cert_len(const certs_cell_cert_t *inp);
 /** Set the value of the cert_len field of the certs_cell_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -242,6 +258,9 @@ size_t certs_cell_cert_getlen_body(const certs_cell_cert_t *inp);
  * body of the certs_cell_cert_t in 'inp'.
  */
 uint8_t certs_cell_cert_get_body(certs_cell_cert_t *inp, size_t idx);
+/** As certs_cell_cert_get_body, but take and return a const pointer
+ */
+uint8_t certs_cell_cert_getconst_body(const certs_cell_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * body of the certs_cell_cert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -254,6 +273,9 @@ int certs_cell_cert_add_body(certs_cell_cert_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field body of 'inp'.
  */
 uint8_t * certs_cell_cert_getarray_body(certs_cell_cert_t *inp);
+/** As certs_cell_cert_get_body, but take and return a const pointer
+ */
+const uint8_t  * certs_cell_cert_getconstarray_body(const certs_cell_cert_t *inp);
 /** Change the length of the variable-length array field body of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -302,7 +324,11 @@ size_t rsa_ed_crosscert_getlen_ed_key(const rsa_ed_crosscert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * ed_key of the rsa_ed_crosscert_t in 'inp'.
  */
-uint8_t rsa_ed_crosscert_get_ed_key(const rsa_ed_crosscert_t *inp, size_t idx);
+uint8_t rsa_ed_crosscert_get_ed_key(rsa_ed_crosscert_t *inp, size_t idx);
+/** As rsa_ed_crosscert_get_ed_key, but take and return a const
+ * pointer
+ */
+uint8_t rsa_ed_crosscert_getconst_ed_key(const rsa_ed_crosscert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * ed_key of the rsa_ed_crosscert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -311,10 +337,14 @@ int rsa_ed_crosscert_set_ed_key(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt
 /** Return a pointer to the 32-element array field ed_key of 'inp'.
  */
 uint8_t * rsa_ed_crosscert_getarray_ed_key(rsa_ed_crosscert_t *inp);
+/** As rsa_ed_crosscert_get_ed_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * rsa_ed_crosscert_getconstarray_ed_key(const rsa_ed_crosscert_t *inp);
 /** Return the value of the expiration field of the rsa_ed_crosscert_t
  * in 'inp'
  */
-uint32_t rsa_ed_crosscert_get_expiration(rsa_ed_crosscert_t *inp);
+uint32_t rsa_ed_crosscert_get_expiration(const rsa_ed_crosscert_t *inp);
 /** Set the value of the expiration field of the rsa_ed_crosscert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -326,7 +356,7 @@ const uint8_t * rsa_ed_crosscert_get_end_of_signed(const rsa_ed_crosscert_t *inp
 /** Return the value of the sig_len field of the rsa_ed_crosscert_t in
  * 'inp'
  */
-uint8_t rsa_ed_crosscert_get_sig_len(rsa_ed_crosscert_t *inp);
+uint8_t rsa_ed_crosscert_get_sig_len(const rsa_ed_crosscert_t *inp);
 /** Set the value of the sig_len field of the rsa_ed_crosscert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -340,6 +370,9 @@ size_t rsa_ed_crosscert_getlen_sig(const rsa_ed_crosscert_t *inp);
  * sig of the rsa_ed_crosscert_t in 'inp'.
  */
 uint8_t rsa_ed_crosscert_get_sig(rsa_ed_crosscert_t *inp, size_t idx);
+/** As rsa_ed_crosscert_get_sig, but take and return a const pointer
+ */
+uint8_t rsa_ed_crosscert_getconst_sig(const rsa_ed_crosscert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * sig of the rsa_ed_crosscert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -352,6 +385,9 @@ int rsa_ed_crosscert_add_sig(rsa_ed_crosscert_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field sig of 'inp'.
  */
 uint8_t * rsa_ed_crosscert_getarray_sig(rsa_ed_crosscert_t *inp);
+/** As rsa_ed_crosscert_get_sig, but take and return a const pointer
+ */
+const uint8_t  * rsa_ed_crosscert_getconstarray_sig(const rsa_ed_crosscert_t *inp);
 /** Change the length of the variable-length array field sig of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -398,7 +434,10 @@ size_t auth1_getlen_type(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field type
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_type(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_type(auth1_t *inp, size_t idx);
+/** As auth1_get_type, but take and return a const pointer
+ */
+uint8_t auth1_getconst_type(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field type
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -406,6 +445,9 @@ int auth1_set_type(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 8-element array field type of 'inp'.
  */
 uint8_t * auth1_getarray_type(auth1_t *inp);
+/** As auth1_get_type, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_type(const auth1_t *inp);
 /** Return the (constant) length of the array holding the cid field of
  * the auth1_t in 'inp'.
  */
@@ -413,7 +455,10 @@ size_t auth1_getlen_cid(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field cid
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_cid(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_cid(auth1_t *inp, size_t idx);
+/** As auth1_get_cid, but take and return a const pointer
+ */
+uint8_t auth1_getconst_cid(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field cid
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -421,6 +466,9 @@ int auth1_set_cid(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field cid of 'inp'.
  */
 uint8_t * auth1_getarray_cid(auth1_t *inp);
+/** As auth1_get_cid, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_cid(const auth1_t *inp);
 /** Return the (constant) length of the array holding the sid field of
  * the auth1_t in 'inp'.
  */
@@ -428,7 +476,10 @@ size_t auth1_getlen_sid(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field sid
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_sid(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_sid(auth1_t *inp, size_t idx);
+/** As auth1_get_sid, but take and return a const pointer
+ */
+uint8_t auth1_getconst_sid(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field sid
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -436,6 +487,9 @@ int auth1_set_sid(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field sid of 'inp'.
  */
 uint8_t * auth1_getarray_sid(auth1_t *inp);
+/** As auth1_get_sid, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_sid(const auth1_t *inp);
 /** Return the (constant) length of the array holding the u1_cid_ed
  * field of the auth1_t in 'inp'.
  */
@@ -443,7 +497,10 @@ size_t auth1_getlen_u1_cid_ed(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * u1_cid_ed of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_u1_cid_ed(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_u1_cid_ed(auth1_t *inp, size_t idx);
+/** As auth1_get_u1_cid_ed, but take and return a const pointer
+ */
+uint8_t auth1_getconst_u1_cid_ed(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * u1_cid_ed of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -452,6 +509,9 @@ int auth1_set_u1_cid_ed(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field u1_cid_ed of 'inp'.
  */
 uint8_t * auth1_getarray_u1_cid_ed(auth1_t *inp);
+/** As auth1_get_u1_cid_ed, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_u1_cid_ed(const auth1_t *inp);
 /** Return the (constant) length of the array holding the u1_sid_ed
  * field of the auth1_t in 'inp'.
  */
@@ -459,7 +519,10 @@ size_t auth1_getlen_u1_sid_ed(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * u1_sid_ed of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_u1_sid_ed(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_u1_sid_ed(auth1_t *inp, size_t idx);
+/** As auth1_get_u1_sid_ed, but take and return a const pointer
+ */
+uint8_t auth1_getconst_u1_sid_ed(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * u1_sid_ed of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -468,6 +531,9 @@ int auth1_set_u1_sid_ed(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field u1_sid_ed of 'inp'.
  */
 uint8_t * auth1_getarray_u1_sid_ed(auth1_t *inp);
+/** As auth1_get_u1_sid_ed, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_u1_sid_ed(const auth1_t *inp);
 /** Return the (constant) length of the array holding the slog field
  * of the auth1_t in 'inp'.
  */
@@ -475,7 +541,10 @@ size_t auth1_getlen_slog(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field slog
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_slog(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_slog(auth1_t *inp, size_t idx);
+/** As auth1_get_slog, but take and return a const pointer
+ */
+uint8_t auth1_getconst_slog(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field slog
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -483,6 +552,9 @@ int auth1_set_slog(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field slog of 'inp'.
  */
 uint8_t * auth1_getarray_slog(auth1_t *inp);
+/** As auth1_get_slog, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_slog(const auth1_t *inp);
 /** Return the (constant) length of the array holding the clog field
  * of the auth1_t in 'inp'.
  */
@@ -490,7 +562,10 @@ size_t auth1_getlen_clog(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field clog
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_clog(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_clog(auth1_t *inp, size_t idx);
+/** As auth1_get_clog, but take and return a const pointer
+ */
+uint8_t auth1_getconst_clog(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field clog
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -498,6 +573,9 @@ int auth1_set_clog(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field clog of 'inp'.
  */
 uint8_t * auth1_getarray_clog(auth1_t *inp);
+/** As auth1_get_clog, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_clog(const auth1_t *inp);
 /** Return the (constant) length of the array holding the scert field
  * of the auth1_t in 'inp'.
  */
@@ -505,7 +583,10 @@ size_t auth1_getlen_scert(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * scert of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_scert(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_scert(auth1_t *inp, size_t idx);
+/** As auth1_get_scert, but take and return a const pointer
+ */
+uint8_t auth1_getconst_scert(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * scert of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -514,6 +595,9 @@ int auth1_set_scert(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field scert of 'inp'.
  */
 uint8_t * auth1_getarray_scert(auth1_t *inp);
+/** As auth1_get_scert, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_scert(const auth1_t *inp);
 /** Return the (constant) length of the array holding the tlssecrets
  * field of the auth1_t in 'inp'.
  */
@@ -521,7 +605,10 @@ size_t auth1_getlen_tlssecrets(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * tlssecrets of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_tlssecrets(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_tlssecrets(auth1_t *inp, size_t idx);
+/** As auth1_get_tlssecrets, but take and return a const pointer
+ */
+uint8_t auth1_getconst_tlssecrets(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * tlssecrets of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -531,6 +618,9 @@ int auth1_set_tlssecrets(auth1_t *inp, size_t idx, uint8_t elt);
  * 'inp'.
  */
 uint8_t * auth1_getarray_tlssecrets(auth1_t *inp);
+/** As auth1_get_tlssecrets, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_tlssecrets(const auth1_t *inp);
 /** Return the position for end_of_fixed_part when we parsed this
  * object
  */
@@ -542,7 +632,10 @@ size_t auth1_getlen_rand(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field rand
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_rand(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_rand(auth1_t *inp, size_t idx);
+/** As auth1_get_rand, but take and return a const pointer
+ */
+uint8_t auth1_getconst_rand(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field rand
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -550,6 +643,9 @@ int auth1_set_rand(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 24-element array field rand of 'inp'.
  */
 uint8_t * auth1_getarray_rand(auth1_t *inp);
+/** As auth1_get_rand, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_rand(const auth1_t *inp);
 /** Return the position for end_of_signed when we parsed this object
  */
 const uint8_t * auth1_get_end_of_signed(const auth1_t *inp);
@@ -561,6 +657,9 @@ size_t auth1_getlen_sig(const auth1_t *inp);
  * sig of the auth1_t in 'inp'.
  */
 uint8_t auth1_get_sig(auth1_t *inp, size_t idx);
+/** As auth1_get_sig, but take and return a const pointer
+ */
+uint8_t auth1_getconst_sig(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * sig of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -572,6 +671,9 @@ int auth1_add_sig(auth1_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field sig of 'inp'.
  */
 uint8_t * auth1_getarray_sig(auth1_t *inp);
+/** As auth1_get_sig, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_sig(const auth1_t *inp);
 /** Change the length of the variable-length array field sig of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -613,7 +715,7 @@ const char *certs_cell_check(const certs_cell_t *obj);
 int certs_cell_clear_errors(certs_cell_t *obj);
 /** Return the value of the n_certs field of the certs_cell_t in 'inp'
  */
-uint8_t certs_cell_get_n_certs(certs_cell_t *inp);
+uint8_t certs_cell_get_n_certs(const certs_cell_t *inp);
 /** Set the value of the n_certs field of the certs_cell_t in 'inp' to
  * 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -627,6 +729,9 @@ size_t certs_cell_getlen_certs(const certs_cell_t *inp);
  * certs of the certs_cell_t in 'inp'.
  */
 struct certs_cell_cert_st * certs_cell_get_certs(certs_cell_t *inp, size_t idx);
+/** As certs_cell_get_certs, but take and return a const pointer
+ */
+ const struct certs_cell_cert_st * certs_cell_getconst_certs(const certs_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * certs of the certs_cell_t in 'inp', so that it will hold the value
  * 'elt'. Free the previous value, if any.
@@ -643,6 +748,9 @@ int certs_cell_add_certs(certs_cell_t *inp, struct certs_cell_cert_st * elt);
  * 'inp'.
  */
 struct certs_cell_cert_st * * certs_cell_getarray_certs(certs_cell_t *inp);
+/** As certs_cell_get_certs, but take and return a const pointer
+ */
+const struct certs_cell_cert_st *  const  * certs_cell_getconstarray_certs(const certs_cell_t *inp);
 /** Change the length of the variable-length array field certs of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index 62662a9369..c356515d36 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.4.6.
+/* pwbox.c -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -13,7 +13,7 @@
   } while (0)
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
-/* If we're runnning a static analysis tool, we don't want it to complain
+/* If we're running a static analysis tool, we don't want it to complain
  * that some of our remaining-bytes checks are dead-code. */
 int pwbox_deadcode_dummy__ = 0;
 #define OR_DEADCODE_DUMMY || pwbox_deadcode_dummy__
@@ -62,7 +62,7 @@ pwbox_encoded_free(pwbox_encoded_t *obj)
 }
 
 uint32_t
-pwbox_encoded_get_fixedbytes0(pwbox_encoded_t *inp)
+pwbox_encoded_get_fixedbytes0(const pwbox_encoded_t *inp)
 {
   return inp->fixedbytes0;
 }
@@ -77,7 +77,7 @@ pwbox_encoded_set_fixedbytes0(pwbox_encoded_t *inp, uint32_t val)
   return 0;
 }
 uint32_t
-pwbox_encoded_get_fixedbytes1(pwbox_encoded_t *inp)
+pwbox_encoded_get_fixedbytes1(const pwbox_encoded_t *inp)
 {
   return inp->fixedbytes1;
 }
@@ -92,7 +92,7 @@ pwbox_encoded_set_fixedbytes1(pwbox_encoded_t *inp, uint32_t val)
   return 0;
 }
 uint8_t
-pwbox_encoded_get_header_len(pwbox_encoded_t *inp)
+pwbox_encoded_get_header_len(const pwbox_encoded_t *inp)
 {
   return inp->header_len;
 }
@@ -114,6 +114,11 @@ pwbox_encoded_get_skey_header(pwbox_encoded_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->skey_header, idx);
 }
 
+uint8_t
+pwbox_encoded_getconst_skey_header(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_skey_header((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_skey_header(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -139,6 +144,11 @@ pwbox_encoded_getarray_skey_header(pwbox_encoded_t *inp)
 {
   return inp->skey_header.elts_;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_skey_header(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_skey_header((pwbox_encoded_t*)inp);
+}
 int
 pwbox_encoded_setlen_skey_header(pwbox_encoded_t *inp, size_t newlen)
 {
@@ -166,12 +176,17 @@ pwbox_encoded_getlen_iv(const pwbox_encoded_t *inp)
 }
 
 uint8_t
-pwbox_encoded_get_iv(const pwbox_encoded_t *inp, size_t idx)
+pwbox_encoded_get_iv(pwbox_encoded_t *inp, size_t idx)
 {
   trunnel_assert(idx < 16);
   return inp->iv[idx];
 }
 
+uint8_t
+pwbox_encoded_getconst_iv(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_iv((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_iv(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -185,6 +200,11 @@ pwbox_encoded_getarray_iv(pwbox_encoded_t *inp)
 {
   return inp->iv;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_iv(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_iv((pwbox_encoded_t*)inp);
+}
 size_t
 pwbox_encoded_getlen_data(const pwbox_encoded_t *inp)
 {
@@ -197,6 +217,11 @@ pwbox_encoded_get_data(pwbox_encoded_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->data, idx);
 }
 
+uint8_t
+pwbox_encoded_getconst_data(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_data((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_data(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -218,6 +243,11 @@ pwbox_encoded_getarray_data(pwbox_encoded_t *inp)
 {
   return inp->data.elts_;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_data(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_data((pwbox_encoded_t*)inp);
+}
 int
 pwbox_encoded_setlen_data(pwbox_encoded_t *inp, size_t newlen)
 {
@@ -241,12 +271,17 @@ pwbox_encoded_getlen_hmac(const pwbox_encoded_t *inp)
 }
 
 uint8_t
-pwbox_encoded_get_hmac(const pwbox_encoded_t *inp, size_t idx)
+pwbox_encoded_get_hmac(pwbox_encoded_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->hmac[idx];
 }
 
+uint8_t
+pwbox_encoded_getconst_hmac(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_hmac((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_hmac(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -260,6 +295,11 @@ pwbox_encoded_getarray_hmac(pwbox_encoded_t *inp)
 {
   return inp->hmac;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_hmac(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_hmac((pwbox_encoded_t*)inp);
+}
 const char *
 pwbox_encoded_check(const pwbox_encoded_t *obj)
 {
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index 77a813d123..a9a421408a 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by by Trunnel v1.4.6.
+/* pwbox.h -- generated by Trunnel v1.5.2.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -61,7 +61,7 @@ int pwbox_encoded_clear_errors(pwbox_encoded_t *obj);
 /** Return the value of the fixedbytes0 field of the pwbox_encoded_t
  * in 'inp'
  */
-uint32_t pwbox_encoded_get_fixedbytes0(pwbox_encoded_t *inp);
+uint32_t pwbox_encoded_get_fixedbytes0(const pwbox_encoded_t *inp);
 /** Set the value of the fixedbytes0 field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -70,7 +70,7 @@ int pwbox_encoded_set_fixedbytes0(pwbox_encoded_t *inp, uint32_t val);
 /** Return the value of the fixedbytes1 field of the pwbox_encoded_t
  * in 'inp'
  */
-uint32_t pwbox_encoded_get_fixedbytes1(pwbox_encoded_t *inp);
+uint32_t pwbox_encoded_get_fixedbytes1(const pwbox_encoded_t *inp);
 /** Set the value of the fixedbytes1 field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -79,7 +79,7 @@ int pwbox_encoded_set_fixedbytes1(pwbox_encoded_t *inp, uint32_t val);
 /** Return the value of the header_len field of the pwbox_encoded_t in
  * 'inp'
  */
-uint8_t pwbox_encoded_get_header_len(pwbox_encoded_t *inp);
+uint8_t pwbox_encoded_get_header_len(const pwbox_encoded_t *inp);
 /** Set the value of the header_len field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -93,6 +93,10 @@ size_t pwbox_encoded_getlen_skey_header(const pwbox_encoded_t *inp);
  * skey_header of the pwbox_encoded_t in 'inp'.
  */
 uint8_t pwbox_encoded_get_skey_header(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_skey_header, but take and return a const
+ * pointer
+ */
+uint8_t pwbox_encoded_getconst_skey_header(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * skey_header of the pwbox_encoded_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -106,6 +110,10 @@ int pwbox_encoded_add_skey_header(pwbox_encoded_t *inp, uint8_t elt);
  * 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_skey_header(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_skey_header, but take and return a const
+ * pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_skey_header(const pwbox_encoded_t *inp);
 /** Change the length of the variable-length array field skey_header
  * of 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on
  * success; return -1 and set the error code on 'inp' on failure.
@@ -118,7 +126,10 @@ size_t pwbox_encoded_getlen_iv(const pwbox_encoded_t *inp);
 /** Return the element at position 'idx' of the fixed array field iv
  * of the pwbox_encoded_t in 'inp'.
  */
-uint8_t pwbox_encoded_get_iv(const pwbox_encoded_t *inp, size_t idx);
+uint8_t pwbox_encoded_get_iv(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_iv, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_iv(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field iv
  * of the pwbox_encoded_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -127,6 +138,9 @@ int pwbox_encoded_set_iv(pwbox_encoded_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 16-element array field iv of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_iv(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_iv, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_iv(const pwbox_encoded_t *inp);
 /** Return the length of the dynamic array holding the data field of
  * the pwbox_encoded_t in 'inp'.
  */
@@ -135,6 +149,9 @@ size_t pwbox_encoded_getlen_data(const pwbox_encoded_t *inp);
  * data of the pwbox_encoded_t in 'inp'.
  */
 uint8_t pwbox_encoded_get_data(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_data, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_data(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * data of the pwbox_encoded_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -147,6 +164,9 @@ int pwbox_encoded_add_data(pwbox_encoded_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field data of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_data(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_data, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_data(const pwbox_encoded_t *inp);
 /** Change the length of the variable-length array field data of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -159,7 +179,10 @@ size_t pwbox_encoded_getlen_hmac(const pwbox_encoded_t *inp);
 /** Return the element at position 'idx' of the fixed array field hmac
  * of the pwbox_encoded_t in 'inp'.
  */
-uint8_t pwbox_encoded_get_hmac(const pwbox_encoded_t *inp, size_t idx);
+uint8_t pwbox_encoded_get_hmac(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_hmac, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_hmac(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field hmac
  * of the pwbox_encoded_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -168,6 +191,9 @@ int pwbox_encoded_set_hmac(pwbox_encoded_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field hmac of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_hmac(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_hmac, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_hmac(const pwbox_encoded_t *inp);
 
 
 #endif
diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h
index b7c2ab98ef..8aa6d0ddaa 100644
--- a/src/trunnel/trunnel-local.h
+++ b/src/trunnel/trunnel-local.h
@@ -4,7 +4,7 @@
 
 #include "util.h"
 #include "compat.h"
-#include "crypto.h"
+#include "crypto_util.h"
 
 #define trunnel_malloc tor_malloc
 #define trunnel_calloc tor_calloc
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index f6d0e7d684..434b6dd4c8 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -218,7 +218,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.2.9.17-dev"
+#define VERSION "0.3.4.11-dev"